@promptbook/website-crawler 0.100.0-0 → 0.100.0-10

package/esm/index.es.js

@@ -29,7 +29,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.100.0-0';
+const PROMPTBOOK_ENGINE_VERSION = '0.100.0-10';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1013,7 +1013,23 @@ async function getScraperIntermediateSource(source, options) {
         .join('/') +
         '.' +
         extension;
-    await mkdir(dirname(cacheFilename), { recursive: true });
+    // Note: Try to create cache directory, but don't fail if filesystem has issues
+    try {
+        await mkdir(dirname(cacheFilename), { recursive: true });
+    }
+    catch (error) {
+        // Note: If we can't create cache directory, continue without it
+        //       This handles read-only filesystems, permission issues, and missing parent directories
+        if (error instanceof Error && (error.message.includes('EROFS') ||
+            error.message.includes('read-only') ||
+            error.message.includes('EACCES') ||
+            error.message.includes('EPERM') ||
+            error.message.includes('ENOENT'))) ;
+        else {
+            // Re-throw other unexpected errors
+            throw error;
+        }
+    }
     let isDestroyed = true;
     const fileHandler = {
         filename: cacheFilename,
@@ -3498,12 +3514,58 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
         const rootDirname = join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
         const filepath = join(...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${basename.substring(0, MAX_FILENAME_LENGTH)}.${mimeTypeToExtension(mimeType)}`);
-        await tools.fs.mkdir(dirname(join(rootDirname, filepath)), { recursive: true });
+        // Note: Try to create cache directory, but don't fail if filesystem has issues
+        try {
+            await tools.fs.mkdir(dirname(join(rootDirname, filepath)), { recursive: true });
+        }
+        catch (error) {
+            // Note: If we can't create cache directory, we'll handle it when trying to write the file
+            //       This handles read-only filesystems, permission issues, and missing parent directories
+            if (error instanceof Error && (error.message.includes('EROFS') ||
+                error.message.includes('read-only') ||
+                error.message.includes('EACCES') ||
+                error.message.includes('EPERM') ||
+                error.message.includes('ENOENT'))) ;
+            else {
+                // Re-throw other unexpected errors
+                throw error;
+            }
+        }
         const fileContent = Buffer.from(await response.arrayBuffer());
         if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) {
             throw new LimitReachedError(`File is too large (${Math.round(fileContent.length / 1024 / 1024)}MB). Maximum allowed size is ${Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024)}MB.`);
         }
-        await tools.fs.writeFile(join(rootDirname, filepath), fileContent);
+        // Note: Try to cache the downloaded file, but don't fail if the filesystem is read-only
+        try {
+            await tools.fs.writeFile(join(rootDirname, filepath), fileContent);
+        }
+        catch (error) {
+            // Note: If we can't write to cache, we'll process the file directly from memory
+            //       This handles read-only filesystems like Vercel
+            if (error instanceof Error && (error.message.includes('EROFS') ||
+                error.message.includes('read-only') ||
+                error.message.includes('EACCES') ||
+                error.message.includes('EPERM') ||
+                error.message.includes('ENOENT'))) {
+                // Return a handler that works directly with the downloaded content
+                return {
+                    source: name,
+                    filename: null,
+                    url,
+                    mimeType,
+                    async asJson() {
+                        return JSON.parse(fileContent.toString('utf-8'));
+                    },
+                    async asText() {
+                        return fileContent.toString('utf-8');
+                    },
+                };
+            }
+            else {
+                // Re-throw other unexpected errors
+                throw error;
+            }
+        }
         // TODO: [💵] Check the file security
         // TODO: [🧹][🧠] Delete the file after the scraping is done
         return makeKnowledgeSourceHandler({ name, knowledgeSourceContent: filepath }, tools, {
@@ -6270,7 +6332,23 @@ class WebsiteScraper {
             extension: 'html',
             isVerbose,
         });
-        await this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8');
+        // Note: Try to cache the scraped content, but don't fail if the filesystem is read-only
+        try {
+            await this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8');
+        }
+        catch (error) {
+            // Note: If we can't write to cache, we'll continue without caching
+            //       This handles read-only filesystems like Vercel
+            if (error instanceof Error && (error.message.includes('EROFS') ||
+                error.message.includes('read-only') ||
+                error.message.includes('EACCES') ||
+                error.message.includes('EPERM') ||
+                error.message.includes('ENOENT'))) ;
+            else {
+                // Re-throw other unexpected errors
+                throw error;
+            }
+        }
         const markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
         return { ...cacheFilehandler, markdown };
     }