@promptbook/website-crawler 0.100.0-1 → 0.100.0-10

package/esm/typings/src/config.d.ts

@@ -138,9 +138,6 @@ export declare const SMALL_NUMBER = 0.001;
 /**
  * Timeout for the connections in milliseconds
  *
- * Note: Increased from 7 seconds to 30 seconds to accommodate OAuth flows
- * like Facebook login which may require user interaction and redirects
- *
  * @private within the repository - too low-level in comparison with other `MAX_...`
  */
 export declare const CONNECTION_TIMEOUT_MS: number;
@@ -150,13 +147,6 @@ export declare const CONNECTION_TIMEOUT_MS: number;
  * @private within the repository - too low-level in comparison with other `MAX_...`
  */
 export declare const CONNECTION_RETRIES_LIMIT = 5;
-/**
- * Timeout specifically for OAuth authentication flows in milliseconds
- * OAuth flows typically require more time due to user interaction and redirects
- *
- * @private within the repository - too low-level in comparison with other `MAX_...`
- */
-export declare const OAUTH_TIMEOUT_MS: number;
 /**
  * Short time interval to prevent race conditions in milliseconds
  *

package/esm/typings/src/version.d.ts

@@ -15,7 +15,7 @@ export declare const BOOK_LANGUAGE_VERSION: string_semantic_version;
 export declare const PROMPTBOOK_ENGINE_VERSION: string_promptbook_version;
 /**
  * Represents the version string of the Promptbook engine.
- * It follows semantic versioning (e.g., `0.100.0-0`).
+ * It follows semantic versioning (e.g., `0.100.0-9`).
  *
  * @generated
  */

package/esm/typings/src/wizard/wizard.d.ts

@@ -6,6 +6,15 @@ import type { InputParameters } from '../types/typeAliases';
 import type { string_filename } from '../types/typeAliases';
 import type { string_parameter_value } from '../types/typeAliases';
 import type { string_pipeline_url } from '../types/typeAliases';
+/**
+ * Options for wizard methods
+ */
+interface WizardOptions {
+    /**
+     * Whether to enable verbose logging
+     */
+    isVerbose?: boolean;
+}
 /**
  * Wizard for simple usage of the Promptbook
  * Look at `wizard` for more details
@@ -26,7 +35,7 @@ declare class Wizard {
      *
      * Note: This works similar to the `ptbk run` command
      */
-    execute(book: string_pipeline_url | string_filename | PipelineString, inputParameters: InputParameters): Promise<{
+    execute(book: string_pipeline_url | string_filename | PipelineString, inputParameters: InputParameters, options?: WizardOptions): Promise<{
         /**
          * Simple result of the execution
          */
@@ -36,9 +45,9 @@ declare class Wizard {
     /**
      * Provides the tools automatically for the Node.js environment
      *
-     * @param pipelineSource
+     * @param options
      */
-    getExecutionTools(): Promise<Required<Pick<ExecutionTools, 'fs' | 'fetch'>>>;
+    getExecutionTools(options?: WizardOptions): Promise<Required<Pick<ExecutionTools, 'fs' | 'fetch'>>>;
     /**
      * Load book from the source
      *
@@ -49,8 +58,9 @@ declare class Wizard {
      * 3) As a string
      *
      * @param pipelineSource
+     * @param options
      */
-    getCompiledBook(pipelineSource: string_filename | string_pipeline_url | PipelineString): Promise<PipelineJson>;
+    getCompiledBook(pipelineSource: string_filename | string_pipeline_url | PipelineString, options?: WizardOptions): Promise<PipelineJson>;
 }
 /**
  * Wizard for simple usage of the Promptbook

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/website-crawler",
-    "version": "0.100.0-1",
+    "version": "0.100.0-10",
     "description": "Promptbook: Run AI apps in plain human language across multiple models and platforms",
     "private": false,
     "sideEffects": false,
@@ -95,7 +95,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.100.0-1"
+        "@promptbook/core": "0.100.0-10"
     },
     "dependencies": {
         "@mozilla/readability": "0.6.0",

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.100.0-1';
+    const PROMPTBOOK_ENGINE_VERSION = '0.100.0-10';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1009,7 +1009,23 @@
             .join('/') +
             '.' +
             extension;
-        await promises.mkdir(path.dirname(cacheFilename), { recursive: true });
+        // Note: Try to create cache directory, but don't fail if filesystem has issues
+        try {
+            await promises.mkdir(path.dirname(cacheFilename), { recursive: true });
+        }
+        catch (error) {
+            // Note: If we can't create cache directory, continue without it
+            //       This handles read-only filesystems, permission issues, and missing parent directories
+            if (error instanceof Error && (error.message.includes('EROFS') ||
+                error.message.includes('read-only') ||
+                error.message.includes('EACCES') ||
+                error.message.includes('EPERM') ||
+                error.message.includes('ENOENT'))) ;
+            else {
+                // Re-throw other unexpected errors
+                throw error;
+            }
+        }
         let isDestroyed = true;
         const fileHandler = {
             filename: cacheFilename,
@@ -3494,12 +3510,58 @@
             //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
             const rootDirname = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
             const filepath = path.join(...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${basename.substring(0, MAX_FILENAME_LENGTH)}.${mimeTypeToExtension(mimeType)}`);
-            await tools.fs.mkdir(path.dirname(path.join(rootDirname, filepath)), { recursive: true });
+            // Note: Try to create cache directory, but don't fail if filesystem has issues
+            try {
+                await tools.fs.mkdir(path.dirname(path.join(rootDirname, filepath)), { recursive: true });
+            }
+            catch (error) {
+                // Note: If we can't create cache directory, we'll handle it when trying to write the file
+                //       This handles read-only filesystems, permission issues, and missing parent directories
+                if (error instanceof Error && (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) ;
+                else {
+                    // Re-throw other unexpected errors
+                    throw error;
+                }
+            }
             const fileContent = Buffer.from(await response.arrayBuffer());
             if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) {
                 throw new LimitReachedError(`File is too large (${Math.round(fileContent.length / 1024 / 1024)}MB). Maximum allowed size is ${Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024)}MB.`);
             }
-            await tools.fs.writeFile(path.join(rootDirname, filepath), fileContent);
+            // Note: Try to cache the downloaded file, but don't fail if the filesystem is read-only
+            try {
+                await tools.fs.writeFile(path.join(rootDirname, filepath), fileContent);
+            }
+            catch (error) {
+                // Note: If we can't write to cache, we'll process the file directly from memory
+                //       This handles read-only filesystems like Vercel
+                if (error instanceof Error && (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) {
+                    // Return a handler that works directly with the downloaded content
+                    return {
+                        source: name,
+                        filename: null,
+                        url,
+                        mimeType,
+                        async asJson() {
+                            return JSON.parse(fileContent.toString('utf-8'));
+                        },
+                        async asText() {
+                            return fileContent.toString('utf-8');
+                        },
+                    };
+                }
+                else {
+                    // Re-throw other unexpected errors
+                    throw error;
+                }
+            }
             // TODO: [💵] Check the file security
             // TODO: [🧹][🧠] Delete the file after the scraping is done
             return makeKnowledgeSourceHandler({ name, knowledgeSourceContent: filepath }, tools, {
@@ -6266,7 +6328,23 @@
                 extension: 'html',
                 isVerbose,
             });
-            await this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8');
+            // Note: Try to cache the scraped content, but don't fail if the filesystem is read-only
+            try {
+                await this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8');
+            }
+            catch (error) {
+                // Note: If we can't write to cache, we'll continue without caching
+                //       This handles read-only filesystems like Vercel
+                if (error instanceof Error && (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) ;
+                else {
+                    // Re-throw other unexpected errors
+                    throw error;
+                }
+            }
             const markdown = this.showdownConverter.makeMarkdown(html, jsdom$1.window.document);
             return { ...cacheFilehandler, markdown };
         }