@promptbook/website-crawler 0.101.0-9 → 0.101.0

package/README.md

@@ -29,10 +29,6 @@ Write AI applications using plain human language across multiple models and plat
 
 
 
-<blockquote style="color: #ff8811">
-    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
-</blockquote>
-
 ## 📦 Package `@promptbook/website-crawler`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).

package/esm/index.es.js

@@ -27,7 +27,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.101.0-9';
+const PROMPTBOOK_ENGINE_VERSION = '0.101.0';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1047,11 +1047,12 @@ async function getScraperIntermediateSource(source, options) {
     catch (error) {
         // Note: If we can't create cache directory, continue without it
         //       This handles read-only filesystems, permission issues, and missing parent directories
-        if (error instanceof Error && (error.message.includes('EROFS') ||
-            error.message.includes('read-only') ||
-            error.message.includes('EACCES') ||
-            error.message.includes('EPERM') ||
-            error.message.includes('ENOENT'))) ;
+        if (error instanceof Error &&
+            (error.message.includes('EROFS') ||
+                error.message.includes('read-only') ||
+                error.message.includes('EACCES') ||
+                error.message.includes('EPERM') ||
+                error.message.includes('ENOENT'))) ;
         else {
             // Re-throw other unexpected errors
             throw error;
@@ -3080,75 +3081,32 @@ function countUsage(llmTools) {
  */
 
 /**
- * Predefined profiles for LLM providers to maintain consistency across the application
- * These profiles represent each provider as a virtual persona in chat interfaces
+ * Takes an item or an array of items and returns an array of items
+ *
+ * 1) Any item except array and undefined returns array with that one item (also null)
+ * 2) Undefined returns empty array
+ * 3) Array returns itself
  *
- * @private !!!!
+ * @private internal utility
  */
-const LLM_PROVIDER_PROFILES = {
-    OPENAI: {
-        name: 'OPENAI',
-        fullname: 'OpenAI GPT',
-        color: '#10a37f', // OpenAI's signature green
-        // Note: avatarSrc could be added when we have provider logos available
-    },
-    ANTHROPIC: {
-        name: 'ANTHROPIC',
-        fullname: 'Anthropic Claude',
-        color: '#d97706', // Anthropic's orange/amber color
-    },
-    AZURE_OPENAI: {
-        name: 'AZURE_OPENAI',
-        fullname: 'Azure OpenAI',
-        color: '#0078d4', // Microsoft Azure blue
-    },
-    GOOGLE: {
-        name: 'GOOGLE',
-        fullname: 'Google Gemini',
-        color: '#4285f4', // Google blue
-    },
-    DEEPSEEK: {
-        name: 'DEEPSEEK',
-        fullname: 'DeepSeek',
-        color: '#7c3aed', // Purple color for DeepSeek
-    },
-    OLLAMA: {
-        name: 'OLLAMA',
-        fullname: 'Ollama',
-        color: '#059669', // Emerald green for local models
-    },
-    REMOTE: {
-        name: 'REMOTE',
-        fullname: 'Remote Server',
-        color: '#6b7280', // Gray for remote/proxy connections
-    },
-    MOCKED_ECHO: {
-        name: 'MOCKED_ECHO',
-        fullname: 'Echo (Test)',
-        color: '#8b5cf6', // Purple for test/mock tools
-    },
-    MOCKED_FAKE: {
-        name: 'MOCKED_FAKE',
-        fullname: 'Fake LLM (Test)',
-        color: '#ec4899', // Pink for fake/test tools
-    },
-    VERCEL: {
-        name: 'VERCEL',
-        fullname: 'Vercel AI',
-        color: '#000000', // Vercel's black
-    },
-    MULTIPLE: {
-        name: 'MULTIPLE',
-        fullname: 'Multiple Providers',
-        color: '#6366f1', // Indigo for combined/multiple providers
-    },
-};
+function arrayableToArray(input) {
+    if (input === undefined) {
+        return [];
+    }
+    if (input instanceof Array) {
+        return input;
+    }
+    return [input];
+}
+
 /**
- * TODO: Refactor this - each profile must be alongside the provider definition
- * TODO: [🕛] Unite `AgentBasicInformation`, `ChatParticipant`, `LlmExecutionTools` +  `LlmToolsMetadata`
- * Note: [💞] Ignore a discrepancy between file name and entity name
+ * Profile for Multiple providers aggregation
  */
-
+const MULTIPLE_PROVIDER_PROFILE = {
+    name: 'MULTIPLE',
+    fullname: 'Multiple Providers',
+    color: '#6366f1',
+};
 /**
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
  *
@@ -3159,12 +3117,10 @@ class MultipleLlmExecutionTools {
     /**
      * Gets array of execution tools in order of priority
      */
-    constructor(...llmExecutionTools) {
+    constructor(title, ...llmExecutionTools) {
+        this.title = title;
         this.llmExecutionTools = llmExecutionTools;
     }
-    get title() {
-        return 'Multiple LLM Providers';
-    }
     get description() {
         const innerModelsTitlesAndDescriptions = this.llmExecutionTools
             .map(({ title, description }, index) => {
@@ -3186,7 +3142,7 @@ class MultipleLlmExecutionTools {
             `);
     }
     get profile() {
-        return LLM_PROVIDER_PROFILES.MULTIPLE;
+        return MULTIPLE_PROVIDER_PROFILE;
     }
     /**
      * Check the configuration of all execution tools
@@ -3250,7 +3206,7 @@ class MultipleLlmExecutionTools {
                         return await llmExecutionTools.callEmbeddingModel(prompt);
                     // <- case [🤖]:
                     default:
-                        throw new UnexpectedError(`Unknown model variant "${prompt.modelRequirements.modelVariant}"`);
+                        throw new UnexpectedError(`Unknown model variant "${prompt.modelRequirements.modelVariant}" in ${llmExecutionTools.title}`);
                 }
             }
             catch (error) {
@@ -3271,7 +3227,7 @@ class MultipleLlmExecutionTools {
             //     2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined
             //     3) ...
             spaceTrim$1((block) => `
-                          All execution tools failed:
+                          All execution tools of ${this.title} failed:
 
                           ${block(errors
                 .map(({ error, llmExecutionTools }, i) => `${i + 1}) **${llmExecutionTools.title}** thrown **${error.name || 'Error'}:** ${error.message}`)
@@ -3280,11 +3236,11 @@ class MultipleLlmExecutionTools {
                     `));
         }
         else if (this.llmExecutionTools.length === 0) {
-            throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\``);
+            throw new PipelineExecutionError(`You have not provided any \`LlmExecutionTools\` into ${this.title}`);
         }
         else {
             throw new PipelineExecutionError(spaceTrim$1((block) => `
-                          You have not provided any \`LlmExecutionTools\` that support model variant "${prompt.modelRequirements.modelVariant}"
+                          You have not provided any \`LlmExecutionTools\` that support model variant "${prompt.modelRequirements.modelVariant}" into ${this.title}
 
                           Available \`LlmExecutionTools\`:
                           ${block(this.description)}
@@ -3314,7 +3270,7 @@ class MultipleLlmExecutionTools {
  *
  * @public exported from `@promptbook/core`
  */
-function joinLlmExecutionTools(...llmExecutionTools) {
+function joinLlmExecutionTools(title, ...llmExecutionTools) {
     if (llmExecutionTools.length === 0) {
         const warningMessage = spaceTrim$1(`
             You have not provided any \`LlmExecutionTools\`
@@ -3346,30 +3302,27 @@ function joinLlmExecutionTools(...llmExecutionTools) {
         };
         */
     }
-    return new MultipleLlmExecutionTools(...llmExecutionTools);
+    return new MultipleLlmExecutionTools(title || 'Multiple LLM Providers joined by `joinLlmExecutionTools`', ...llmExecutionTools);
 }
 /**
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
  */
 
 /**
- * Takes an item or an array of items and returns an array of items
- *
- * 1) Any item except array and undefined returns array with that one item (also null)
- * 2) Undefined returns empty array
- * 3) Array returns itself
+ * Just returns the given `LlmExecutionTools` or joins multiple into one
  *
- * @private internal utility
+ * @public exported from `@promptbook/core`
  */
-function arrayableToArray(input) {
-    if (input === undefined) {
-        return [];
-    }
-    if (input instanceof Array) {
-        return input;
-    }
-    return [input];
+function getSingleLlmExecutionTools(oneOrMoreLlmExecutionTools) {
+    const _llms = arrayableToArray(oneOrMoreLlmExecutionTools);
+    const llmTools = _llms.length === 1
+        ? _llms[0]
+        : joinLlmExecutionTools('Multiple LLM Providers joined by `getSingleLlmExecutionTools`', ..._llms);
+    return llmTools;
 }
+/**
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
+ */
 
 /**
  * Prepares the persona for the pipeline
@@ -3388,8 +3341,7 @@ async function preparePersona(personaDescription, tools, options) {
         pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
         tools,
     });
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     const availableModels = (await llmTools.listModels())
         .filter(({ modelVariant }) => modelVariant === 'CHAT')
         .map(({ modelName, modelDescription }) => ({
@@ -3433,6 +3385,7 @@ async function preparePersona(personaDescription, tools, options) {
     };
 }
 /**
+ * TODO: [😩] DRY `preparePersona` and `selectBestModelFromAvailable`
  * TODO: [🔃][main] If the persona was prepared with different version or different set of models, prepare it once again
  * TODO: [🏢] Check validity of `modelName` in pipeline
  * TODO: [🏢] Check validity of `systemMessage` in pipeline
@@ -4041,9 +3994,7 @@ async function preparePipeline(pipeline, tools, options) {
     if (tools === undefined || tools.llm === undefined) {
         throw new MissingToolsError('LLM tools are required for preparing the pipeline');
     }
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     const llmToolsWithUsage = countUsage(llmTools);
     //    <- TODO: [🌯]
     /*
@@ -5186,9 +5137,7 @@ async function executeAttempts(options) {
         $scriptPipelineExecutionErrors: [],
         $failedResults: [], // Track all failed attempts
     };
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+    const llmTools = getSingleLlmExecutionTools(tools.llm);
     attempts: for (let attemptIndex = -jokerParameterNames.length; attemptIndex < maxAttempts; attemptIndex++) {
         const isJokerAttempt = attemptIndex < 0;
         const jokerParameterName = jokerParameterNames[jokerParameterNames.length + attemptIndex];
@@ -5708,9 +5657,7 @@ async function getKnowledgeForTask(options) {
         return ''; // <- Note: Np knowledge present, return empty string
     }
     try {
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(tools.llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const llmTools = getSingleLlmExecutionTools(tools.llm);
         const taskEmbeddingPrompt = {
             title: 'Knowledge Search',
             modelRequirements: {
@@ -6311,13 +6258,13 @@ function createPipelineExecutor(options) {
                     // Calculate and update tldr based on pipeline progress
                     const cv = newOngoingResult;
                     // Calculate progress based on parameters resolved vs total parameters
-                    const totalParameters = pipeline.parameters.filter(p => !p.isInput).length;
+                    const totalParameters = pipeline.parameters.filter((p) => !p.isInput).length;
                     let resolvedParameters = 0;
                     let currentTaskTitle = '';
                     // Get the resolved parameters from output parameters
                     if (cv === null || cv === void 0 ? void 0 : cv.outputParameters) {
                         // Count how many output parameters have non-empty values
-                        resolvedParameters = Object.values(cv.outputParameters).filter(value => value !== undefined && value !== null && String(value).trim() !== '').length;
+                        resolvedParameters = Object.values(cv.outputParameters).filter((value) => value !== undefined && value !== null && String(value).trim() !== '').length;
                     }
                     // Try to determine current task from execution report
                     if (((_a = cv === null || cv === void 0 ? void 0 : cv.executionReport) === null || _a === void 0 ? void 0 : _a.promptExecutions) && cv.executionReport.promptExecutions.length > 0) {
@@ -6427,9 +6374,7 @@ class MarkdownScraper {
             throw new MissingToolsError('LLM tools are required for scraping external files');
             // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
         }
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const llmTools = getSingleLlmExecutionTools(llm);
         // TODO: [🌼] In future use `ptbk make` and made getPipelineCollection
         const collection = createCollectionFromJson(...PipelineCollection);
         const prepareKnowledgeFromMarkdownExecutor = createPipelineExecutor({
@@ -6629,11 +6574,12 @@ class WebsiteScraper {
         catch (error) {
             // Note: If we can't write to cache, we'll continue without caching
             //       This handles read-only filesystems like Vercel
-            if (error instanceof Error && (error.message.includes('EROFS') ||
-                error.message.includes('read-only') ||
-                error.message.includes('EACCES') ||
-                error.message.includes('EPERM') ||
-                error.message.includes('ENOENT'))) ;
+            if (error instanceof Error &&
+                (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) ;
             else {
                 // Re-throw other unexpected errors
                 throw error;