@promptbook/node 0.88.0 → 0.89.0-11

package/umd/index.umd.js

@@ -46,7 +46,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.88.0';
+    const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -175,6 +175,7 @@
      */
     const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
     // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
+    // TODO: !!!!!! Just .promptbook dir, hardocode others
     /**
      * Where to store the temporary downloads
      *
@@ -191,6 +192,15 @@
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
+    /*
+    TODO: [🌃]
+    /**
+     * Id of application for the wizzard when using remote server
+     *
+     * @public exported from `@promptbook/core`
+     * /
+    ex-port const WIZZARD_APP_ID: string_app_id = 'wizzard';
+    */
     /**
      * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
      *
@@ -232,7 +242,7 @@
     true);
     /**
      * Note: [💞] Ignore a discrepancy between file name and entity name
-     * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value
+     * TODO: [🧠][🧜‍♂️] Maybe join remoteServerUrl and path into single value
      */
 
     /**
@@ -1700,6 +1710,19 @@
         }
     }
 
+    /**
+     * AuthenticationError is thrown from login function which is dependency of remote server
+     *
+     * @public exported from `@promptbook/core`
+     */
+    class AuthenticationError extends Error {
+        constructor(message) {
+            super(message);
+            this.name = 'AuthenticationError';
+            Object.setPrototypeOf(this, AuthenticationError.prototype);
+        }
+    }
+
     /**
      * This error indicates that the pipeline collection cannot be propperly loaded
      *
@@ -1791,6 +1814,19 @@
         }
     }
 
+    /**
+     * Error thrown when a fetch request fails
+     *
+     * @public exported from `@promptbook/core`
+     */
+    class PromptbookFetchError extends Error {
+        constructor(message) {
+            super(message);
+            this.name = 'PromptbookFetchError';
+            Object.setPrototypeOf(this, PromptbookFetchError.prototype);
+        }
+    }
+
     /**
      * Index of all custom errors
      *
@@ -1828,6 +1864,8 @@
         TypeError,
         URIError,
         AggregateError,
+        AuthenticationError,
+        PromptbookFetchError,
         /*
       Note: Not widely supported
       > InternalError,
@@ -1889,6 +1927,7 @@
         const { isSuccessful, errors, warnings } = executionResult;
         for (const warning of warnings) {
             console.warn(warning.message);
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
         if (isSuccessful === true) {
             return;
@@ -2026,6 +2065,10 @@
 
                     Cannot serialize error with name "${name}"
 
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
                     ${block(stack || message)}
 
                 `));
@@ -2113,30 +2156,42 @@
         }
     }
 
+    /**
+     * Represents the uncertain value
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const ZERO_VALUE = $deepFreeze({ value: 0 });
+    /**
+     * Represents the uncertain value
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
     /**
      * Represents the usage with no resources consumed
      *
      * @public exported from `@promptbook/core`
      */
     const ZERO_USAGE = $deepFreeze({
-        price: { value: 0 },
+        price: ZERO_VALUE,
         input: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
+            tokensCount: ZERO_VALUE,
+            charactersCount: ZERO_VALUE,
+            wordsCount: ZERO_VALUE,
+            sentencesCount: ZERO_VALUE,
+            linesCount: ZERO_VALUE,
+            paragraphsCount: ZERO_VALUE,
+            pagesCount: ZERO_VALUE,
         },
         output: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
+            tokensCount: ZERO_VALUE,
+            charactersCount: ZERO_VALUE,
+            wordsCount: ZERO_VALUE,
+            sentencesCount: ZERO_VALUE,
+            linesCount: ZERO_VALUE,
+            paragraphsCount: ZERO_VALUE,
+            pagesCount: ZERO_VALUE,
         },
     });
     /**
@@ -2145,24 +2200,24 @@
      * @public exported from `@promptbook/core`
      */
     $deepFreeze({
-        price: { value: 0, isUncertain: true },
+        price: UNCERTAIN_ZERO_VALUE,
         input: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
+            tokensCount: UNCERTAIN_ZERO_VALUE,
+            charactersCount: UNCERTAIN_ZERO_VALUE,
+            wordsCount: UNCERTAIN_ZERO_VALUE,
+            sentencesCount: UNCERTAIN_ZERO_VALUE,
+            linesCount: UNCERTAIN_ZERO_VALUE,
+            paragraphsCount: UNCERTAIN_ZERO_VALUE,
+            pagesCount: UNCERTAIN_ZERO_VALUE,
         },
         output: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
+            tokensCount: UNCERTAIN_ZERO_VALUE,
+            charactersCount: UNCERTAIN_ZERO_VALUE,
+            wordsCount: UNCERTAIN_ZERO_VALUE,
+            sentencesCount: UNCERTAIN_ZERO_VALUE,
+            linesCount: UNCERTAIN_ZERO_VALUE,
+            paragraphsCount: UNCERTAIN_ZERO_VALUE,
+            pagesCount: UNCERTAIN_ZERO_VALUE,
         },
     });
     /**
@@ -2333,7 +2388,7 @@
                 if (parameterNames.has(subparameterName)) {
                     parameterNames.delete(subparameterName);
                     parameterNames.add(foreach.parameterName);
-                    // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
+                    // <- TODO: [🏮] Warn/logic error when `subparameterName` not used
                 }
             }
         }
@@ -2827,6 +2882,7 @@
         `);
             // TODO: [🟥] Detect browser / node and make it colorfull
             console.warn(warningMessage);
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
             /*
             return {
                 async listModels() {
@@ -4416,6 +4472,7 @@
 
                     @see more at https://ptbk.io/prepare-pipeline
                 `));
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
         let runCount = 0;
         const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
@@ -4492,8 +4549,9 @@
      * @returns LLM tools with same functionality with added total cost counting
      * @public exported from `@promptbook/core`
      */
-    function countTotalUsage(llmTools) {
+    function countUsage(llmTools) {
         let totalUsage = ZERO_USAGE;
+        const spending = new rxjs.Subject();
         const proxyTools = {
             get title() {
                 // TODO: [🧠] Maybe put here some suffix
@@ -4503,12 +4561,15 @@
                 // TODO: [🧠] Maybe put here some suffix
                 return llmTools.description;
             },
-            async checkConfiguration() {
+            checkConfiguration() {
                 return /* not await */ llmTools.checkConfiguration();
             },
             listModels() {
                 return /* not await */ llmTools.listModels();
             },
+            spending() {
+                return spending.asObservable();
+            },
             getTotalUsage() {
                 // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
                 return totalUsage;
@@ -4519,6 +4580,7 @@
                 // console.info('[🚕] callChatModel through countTotalUsage');
                 const promptResult = await llmTools.callChatModel(prompt);
                 totalUsage = addUsage(totalUsage, promptResult.usage);
+                spending.next(promptResult.usage);
                 return promptResult;
             };
         }
@@ -4527,6 +4589,7 @@
                 // console.info('[🚕] callCompletionModel through countTotalUsage');
                 const promptResult = await llmTools.callCompletionModel(prompt);
                 totalUsage = addUsage(totalUsage, promptResult.usage);
+                spending.next(promptResult.usage);
                 return promptResult;
             };
         }
@@ -4535,6 +4598,7 @@
                 // console.info('[🚕] callEmbeddingModel through countTotalUsage');
                 const promptResult = await llmTools.callEmbeddingModel(prompt);
                 totalUsage = addUsage(totalUsage, promptResult.usage);
+                spending.next(promptResult.usage);
                 return promptResult;
             };
         }
@@ -5013,17 +5077,24 @@
     /**
      * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
      *
-     * @private as default `fetch` function used in Promptbook scrapers
+     * @public exported from `@promptbook/core`
      */
-    const scraperFetch = async (url, init) => {
+    const promptbookFetch = async (urlOrRequest, init) => {
         try {
-            return await fetch(url, init);
+            return await fetch(urlOrRequest, init);
         }
         catch (error) {
             if (!(error instanceof Error)) {
                 throw error;
             }
-            throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
+            let url;
+            if (typeof urlOrRequest === 'string') {
+                url = urlOrRequest;
+            }
+            else if (urlOrRequest instanceof Request) {
+                url = urlOrRequest.url;
+            }
+            throw new PromptbookFetchError(spaceTrim__default["default"]((block) => `
                     Can not fetch "${url}"
 
                     Fetch error:
@@ -5044,7 +5115,7 @@
     async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         // console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
         var _a;
-        const { fetch = scraperFetch } = tools;
+        const { fetch = promptbookFetch } = tools;
         const { knowledgeSourceContent } = knowledgeSource;
         let { name } = knowledgeSource;
         const { rootDirname = null, 
@@ -5185,63 +5256,73 @@
         const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
         const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
         await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
-            let partialPieces = null;
-            const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
-            const scrapers = arrayableToArray(tools.scrapers);
-            for (const scraper of scrapers) {
-                if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                // <- TODO: [🦔] Implement mime-type wildcards
-                ) {
-                    continue;
-                }
-                const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
-                if (partialPiecesUnchecked !== null) {
-                    partialPieces = [...partialPiecesUnchecked];
-                    // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
-                    break;
-                }
-                console.warn(spaceTrim__default["default"]((block) => `
-                        Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
+            try {
+                let partialPieces = null;
+                const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
+                const scrapers = arrayableToArray(tools.scrapers);
+                for (const scraper of scrapers) {
+                    if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                    // <- TODO: [🦔] Implement mime-type wildcards
+                    ) {
+                        continue;
+                    }
+                    const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
+                    if (partialPiecesUnchecked !== null) {
+                        partialPieces = [...partialPiecesUnchecked];
+                        // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                        break;
+                    }
+                    console.warn(spaceTrim__default["default"]((block) => `
+                            Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
 
-                        The source:
-                        ${block(knowledgeSource.knowledgeSourceContent
-                .split('\n')
-                .map((line) => `> ${line}`)
-                .join('\n'))}
+                            The source:
+                            ${block(knowledgeSource.knowledgeSourceContent
+                    .split('\n')
+                    .map((line) => `> ${line}`)
+                    .join('\n'))}
 
-                        ${block($registeredScrapersMessage(scrapers))}
+                            ${block($registeredScrapersMessage(scrapers))}
 
 
-                    `));
-            }
-            if (partialPieces === null) {
-                throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
-                        Cannot scrape knowledge
+                        `));
+                    // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
+                }
+                if (partialPieces === null) {
+                    throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
+                            Cannot scrape knowledge
 
-                        The source:
-                        > ${block(knowledgeSource.knowledgeSourceContent
-                .split('\n')
-                .map((line) => `> ${line}`)
-                .join('\n'))}
+                            The source:
+                            > ${block(knowledgeSource.knowledgeSourceContent
+                    .split('\n')
+                    .map((line) => `> ${line}`)
+                    .join('\n'))}
 
-                        No scraper found for the mime type "${sourceHandler.mimeType}"
+                            No scraper found for the mime type "${sourceHandler.mimeType}"
 
-                        ${block($registeredScrapersMessage(scrapers))}
+                            ${block($registeredScrapersMessage(scrapers))}
 
 
-                    `));
+                        `));
+                }
+                const pieces = partialPieces.map((partialPiece) => ({
+                    ...partialPiece,
+                    sources: [
+                        {
+                            name: knowledgeSource.name,
+                            // line, column <- TODO: [☀]
+                            // <- TODO: [❎]
+                        },
+                    ],
+                }));
+                knowledgePreparedUnflatten[index] = pieces;
+            }
+            catch (error) {
+                if (!(error instanceof Error)) {
+                    throw error;
+                }
+                console.warn(error);
+                // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
             }
-            const pieces = partialPieces.map((partialPiece) => ({
-                ...partialPiece,
-                sources: [
-                    {
-                        name: knowledgeSource.name,
-                        // line, column <- TODO: [☀]
-                        // <- TODO: [❎]
-                    },
-                ],
-            }));
-            knowledgePreparedUnflatten[index] = pieces;
         });
         const knowledgePrepared = knowledgePreparedUnflatten.flat();
         return knowledgePrepared;
@@ -5347,7 +5428,7 @@
         // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
         const _llms = arrayableToArray(tools.llm);
         const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
-        const llmToolsWithUsage = countTotalUsage(llmTools);
+        const llmToolsWithUsage = countUsage(llmTools);
         //    <- TODO: [🌯]
         /*
         TODO: [🧠][🪑][🔃] Should this be done or not
@@ -7066,7 +7147,8 @@
             if ($pipelineJson.defaultModelRequirements[command.key] !== undefined) {
                 if ($pipelineJson.defaultModelRequirements[command.key] === command.value) {
                     console.warn(`Multiple commands \`MODEL ${command.key} ${command.value}\` in the pipeline head`);
-                    // <- TODO: [🚎][💩] Some better way how to get warnings from pipeline parsing / logic
+                    // <- TODO: [🏮] Some better way how to get warnings from pipeline parsing / logic
+                    // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                 }
                 else {
                     throw new ParseError(spaceTrim__default["default"](`
@@ -7098,6 +7180,7 @@
                     modelVariant: 'VARIANT',
                     maxTokens: '???',
                 }[command.key]} ${command.value}\` in the task "${$taskJson.title || $taskJson.name}"`);
+                    // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                 }
                 else {
                     throw new ParseError(spaceTrim__default["default"](`
@@ -7377,15 +7460,15 @@
         }
         console.warn(spaceTrim__default["default"](`
 
-         Persona "${personaName}" is defined multiple times with different description:
+            Persona "${personaName}" is defined multiple times with different description:
 
-         First definition:
-         ${persona.description}
+            First definition:
+            ${persona.description}
 
-         Second definition:
-         ${personaDescription}
+            Second definition:
+            ${personaDescription}
 
-     `));
+        `));
         persona.description += spaceTrim__default["default"]('\n\n' + personaDescription);
     }
 
@@ -9021,6 +9104,7 @@
                     }
                     else {
                         console.warn(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`);
+                        // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                         resolve('Command exceeded time limit');
                     }
                 });
@@ -9046,6 +9130,7 @@
                     output.push(stderr.toString());
                     if (isVerbose && stderr.toString().trim()) {
                         console.warn(stderr.toString());
+                        // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                     }
                 });
                 const finishWithCode = (code) => {
@@ -9057,6 +9142,7 @@
                         else {
                             if (isVerbose) {
                                 console.warn(`Command "${humanReadableCommand}" exited with code ${code}`);
+                                // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                             }
                             resolve(spaceTrim.spaceTrim(output.join('\n')));
                         }
@@ -9078,6 +9164,7 @@
                     else {
                         if (isVerbose) {
                             console.warn(error);
+                            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
                         }
                         resolve(spaceTrim.spaceTrim(output.join('\n')));
                     }
@@ -9508,21 +9595,15 @@
      */
 
     /**
-     * @@@
-     *
-     * @@@ .env
+     * Provides the path to the `.env` file
      *
-     * It looks for environment variables:
-     * - `process.env.OPENAI_API_KEY`
-     * - `process.env.ANTHROPIC_CLAUDE_API_KEY`
-     * - ...
+     * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
      *
-     * @returns @@@
-     * @public exported from `@promptbook/node`
+     * @private within the repository - for CLI utils
      */
-    async function $provideLlmToolsConfigurationFromEnv() {
+    async function $provideEnvFilename() {
         if (!$isRunningInNode()) {
-            throw new EnvironmentMismatchError('Function `$provideLlmToolsFromEnv` works only in Node.js environment');
+            throw new EnvironmentMismatchError('Function `$provideEnvFilename` works only in Node.js environment');
         }
         const envFilePatterns = [
             '.env',
@@ -9542,8 +9623,7 @@
                 const envFilename = path.join(rootDirname, pattern);
                 if (await isFileExisting(envFilename, $provideFilesystemForNode())) {
                     $setUsedEnvFilename(envFilename);
-                    dotenv__namespace.config({ path: envFilename });
-                    break up_to_root;
+                    return envFilename;
                 }
             }
             if (isRootPath(rootDirname)) {
@@ -9552,6 +9632,34 @@
             // Note: If the directory does not exist, try the parent directory
             rootDirname = path.join(rootDirname, '..');
         }
+        return null;
+    }
+    /**
+     * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+     */
+
+    /**
+     * @@@
+     *
+     * @@@ .env
+     * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
+     *
+     * It looks for environment variables:
+     * - `process.env.OPENAI_API_KEY`
+     * - `process.env.ANTHROPIC_CLAUDE_API_KEY`
+     * - ...
+     *
+     * @returns @@@
+     * @public exported from `@promptbook/node`
+     */
+    async function $provideLlmToolsConfigurationFromEnv() {
+        if (!$isRunningInNode()) {
+            throw new EnvironmentMismatchError('Function `$provideLlmToolsFromEnv` works only in Node.js environment');
+        }
+        const envFilepath = await $provideEnvFilename();
+        if (envFilepath !== null) {
+            dotenv__namespace.config({ path: envFilepath });
+        }
         const llmToolsConfiguration = $llmToolsMetadataRegister
             .list()
             .map((metadata) => metadata.createConfigurationFromEnv(process.env))
@@ -9559,15 +9667,8 @@
         return llmToolsConfiguration;
     }
     /**
-     * TODO: [🧠][🪁] Maybe do allow to do auto-install if package not registered and not found
-     * TODO: Add Azure OpenAI
-     * TODO: [🧠][🍛]
-     * TODO: [🧠] Is there some meaningfull way how to test this util
      * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
-     * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
-     * TODO: This should be maybe not under `_common` but under `utils`
-     * TODO: [🧠][⚛] Maybe pass env as argument
-     * TODO: [®] DRY Register logic */
+     */
 
     /**
      * @@@
@@ -9626,6 +9727,7 @@
      * Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
      *
      * @@@ .env
+     * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
      *
      * It looks for environment variables:
      * - `process.env.OPENAI_API_KEY`