@promptbook/markdown-utils 0.88.0 → 0.89.0-2

package/esm/typings/src/playground/playground.d.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env ts-node
+export {};
+/**
+ * Note: [⚫] Code in this file should never be published in any package
+ */

package/esm/typings/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.d.ts

@@ -37,7 +37,8 @@ export type PromptbookServer_AnonymousIdentification = {
     /**
      * Identifier of the end user
      *
-     * Note: this is passed to the certain model providers to identify misuse
+     * Note: This can be either some id or email or any other identifier
+     * Note: In anonymous mode, this is passed to the certain model providers to identify misuse
      * Note: In anonymous mode, there is no need to identify yourself, nor does it change the actual configuration of LLM Tools (unlike in application mode)
      */
     readonly userId?: string_user_id;

package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts

@@ -58,15 +58,27 @@ export type ApplicationRemoteServerOptions<TCustomOptions> = {
 };
 export type ApplicationRemoteServerClientOptions<TCustomOptions> = {
     /**
-     * @@@
+     * Identifier of the application
+     *
+     * Note: This is usefull when you use Promptbook remote server for multiple apps/frontends, if its used just for single app, use here just "app" or "your-app-name"
+     * Note: This can be some id or some semantic name like "email-agent"
      */
     readonly appId: string_app_id | null;
     /**
-     * @@@
+     * Identifier of the end user
+     *
+     * Note: This can be either some id or email or any other identifier
+     * Note: This is also passed to the certain model providers to identify misuse
      */
     readonly userId?: string_user_id;
     /**
-     * @@@
+     * Token of the user to verify its identity
+     *
+     * Note: This is passed for example to `createLlmExecutionTools`
+     */
+    readonly userToken?: string_user_id;
+    /**
+     * Additional arbitrary options to identify the client or to pass custom metadata
      */
     readonly customOptions?: TCustomOptions;
 };

package/esm/typings/src/types/typeAliases.d.ts

@@ -242,6 +242,12 @@ export type string_promptbook_documentation_url = `https://github.com/webgptorg/
  * For example `"towns.cz"`
  */
 export type string_domain = string;
+/**
+ * Semantic helper
+ *
+ * For example `"https://*.pavolhejny.com/*"`
+ */
+export type string_origin = string;
 /**
  * Semantic helper
  *
@@ -433,13 +439,13 @@ export type string_uuid = string & {
  *
  * @@@
  */
-export type string_app_id = id;
+export type string_app_id = id | 'app';
 /**
  * End user identifier
  *
  * @@@
  */
-export type string_user_id = id;
+export type string_user_id = id | string_email;
 /**
  * Semantic helper
  *

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/markdown-utils",
-    "version": "0.88.0",
+    "version": "0.89.0-2",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.88.0';
+    const PROMPTBOOK_ENGINE_VERSION = '0.89.0-2';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1996,6 +1996,7 @@
         const { isSuccessful, errors, warnings } = executionResult;
         for (const warning of warnings) {
             console.warn(warning.message);
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
         if (isSuccessful === true) {
             return;
@@ -2174,30 +2175,42 @@
         await Promise.all(tasks);
     }
 
+    /**
+     * Represents the uncertain value
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const ZERO_VALUE = $deepFreeze({ value: 0 });
+    /**
+     * Represents the uncertain value
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
     /**
      * Represents the usage with no resources consumed
      *
      * @public exported from `@promptbook/core`
      */
     const ZERO_USAGE = $deepFreeze({
-        price: { value: 0 },
+        price: ZERO_VALUE,
         input: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
+            tokensCount: ZERO_VALUE,
+            charactersCount: ZERO_VALUE,
+            wordsCount: ZERO_VALUE,
+            sentencesCount: ZERO_VALUE,
+            linesCount: ZERO_VALUE,
+            paragraphsCount: ZERO_VALUE,
+            pagesCount: ZERO_VALUE,
         },
         output: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
+            tokensCount: ZERO_VALUE,
+            charactersCount: ZERO_VALUE,
+            wordsCount: ZERO_VALUE,
+            sentencesCount: ZERO_VALUE,
+            linesCount: ZERO_VALUE,
+            paragraphsCount: ZERO_VALUE,
+            pagesCount: ZERO_VALUE,
         },
     });
     /**
@@ -2206,24 +2219,24 @@
      * @public exported from `@promptbook/core`
      */
     $deepFreeze({
-        price: { value: 0, isUncertain: true },
+        price: UNCERTAIN_ZERO_VALUE,
         input: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
+            tokensCount: UNCERTAIN_ZERO_VALUE,
+            charactersCount: UNCERTAIN_ZERO_VALUE,
+            wordsCount: UNCERTAIN_ZERO_VALUE,
+            sentencesCount: UNCERTAIN_ZERO_VALUE,
+            linesCount: UNCERTAIN_ZERO_VALUE,
+            paragraphsCount: UNCERTAIN_ZERO_VALUE,
+            pagesCount: UNCERTAIN_ZERO_VALUE,
         },
         output: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
+            tokensCount: UNCERTAIN_ZERO_VALUE,
+            charactersCount: UNCERTAIN_ZERO_VALUE,
+            wordsCount: UNCERTAIN_ZERO_VALUE,
+            sentencesCount: UNCERTAIN_ZERO_VALUE,
+            linesCount: UNCERTAIN_ZERO_VALUE,
+            paragraphsCount: UNCERTAIN_ZERO_VALUE,
+            pagesCount: UNCERTAIN_ZERO_VALUE,
         },
     });
     /**
@@ -2284,8 +2297,9 @@
      * @returns LLM tools with same functionality with added total cost counting
      * @public exported from `@promptbook/core`
      */
-    function countTotalUsage(llmTools) {
+    function countUsage(llmTools) {
         let totalUsage = ZERO_USAGE;
+        const spending = new rxjs.Subject();
         const proxyTools = {
             get title() {
                 // TODO: [🧠] Maybe put here some suffix
@@ -2295,12 +2309,15 @@
                 // TODO: [🧠] Maybe put here some suffix
                 return llmTools.description;
             },
-            async checkConfiguration() {
+            checkConfiguration() {
                 return /* not await */ llmTools.checkConfiguration();
             },
             listModels() {
                 return /* not await */ llmTools.listModels();
             },
+            spending() {
+                return spending.asObservable();
+            },
             getTotalUsage() {
                 // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
                 return totalUsage;
@@ -2311,6 +2328,7 @@
                 // console.info('[🚕] callChatModel through countTotalUsage');
                 const promptResult = await llmTools.callChatModel(prompt);
                 totalUsage = addUsage(totalUsage, promptResult.usage);
+                spending.next(promptResult.usage);
                 return promptResult;
             };
         }
@@ -2319,6 +2337,7 @@
                 // console.info('[🚕] callCompletionModel through countTotalUsage');
                 const promptResult = await llmTools.callCompletionModel(prompt);
                 totalUsage = addUsage(totalUsage, promptResult.usage);
+                spending.next(promptResult.usage);
                 return promptResult;
             };
         }
@@ -2327,6 +2346,7 @@
                 // console.info('[🚕] callEmbeddingModel through countTotalUsage');
                 const promptResult = await llmTools.callEmbeddingModel(prompt);
                 totalUsage = addUsage(totalUsage, promptResult.usage);
+                spending.next(promptResult.usage);
                 return promptResult;
             };
         }
@@ -2504,6 +2524,7 @@
         `);
             // TODO: [🟥] Detect browser / node and make it colorfull
             console.warn(warningMessage);
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
             /*
             return {
                 async listModels() {
@@ -3447,63 +3468,73 @@
         const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
         const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
         await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
-            let partialPieces = null;
-            const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
-            const scrapers = arrayableToArray(tools.scrapers);
-            for (const scraper of scrapers) {
-                if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                // <- TODO: [🦔] Implement mime-type wildcards
-                ) {
-                    continue;
-                }
-                const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
-                if (partialPiecesUnchecked !== null) {
-                    partialPieces = [...partialPiecesUnchecked];
-                    // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
-                    break;
-                }
-                console.warn(spaceTrim__default["default"]((block) => `
-                        Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
+            try {
+                let partialPieces = null;
+                const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
+                const scrapers = arrayableToArray(tools.scrapers);
+                for (const scraper of scrapers) {
+                    if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                    // <- TODO: [🦔] Implement mime-type wildcards
+                    ) {
+                        continue;
+                    }
+                    const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
+                    if (partialPiecesUnchecked !== null) {
+                        partialPieces = [...partialPiecesUnchecked];
+                        // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                        break;
+                    }
+                    console.warn(spaceTrim__default["default"]((block) => `
+                            Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
 
-                        The source:
-                        ${block(knowledgeSource.knowledgeSourceContent
-                .split('\n')
-                .map((line) => `> ${line}`)
-                .join('\n'))}
+                            The source:
+                            ${block(knowledgeSource.knowledgeSourceContent
+                    .split('\n')
+                    .map((line) => `> ${line}`)
+                    .join('\n'))}
 
-                        ${block($registeredScrapersMessage(scrapers))}
+                            ${block($registeredScrapersMessage(scrapers))}
 
 
-                    `));
-            }
-            if (partialPieces === null) {
-                throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
-                        Cannot scrape knowledge
+                        `));
+                    // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
+                }
+                if (partialPieces === null) {
+                    throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
+                            Cannot scrape knowledge
 
-                        The source:
-                        > ${block(knowledgeSource.knowledgeSourceContent
-                .split('\n')
-                .map((line) => `> ${line}`)
-                .join('\n'))}
+                            The source:
+                            > ${block(knowledgeSource.knowledgeSourceContent
+                    .split('\n')
+                    .map((line) => `> ${line}`)
+                    .join('\n'))}
 
-                        No scraper found for the mime type "${sourceHandler.mimeType}"
+                            No scraper found for the mime type "${sourceHandler.mimeType}"
 
-                        ${block($registeredScrapersMessage(scrapers))}
+                            ${block($registeredScrapersMessage(scrapers))}
 
 
-                    `));
+                        `));
+                }
+                const pieces = partialPieces.map((partialPiece) => ({
+                    ...partialPiece,
+                    sources: [
+                        {
+                            name: knowledgeSource.name,
+                            // line, column <- TODO: [☀]
+                            // <- TODO: [❎]
+                        },
+                    ],
+                }));
+                knowledgePreparedUnflatten[index] = pieces;
+            }
+            catch (error) {
+                if (!(error instanceof Error)) {
+                    throw error;
+                }
+                console.warn(error);
+                // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
             }
-            const pieces = partialPieces.map((partialPiece) => ({
-                ...partialPiece,
-                sources: [
-                    {
-                        name: knowledgeSource.name,
-                        // line, column <- TODO: [☀]
-                        // <- TODO: [❎]
-                    },
-                ],
-            }));
-            knowledgePreparedUnflatten[index] = pieces;
         });
         const knowledgePrepared = knowledgePreparedUnflatten.flat();
         return knowledgePrepared;
@@ -3609,7 +3640,7 @@
         // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
         const _llms = arrayableToArray(tools.llm);
         const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
-        const llmToolsWithUsage = countTotalUsage(llmTools);
+        const llmToolsWithUsage = countUsage(llmTools);
         //    <- TODO: [🌯]
         /*
         TODO: [🧠][🪑][🔃] Should this be done or not
@@ -3921,7 +3952,7 @@
                 if (parameterNames.has(subparameterName)) {
                     parameterNames.delete(subparameterName);
                     parameterNames.add(foreach.parameterName);
-                    // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
+                    // <- TODO: [🏮] Warn/logic error when `subparameterName` not used
                 }
             }
         }
@@ -5415,6 +5446,7 @@
 
                     @see more at https://ptbk.io/prepare-pipeline
                 `));
+            // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
         let runCount = 0;
         const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
@@ -5689,7 +5721,8 @@
                 `));
         }
         console.warn(`No place where to put the section <!--${sectionName}-->, using the end of the file`);
-        // <- TODO: [🚎][💩] Some better way how to get warnings from pipeline parsing / logic
+        // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
+        // <- TODO: [🏮] Some better way how to get warnings from pipeline parsing / logic
         return spaceTrim.spaceTrim((block) => `
             ${block(content)}