@promptbook/website-crawler 0.92.0-11 → 0.92.0-13

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-11';
+    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-13';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2271,75 +2271,6 @@
      *     - [♨] Are tasks prepared
      */
 
-    /**
-     * Converts a JavaScript Object Notation (JSON) string into an object.
-     *
-     * Note: This is wrapper around `JSON.parse()` with better error and type handling
-     *
-     * @public exported from `@promptbook/utils`
-     */
-    function jsonParse(value) {
-        if (value === undefined) {
-            throw new Error(`Can not parse JSON from undefined value.`);
-        }
-        else if (typeof value !== 'string') {
-            console.error('Can not parse JSON from non-string value.', { text: value });
-            throw new Error(spaceTrim__default["default"](`
-                    Can not parse JSON from non-string value.
-
-                    The value type: ${typeof value}
-                    See more in console.
-                `));
-        }
-        try {
-            return JSON.parse(value);
-        }
-        catch (error) {
-            if (!(error instanceof Error)) {
-                throw error;
-            }
-            throw new Error(spaceTrim__default["default"]((block) => `
-                    ${block(error.message)}
-            
-                    The JSON text:
-                    ${block(value)}
-                `));
-        }
-    }
-    /**
-     * TODO: !!!! Use in Promptbook.studio
-     */
-
-    /**
-     * Recursively converts JSON strings to JSON objects
-
-     * @public exported from `@promptbook/utils`
-     */
-    function jsonStringsToJsons(object) {
-        if (object === null) {
-            return object;
-        }
-        if (Array.isArray(object)) {
-            return object.map(jsonStringsToJsons);
-        }
-        if (typeof object !== 'object') {
-            return object;
-        }
-        const newObject = { ...object };
-        for (const [key, value] of Object.entries(object)) {
-            if (typeof value === 'string' && isValidJsonString(value)) {
-                newObject[key] = jsonParse(value);
-            }
-            else {
-                newObject[key] = jsonStringsToJsons(value);
-            }
-        }
-        return newObject;
-    }
-    /**
-     * TODO: Type the return type correctly
-     */
-
     /**
      * This error indicates problems parsing the format value
      *
@@ -2500,6 +2431,104 @@
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
+    /**
+     * Serializes an error into a [🚉] JSON-serializable object
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    function serializeError(error) {
+        const { name, message, stack } = error;
+        const { id } = error;
+        if (!Object.keys(ALL_ERRORS).includes(name)) {
+            console.error(spaceTrim__default["default"]((block) => `
+
+                    Cannot serialize error with name "${name}"
+
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
+                    ${block(stack || message)}
+
+                `));
+        }
+        return {
+            name: name,
+            message,
+            stack,
+            id, // Include id in the serialized object
+        };
+    }
+
+    /**
+     * Converts a JavaScript Object Notation (JSON) string into an object.
+     *
+     * Note: This is wrapper around `JSON.parse()` with better error and type handling
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    function jsonParse(value) {
+        if (value === undefined) {
+            throw new Error(`Can not parse JSON from undefined value.`);
+        }
+        else if (typeof value !== 'string') {
+            console.error('Can not parse JSON from non-string value.', { text: value });
+            throw new Error(spaceTrim__default["default"](`
+                    Can not parse JSON from non-string value.
+
+                    The value type: ${typeof value}
+                    See more in console.
+                `));
+        }
+        try {
+            return JSON.parse(value);
+        }
+        catch (error) {
+            if (!(error instanceof Error)) {
+                throw error;
+            }
+            throw new Error(spaceTrim__default["default"]((block) => `
+                    ${block(error.message)}
+            
+                    The JSON text:
+                    ${block(value)}
+                `));
+        }
+    }
+    /**
+     * TODO: !!!! Use in Promptbook.studio
+     */
+
+    /**
+     * Recursively converts JSON strings to JSON objects
+
+     * @public exported from `@promptbook/utils`
+     */
+    function jsonStringsToJsons(object) {
+        if (object === null) {
+            return object;
+        }
+        if (Array.isArray(object)) {
+            return object.map(jsonStringsToJsons);
+        }
+        if (typeof object !== 'object') {
+            return object;
+        }
+        const newObject = { ...object };
+        for (const [key, value] of Object.entries(object)) {
+            if (typeof value === 'string' && isValidJsonString(value)) {
+                newObject[key] = jsonParse(value);
+            }
+            else {
+                newObject[key] = jsonStringsToJsons(value);
+            }
+        }
+        return newObject;
+    }
+    /**
+     * TODO: Type the return type correctly
+     */
+
     /**
      * Deserializes the error object
      *
@@ -2665,64 +2694,6 @@
      * TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
      */
 
-    /**
-     * Serializes an error into a [🚉] JSON-serializable object
-     *
-     * @public exported from `@promptbook/utils`
-     */
-    function serializeError(error) {
-        const { name, message, stack } = error;
-        const { id } = error;
-        if (!Object.keys(ALL_ERRORS).includes(name)) {
-            console.error(spaceTrim__default["default"]((block) => `
-
-                    Cannot serialize error with name "${name}"
-
-                    Authors of Promptbook probably forgot to add this error into the list of errors:
-                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
-
-
-                    ${block(stack || message)}
-
-                `));
-        }
-        return {
-            name: name,
-            message,
-            stack,
-            id, // Include id in the serialized object
-        };
-    }
-
-    /**
-     * Async version of Array.forEach
-     *
-     * @param array - Array to iterate over
-     * @param options - Options for the function
-     * @param callbackfunction - Function to call for each item
-     * @public exported from `@promptbook/utils`
-     * @deprecated [🪂] Use queues instead
-     */
-    async function forEachAsync(array, options, callbackfunction) {
-        const { maxParallelCount = Infinity } = options;
-        let index = 0;
-        let runningTasks = [];
-        const tasks = [];
-        for (const item of array) {
-            const currentIndex = index++;
-            const task = callbackfunction(item, currentIndex, array);
-            tasks.push(task);
-            runningTasks.push(task);
-            /* not await */ Promise.resolve(task).then(() => {
-                runningTasks = runningTasks.filter((t) => t !== task);
-            });
-            if (maxParallelCount < runningTasks.length) {
-                await Promise.race(runningTasks);
-            }
-        }
-        await Promise.all(tasks);
-    }
-
     /**
      * Represents the uncertain value
      *
@@ -2766,7 +2737,7 @@
      *
      * @public exported from `@promptbook/core`
      */
-    $deepFreeze({
+    const UNCERTAIN_USAGE = $deepFreeze({
         price: UNCERTAIN_ZERO_VALUE,
         input: {
             tokensCount: UNCERTAIN_ZERO_VALUE,
@@ -2791,6 +2762,35 @@
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
+    /**
+     * Async version of Array.forEach
+     *
+     * @param array - Array to iterate over
+     * @param options - Options for the function
+     * @param callbackfunction - Function to call for each item
+     * @public exported from `@promptbook/utils`
+     * @deprecated [🪂] Use queues instead
+     */
+    async function forEachAsync(array, options, callbackfunction) {
+        const { maxParallelCount = Infinity } = options;
+        let index = 0;
+        let runningTasks = [];
+        const tasks = [];
+        for (const item of array) {
+            const currentIndex = index++;
+            const task = callbackfunction(item, currentIndex, array);
+            tasks.push(task);
+            runningTasks.push(task);
+            /* not await */ Promise.resolve(task).then(() => {
+                runningTasks = runningTasks.filter((t) => t !== task);
+            });
+            if (maxParallelCount < runningTasks.length) {
+                await Promise.race(runningTasks);
+            }
+        }
+        await Promise.all(tasks);
+    }
+
     /**
      * Function `addUsage` will add multiple usages into one
      *
@@ -4084,6 +4084,24 @@
         // encoding: 'utf-8',
     });
 
+    /**
+     * Converts a CSV string into an object
+     *
+     * Note: This is wrapper around `papaparse.parse()` with better autohealing
+     *
+     * @private - for now until `@promptbook/csv` is released
+     */
+    function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
+        settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
+        // Note: Autoheal invalid '\n' characters
+        if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
+            console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
+            value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+        }
+        const csv = papaparse.parse(value, settings);
+        return csv;
+    }
+
     /**
      * Function to check if a string is valid CSV
      *
@@ -4106,31 +4124,13 @@
         }
     }
 
-    /**
-     * Converts a CSV string into an object
-     *
-     * Note: This is wrapper around `papaparse.parse()` with better autohealing
-     *
-     * @private - for now until `@promptbook/csv` is released
-     */
-    function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
-        settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
-        // Note: Autoheal invalid '\n' characters
-        if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
-            console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
-            value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
-        }
-        const csv = papaparse.parse(value, settings);
-        return csv;
-    }
-
     /**
      * Definition for CSV spreadsheet
      *
      * @public exported from `@promptbook/core`
      *                          <- TODO: [🏢] Export from package `@promptbook/csv`
      */
-    const CsvFormatDefinition = {
+    const CsvFormatParser = {
         formatName: 'CSV',
         aliases: ['SPREADSHEET', 'TABLE'],
         isValid(value, settings, schema) {
@@ -4142,7 +4142,7 @@
         heal(value, settings, schema) {
             throw new Error('Not implemented');
         },
-        subvalueDefinitions: [
+        subvalueParsers: [
             {
                 subvalueName: 'ROW',
                 async mapValues(value, outputParameterName, settings, mapCallback) {
@@ -4203,10 +4203,10 @@
         ],
     };
     /**
-     * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `CsvFormatDefinition` implement `heal
-     * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `CsvFormatParser` implement `heal
+     * TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside CSV objects and other formats
      */
 
@@ -4215,7 +4215,7 @@
      *
      * @private still in development [🏢]
      */
-    const JsonFormatDefinition = {
+    const JsonFormatParser = {
         formatName: 'JSON',
         mimeType: 'application/json',
         isValid(value, settings, schema) {
@@ -4227,28 +4227,28 @@
         heal(value, settings, schema) {
             throw new Error('Not implemented');
         },
-        subvalueDefinitions: [],
+        subvalueParsers: [],
     };
     /**
      * TODO: [🧠] Maybe propper instance of object
      * TODO: [0] Make string_serialized_json
      * TODO: [1] Make type for JSON Settings and Schema
      * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
-     * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `JsonFormatDefinition` implement `heal
-     * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `JsonFormatParser` implement `heal
+     * TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside JSON objects and other formats
      */
 
     /**
      * Definition for any text - this will be always valid
      *
-     * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
+     * Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
      *
      * @public exported from `@promptbook/core`
      */
-    const TextFormatDefinition = {
+    const TextFormatParser = {
         formatName: 'TEXT',
         isValid(value) {
             return typeof value === 'string';
@@ -4257,9 +4257,9 @@
             return typeof partialValue === 'string';
         },
         heal() {
-            throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
+            throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
         },
-        subvalueDefinitions: [
+        subvalueParsers: [
             {
                 subvalueName: 'LINE',
                 async mapValues(value, outputParameterName, settings, mapCallback) {
@@ -4279,10 +4279,10 @@
     /**
      * TODO: [1] Make type for XML Text and Schema
      * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
-     * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `TextFormatDefinition` implement `heal
-     * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `TextFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `TextFormatParser` implement `heal
+     * TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside each item of list and other formats
      */
 
@@ -4315,7 +4315,7 @@
      *
      * @private still in development [🏢]
      */
-    const XmlFormatDefinition = {
+    const XmlFormatParser = {
         formatName: 'XML',
         mimeType: 'application/xml',
         isValid(value, settings, schema) {
@@ -4327,17 +4327,17 @@
         heal(value, settings, schema) {
             throw new Error('Not implemented');
         },
-        subvalueDefinitions: [],
+        subvalueParsers: [],
     };
     /**
      * TODO: [🧠] Maybe propper instance of object
      * TODO: [0] Make string_serialized_xml
      * TODO: [1] Make type for XML Settings and Schema
      * TODO: [🧠] What to use for validating XMLs - XSD,...
-     * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `XmlFormatDefinition` implement `heal
-     * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `XmlFormatParser` implement `heal
+     * TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside XML and other formats
      */
 
@@ -4346,12 +4346,7 @@
      *
      * @private internal index of `...` <- TODO [🏢]
      */
-    const FORMAT_DEFINITIONS = [
-        JsonFormatDefinition,
-        XmlFormatDefinition,
-        TextFormatDefinition,
-        CsvFormatDefinition,
-    ];
+    const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
     /**
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
@@ -4521,7 +4516,7 @@
     }
     /**
      * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
-     * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
+     * TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
      */
 
     /**
@@ -4723,7 +4718,7 @@
         PAGES: countPages,
     };
     /**
-     * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
+     * TODO: [🧠][🤠] This should be probbably as part of `TextFormatParser`
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
@@ -4751,7 +4746,7 @@
     }
     /**
      * TODO: [💝] Unite object for expecting amount and format
-     * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
+     * TODO: [🧠][🤠] This should be part of `TextFormatParser`
      * Note: [💝] and [🤠] are interconnected together
      */
 
@@ -4979,7 +4974,7 @@
                 if (task.format) {
                     if (task.format === 'JSON') {
                         if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
-                            // TODO: [🏢] Do more universally via `FormatDefinition`
+                            // TODO: [🏢] Do more universally via `FormatParser`
                             try {
                                 $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
                             }
@@ -5117,16 +5112,16 @@
                     ${block(pipelineIdentification)}
                 `));
         }
-        const subvalueDefinition = formatDefinition.subvalueDefinitions.find((subvalueDefinition) => [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(task.foreach.subformatName));
-        if (subvalueDefinition === undefined) {
+        const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
+        if (subvalueParser === undefined) {
             throw new UnexpectedError(
             // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
             spaceTrim__default["default"]((block) => `
                     Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
 
                     Available subformat names for format "${formatDefinition.formatName}":
-                    ${block(formatDefinition.subvalueDefinitions
-            .map((subvalueDefinition) => subvalueDefinition.subvalueName)
+                    ${block(formatDefinition.subvalueParsers
+            .map((subvalueParser) => subvalueParser.subvalueName)
             .map((subvalueName) => `- ${subvalueName}`)
             .join('\n'))}
 
@@ -5140,7 +5135,7 @@
             formatSettings = csvSettings;
             // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
         }
-        const resultString = await subvalueDefinition.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
+        const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
             let mappedParameters;
             // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
             // TODO: When done [🐚] Report progress also for each subvalue here
@@ -5202,6 +5197,27 @@
         return RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */;
     }
 
+    /**
+     * Computes the cosine similarity between two embedding vectors
+     *
+     * Note: This is helping function for RAG (retrieval-augmented generation)
+     *
+     * @param embeddingVector1
+     * @param embeddingVector2
+     * @returns Cosine similarity between the two vectors
+     *
+     * @public exported from `@promptbook/core`
+     */
+    function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
+        if (embeddingVector1.length !== embeddingVector2.length) {
+            throw new TypeError('Embedding vectors must have the same length');
+        }
+        const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
+        const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
+        const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
+        return 1 - dotProduct / (magnitude1 * magnitude2);
+    }
+
     /**
      * @@@
      *
@@ -5228,7 +5244,7 @@
             },
             content: task.content,
             parameters: {
-            /* !!!!!!!! */
+            /* !!!! */
             },
         };
         const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
@@ -5263,16 +5279,6 @@
         return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
         //                                                      <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
     }
-    // TODO: !!!!!! Annotate + to new file
-    function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
-        if (embeddingVector1.length !== embeddingVector2.length) {
-            throw new TypeError('Embedding vectors must have the same length');
-        }
-        const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
-        const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
-        const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
-        return 1 - dotProduct / (magnitude1 * magnitude2);
-    }
     /**
      * TODO: !!!! Verify if this is working
      * TODO: [♨] Implement Better - use keyword search
@@ -5481,6 +5487,7 @@
          * Note: This is a flag to prevent `onProgress` call after the pipeline execution is finished
          */
         let isReturned = false;
+        console.log(`!!! preparedPipeline`, preparedPipeline);
         // Note: Check that all input input parameters are defined
         for (const parameter of preparedPipeline.parameters.filter(({ isInput }) => isInput)) {
             if (inputParameters[parameter.name] === undefined) {
@@ -5775,6 +5782,22 @@
                 cacheDirname,
                 intermediateFilesStrategy,
                 isAutoInstalled,
+            }).catch((error) => {
+                assertsError(error);
+                return exportJson({
+                    name: 'pipelineExecutorResult',
+                    message: `Unuccessful PipelineExecutorResult, last catch`,
+                    order: [],
+                    value: {
+                        isSuccessful: false,
+                        errors: [serializeError(error)],
+                        warnings: [],
+                        usage: UNCERTAIN_USAGE,
+                        executionReport: null,
+                        outputParameters: {},
+                        preparedPipeline,
+                    },
+                });
             });
         };
         const pipelineExecutor = (inputParameters) => createTask({