@promptbook/documents 0.92.0-22 → 0.92.0-23

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/documents",
-    "version": "0.92.0-22",
+    "version": "0.92.0-23",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,
@@ -51,7 +51,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/documents.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.92.0-22"
+        "@promptbook/core": "0.92.0-23"
     },
     "dependencies": {
         "colors": "1.4.0",

package/umd/index.umd.js

@@ -26,7 +26,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-22';
+    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-23';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -102,6 +102,12 @@
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+    /**
+     * @@@
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const BIG_DATASET_TRESHOLD = 50;
     // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
     /**
      * The maximum number of iterations for a loops
@@ -181,7 +187,7 @@
     const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
     // <- TODO: [🧜‍♂️]
     /**
-     * @@@
+     * Default settings for parsing and generating CSV files in Promptbook.
      *
      * @public exported from `@promptbook/core`
      */
@@ -192,19 +198,19 @@
         skipEmptyLines: true,
     });
     /**
-     * @@@
+     * Controls whether verbose logging is enabled by default throughout the application.
      *
      * @public exported from `@promptbook/core`
      */
     let DEFAULT_IS_VERBOSE = false;
     /**
-     * @@@
+     * Controls whether auto-installation of dependencies is enabled by default.
      *
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_IS_AUTO_INSTALLED = false;
     /**
-     * @@@
+     * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
      *
      * @private within the repository
      */
@@ -1122,7 +1128,7 @@
      * Function isValidJsonString will tell you if the string is valid JSON or not
      *
      * @param value The string to check
-     * @returns True if the string is a valid JSON string, false otherwise
+     * @returns `true` if the string is a valid JSON string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -1533,8 +1539,12 @@
      */
 
     /**
-     * @@@
+     * Creates a deep clone of the given object
+     *
+     * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
      *
+     * @param objectValue The object to clone.
+     * @returns A deep, writable clone of the input object.
      * @public exported from `@promptbook/utils`
      */
     function deepClone(objectValue) {
@@ -3319,11 +3329,11 @@
     }
 
     /**
-     * Register is @@@
+     * Global registry for storing and managing registered entities of a given type.
      *
      * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
      *
-     * @private internal utility, exported are only signleton instances of this class
+     * @private internal utility, exported are only singleton instances of this class
      */
     class $Register {
         constructor(registerName) {
@@ -3367,10 +3377,10 @@
     }
 
     /**
-     * @@@
+     * Global registry for storing metadata about all available scrapers and converters.
      *
-     * Note: `$` is used to indicate that this interacts with the global scope
-     * @singleton Only one instance of each register is created per build, but thare can be more @@@
+     * Note: `$` is used to indicate that this interacts with the global scope.
+     * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
      * @public exported from `@promptbook/core`
      */
     const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -4261,7 +4271,7 @@
      * Function to check if a string is valid CSV
      *
      * @param value The string to check
-     * @returns True if the string is a valid CSV string, false otherwise
+     * @returns `true` if the string is a valid CSV string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -4318,14 +4328,15 @@
                             `));
                     }
                     const mappedData = [];
-                    for (let index = 0; index < csv.data.length; index++) {
+                    const length = csv.data.length;
+                    for (let index = 0; index < length; index++) {
                         const row = csv.data[index];
                         if (row[outputParameterName]) {
                             throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                         }
                         const mappedRow = {
                             ...row,
-                            [outputParameterName]: await mapCallback(row, index),
+                            [outputParameterName]: await mapCallback(row, index, length),
                         };
                         mappedData.push(mappedRow);
                         if (onProgress) {
@@ -4356,9 +4367,9 @@
                             `));
                     }
                     const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                             const index = rowIndex * Object.keys(row).length + columnIndex;
-                            return /* not await */ mapCallback({ [key]: value }, index);
+                            return /* not await */ mapCallback({ [key]: value }, index, array.length);
                         }));
                     }));
                     return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -4429,12 +4440,12 @@
                 async mapValues(options) {
                     const { value, mapCallback, onProgress } = options;
                     const lines = value.split('\n');
-                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                     // TODO: [🧠] Maybe option to skip empty line
                     /* not await */ mapCallback({
                         lineContent,
                         // TODO: [🧠] Maybe also put here `lineNumber`
-                    }, lineNumber)));
+                    }, lineNumber, array.length)));
                     return mappedLines.join('\n');
                 },
             },
@@ -4455,7 +4466,7 @@
      * Function to check if a string is valid XML
      *
      * @param value
-     * @returns True if the string is a valid XML string, false otherwise
+     * @returns `true` if the string is a valid XML string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -4517,13 +4528,13 @@
      */
 
     /**
-     * Maps available parameters to expected parameters
+     * Maps available parameters to expected parameters for a pipeline task.
      *
      * The strategy is:
-     * 1) @@@
-     * 2) @@@
+     * 1) First, match parameters by name where both available and expected.
+     * 2) Then, if there are unmatched expected and available parameters, map them by order.
      *
-     * @throws {PipelineExecutionError} @@@
+     * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
      * @private within the repository used in `createPipelineExecutor`
      */
     function mapAvailableToExpectedParameters(options) {
@@ -5243,7 +5254,11 @@
      */
 
     /**
-     * @@@
+     * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+     * Handles format and subformat resolution, error handling, and progress reporting.
+     *
+     * @param options - Options for execution, including task details and progress callback.
+     * @returns The result of the subvalue mapping or execution attempts.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5308,15 +5323,11 @@
             settings: formatSettings,
             onProgress(partialResultString) {
                 return onProgress(Object.freeze({
-                    [task.resultingParameterName]: 
-                    // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
-                    partialResultString,
+                    [task.resultingParameterName]: partialResultString,
                 }));
             },
-            async mapCallback(subparameters, index) {
+            async mapCallback(subparameters, index, length) {
                 let mappedParameters;
-                // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-                // TODO: When done [🐚] Report progress also for each subvalue here
                 try {
                     mappedParameters = mapAvailableToExpectedParameters({
                         expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
@@ -5327,32 +5338,52 @@
                     if (!(error instanceof PipelineExecutionError)) {
                         throw error;
                     }
-                    throw new PipelineExecutionError(spaceTrim__default["default"]((block) => `
-                        ${error.message}
+                    const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping data
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `));
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(highLevelError);
+                        return '~';
+                    }
+                    throw highLevelError;
                 }
                 const allSubparameters = {
                     ...parameters,
                     ...mappedParameters,
                 };
-                // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
                 Object.freeze(allSubparameters);
-                const subresultString = await executeAttempts({
-                    ...options,
-                    priority: priority + index,
-                    parameters: allSubparameters,
-                    pipelineIdentification: spaceTrim__default["default"]((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-                });
-                return subresultString;
+                try {
+                    const subresultString = await executeAttempts({
+                        ...options,
+                        priority: priority + index,
+                        parameters: allSubparameters,
+                        pipelineIdentification: spaceTrim__default["default"]((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                    });
+                    return subresultString;
+                }
+                catch (error) {
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(spaceTrim__default["default"]((block) => `
+                              Error in FOREACH command:
+
+                              ${block(pipelineIdentification)}
+
+                              ${block(pipelineIdentification)}
+                              Subparameter index: ${index}
+                          `));
+                        return '~';
+                    }
+                    throw error;
+                }
             },
         });
         return resultString;
@@ -5487,7 +5518,11 @@
      */
 
     /**
-     * @@@
+     * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+     * Ensures all reserved parameters are defined and throws if any are missing.
+     *
+     * @param options - Options including tools, pipeline, task, and context.
+     * @returns An object containing all reserved parameters for the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5520,7 +5555,10 @@
     }
 
     /**
-     * @@@
+     * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+     *
+     * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+     * @returns The output parameters produced by the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5654,9 +5692,12 @@
     }
 
     /**
-     * @@@
+     * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
+     *
+     * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
      *
-     * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+     * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+     * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
      *
      * @private internal utility of `createPipelineExecutor`
      */