@promptbook/node 0.92.0-21 → 0.92.0-23

package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts

@@ -1,10 +1,11 @@
 import type { string_parameter_name } from '../../../types/typeAliases';
 /**
- * Function `validateParameterName` will @@@
+ * Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
+ * It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
  *
- * @param parameterName @@@
- * @returns @@@
- * @throws {ParseError} @@@
+ * @param parameterName The parameter name to validate and normalize.
+ * @returns The validated and normalized parameter name.
+ * @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
  * @private within the repository
  */
 export declare function validateParameterName(parameterName: string): string_parameter_name;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/node",
-    "version": "0.92.0-21",
+    "version": "0.92.0-23",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,
@@ -51,7 +51,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/node.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.92.0-21"
+        "@promptbook/core": "0.92.0-23"
     },
     "dependencies": {
         "colors": "1.4.0",

package/umd/index.umd.js

@@ -46,7 +46,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-21';
+    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-23';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -117,6 +117,12 @@
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+    /**
+     * @@@
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const BIG_DATASET_TRESHOLD = 50;
     // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
     /**
      * The maximum number of iterations for a loops
@@ -211,7 +217,7 @@
     const DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = `index`;
     // <- TODO: [🧜‍♂️]
     /**
-     * @@@
+     * Default settings for parsing and generating CSV files in Promptbook.
      *
      * @public exported from `@promptbook/core`
      */
@@ -222,19 +228,19 @@
         skipEmptyLines: true,
     });
     /**
-     * @@@
+     * Controls whether verbose logging is enabled by default throughout the application.
      *
      * @public exported from `@promptbook/core`
      */
     let DEFAULT_IS_VERBOSE = false;
     /**
-     * @@@
+     * Controls whether auto-installation of dependencies is enabled by default.
      *
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_IS_AUTO_INSTALLED = false;
     /**
-     * @@@
+     * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
      *
      * @private within the repository
      */
@@ -580,8 +586,12 @@
      */
 
     /**
-     * @@@
+     * Creates a deep clone of the given object
      *
+     * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
+     *
+     * @param objectValue The object to clone.
+     * @returns A deep, writable clone of the input object.
      * @public exported from `@promptbook/utils`
      */
     function deepClone(objectValue) {
@@ -1220,7 +1230,7 @@
      * Function isValidJsonString will tell you if the string is valid JSON or not
      *
      * @param value The string to check
-     * @returns True if the string is a valid JSON string, false otherwise
+     * @returns `true` if the string is a valid JSON string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -2562,7 +2572,7 @@
      * Function to check if a string is valid CSV
      *
      * @param value The string to check
-     * @returns True if the string is a valid CSV string, false otherwise
+     * @returns `true` if the string is a valid CSV string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -2601,7 +2611,8 @@
         subvalueParsers: [
             {
                 subvalueName: 'ROW',
-                async mapValues(value, outputParameterName, settings, mapCallback) {
+                async mapValues(options) {
+                    const { value, outputParameterName, settings, mapCallback, onProgress } = options;
                     const csv = csvParse(value, settings);
                     if (csv.errors.length !== 0) {
                         throw new CsvFormatError(spaceTrim__default["default"]((block) => `
@@ -2617,21 +2628,30 @@
                                 ${block(value)}
                             `));
                     }
-                    const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                    const mappedData = [];
+                    const length = csv.data.length;
+                    for (let index = 0; index < length; index++) {
+                        const row = csv.data[index];
                         if (row[outputParameterName]) {
                             throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                         }
-                        return {
+                        const mappedRow = {
                             ...row,
-                            [outputParameterName]: await mapCallback(row, index),
+                            [outputParameterName]: await mapCallback(row, index, length),
                         };
-                    }));
+                        mappedData.push(mappedRow);
+                        if (onProgress) {
+                            // Note: Report the CSV with all rows mapped so far
+                            await onProgress(papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                        }
+                    }
                     return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
                 },
             },
             {
                 subvalueName: 'CELL',
-                async mapValues(value, outputParameterName, settings, mapCallback) {
+                async mapValues(options) {
+                    const { value, settings, mapCallback, onProgress } = options;
                     const csv = csvParse(value, settings);
                     if (csv.errors.length !== 0) {
                         throw new CsvFormatError(spaceTrim__default["default"]((block) => `
@@ -2648,9 +2668,9 @@
                             `));
                     }
                     const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                             const index = rowIndex * Object.keys(row).length + columnIndex;
-                            return /* not await */ mapCallback({ [key]: value }, index);
+                            return /* not await */ mapCallback({ [key]: value }, index, array.length);
                         }));
                     }));
                     return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -2718,14 +2738,15 @@
         subvalueParsers: [
             {
                 subvalueName: 'LINE',
-                async mapValues(value, outputParameterName, settings, mapCallback) {
+                async mapValues(options) {
+                    const { value, mapCallback, onProgress } = options;
                     const lines = value.split('\n');
-                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                     // TODO: [🧠] Maybe option to skip empty line
                     /* not await */ mapCallback({
                         lineContent,
                         // TODO: [🧠] Maybe also put here `lineNumber`
-                    }, lineNumber)));
+                    }, lineNumber, array.length)));
                     return mappedLines.join('\n');
                 },
             },
@@ -2746,7 +2767,7 @@
      * Function to check if a string is valid XML
      *
      * @param value
-     * @returns True if the string is a valid XML string, false otherwise
+     * @returns `true` if the string is a valid XML string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -2808,13 +2829,13 @@
      */
 
     /**
-     * Maps available parameters to expected parameters
+     * Maps available parameters to expected parameters for a pipeline task.
      *
      * The strategy is:
-     * 1) @@@
-     * 2) @@@
+     * 1) First, match parameters by name where both available and expected.
+     * 2) Then, if there are unmatched expected and available parameters, map them by order.
      *
-     * @throws {PipelineExecutionError} @@@
+     * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
      * @private within the repository used in `createPipelineExecutor`
      */
     function mapAvailableToExpectedParameters(options) {
@@ -4023,12 +4044,16 @@
      */
 
     /**
-     * @@@
+     * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+     * Handles format and subformat resolution, error handling, and progress reporting.
+     *
+     * @param options - Options for execution, including task details and progress callback.
+     * @returns The result of the subvalue mapping or execution attempts.
      *
      * @private internal utility of `createPipelineExecutor`
      */
     async function executeFormatSubvalues(options) {
-        const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+        const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
         if (task.foreach === undefined) {
             return /* not await */ executeAttempts(options);
         }
@@ -4082,46 +4107,74 @@
             formatSettings = csvSettings;
             // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
         }
-        const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-            let mappedParameters;
-            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-            // TODO: When done [🐚] Report progress also for each subvalue here
-            try {
-                mappedParameters = mapAvailableToExpectedParameters({
-                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                    availableParameters: subparameters,
-                });
-            }
-            catch (error) {
-                if (!(error instanceof PipelineExecutionError)) {
-                    throw error;
+        const resultString = await subvalueParser.mapValues({
+            value: parameterValue,
+            outputParameterName: task.foreach.outputSubparameterName,
+            settings: formatSettings,
+            onProgress(partialResultString) {
+                return onProgress(Object.freeze({
+                    [task.resultingParameterName]: partialResultString,
+                }));
+            },
+            async mapCallback(subparameters, index, length) {
+                let mappedParameters;
+                try {
+                    mappedParameters = mapAvailableToExpectedParameters({
+                        expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                        availableParameters: subparameters,
+                    });
                 }
-                throw new PipelineExecutionError(spaceTrim__default["default"]((block) => `
-                        ${error.message}
+                catch (error) {
+                    if (!(error instanceof PipelineExecutionError)) {
+                        throw error;
+                    }
+                    const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping data
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
-            }
-            const allSubparameters = {
-                ...parameters,
-                ...mappedParameters,
-            };
-            // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-            Object.freeze(allSubparameters);
-            const subresultString = await executeAttempts({
-                ...options,
-                priority: priority + index,
-                parameters: allSubparameters,
-                pipelineIdentification: spaceTrim__default["default"]((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-            });
-            return subresultString;
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `));
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(highLevelError);
+                        return '~';
+                    }
+                    throw highLevelError;
+                }
+                const allSubparameters = {
+                    ...parameters,
+                    ...mappedParameters,
+                };
+                Object.freeze(allSubparameters);
+                try {
+                    const subresultString = await executeAttempts({
+                        ...options,
+                        priority: priority + index,
+                        parameters: allSubparameters,
+                        pipelineIdentification: spaceTrim__default["default"]((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                    });
+                    return subresultString;
+                }
+                catch (error) {
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(spaceTrim__default["default"]((block) => `
+                              Error in FOREACH command:
+
+                              ${block(pipelineIdentification)}
+
+                              ${block(pipelineIdentification)}
+                              Subparameter index: ${index}
+                          `));
+                        return '~';
+                    }
+                    throw error;
+                }
+            },
         });
         return resultString;
     }
@@ -4255,7 +4308,11 @@
      */
 
     /**
-     * @@@
+     * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+     * Ensures all reserved parameters are defined and throws if any are missing.
+     *
+     * @param options - Options including tools, pipeline, task, and context.
+     * @returns An object containing all reserved parameters for the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -4288,18 +4345,16 @@
     }
 
     /**
-     * @@@
+     * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+     *
+     * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+     * @returns The output parameters produced by the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
     async function executeTask(options) {
         const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
         const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-        await onProgress({
-            outputParameters: {
-                [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-            },
-        });
         // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
         const usedParameterNames = extractParameterNamesFromTask(currentTask);
         const dependentParameterNames = new Set(currentTask.dependentParameterNames);
@@ -4374,6 +4429,7 @@
             preparedPipeline,
             tools,
             $executionReport,
+            onProgress,
             pipelineIdentification,
             maxExecutionAttempts,
             maxParallelCount,
@@ -4426,9 +4482,12 @@
     }
 
     /**
-     * @@@
+     * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
+     *
+     * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
      *
-     * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+     * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+     * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5029,11 +5088,11 @@
     }
 
     /**
-     * Register is @@@
+     * Global registry for storing and managing registered entities of a given type.
      *
      * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
      *
-     * @private internal utility, exported are only signleton instances of this class
+     * @private internal utility, exported are only singleton instances of this class
      */
     class $Register {
         constructor(registerName) {
@@ -5077,10 +5136,10 @@
     }
 
     /**
-     * @@@
+     * Global registry for storing metadata about all available scrapers and converters.
      *
-     * Note: `$` is used to indicate that this interacts with the global scope
-     * @singleton Only one instance of each register is created per build, but thare can be more @@@
+     * Note: `$` is used to indicate that this interacts with the global scope.
+     * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
      * @public exported from `@promptbook/core`
      */
     const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -6619,11 +6678,12 @@
     }
 
     /**
-     * Function `validateParameterName` will @@@
+     * Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
+     * It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
      *
-     * @param parameterName @@@
-     * @returns @@@
-     * @throws {ParseError} @@@
+     * @param parameterName The parameter name to validate and normalize.
+     * @returns The validated and normalized parameter name.
+     * @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
      * @private within the repository
      */
     function validateParameterName(parameterName) {
@@ -7797,10 +7857,10 @@
     }
 
     /**
-     * @@@
+     * Checks if the given value is a valid JavaScript identifier name.
      *
-     * @param javascriptName @@@
-     * @returns @@@
+     * @param javascriptName The value to check for JavaScript identifier validity.
+     * @returns `true` if the value is a valid JavaScript name, false otherwise.
      * @public exported from `@promptbook/utils`
      */
     function isValidJavascriptName(javascriptName) {