@promptbook/cli 0.92.0-21 → 0.92.0-23

package/esm/index.es.js

@@ -47,7 +47,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-21';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-23';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -163,6 +163,13 @@ const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/core`
+ */
+const BIG_DATASET_TRESHOLD = 50;
+// <- TODO: !!!! Use
 /**
  * Warning message for the generated sections and files files
  *
@@ -247,6 +254,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
+// <- TODO: [🐝]
 /**
  * Where to store your books
  * This is kind of a "src" for your books
@@ -318,7 +326,7 @@ const MOMENT_ARG_THRESHOLDS = {
 const DEFAULT_REMOTE_SERVER_URL = REMOTE_SERVER_URLS[0].urls[0];
 // <- TODO: [🧜‍♂️]
 /**
- * @@@
+ * Default settings for parsing and generating CSV files in Promptbook.
  *
  * @public exported from `@promptbook/core`
  */
@@ -329,13 +337,13 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
     skipEmptyLines: true,
 });
 /**
- * @@@
+ * Controls whether verbose logging is enabled by default throughout the application.
  *
  * @public exported from `@promptbook/core`
  */
 let DEFAULT_IS_VERBOSE = false;
 /**
- * @@@
+ * Controls whether auto-installation of dependencies is enabled by default.
  *
  * @public exported from `@promptbook/core`
  */
@@ -355,7 +363,7 @@ const DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = `getPipelineCollection`;
  */
 const DEFAULT_RPM$1 = 60;
 /**
- * @@@
+ * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
  *
  * @private within the repository
  */
@@ -774,11 +782,11 @@ function normalizeTo_snake_case(text) {
 }
 
 /**
- * Register is @@@
+ * Global registry for storing and managing registered entities of a given type.
  *
  * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
  *
- * @private internal utility, exported are only signleton instances of this class
+ * @private internal utility, exported are only singleton instances of this class
  */
 class $Register {
     constructor(registerName) {
@@ -1472,8 +1480,12 @@ function checkSerializableAsJson(options) {
  */
 
 /**
- * @@@
+ * Creates a deep clone of the given object
  *
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
+ *
+ * @param objectValue The object to clone.
+ * @returns A deep, writable clone of the input object.
  * @public exported from `@promptbook/utils`
  */
 function deepClone(objectValue) {
@@ -2720,8 +2732,10 @@ function cacheLlmTools(llmTools, options = {}) {
             sha256(hexEncoder.parse(JSON.stringify({ parameters, content, modelRequirements }))).toString( /* hex */));
         const cacheItem = !isCacheReloaded ? await storage.getItem(key) : null;
         if (cacheItem) {
+            console.log('!!! Cache hit for key:', key);
             return cacheItem.promptResult;
         }
+        console.log('!!! Cache miss for key:', key);
         let promptResult;
         variant: switch (prompt.modelRequirements.modelVariant) {
             case 'CHAT':
@@ -3929,10 +3943,10 @@ async function $provideScrapersForNode(tools, options) {
  */
 
 /**
- * @@@
+ * Global registry for storing metadata about all available scrapers and converters.
  *
- * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * Note: `$` is used to indicate that this interacts with the global scope.
+ * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
  * @public exported from `@promptbook/core`
  */
 const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -4512,7 +4526,7 @@ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"
  * Function isValidJsonString will tell you if the string is valid JSON or not
  *
  * @param value The string to check
- * @returns True if the string is a valid JSON string, false otherwise
+ * @returns `true` if the string is a valid JSON string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -5426,7 +5440,7 @@ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO:
  * Function to check if a string is valid CSV
  *
  * @param value The string to check
- * @returns True if the string is a valid CSV string, false otherwise
+ * @returns `true` if the string is a valid CSV string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -5465,7 +5479,8 @@ const CsvFormatParser = {
     subvalueParsers: [
         {
             subvalueName: 'ROW',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, outputParameterName, settings, mapCallback, onProgress } = options;
                 const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim((block) => `
@@ -5481,21 +5496,30 @@ const CsvFormatParser = {
                                 ${block(value)}
                             `));
                 }
-                const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                const mappedData = [];
+                const length = csv.data.length;
+                for (let index = 0; index < length; index++) {
+                    const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
-                    return {
+                    const mappedRow = {
                         ...row,
-                        [outputParameterName]: await mapCallback(row, index),
+                        [outputParameterName]: await mapCallback(row, index, length),
                     };
-                }));
+                    mappedData.push(mappedRow);
+                    if (onProgress) {
+                        // Note: Report the CSV with all rows mapped so far
+                        await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                    }
+                }
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
             },
         },
         {
             subvalueName: 'CELL',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, settings, mapCallback, onProgress } = options;
                 const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim((block) => `
@@ -5512,9 +5536,9 @@ const CsvFormatParser = {
                             `));
                 }
                 const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                         const index = rowIndex * Object.keys(row).length + columnIndex;
-                        return /* not await */ mapCallback({ [key]: value }, index);
+                        return /* not await */ mapCallback({ [key]: value }, index, array.length);
                     }));
                 }));
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -5582,14 +5606,15 @@ const TextFormatParser = {
     subvalueParsers: [
         {
             subvalueName: 'LINE',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
-                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                 // TODO: [🧠] Maybe option to skip empty line
                 /* not await */ mapCallback({
                     lineContent,
                     // TODO: [🧠] Maybe also put here `lineNumber`
-                }, lineNumber)));
+                }, lineNumber, array.length)));
                 return mappedLines.join('\n');
             },
         },
@@ -5610,7 +5635,7 @@ const TextFormatParser = {
  * Function to check if a string is valid XML
  *
  * @param value
- * @returns True if the string is a valid XML string, false otherwise
+ * @returns `true` if the string is a valid XML string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -5672,13 +5697,13 @@ const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser,
  */
 
 /**
- * Maps available parameters to expected parameters
+ * Maps available parameters to expected parameters for a pipeline task.
  *
  * The strategy is:
- * 1) @@@
- * 2) @@@
+ * 1) First, match parameters by name where both available and expected.
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
  *
- * @throws {PipelineExecutionError} @@@
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
  * @private within the repository used in `createPipelineExecutor`
  */
 function mapAvailableToExpectedParameters(options) {
@@ -6417,12 +6442,16 @@ async function executeAttempts(options) {
  */
 
 /**
- * @@@
+ * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+ * Handles format and subformat resolution, error handling, and progress reporting.
+ *
+ * @param options - Options for execution, including task details and progress callback.
+ * @returns The result of the subvalue mapping or execution attempts.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeFormatSubvalues(options) {
-    const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+    const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
     if (task.foreach === undefined) {
         return /* not await */ executeAttempts(options);
     }
@@ -6476,46 +6505,74 @@ async function executeFormatSubvalues(options) {
         formatSettings = csvSettings;
         // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
     }
-    const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-        let mappedParameters;
-        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-        // TODO: When done [🐚] Report progress also for each subvalue here
-        try {
-            mappedParameters = mapAvailableToExpectedParameters({
-                expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                availableParameters: subparameters,
-            });
-        }
-        catch (error) {
-            if (!(error instanceof PipelineExecutionError)) {
-                throw error;
+    const resultString = await subvalueParser.mapValues({
+        value: parameterValue,
+        outputParameterName: task.foreach.outputSubparameterName,
+        settings: formatSettings,
+        onProgress(partialResultString) {
+            return onProgress(Object.freeze({
+                [task.resultingParameterName]: partialResultString,
+            }));
+        },
+        async mapCallback(subparameters, index, length) {
+            let mappedParameters;
+            try {
+                mappedParameters = mapAvailableToExpectedParameters({
+                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                    availableParameters: subparameters,
+                });
             }
-            throw new PipelineExecutionError(spaceTrim((block) => `
-                        ${error.message}
+            catch (error) {
+                if (!(error instanceof PipelineExecutionError)) {
+                    throw error;
+                }
+                const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping data
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
-        }
-        const allSubparameters = {
-            ...parameters,
-            ...mappedParameters,
-        };
-        // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-        Object.freeze(allSubparameters);
-        const subresultString = await executeAttempts({
-            ...options,
-            priority: priority + index,
-            parameters: allSubparameters,
-            pipelineIdentification: spaceTrim((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-        });
-        return subresultString;
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `));
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(highLevelError);
+                    return '~';
+                }
+                throw highLevelError;
+            }
+            const allSubparameters = {
+                ...parameters,
+                ...mappedParameters,
+            };
+            Object.freeze(allSubparameters);
+            try {
+                const subresultString = await executeAttempts({
+                    ...options,
+                    priority: priority + index,
+                    parameters: allSubparameters,
+                    pipelineIdentification: spaceTrim((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                });
+                return subresultString;
+            }
+            catch (error) {
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(spaceTrim((block) => `
+                              Error in FOREACH command:
+
+                              ${block(pipelineIdentification)}
+
+                              ${block(pipelineIdentification)}
+                              Subparameter index: ${index}
+                          `));
+                    return '~';
+                }
+                throw error;
+            }
+        },
     });
     return resultString;
 }
@@ -6649,7 +6706,11 @@ async function getKnowledgeForTask(options) {
  */
 
 /**
- * @@@
+ * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+ * Ensures all reserved parameters are defined and throws if any are missing.
+ *
+ * @param options - Options including tools, pipeline, task, and context.
+ * @returns An object containing all reserved parameters for the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -6682,18 +6743,16 @@ async function getReservedParametersForTask(options) {
 }
 
 /**
- * @@@
+ * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+ *
+ * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+ * @returns The output parameters produced by the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeTask(options) {
     const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
     const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-    await onProgress({
-        outputParameters: {
-            [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-        },
-    });
     // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
     const usedParameterNames = extractParameterNamesFromTask(currentTask);
     const dependentParameterNames = new Set(currentTask.dependentParameterNames);
@@ -6768,6 +6827,7 @@ async function executeTask(options) {
         preparedPipeline,
         tools,
         $executionReport,
+        onProgress,
         pipelineIdentification,
         maxExecutionAttempts,
         maxParallelCount,
@@ -6820,9 +6880,12 @@ function filterJustOutputParameters(options) {
 }
 
 /**
- * @@@
+ * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
  *
- * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+ * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
+ *
+ * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+ * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -8548,11 +8611,12 @@ function removeQuotes(text) {
 }
 
 /**
- * Function `validateParameterName` will @@@
+ * Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
+ * It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
  *
- * @param parameterName @@@
- * @returns @@@
- * @throws {ParseError} @@@
+ * @param parameterName The parameter name to validate and normalize.
+ * @returns The validated and normalized parameter name.
+ * @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
  * @private within the repository
  */
 function validateParameterName(parameterName) {
@@ -9726,10 +9790,10 @@ function $applyToTaskJson(command, $taskJson, $pipelineJson) {
 }
 
 /**
- * @@@
+ * Checks if the given value is a valid JavaScript identifier name.
  *
- * @param javascriptName @@@
- * @returns @@@
+ * @param javascriptName The value to check for JavaScript identifier validity.
+ * @returns `true` if the value is a valid JavaScript name, false otherwise.
  * @public exported from `@promptbook/utils`
  */
 function isValidJavascriptName(javascriptName) {