@promptbook/documents 0.92.0-22 → 0.92.0-24

package/esm/typings/src/utils/environment/$getGlobalScope.d.ts

@@ -1,6 +1,7 @@
 import type { really_any } from '../organization/really_any';
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *

package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts

@@ -1,26 +1,26 @@
 import type { string_parameter_name } from '../../types/typeAliases';
 import type { string_parameter_value } from '../../types/typeAliases';
 /**
- * @@@
+ * Options for mapping available parameters to expected parameters in a pipeline task.
  */
 type MakeapAvailableToExpectedParametersOptions = {
     /**
-     * @@@
+     * The set of expected parameter names (keys) for the task, all values are null.
      */
     readonly expectedParameters: Readonly<Record<string_parameter_name, null>>;
     /**
-     * @@@
+     * The set of available parameters (name-value pairs) to map to the expected parameters.
      */
     readonly availableParameters: Readonly<Record<string_parameter_name, string_parameter_value>>;
 };
 /**
- * Maps available parameters to expected parameters
+ * Maps available parameters to expected parameters for a pipeline task.
  *
  * The strategy is:
- * 1) @@@
- * 2) @@@
+ * 1) First, match parameters by name where both available and expected.
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
  *
- * @throws {PipelineExecutionError} @@@
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
  * @private within the repository used in `createPipelineExecutor`
  */
 export declare function mapAvailableToExpectedParameters(options: MakeapAvailableToExpectedParametersOptions): Readonly<Record<string_parameter_name, string_parameter_value>>;

package/esm/typings/src/utils/serialization/clonePipeline.d.ts

@@ -1,10 +1,11 @@
 import type { PipelineJson } from '../../pipeline/PipelineJson/PipelineJson';
 /**
- * @@@
+ * Creates a deep clone of a PipelineJson object, copying all properties explicitly.
  *
- * Note: It is usefull @@@
+ * Note: It is useful for ensuring that modifications to the returned pipeline do not affect the original.
  *
- * @param pipeline
+ * @param pipeline The pipeline to clone.
+ * @returns A new PipelineJson object with the same properties as the input.
  * @public exported from `@promptbook/utils`
  */
 export declare function clonePipeline(pipeline: PipelineJson): PipelineJson;

package/esm/typings/src/utils/serialization/deepClone.d.ts

@@ -1,7 +1,11 @@
 import type { WritableDeep } from 'type-fest';
 /**
- * @@@
+ * Creates a deep clone of the given object
  *
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
+ *
+ * @param objectValue The object to clone.
+ * @returns A deep, writable clone of the input object.
  * @public exported from `@promptbook/utils`
  */
 export declare function deepClone<TObject>(objectValue: TObject): WritableDeep<TObject>;

package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts

@@ -1,10 +1,10 @@
 import type { string_javascript_name } from '../../../types/typeAliases';
 import type { really_unknown } from '../../organization/really_unknown';
 /**
- * @@@
+ * Checks if the given value is a valid JavaScript identifier name.
  *
- * @param javascriptName @@@
- * @returns @@@
+ * @param javascriptName The value to check for JavaScript identifier validity.
+ * @returns `true` if the value is a valid JavaScript name, false otherwise.
  * @public exported from `@promptbook/utils`
  */
 export declare function isValidJavascriptName(javascriptName: really_unknown): javascriptName is string_javascript_name;

package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts

@@ -1,10 +1,11 @@
 import type { string_parameter_name } from '../../../types/typeAliases';
 /**
- * Function `validateParameterName` will @@@
+ * Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
+ * It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
  *
- * @param parameterName @@@
- * @returns @@@
- * @throws {ParseError} @@@
+ * @param parameterName The parameter name to validate and normalize.
+ * @returns The validated and normalized parameter name.
+ * @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
  * @private within the repository
  */
 export declare function validateParameterName(parameterName: string): string_parameter_name;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/documents",
-    "version": "0.92.0-22",
+    "version": "0.92.0-24",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,
@@ -51,7 +51,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/documents.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.92.0-22"
+        "@promptbook/core": "0.92.0-24"
     },
     "dependencies": {
         "colors": "1.4.0",

package/umd/index.umd.js

@@ -26,7 +26,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-22';
+    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-24';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -102,6 +102,21 @@
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+    /**
+     * Threshold value that determines when a dataset is considered "big"
+     * and may require special handling or optimizations
+     *
+     * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const BIG_DATASET_TRESHOLD = 50;
+    /**
+     * Placeholder text used to represent a placeholder value of failed operation
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const FAILED_VALUE_PLACEHOLDER = '!?';
     // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
     /**
      * The maximum number of iterations for a loops
@@ -181,7 +196,7 @@
     const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
     // <- TODO: [🧜‍♂️]
     /**
-     * @@@
+     * Default settings for parsing and generating CSV files in Promptbook.
      *
      * @public exported from `@promptbook/core`
      */
@@ -192,19 +207,19 @@
         skipEmptyLines: true,
     });
     /**
-     * @@@
+     * Controls whether verbose logging is enabled by default throughout the application.
      *
      * @public exported from `@promptbook/core`
      */
     let DEFAULT_IS_VERBOSE = false;
     /**
-     * @@@
+     * Controls whether auto-installation of dependencies is enabled by default.
      *
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_IS_AUTO_INSTALLED = false;
     /**
-     * @@@
+     * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
      *
      * @private within the repository
      */
@@ -1122,7 +1137,7 @@
      * Function isValidJsonString will tell you if the string is valid JSON or not
      *
      * @param value The string to check
-     * @returns True if the string is a valid JSON string, false otherwise
+     * @returns `true` if the string is a valid JSON string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -1533,8 +1548,12 @@
      */
 
     /**
-     * @@@
+     * Creates a deep clone of the given object
+     *
+     * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
      *
+     * @param objectValue The object to clone.
+     * @returns A deep, writable clone of the input object.
      * @public exported from `@promptbook/utils`
      */
     function deepClone(objectValue) {
@@ -2121,7 +2140,7 @@
         /**
          * Constructs a pipeline collection from pipelines
          *
-         * @param pipelines @@@
+         * @param pipelines Array of pipeline JSON objects to include in the collection
          *
          * Note: During the construction logic of all pipelines are validated
          * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -3240,7 +3259,8 @@
      */
 
     /**
-     * @@@
+     * Safely retrieves the global scope object (window in browser, global in Node.js)
+     * regardless of the JavaScript environment in which the code is running
      *
      * Note: `$` is used to indicate that this function is not a pure function - it access global scope
      *
@@ -3319,11 +3339,11 @@
     }
 
     /**
-     * Register is @@@
+     * Global registry for storing and managing registered entities of a given type.
      *
      * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
      *
-     * @private internal utility, exported are only signleton instances of this class
+     * @private internal utility, exported are only singleton instances of this class
      */
     class $Register {
         constructor(registerName) {
@@ -3367,10 +3387,10 @@
     }
 
     /**
-     * @@@
+     * Global registry for storing metadata about all available scrapers and converters.
      *
-     * Note: `$` is used to indicate that this interacts with the global scope
-     * @singleton Only one instance of each register is created per build, but thare can be more @@@
+     * Note: `$` is used to indicate that this interacts with the global scope.
+     * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
      * @public exported from `@promptbook/core`
      */
     const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -4261,7 +4281,7 @@
      * Function to check if a string is valid CSV
      *
      * @param value The string to check
-     * @returns True if the string is a valid CSV string, false otherwise
+     * @returns `true` if the string is a valid CSV string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -4318,18 +4338,28 @@
                             `));
                     }
                     const mappedData = [];
-                    for (let index = 0; index < csv.data.length; index++) {
+                    const length = csv.data.length;
+                    for (let index = 0; index < length; index++) {
                         const row = csv.data[index];
                         if (row[outputParameterName]) {
                             throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                         }
                         const mappedRow = {
                             ...row,
-                            [outputParameterName]: await mapCallback(row, index),
+                            [outputParameterName]: await mapCallback(row, index, length),
                         };
                         mappedData.push(mappedRow);
                         if (onProgress) {
                             // Note: Report the CSV with all rows mapped so far
+                            /*
+                            !!!!
+                             // Report progress with updated value
+                                const progressData = mappedData.map((row, i) =>
+                                  i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                              );
+
+
+                            */
                             await onProgress(papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
                         }
                     }
@@ -4356,9 +4386,9 @@
                             `));
                     }
                     const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                             const index = rowIndex * Object.keys(row).length + columnIndex;
-                            return /* not await */ mapCallback({ [key]: value }, index);
+                            return /* not await */ mapCallback({ [key]: value }, index, array.length);
                         }));
                     }));
                     return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -4429,12 +4459,12 @@
                 async mapValues(options) {
                     const { value, mapCallback, onProgress } = options;
                     const lines = value.split('\n');
-                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                     // TODO: [🧠] Maybe option to skip empty line
                     /* not await */ mapCallback({
                         lineContent,
                         // TODO: [🧠] Maybe also put here `lineNumber`
-                    }, lineNumber)));
+                    }, lineNumber, array.length)));
                     return mappedLines.join('\n');
                 },
             },
@@ -4455,7 +4485,7 @@
      * Function to check if a string is valid XML
      *
      * @param value
-     * @returns True if the string is a valid XML string, false otherwise
+     * @returns `true` if the string is a valid XML string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -4517,13 +4547,13 @@
      */
 
     /**
-     * Maps available parameters to expected parameters
+     * Maps available parameters to expected parameters for a pipeline task.
      *
      * The strategy is:
-     * 1) @@@
-     * 2) @@@
+     * 1) First, match parameters by name where both available and expected.
+     * 2) Then, if there are unmatched expected and available parameters, map them by order.
      *
-     * @throws {PipelineExecutionError} @@@
+     * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
      * @private within the repository used in `createPipelineExecutor`
      */
     function mapAvailableToExpectedParameters(options) {
@@ -5243,7 +5273,11 @@
      */
 
     /**
-     * @@@
+     * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+     * Handles format and subformat resolution, error handling, and progress reporting.
+     *
+     * @param options - Options for execution, including task details and progress callback.
+     * @returns The result of the subvalue mapping or execution attempts.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5308,15 +5342,11 @@
             settings: formatSettings,
             onProgress(partialResultString) {
                 return onProgress(Object.freeze({
-                    [task.resultingParameterName]: 
-                    // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
-                    partialResultString,
+                    [task.resultingParameterName]: partialResultString,
                 }));
             },
-            async mapCallback(subparameters, index) {
+            async mapCallback(subparameters, index, length) {
                 let mappedParameters;
-                // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-                // TODO: When done [🐚] Report progress also for each subvalue here
                 try {
                     mappedParameters = mapAvailableToExpectedParameters({
                         expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
@@ -5327,32 +5357,52 @@
                     if (!(error instanceof PipelineExecutionError)) {
                         throw error;
                     }
-                    throw new PipelineExecutionError(spaceTrim__default["default"]((block) => `
-                        ${error.message}
+                    const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping data
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `));
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(highLevelError);
+                        return FAILED_VALUE_PLACEHOLDER;
+                    }
+                    throw highLevelError;
                 }
                 const allSubparameters = {
                     ...parameters,
                     ...mappedParameters,
                 };
-                // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
                 Object.freeze(allSubparameters);
-                const subresultString = await executeAttempts({
-                    ...options,
-                    priority: priority + index,
-                    parameters: allSubparameters,
-                    pipelineIdentification: spaceTrim__default["default"]((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-                });
-                return subresultString;
+                try {
+                    const subresultString = await executeAttempts({
+                        ...options,
+                        priority: priority + index,
+                        parameters: allSubparameters,
+                        pipelineIdentification: spaceTrim__default["default"]((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                    });
+                    return subresultString;
+                }
+                catch (error) {
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(spaceTrim__default["default"]((block) => `
+                              Error in FOREACH command:
+
+                              ${block(pipelineIdentification)}
+
+                              ${block(pipelineIdentification)}
+                              Subparameter index: ${index}
+                          `));
+                        return FAILED_VALUE_PLACEHOLDER;
+                    }
+                    throw error;
+                }
             },
         });
         return resultString;
@@ -5487,7 +5537,11 @@
      */
 
     /**
-     * @@@
+     * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+     * Ensures all reserved parameters are defined and throws if any are missing.
+     *
+     * @param options - Options including tools, pipeline, task, and context.
+     * @returns An object containing all reserved parameters for the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5520,7 +5574,10 @@
     }
 
     /**
-     * @@@
+     * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+     *
+     * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+     * @returns The output parameters produced by the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5654,9 +5711,12 @@
     }
 
     /**
-     * @@@
+     * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
+     *
+     * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
      *
-     * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+     * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+     * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
      *
      * @private internal utility of `createPipelineExecutor`
      */