@promptbook/markdown-utils 0.92.0-9 → 0.92.0

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.92.0-9';
+    const PROMPTBOOK_ENGINE_VERSION = '0.92.0';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -59,8 +59,8 @@
     /**
      * Extracts all code blocks from markdown.
      *
-     * Note: There are multiple simmilar function:
-     * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
+     * Note: There are multiple similar functions:
+     * - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
      * - `extractJsonBlock` extracts exactly one valid JSON code block
      * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
      * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
@@ -110,7 +110,7 @@
                 if (currentCodeBlock.content !== '') {
                     currentCodeBlock.content += '\n';
                 }
-                currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
+                currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
             }
         }
         if (currentCodeBlock !== null) {
@@ -127,8 +127,8 @@
      *
      * - When there are multiple or no code blocks the function throws a `ParseError`
      *
-     * Note: There are multiple simmilar function:
-     * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
+     * Note: There are multiple similar functions:
+     * - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
      * - `extractJsonBlock` extracts exactly one valid JSON code block
      * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
      * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
@@ -158,7 +158,7 @@
      *
      * - When there are multiple or no code blocks the function throws a `ParseError`
      *
-     * Note: There are multiple simmilar function:
+     * Note: There are multiple similar function:
      * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
      * - `extractJsonBlock` extracts exactly one valid JSON code block
      * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
@@ -176,7 +176,7 @@
      * Returns the same value that is passed as argument.
      * No side effects.
      *
-     * Note: It can be usefull for:
+     * Note: It can be useful for:
      *
      * 1) Leveling indentation
      * 2) Putting always-true or always-false conditions without getting eslint errors
@@ -231,6 +231,21 @@
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+    /**
+     * Threshold value that determines when a dataset is considered "big"
+     * and may require special handling or optimizations
+     *
+     * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const BIG_DATASET_TRESHOLD = 50;
+    /**
+     * Placeholder text used to represent a placeholder value of failed operation
+     *
+     * @public exported from `@promptbook/core`
+     */
+    const FAILED_VALUE_PLACEHOLDER = '!?';
     // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
     /**
      * The maximum number of iterations for a loops
@@ -310,7 +325,7 @@
     const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
     // <- TODO: [🧜‍♂️]
     /**
-     * @@@
+     * Default settings for parsing and generating CSV files in Promptbook.
      *
      * @public exported from `@promptbook/core`
      */
@@ -321,19 +336,19 @@
         skipEmptyLines: true,
     });
     /**
-     * @@@
+     * Controls whether verbose logging is enabled by default throughout the application.
      *
      * @public exported from `@promptbook/core`
      */
     let DEFAULT_IS_VERBOSE = false;
     /**
-     * @@@
+     * Controls whether auto-installation of dependencies is enabled by default.
      *
      * @public exported from `@promptbook/core`
      */
     const DEFAULT_IS_AUTO_INSTALLED = false;
     /**
-     * @@@
+     * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
      *
      * @private within the repository
      */
@@ -401,7 +416,7 @@
                     ${block(message)}
 
                     Note: This error should not happen.
-                    It's probbably a bug in the pipeline collection
+                    It's probably a bug in the pipeline collection
 
                     Please report issue:
                     ${block(getErrorReportUrl(new Error(message)).href)}
@@ -466,7 +481,7 @@
      * Function isValidJsonString will tell you if the string is valid JSON or not
      *
      * @param value The string to check
-     * @returns True if the string is a valid JSON string, false otherwise
+     * @returns `true` if the string is a valid JSON string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -492,7 +507,7 @@
      * - When there are multiple JSON code blocks the function throws a `ParseError`
      *
      * Note: It is not important if marked as ```json BUT if it is VALID JSON
-     * Note: There are multiple simmilar function:
+     * Note: There are multiple similar function:
      * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
      * - `extractJsonBlock` extracts exactly one valid JSON code block
      * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
@@ -517,14 +532,14 @@
     }
     /**
      * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
-     * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
+     * TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
      */
 
     /**
      * Just says that the variable is not used but should be kept
      * No side effects.
      *
-     * Note: It can be usefull for:
+     * Note: It can be useful for:
      *
      * 1) Suppressing eager optimization of unused imports
      * 2) Suppressing eslint errors of unused variables in the tests
@@ -607,7 +622,7 @@
      * Tests if given string is valid URL.
      *
      * Note: Dataurl are considered perfectly valid.
-     * Note: There are two simmilar functions:
+     * Note: There are two similar functions:
      * - `isValidUrl` which tests any URL
      * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
      *
@@ -717,7 +732,7 @@
         if (bookVersion !== `undefined`) {
             commands.push(`BOOK VERSION ${bookVersion}`);
         }
-        // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary
+        // TODO: [main] !!5 This increases size of the bundle and is probably not necessary
         pipelineString = prettifyMarkdown(pipelineString);
         for (const parameter of parameters.filter(({ isInput }) => isInput)) {
             commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
@@ -1016,8 +1031,12 @@
      */
 
     /**
-     * @@@
+     * Creates a deep clone of the given object
+     *
+     * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
      *
+     * @param objectValue The object to clone.
+     * @returns A deep, writable clone of the input object.
      * @public exported from `@promptbook/utils`
      */
     function deepClone(objectValue) {
@@ -1099,13 +1118,13 @@
      */
     const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
     /**
-     * @@@
+     * Placeholder value indicating a parameter is missing its value.
      *
      * @private within the repository
      */
     const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
     /**
-     * @@@
+     * Placeholder value indicating a parameter is restricted and cannot be used directly.
      *
      * @private within the repository
      */
@@ -1150,7 +1169,7 @@
     /**
      * Tests if given string is valid semantic version
      *
-     * Note: There are two simmilar functions:
+     * Note: There are two similar functions:
      * - `isValidSemanticVersion` which tests any semantic version
      * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
      *
@@ -1172,7 +1191,7 @@
      *
      * @see https://www.npmjs.com/package/promptbook?activeTab=versions
      * Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
-     * Note: There are two simmilar functions:
+     * Note: There are two similar functions:
      * - `isValidSemanticVersion` which tests any semantic version
      * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
      *
@@ -1192,7 +1211,7 @@
     /**
      * Tests if given string is valid pipeline URL URL.
      *
-     * Note: There are two simmilar functions:
+     * Note: There are two similar functions:
      * - `isValidUrl` which tests any URL
      * - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
      *
@@ -1289,7 +1308,7 @@
                     ${block(pipelineIdentification)}
                 `));
         }
-        // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
+        // TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
         if (!Array.isArray(pipeline.parameters)) {
             // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
             throw new ParseError(spaceTrim.spaceTrim((block) => `
@@ -1300,7 +1319,7 @@
                     ${block(pipelineIdentification)}
                 `));
         }
-        // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
+        // TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
         if (!Array.isArray(pipeline.tasks)) {
             // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
             throw new ParseError(spaceTrim.spaceTrim((block) => `
@@ -1604,7 +1623,7 @@
         /**
          * Constructs a pipeline collection from pipelines
          *
-         * @param pipelines @@@
+         * @param pipelines Array of pipeline JSON objects to include in the collection
          *
          * Note: During the construction logic of all pipelines are validated
          * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -1718,7 +1737,7 @@
             super(spaceTrim.spaceTrim((block) => `
                     ${block(message)}
 
-                    Note: You have probbably forgot to provide some tools for pipeline execution or preparation
+                    Note: You have probably forgot to provide some tools for pipeline execution or preparation
 
                 `));
             this.name = 'MissingToolsError';
@@ -1752,7 +1771,7 @@
             super(message);
             this.name = 'PipelineExecutionError';
             // TODO: [🐙] DRY - Maybe $randomId
-            this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
+            this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts   */)}`;
             Object.setPrototypeOf(this, PipelineExecutionError.prototype);
         }
     }
@@ -1768,15 +1787,18 @@
      * @public exported from `@promptbook/core`
      */
     function isPipelinePrepared(pipeline) {
-        // Note: Ignoring `pipeline.preparations` @@@
-        // Note: Ignoring `pipeline.knowledgePieces` @@@
+        // Note: Ignoring `pipeline.preparations`
+        // Note: Ignoring `pipeline.knowledgePieces`
         if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
+            // console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
             return false;
         }
         if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
+            // console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
             return false;
         }
         if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
+            //console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
             return false;
         }
         /*
@@ -1797,75 +1819,6 @@
      *     - [♨] Are tasks prepared
      */
 
-    /**
-     * Converts a JavaScript Object Notation (JSON) string into an object.
-     *
-     * Note: This is wrapper around `JSON.parse()` with better error and type handling
-     *
-     * @public exported from `@promptbook/utils`
-     */
-    function jsonParse(value) {
-        if (value === undefined) {
-            throw new Error(`Can not parse JSON from undefined value.`);
-        }
-        else if (typeof value !== 'string') {
-            console.error('Can not parse JSON from non-string value.', { text: value });
-            throw new Error(spaceTrim__default["default"](`
-                    Can not parse JSON from non-string value.
-
-                    The value type: ${typeof value}
-                    See more in console.
-                `));
-        }
-        try {
-            return JSON.parse(value);
-        }
-        catch (error) {
-            if (!(error instanceof Error)) {
-                throw error;
-            }
-            throw new Error(spaceTrim__default["default"]((block) => `
-                    ${block(error.message)}
-            
-                    The JSON text:
-                    ${block(value)}
-                `));
-        }
-    }
-    /**
-     * TODO: !!!! Use in Promptbook.studio
-     */
-
-    /**
-     * Recursively converts JSON strings to JSON objects
-
-     * @public exported from `@promptbook/utils`
-     */
-    function jsonStringsToJsons(object) {
-        if (object === null) {
-            return object;
-        }
-        if (Array.isArray(object)) {
-            return object.map(jsonStringsToJsons);
-        }
-        if (typeof object !== 'object') {
-            return object;
-        }
-        const newObject = { ...object };
-        for (const [key, value] of Object.entries(object)) {
-            if (typeof value === 'string' && isValidJsonString(value)) {
-                newObject[key] = jsonParse(value);
-            }
-            else {
-                newObject[key] = jsonStringsToJsons(value);
-            }
-        }
-        return newObject;
-    }
-    /**
-     * TODO: Type the return type correctly
-     */
-
     /**
      * This error indicates problems parsing the format value
      *
@@ -1910,7 +1863,7 @@
     }
 
     /**
-     * This error indicates that the pipeline collection cannot be propperly loaded
+     * This error indicates that the pipeline collection cannot be properly loaded
      *
      * @public exported from `@promptbook/core`
      */
@@ -2075,6 +2028,101 @@
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
+    /**
+     * Serializes an error into a [🚉] JSON-serializable object
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    function serializeError(error) {
+        const { name, message, stack } = error;
+        const { id } = error;
+        if (!Object.keys(ALL_ERRORS).includes(name)) {
+            console.error(spaceTrim__default["default"]((block) => `
+
+                    Cannot serialize error with name "${name}"
+
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
+                    ${block(stack || message)}
+
+                `));
+        }
+        return {
+            name: name,
+            message,
+            stack,
+            id, // Include id in the serialized object
+        };
+    }
+
+    /**
+     * Converts a JavaScript Object Notation (JSON) string into an object.
+     *
+     * Note: This is wrapper around `JSON.parse()` with better error and type handling
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    function jsonParse(value) {
+        if (value === undefined) {
+            throw new Error(`Can not parse JSON from undefined value.`);
+        }
+        else if (typeof value !== 'string') {
+            console.error('Can not parse JSON from non-string value.', { text: value });
+            throw new Error(spaceTrim__default["default"](`
+                    Can not parse JSON from non-string value.
+
+                    The value type: ${typeof value}
+                    See more in console.
+                `));
+        }
+        try {
+            return JSON.parse(value);
+        }
+        catch (error) {
+            if (!(error instanceof Error)) {
+                throw error;
+            }
+            throw new Error(spaceTrim__default["default"]((block) => `
+                    ${block(error.message)}
+
+                    The JSON text:
+                    ${block(value)}
+                `));
+        }
+    }
+
+    /**
+     * Recursively converts JSON strings to JSON objects
+
+     * @public exported from `@promptbook/utils`
+     */
+    function jsonStringsToJsons(object) {
+        if (object === null) {
+            return object;
+        }
+        if (Array.isArray(object)) {
+            return object.map(jsonStringsToJsons);
+        }
+        if (typeof object !== 'object') {
+            return object;
+        }
+        const newObject = { ...object };
+        for (const [key, value] of Object.entries(object)) {
+            if (typeof value === 'string' && isValidJsonString(value)) {
+                newObject[key] = jsonParse(value);
+            }
+            else {
+                newObject[key] = jsonStringsToJsons(value);
+            }
+        }
+        return newObject;
+    }
+    /**
+     * TODO: Type the return type correctly
+     */
+
     /**
      * Deserializes the error object
      *
@@ -2149,8 +2197,9 @@
      */
     function createTask(options) {
         const { taskType, taskProcessCallback } = options;
+        let { title } = options;
         // TODO: [🐙] DRY
-        const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
+        const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts   */)}`;
         let status = 'RUNNING';
         const createdAt = new Date();
         let updatedAt = createdAt;
@@ -2160,6 +2209,10 @@
         const partialResultSubject = new rxjs.Subject();
         // <- Note: Not using `BehaviorSubject` because on error we can't access the last value
         const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
+            if (newOngoingResult.title) {
+                title = newOngoingResult.title;
+            }
+            updatedAt = new Date();
             Object.assign(currentValue, newOngoingResult);
             // <- TODO: assign deep
             partialResultSubject.next(newOngoingResult);
@@ -2205,17 +2258,24 @@
         return {
             taskType,
             taskId,
+            get promptbookVersion() {
+                return PROMPTBOOK_ENGINE_VERSION;
+            },
+            get title() {
+                return title;
+                // <- Note: [1] These must be getters to allow changing the value in the future
+            },
             get status() {
                 return status;
-                // <- Note: [1] Theese must be getters to allow changing the value in the future
+                // <- Note: [1] --||--
             },
             get createdAt() {
                 return createdAt;
-                // <- Note: [1]
+                // <- Note: [1] --||--
             },
             get updatedAt() {
                 return updatedAt;
-                // <- Note: [1]
+                // <- Note: [1] --||--
             },
             asPromise,
             asObservable() {
@@ -2223,15 +2283,15 @@
             },
             get errors() {
                 return errors;
-                // <- Note: [1]
+                // <- Note: [1] --||--
             },
             get warnings() {
                 return warnings;
-                // <- Note: [1]
+                // <- Note: [1] --||--
             },
             get currentValue() {
                 return currentValue;
-                // <- Note: [1]
+                // <- Note: [1] --||--
             },
         };
     }
@@ -2240,64 +2300,6 @@
      * TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
      */
 
-    /**
-     * Serializes an error into a [🚉] JSON-serializable object
-     *
-     * @public exported from `@promptbook/utils`
-     */
-    function serializeError(error) {
-        const { name, message, stack } = error;
-        const { id } = error;
-        if (!Object.keys(ALL_ERRORS).includes(name)) {
-            console.error(spaceTrim__default["default"]((block) => `
-
-                    Cannot serialize error with name "${name}"
-
-                    Authors of Promptbook probably forgot to add this error into the list of errors:
-                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
-
-
-                    ${block(stack || message)}
-
-                `));
-        }
-        return {
-            name: name,
-            message,
-            stack,
-            id, // Include id in the serialized object
-        };
-    }
-
-    /**
-     * Async version of Array.forEach
-     *
-     * @param array - Array to iterate over
-     * @param options - Options for the function
-     * @param callbackfunction - Function to call for each item
-     * @public exported from `@promptbook/utils`
-     * @deprecated [🪂] Use queues instead
-     */
-    async function forEachAsync(array, options, callbackfunction) {
-        const { maxParallelCount = Infinity } = options;
-        let index = 0;
-        let runningTasks = [];
-        const tasks = [];
-        for (const item of array) {
-            const currentIndex = index++;
-            const task = callbackfunction(item, currentIndex, array);
-            tasks.push(task);
-            runningTasks.push(task);
-            /* not await */ Promise.resolve(task).then(() => {
-                runningTasks = runningTasks.filter((t) => t !== task);
-            });
-            if (maxParallelCount < runningTasks.length) {
-                await Promise.race(runningTasks);
-            }
-        }
-        await Promise.all(tasks);
-    }
-
     /**
      * Represents the uncertain value
      *
@@ -2341,7 +2343,7 @@
      *
      * @public exported from `@promptbook/core`
      */
-    $deepFreeze({
+    const UNCERTAIN_USAGE = $deepFreeze({
         price: UNCERTAIN_ZERO_VALUE,
         input: {
             tokensCount: UNCERTAIN_ZERO_VALUE,
@@ -2366,6 +2368,35 @@
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
+    /**
+     * Async version of Array.forEach
+     *
+     * @param array - Array to iterate over
+     * @param options - Options for the function
+     * @param callbackfunction - Function to call for each item
+     * @public exported from `@promptbook/utils`
+     * @deprecated [🪂] Use queues instead
+     */
+    async function forEachAsync(array, options, callbackfunction) {
+        const { maxParallelCount = Infinity } = options;
+        let index = 0;
+        let runningTasks = [];
+        const tasks = [];
+        for (const item of array) {
+            const currentIndex = index++;
+            const task = callbackfunction(item, currentIndex, array);
+            tasks.push(task);
+            runningTasks.push(task);
+            /* not await */ Promise.resolve(task).then(() => {
+                runningTasks = runningTasks.filter((t) => t !== task);
+            });
+            if (maxParallelCount < runningTasks.length) {
+                await Promise.race(runningTasks);
+            }
+        }
+        await Promise.all(tasks);
+    }
+
     /**
      * Function `addUsage` will add multiple usages into one
      *
@@ -2425,12 +2456,14 @@
         const spending = new rxjs.Subject();
         const proxyTools = {
             get title() {
-                // TODO: [🧠] Maybe put here some suffix
-                return llmTools.title;
+                return `${llmTools.title} (+usage)`;
+                // <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
+                // <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
             },
             get description() {
-                // TODO: [🧠] Maybe put here some suffix
-                return llmTools.description;
+                return `${llmTools.description} (+usage)`;
+                // <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
+                // <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
             },
             checkConfiguration() {
                 return /* not await */ llmTools.checkConfiguration();
@@ -2501,29 +2534,40 @@
             return 'Multiple LLM Providers';
         }
         get description() {
-            return this.llmExecutionTools.map(({ title }, index) => `${index + 1}) \`${title}\``).join('\n');
+            const innerModelsTitlesAndDescriptions = this.llmExecutionTools
+                .map(({ title, description }, index) => {
+                const headLine = `${index + 1}) \`${title}\``;
+                if (description === undefined) {
+                    return headLine;
+                }
+                return spaceTrim__default["default"]((block) => `
+                        ${headLine}
+                        
+                          ${ /* <- Note: Indenting the description: */block(description)}
+                    `);
+            })
+                .join('\n\n');
+            return spaceTrim__default["default"]((block) => `
+                Multiple LLM Providers:
+
+                ${block(innerModelsTitlesAndDescriptions)}
+            `);
         }
         /**
          * Check the configuration of all execution tools
          */
         async checkConfiguration() {
-            // TODO: Maybe do it in parallel
-            for (const llmExecutionTools of this.llmExecutionTools) {
-                await llmExecutionTools.checkConfiguration();
-            }
+            // Note: Run checks in parallel
+            await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
         }
         /**
          * List all available models that can be used
          * This lists is a combination of all available models from all execution tools
          */
         async listModels() {
-            const availableModels = [];
-            for (const llmExecutionTools of this.llmExecutionTools) {
-                // TODO: [🪂] Obtain models in parallel
-                const models = await llmExecutionTools.listModels();
-                availableModels.push(...models);
-            }
-            return availableModels;
+            // Obtain all models in parallel and flatten
+            const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
+            return modelArrays.flat();
         }
         /**
          * Calls the best available chat model
@@ -2764,7 +2808,8 @@
      */
 
     /**
-     * @@@
+     * Safely retrieves the global scope object (window in browser, global in Node.js)
+     * regardless of the JavaScript environment in which the code is running
      *
      * Note: `$` is used to indicate that this function is not a pure function - it access global scope
      *
@@ -2775,10 +2820,10 @@
     }
 
     /**
-     * @@@
+     * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
      *
-     * @param text @@@
-     * @returns @@@
+     * @param text The text string to be converted to SCREAMING_CASE format.
+     * @returns The normalized text in SCREAMING_CASE format.
      * @example 'HELLO_WORLD'
      * @example 'I_LOVE_PROMPTBOOK'
      * @public exported from `@promptbook/utils`
@@ -2830,10 +2875,10 @@
      */
 
     /**
-     * @@@
+     * Normalizes a text string to snake_case format.
      *
-     * @param text @@@
-     * @returns @@@
+     * @param text The text string to be converted to snake_case format.
+     * @returns The normalized text in snake_case format.
      * @example 'hello_world'
      * @example 'i_love_promptbook'
      * @public exported from `@promptbook/utils`
@@ -2843,11 +2888,11 @@
     }
 
     /**
-     * Register is @@@
+     * Global registry for storing and managing registered entities of a given type.
      *
      * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
      *
-     * @private internal utility, exported are only signleton instances of this class
+     * @private internal utility, exported are only singleton instances of this class
      */
     class $Register {
         constructor(registerName) {
@@ -2891,10 +2936,10 @@
     }
 
     /**
-     * @@@
+     * Global registry for storing metadata about all available scrapers and converters.
      *
-     * Note: `$` is used to indicate that this interacts with the global scope
-     * @singleton Only one instance of each register is created per build, but thare can be more @@@
+     * Note: `$` is used to indicate that this interacts with the global scope.
+     * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
      * @public exported from `@promptbook/core`
      */
     const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -2903,10 +2948,11 @@
      */
 
     /**
-     * @@@
+     * Registry for all available scrapers in the system.
+     * Central point for registering and accessing different types of content scrapers.
      *
      * Note: `$` is used to indicate that this interacts with the global scope
-     * @singleton Only one instance of each register is created per build, but thare can be more @@@
+     * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
      * @public exported from `@promptbook/core`
      */
     const $scrapersRegister = new $Register('scraper_constructors');
@@ -3246,10 +3292,10 @@
     */
 
     /**
-     * @@@
+     * Removes diacritic marks (accents) from characters in a string.
      *
-     * @param input @@@
-     * @returns @@@
+     * @param input The string containing diacritics to be normalized.
+     * @returns The string with diacritics removed or normalized.
      * @public exported from `@promptbook/utils`
      */
     function removeDiacritics(input) {
@@ -3263,10 +3309,10 @@
      */
 
     /**
-     * @@@
+     * Converts a given text to kebab-case format.
      *
-     * @param text @@@
-     * @returns @@@
+     * @param text The text to be converted.
+     * @returns The kebab-case formatted string.
      * @example 'hello-world'
      * @example 'i-love-promptbook'
      * @public exported from `@promptbook/utils`
@@ -3335,7 +3381,8 @@
      */
 
     /**
-     * @@@
+     * Converts a name to a properly formatted subfolder path for cache storage.
+     * Handles normalization and path formatting to create consistent cache directory structures.
      *
      * @private for `FileCacheStorage`
      */
@@ -3415,11 +3462,11 @@
     }
 
     /**
-     * @@@
+     * Converts a title string into a normalized name.
      *
-     * @param value @@@
-     * @returns @@@
-     * @example @@@
+     * @param value The title string to be converted to a name.
+     * @returns A normalized name derived from the input title.
+     * @example 'Hello World!' -> 'hello-world'
      * @public exported from `@promptbook/utils`
      */
     function titleToName(value) {
@@ -3470,7 +3517,9 @@
      */
 
     /**
-     * @@@
+     * Factory function that creates a handler for processing knowledge sources.
+     * Provides standardized processing of different types of knowledge sources
+     * across various scraper implementations.
      *
      * @public exported from `@promptbook/core`
      */
@@ -3609,7 +3658,7 @@
     }
 
     /**
-     * Prepares the knowle
+     * Prepares the knowledge pieces
      *
      * @see https://github.com/webgptorg/promptbook/discussions/41
      * @public exported from `@promptbook/core`
@@ -3705,15 +3754,18 @@
      * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
      *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
      * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
-     * TODO: [🧠][❎] Do here propper M:N mapping
+     * TODO: [🧠][❎] Do here proper M:N mapping
      *       [x] One source can make multiple pieces
      *       [ ] One piece can have multiple sources
      */
 
     /**
-     * @@@
+     * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
      *
-     * @public exported from `@promptbook/core`
+     * @param tasks Sequence of tasks that are chained together to form a pipeline
+     * @returns A promise that resolves to the prepared tasks.
+     *
+     * @private internal utility of `preparePipeline`
      */
     async function prepareTasks(pipeline, tools, options) {
         const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -3751,7 +3803,7 @@
         return { tasksPrepared };
     }
     /**
-     * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, simmilar thing with expectations to sync high-level abstractions
+     * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
      * TODO: [🧠] Add context to each task (if missing)
      * TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
      * TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
@@ -3887,7 +3939,7 @@
             order: ORDER_OF_PIPELINE_JSON,
             value: {
                 ...pipeline,
-                // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects
+                // <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
                 title,
                 knowledgeSources: knowledgeSourcesPrepared,
                 knowledgePieces: knowledgePiecesPrepared,
@@ -4150,7 +4202,7 @@
     }
 
     /**
-     * @@@
+     * Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
      *
      * @public exported from `@promptbook/core`
      */
@@ -4159,11 +4211,29 @@
         // encoding: 'utf-8',
     });
 
+    /**
+     * Converts a CSV string into an object
+     *
+     * Note: This is wrapper around `papaparse.parse()` with better autohealing
+     *
+     * @private - for now until `@promptbook/csv` is released
+     */
+    function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
+        settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
+        // Note: Autoheal invalid '\n' characters
+        if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
+            console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
+            value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+        }
+        const csv = papaparse.parse(value, settings);
+        return csv;
+    }
+
     /**
      * Function to check if a string is valid CSV
      *
      * @param value The string to check
-     * @returns True if the string is a valid CSV string, false otherwise
+     * @returns `true` if the string is a valid CSV string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -4187,7 +4257,7 @@
      * @public exported from `@promptbook/core`
      *                          <- TODO: [🏢] Export from package `@promptbook/csv`
      */
-    const CsvFormatDefinition = {
+    const CsvFormatParser = {
         formatName: 'CSV',
         aliases: ['SPREADSHEET', 'TABLE'],
         isValid(value, settings, schema) {
@@ -4199,12 +4269,12 @@
         heal(value, settings, schema) {
             throw new Error('Not implemented');
         },
-        subvalueDefinitions: [
+        subvalueParsers: [
             {
                 subvalueName: 'ROW',
-                async mapValues(value, outputParameterName, settings, mapCallback) {
-                    // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
-                    const csv = papaparse.parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
+                async mapValues(options) {
+                    const { value, outputParameterName, settings, mapCallback, onProgress } = options;
+                    const csv = csvParse(value, settings);
                     if (csv.errors.length !== 0) {
                         throw new CsvFormatError(spaceTrim__default["default"]((block) => `
                                 CSV parsing error
@@ -4219,23 +4289,37 @@
                                 ${block(value)}
                             `));
                     }
-                    const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                    const mappedData = [];
+                    const length = csv.data.length;
+                    for (let index = 0; index < length; index++) {
+                        const row = csv.data[index];
                         if (row[outputParameterName]) {
                             throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                         }
-                        return {
+                        const mappedRow = {
                             ...row,
-                            [outputParameterName]: await mapCallback(row, index),
+                            [outputParameterName]: await mapCallback(row, index, length),
                         };
-                    }));
+                        mappedData.push(mappedRow);
+                        if (onProgress) {
+                            // Note: Report the CSV with all rows mapped so far
+                            /*
+                            // TODO: [🛕] Report progress with all the rows including the pending ones
+                              const progressData = mappedData.map((row, i) =>
+                                i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                            );
+                            */
+                            await onProgress(papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                        }
+                    }
                     return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
                 },
             },
             {
                 subvalueName: 'CELL',
-                async mapValues(value, outputParameterName, settings, mapCallback) {
-                    // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
-                    const csv = papaparse.parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
+                async mapValues(options) {
+                    const { value, settings, mapCallback, onProgress } = options;
+                    const csv = csvParse(value, settings);
                     if (csv.errors.length !== 0) {
                         throw new CsvFormatError(spaceTrim__default["default"]((block) => `
                                 CSV parsing error
@@ -4251,9 +4335,9 @@
                             `));
                     }
                     const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                        return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                             const index = rowIndex * Object.keys(row).length + columnIndex;
-                            return /* not await */ mapCallback({ [key]: value }, index);
+                            return /* not await */ mapCallback({ [key]: value }, index, array.length);
                         }));
                     }));
                     return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -4262,10 +4346,10 @@
         ],
     };
     /**
-     * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `CsvFormatDefinition` implement `heal
-     * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `CsvFormatParser` implement `heal
+     * TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside CSV objects and other formats
      */
 
@@ -4274,7 +4358,7 @@
      *
      * @private still in development [🏢]
      */
-    const JsonFormatDefinition = {
+    const JsonFormatParser = {
         formatName: 'JSON',
         mimeType: 'application/json',
         isValid(value, settings, schema) {
@@ -4286,28 +4370,28 @@
         heal(value, settings, schema) {
             throw new Error('Not implemented');
         },
-        subvalueDefinitions: [],
+        subvalueParsers: [],
     };
     /**
-     * TODO: [🧠] Maybe propper instance of object
+     * TODO: [🧠] Maybe proper instance of object
      * TODO: [0] Make string_serialized_json
      * TODO: [1] Make type for JSON Settings and Schema
      * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
-     * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `JsonFormatDefinition` implement `heal
-     * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `JsonFormatParser` implement `heal
+     * TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside JSON objects and other formats
      */
 
     /**
      * Definition for any text - this will be always valid
      *
-     * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
+     * Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
      *
      * @public exported from `@promptbook/core`
      */
-    const TextFormatDefinition = {
+    const TextFormatParser = {
         formatName: 'TEXT',
         isValid(value) {
             return typeof value === 'string';
@@ -4316,19 +4400,20 @@
             return typeof partialValue === 'string';
         },
         heal() {
-            throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
+            throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
         },
-        subvalueDefinitions: [
+        subvalueParsers: [
             {
                 subvalueName: 'LINE',
-                async mapValues(value, outputParameterName, settings, mapCallback) {
+                async mapValues(options) {
+                    const { value, mapCallback, onProgress } = options;
                     const lines = value.split('\n');
-                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                    const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                     // TODO: [🧠] Maybe option to skip empty line
                     /* not await */ mapCallback({
                         lineContent,
                         // TODO: [🧠] Maybe also put here `lineNumber`
-                    }, lineNumber)));
+                    }, lineNumber, array.length)));
                     return mappedLines.join('\n');
                 },
             },
@@ -4338,10 +4423,10 @@
     /**
      * TODO: [1] Make type for XML Text and Schema
      * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
-     * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `TextFormatDefinition` implement `heal
-     * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `TextFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `TextFormatParser` implement `heal
+     * TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside each item of list and other formats
      */
 
@@ -4349,7 +4434,7 @@
      * Function to check if a string is valid XML
      *
      * @param value
-     * @returns True if the string is a valid XML string, false otherwise
+     * @returns `true` if the string is a valid XML string, false otherwise
      *
      * @public exported from `@promptbook/utils`
      */
@@ -4374,7 +4459,7 @@
      *
      * @private still in development [🏢]
      */
-    const XmlFormatDefinition = {
+    const XmlFormatParser = {
         formatName: 'XML',
         mimeType: 'application/xml',
         isValid(value, settings, schema) {
@@ -4386,17 +4471,17 @@
         heal(value, settings, schema) {
             throw new Error('Not implemented');
         },
-        subvalueDefinitions: [],
+        subvalueParsers: [],
     };
     /**
-     * TODO: [🧠] Maybe propper instance of object
+     * TODO: [🧠] Maybe proper instance of object
      * TODO: [0] Make string_serialized_xml
      * TODO: [1] Make type for XML Settings and Schema
      * TODO: [🧠] What to use for validating XMLs - XSD,...
-     * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
-     * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
-     * TODO: [🍓] In `XmlFormatDefinition` implement `heal
-     * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
+     * TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
+     * TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
+     * TODO: [🍓] In `XmlFormatParser` implement `heal
+     * TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
      * TODO: [🏢] Allow to expect something inside XML and other formats
      */
 
@@ -4405,24 +4490,19 @@
      *
      * @private internal index of `...` <- TODO [🏢]
      */
-    const FORMAT_DEFINITIONS = [
-        JsonFormatDefinition,
-        XmlFormatDefinition,
-        TextFormatDefinition,
-        CsvFormatDefinition,
-    ];
+    const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
     /**
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
     /**
-     * Maps available parameters to expected parameters
+     * Maps available parameters to expected parameters for a pipeline task.
      *
      * The strategy is:
-     * 1) @@@
-     * 2) @@@
+     * 1) First, match parameters by name where both available and expected.
+     * 2) Then, if there are unmatched expected and available parameters, map them by order.
      *
-     * @throws {PipelineExecutionError} @@@
+     * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
      * @private within the repository used in `createPipelineExecutor`
      */
     function mapAvailableToExpectedParameters(options) {
@@ -4445,7 +4525,7 @@
             else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
         }
         if (expectedParameterNames.size === 0) {
-            // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+            // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
             Object.freeze(mappedParameters);
             return mappedParameters;
         }
@@ -4476,7 +4556,7 @@
         for (let i = 0; i < expectedParameterNames.size; i++) {
             mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
         }
-        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
         Object.freeze(mappedParameters);
         return mappedParameters;
     }
@@ -4680,14 +4760,14 @@
         PAGES: countPages,
     };
     /**
-     * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
+     * TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
      * Note: [💞] Ignore a discrepancy between file name and entity name
      */
 
     /**
      * Function checkExpectations will check if the expectations on given value are met
      *
-     * Note: There are two simmilar functions:
+     * Note: There are two similar functions:
      * - `checkExpectations` which throws an error if the expectations are not met
      * - `isPassingExpectations` which returns a boolean
      *
@@ -4708,13 +4788,17 @@
     }
     /**
      * TODO: [💝] Unite object for expecting amount and format
-     * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
+     * TODO: [🧠][🤠] This should be part of `TextFormatParser`
      * Note: [💝] and [🤠] are interconnected together
      */
 
     /**
-     * @@@
+     * Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
+     * (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
+     * Throws errors if execution fails after all attempts.
      *
+     * @param options - The options for execution, including task, parameters, pipeline, and configuration.
+     * @returns The result string of the executed task.
      * @private internal utility of `createPipelineExecutor`
      */
     async function executeAttempts(options) {
@@ -4936,7 +5020,7 @@
                 if (task.format) {
                     if (task.format === 'JSON') {
                         if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
-                            // TODO: [🏢] Do more universally via `FormatDefinition`
+                            // TODO: [🏢] Do more universally via `FormatParser`
                             try {
                                 $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
                             }
@@ -5038,12 +5122,16 @@
      */
 
     /**
-     * @@@
+     * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+     * Handles format and subformat resolution, error handling, and progress reporting.
+     *
+     * @param options - Options for execution, including task details and progress callback.
+     * @returns The result of the subvalue mapping or execution attempts.
      *
      * @private internal utility of `createPipelineExecutor`
      */
     async function executeFormatSubvalues(options) {
-        const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+        const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
         if (task.foreach === undefined) {
             return /* not await */ executeAttempts(options);
         }
@@ -5074,16 +5162,16 @@
                     ${block(pipelineIdentification)}
                 `));
         }
-        const subvalueDefinition = formatDefinition.subvalueDefinitions.find((subvalueDefinition) => [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(task.foreach.subformatName));
-        if (subvalueDefinition === undefined) {
+        const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
+        if (subvalueParser === undefined) {
             throw new UnexpectedError(
             // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
             spaceTrim__default["default"]((block) => `
                     Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
 
                     Available subformat names for format "${formatDefinition.formatName}":
-                    ${block(formatDefinition.subvalueDefinitions
-            .map((subvalueDefinition) => subvalueDefinition.subvalueName)
+                    ${block(formatDefinition.subvalueParsers
+            .map((subvalueParser) => subvalueParser.subvalueName)
             .map((subvalueName) => `- ${subvalueName}`)
             .join('\n'))}
 
@@ -5095,55 +5183,85 @@
         let formatSettings;
         if (formatDefinition.formatName === 'CSV') {
             formatSettings = csvSettings;
-            // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
-        }
-        const resultString = await subvalueDefinition.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-            let mappedParameters;
-            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-            // TODO: When done [🐚] Report progress also for each subvalue here
-            try {
-                mappedParameters = mapAvailableToExpectedParameters({
-                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                    availableParameters: subparameters,
-                });
-            }
-            catch (error) {
-                if (!(error instanceof PipelineExecutionError)) {
-                    throw error;
+            // <- TODO: [🤹‍♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
+        }
+        const resultString = await subvalueParser.mapValues({
+            value: parameterValue,
+            outputParameterName: task.foreach.outputSubparameterName,
+            settings: formatSettings,
+            onProgress(partialResultString) {
+                return onProgress(Object.freeze({
+                    [task.resultingParameterName]: partialResultString,
+                }));
+            },
+            async mapCallback(subparameters, index, length) {
+                let mappedParameters;
+                try {
+                    mappedParameters = mapAvailableToExpectedParameters({
+                        expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                        availableParameters: subparameters,
+                    });
                 }
-                throw new PipelineExecutionError(spaceTrim__default["default"]((block) => `
-                        ${error.message}
+                catch (error) {
+                    if (!(error instanceof PipelineExecutionError)) {
+                        throw error;
+                    }
+                    const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
+                            You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
-            }
-            const allSubparameters = {
-                ...parameters,
-                ...mappedParameters,
-            };
-            // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-            Object.freeze(allSubparameters);
-            const subresultString = await executeAttempts({
-                ...options,
-                priority: priority + index,
-                parameters: allSubparameters,
-                pipelineIdentification: spaceTrim__default["default"]((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-            });
-            return subresultString;
+                            ${block(pipelineIdentification)}
+                        `));
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(highLevelError);
+                        return FAILED_VALUE_PLACEHOLDER;
+                    }
+                    throw highLevelError;
+                }
+                const allSubparameters = {
+                    ...parameters,
+                    ...mappedParameters,
+                };
+                Object.freeze(allSubparameters);
+                try {
+                    const subresultString = await executeAttempts({
+                        ...options,
+                        priority: priority + index,
+                        parameters: allSubparameters,
+                        pipelineIdentification: spaceTrim__default["default"]((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                    });
+                    return subresultString;
+                }
+                catch (error) {
+                    if (length > BIG_DATASET_TRESHOLD) {
+                        console.error(spaceTrim__default["default"]((block) => `
+                              ${error.message}
+
+                              This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
+
+                              ${block(pipelineIdentification)}
+                          `));
+                        return FAILED_VALUE_PLACEHOLDER;
+                    }
+                    throw error;
+                }
+            },
         });
         return resultString;
     }
 
     /**
-     * @@@
+     * Returns the context for a given task, typically used to provide additional information or variables
+     * required for the execution of the task within a pipeline. The context is returned as a string value
+     * that may include markdown formatting.
      *
+     * @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
+     * @returns The context as a string, formatted as markdown and parameter value.
      * @private internal utility of `createPipelineExecutor`
      */
     async function getContextForTask(task) {
@@ -5151,7 +5269,7 @@
     }
 
     /**
-     * @@@
+     * Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5160,91 +5278,128 @@
     }
 
     /**
-     * @@@
+     * Computes the cosine similarity between two embedding vectors
+     *
+     * Note: This is helping function for RAG (retrieval-augmented generation)
      *
-     * Here is the place where RAG (retrieval-augmented generation) happens
+     * @param embeddingVector1
+     * @param embeddingVector2
+     * @returns Cosine similarity between the two vectors
+     *
+     * @public exported from `@promptbook/core`
+     */
+    function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
+        if (embeddingVector1.length !== embeddingVector2.length) {
+            throw new TypeError('Embedding vectors must have the same length');
+        }
+        const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
+        const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
+        const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
+        return 1 - dotProduct / (magnitude1 * magnitude2);
+    }
+
+    /**
+     *
+     * @param knowledgePieces
+     * @returns
+     *
+     * @private internal utility of `createPipelineExecutor`
+     */
+    function knowledgePiecesToString(knowledgePieces) {
+        return knowledgePieces
+            .map((knowledgePiece) => {
+            const { content } = knowledgePiece;
+            return `- ${content}`;
+        })
+            .join('\n');
+        // <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
+    }
+
+    /**
+     * Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
+     * This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
      *
      * @private internal utility of `createPipelineExecutor`
      */
     async function getKnowledgeForTask(options) {
-        const { tools, preparedPipeline, task } = options;
+        const { tools, preparedPipeline, task, parameters } = options;
         const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
         const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
         // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
         if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
-            return 'No knowledge pieces found';
+            return ''; // <- Note: Np knowledge present, return empty string
         }
-        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-        const _llms = arrayableToArray(tools.llm);
-        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
-        const taskEmbeddingPrompt = {
-            title: 'Knowledge Search',
-            modelRequirements: {
-                modelVariant: 'EMBEDDING',
-                modelName: firstKnowlegeIndex.modelName,
-            },
-            content: task.content,
-            parameters: {
-            /* !!!!!!!! */
-            },
-        };
-        const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
-        const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
-            const { index } = knowledgePiece;
-            const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
-            // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
-            if (knowledgePieceIndex === undefined) {
+        try {
+            // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
+            const _llms = arrayableToArray(tools.llm);
+            const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+            const taskEmbeddingPrompt = {
+                title: 'Knowledge Search',
+                modelRequirements: {
+                    modelVariant: 'EMBEDDING',
+                    modelName: firstKnowlegeIndex.modelName,
+                },
+                content: task.content,
+                parameters,
+            };
+            const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
+            const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
+                const { index } = knowledgePiece;
+                const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
+                // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
+                if (knowledgePieceIndex === undefined) {
+                    return {
+                        content: knowledgePiece.content,
+                        relevance: 0,
+                    };
+                }
+                const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
                 return {
                     content: knowledgePiece.content,
-                    relevance: 0,
+                    relevance,
                 };
-            }
-            const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
-            return {
-                content: knowledgePiece.content,
-                relevance,
-            };
-        });
-        const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
-        const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
-        console.log('!!! Embedding', {
-            task,
-            taskEmbeddingPrompt,
-            taskEmbeddingResult,
-            firstKnowlegePiece,
-            firstKnowlegeIndex,
-            knowledgePiecesWithRelevance,
-            knowledgePiecesSorted,
-            knowledgePiecesLimited,
-        });
-        return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
-        //                                                      <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
-    }
-    // TODO: !!!!!! Annotate + to new file
-    function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
-        if (embeddingVector1.length !== embeddingVector2.length) {
-            throw new TypeError('Embedding vectors must have the same length');
+            });
+            const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
+            const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
+            /*
+            console.log('`getKnowledgeForTask` Embedding', {
+                task,
+                taskEmbeddingPrompt,
+                taskEmbeddingResult,
+                firstKnowlegePiece,
+                firstKnowlegeIndex,
+                knowledgePiecesWithRelevance,
+                knowledgePiecesSorted,
+                knowledgePiecesLimited,
+            });
+            */
+            return knowledgePiecesToString(knowledgePiecesLimited);
+        }
+        catch (error) {
+            assertsError(error);
+            console.error('Error in `getKnowledgeForTask`', error);
+            // Note: If the LLM fails, just return all knowledge pieces
+            return knowledgePiecesToString(preparedPipeline.knowledgePieces);
         }
-        const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
-        const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
-        const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
-        return 1 - dotProduct / (magnitude1 * magnitude2);
     }
     /**
-     * TODO: !!!! Verify if this is working
      * TODO: [♨] Implement Better - use keyword search
      * TODO: [♨] Examples of values
      */
 
     /**
-     * @@@
+     * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+     * Ensures all reserved parameters are defined and throws if any are missing.
+     *
+     * @param options - Options including tools, pipeline, task, and context.
+     * @returns An object containing all reserved parameters for the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
     async function getReservedParametersForTask(options) {
-        const { tools, preparedPipeline, task, pipelineIdentification } = options;
+        const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
         const context = await getContextForTask(); // <- [🏍]
-        const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
+        const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
         const examples = await getExamplesForTask();
         const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
         const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5256,6 +5411,9 @@
             currentDate,
             modelName,
         };
+        if (isVerbose) {
+            console.info('Reserved parameters for task:', { options, reservedParameters });
+        }
         // Note: Doublecheck that ALL reserved parameters are defined:
         for (const parameterName of RESERVED_PARAMETER_NAMES) {
             if (reservedParameters[parameterName] === undefined) {
@@ -5270,23 +5428,21 @@
     }
 
     /**
-     * @@@
+     * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+     *
+     * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+     * @returns The output parameters produced by the task.
      *
      * @private internal utility of `createPipelineExecutor`
      */
     async function executeTask(options) {
         const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
         const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-        await onProgress({
-            outputParameters: {
-                [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-            },
-        });
         // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
         const usedParameterNames = extractParameterNamesFromTask(currentTask);
         const dependentParameterNames = new Set(currentTask.dependentParameterNames);
         // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
-        if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
+        if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
             throw new UnexpectedError(spaceTrim.spaceTrim((block) => `
                     Dependent parameters are not consistent with used parameters:
 
@@ -5304,13 +5460,16 @@
 
                 `));
         }
+        const reservedParameters = await getReservedParametersForTask({
+            tools,
+            preparedPipeline,
+            task: currentTask,
+            pipelineIdentification,
+            parameters: parametersToPass,
+            isVerbose,
+        });
         const definedParameters = Object.freeze({
-            ...(await getReservedParametersForTask({
-                tools,
-                preparedPipeline,
-                task: currentTask,
-                pipelineIdentification,
-            })),
+            ...reservedParameters,
             ...parametersToPass,
         });
         const definedParameterNames = new Set(Object.keys(definedParameters));
@@ -5355,6 +5514,7 @@
             preparedPipeline,
             tools,
             $executionReport,
+            onProgress,
             pipelineIdentification,
             maxExecutionAttempts,
             maxParallelCount,
@@ -5382,7 +5542,8 @@
      */
 
     /**
-     * @@@
+     * Filters and returns only the output parameters from the provided pipeline execution options.
+     * Adds warnings for any expected output parameters that are missing.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5407,9 +5568,12 @@
     }
 
     /**
-     * @@@
+     * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
      *
-     * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+     * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
+     *
+     * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+     * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
      *
      * @private internal utility of `createPipelineExecutor`
      */
@@ -5732,10 +5896,27 @@
                 cacheDirname,
                 intermediateFilesStrategy,
                 isAutoInstalled,
+            }).catch((error) => {
+                assertsError(error);
+                return exportJson({
+                    name: 'pipelineExecutorResult',
+                    message: `Unuccessful PipelineExecutorResult, last catch`,
+                    order: [],
+                    value: {
+                        isSuccessful: false,
+                        errors: [serializeError(error)],
+                        warnings: [],
+                        usage: UNCERTAIN_USAGE,
+                        executionReport: null,
+                        outputParameters: {},
+                        preparedPipeline,
+                    },
+                });
             });
         };
         const pipelineExecutor = (inputParameters) => createTask({
             taskType: 'EXECUTION',
+            title: pipeline.title,
             taskProcessCallback(updateOngoingResult) {
                 return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
                     updateOngoingResult(newOngoingResult);
@@ -5838,7 +6019,7 @@
             const knowledge = await Promise.all(
             // TODO: [🪂] Do not send all at once but in chunks
             knowledgeTextPieces.map(async (knowledgeTextPiece, i) => {
-                // Note: Theese are just default values, they will be overwritten by the actual values:
+                // Note: These are just default values, they will be overwritten by the actual values:
                 let name = `piece-${i}`;
                 let title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
                 const knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
@@ -5913,7 +6094,7 @@
      */
 
     /**
-     * @@@
+     * Creates a scraper for markdown content.
      *
      * @public exported from `@promptbook/markdown-utils`
      */
@@ -6068,7 +6249,7 @@
      * Note: It can not work with html syntax and comments
      *
      * @param markdown any valid markdown
-     * @returns @@@
+     * @returns An array of strings, each representing an individual list item found in the markdown
      * @public exported from `@promptbook/markdown-utils`
      */
     function extractAllListItemsFromMarkdown(markdown) {