@promptbook/legacy-documents 0.92.0-9 → 0.93.0

package/esm/index.es.js

@@ -28,7 +28,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-9';
+const PROMPTBOOK_ENGINE_VERSION = '0.93.0';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -38,7 +38,7 @@ const PROMPTBOOK_ENGINE_VERSION = '0.92.0-9';
  * Just says that the variable is not used but should be kept
  * No side effects.
  *
- * Note: It can be usefull for:
+ * Note: It can be useful for:
  *
  * 1) Suppressing eager optimization of unused imports
  * 2) Suppressing eslint errors of unused variables in the tests
@@ -55,7 +55,7 @@ function keepUnused(...valuesToKeep) {
  * Returns the same value that is passed as argument.
  * No side effects.
  *
- * Note: It can be usefull for:
+ * Note: It can be useful for:
  *
  * 1) Leveling indentation
  * 2) Putting always-true or always-false conditions without getting eslint errors
@@ -104,6 +104,21 @@ const DEFAULT_BOOK_TITLE = `✨ Untitled Book`;
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+/**
+ * Threshold value that determines when a dataset is considered "big"
+ * and may require special handling or optimizations
+ *
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
+ *
+ * @public exported from `@promptbook/core`
+ */
+const BIG_DATASET_TRESHOLD = 50;
+/**
+ * Placeholder text used to represent a placeholder value of failed operation
+ *
+ * @public exported from `@promptbook/core`
+ */
+const FAILED_VALUE_PLACEHOLDER = '!?';
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -183,7 +198,7 @@ const DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
 const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
 // <- TODO: [🧜‍♂️]
 /**
- * @@@
+ * Default settings for parsing and generating CSV files in Promptbook.
  *
  * @public exported from `@promptbook/core`
  */
@@ -194,19 +209,19 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
     skipEmptyLines: true,
 });
 /**
- * @@@
+ * Controls whether verbose logging is enabled by default throughout the application.
  *
  * @public exported from `@promptbook/core`
  */
 let DEFAULT_IS_VERBOSE = false;
 /**
- * @@@
+ * Controls whether auto-installation of dependencies is enabled by default.
  *
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_IS_AUTO_INSTALLED = false;
 /**
- * @@@
+ * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
  *
  * @private within the repository
  */
@@ -255,7 +270,7 @@ class MissingToolsError extends Error {
         super(spaceTrim((block) => `
                     ${block(message)}
 
-                    Note: You have probbably forgot to provide some tools for pipeline execution or preparation
+                    Note: You have probably forgot to provide some tools for pipeline execution or preparation
 
                 `));
         this.name = 'MissingToolsError';
@@ -318,7 +333,7 @@ class UnexpectedError extends Error {
                     ${block(message)}
 
                     Note: This error should not happen.
-                    It's probbably a bug in the pipeline collection
+                    It's probably a bug in the pipeline collection
 
                     Please report issue:
                     ${block(getErrorReportUrl(new Error(message)).href)}
@@ -547,7 +562,8 @@ async function isFileExisting(filename, fs) {
  */
 
 /**
- * @@@
+ * Converts a name to a properly formatted subfolder path for cache storage.
+ * Handles normalization and path formatting to create consistent cache directory structures.
  *
  * @private for `FileCacheStorage`
  */
@@ -800,10 +816,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
 */
 
 /**
- * @@@
+ * Removes diacritic marks (accents) from characters in a string.
  *
- * @param input @@@
- * @returns @@@
+ * @param input The string containing diacritics to be normalized.
+ * @returns The string with diacritics removed or normalized.
  * @public exported from `@promptbook/utils`
  */
 function removeDiacritics(input) {
@@ -817,10 +833,10 @@ function removeDiacritics(input) {
  */
 
 /**
- * @@@
+ * Converts a given text to kebab-case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text to be converted.
+ * @returns The kebab-case formatted string.
  * @example 'hello-world'
  * @example 'i-love-promptbook'
  * @public exported from `@promptbook/utils`
@@ -936,7 +952,7 @@ function isValidFilePath(filename) {
  * Tests if given string is valid URL.
  *
  * Note: Dataurl are considered perfectly valid.
- * Note: There are two simmilar functions:
+ * Note: There are two similar functions:
  * - `isValidUrl` which tests any URL
  * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
  *
@@ -962,11 +978,11 @@ function isValidUrl(url) {
 }
 
 /**
- * @@@
+ * Converts a title string into a normalized name.
  *
- * @param value @@@
- * @returns @@@
- * @example @@@
+ * @param value The title string to be converted to a name.
+ * @returns A normalized name derived from the input title.
+ * @example 'Hello World!' -> 'hello-world'
  * @public exported from `@promptbook/utils`
  */
 function titleToName(value) {
@@ -986,9 +1002,8 @@ function titleToName(value) {
 }
 
 /**
- * Create a filename for intermediate cache for scrapers
- *
- * Note: It also checks if directory exists and creates it if not
+ * Retrieves an intermediate source for a scraper based on the knowledge source.
+ * Manages the caching and retrieval of intermediate scraper results for optimized performance.
  *
  * @private as internal utility for scrapers
  */
@@ -1124,7 +1139,7 @@ function assertsError(whatWasThrown) {
  * Function isValidJsonString will tell you if the string is valid JSON or not
  *
  * @param value The string to check
- * @returns True if the string is a valid JSON string, false otherwise
+ * @returns `true` if the string is a valid JSON string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -1236,7 +1251,7 @@ function pipelineJsonToString(pipelineJson) {
     if (bookVersion !== `undefined`) {
         commands.push(`BOOK VERSION ${bookVersion}`);
     }
-    // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary
+    // TODO: [main] !!5 This increases size of the bundle and is probably not necessary
     pipelineString = prettifyMarkdown(pipelineString);
     for (const parameter of parameters.filter(({ isInput }) => isInput)) {
         commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
@@ -1535,8 +1550,12 @@ function checkSerializableAsJson(options) {
  */
 
 /**
- * @@@
+ * Creates a deep clone of the given object
  *
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
+ *
+ * @param objectValue The object to clone.
+ * @returns A deep, writable clone of the input object.
  * @public exported from `@promptbook/utils`
  */
 function deepClone(objectValue) {
@@ -1618,13 +1637,13 @@ const ORDER_OF_PIPELINE_JSON = [
  */
 const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
- * @@@
+ * Placeholder value indicating a parameter is missing its value.
  *
  * @private within the repository
  */
 const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
 /**
- * @@@
+ * Placeholder value indicating a parameter is restricted and cannot be used directly.
  *
  * @private within the repository
  */
@@ -1669,7 +1688,7 @@ class PipelineLogicError extends Error {
 /**
  * Tests if given string is valid semantic version
  *
- * Note: There are two simmilar functions:
+ * Note: There are two similar functions:
  * - `isValidSemanticVersion` which tests any semantic version
  * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
  *
@@ -1691,7 +1710,7 @@ function isValidSemanticVersion(version) {
  *
  * @see https://www.npmjs.com/package/promptbook?activeTab=versions
  * Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
- * Note: There are two simmilar functions:
+ * Note: There are two similar functions:
  * - `isValidSemanticVersion` which tests any semantic version
  * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
  *
@@ -1711,7 +1730,7 @@ function isValidPromptbookVersion(version) {
 /**
  * Tests if given string is valid pipeline URL URL.
  *
- * Note: There are two simmilar functions:
+ * Note: There are two similar functions:
  * - `isValidUrl` which tests any URL
  * - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
  *
@@ -1808,7 +1827,7 @@ function validatePipeline_InnerFunction(pipeline) {
                     ${block(pipelineIdentification)}
                 `));
     }
-    // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
+    // TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
     if (!Array.isArray(pipeline.parameters)) {
         // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
         throw new ParseError(spaceTrim((block) => `
@@ -1819,7 +1838,7 @@ function validatePipeline_InnerFunction(pipeline) {
                     ${block(pipelineIdentification)}
                 `));
     }
-    // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
+    // TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
     if (!Array.isArray(pipeline.tasks)) {
         // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
         throw new ParseError(spaceTrim((block) => `
@@ -2123,7 +2142,7 @@ class SimplePipelineCollection {
     /**
      * Constructs a pipeline collection from pipelines
      *
-     * @param pipelines @@@
+     * @param pipelines Array of pipeline JSON objects to include in the collection
      *
      * Note: During the construction logic of all pipelines are validated
      * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -2253,7 +2272,7 @@ class PipelineExecutionError extends Error {
         super(message);
         this.name = 'PipelineExecutionError';
         // TODO: [🐙] DRY - Maybe $randomId
-        this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
+        this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts   */)}`;
         Object.setPrototypeOf(this, PipelineExecutionError.prototype);
     }
 }
@@ -2269,15 +2288,18 @@ class PipelineExecutionError extends Error {
  * @public exported from `@promptbook/core`
  */
 function isPipelinePrepared(pipeline) {
-    // Note: Ignoring `pipeline.preparations` @@@
-    // Note: Ignoring `pipeline.knowledgePieces` @@@
+    // Note: Ignoring `pipeline.preparations`
+    // Note: Ignoring `pipeline.knowledgePieces`
     if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
+        // console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
         return false;
     }
     if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
+        // console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
         return false;
     }
     if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
+        //console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
         return false;
     }
     /*
@@ -2298,75 +2320,6 @@ function isPipelinePrepared(pipeline) {
  *     - [♨] Are tasks prepared
  */
 
-/**
- * Converts a JavaScript Object Notation (JSON) string into an object.
- *
- * Note: This is wrapper around `JSON.parse()` with better error and type handling
- *
- * @public exported from `@promptbook/utils`
- */
-function jsonParse(value) {
-    if (value === undefined) {
-        throw new Error(`Can not parse JSON from undefined value.`);
-    }
-    else if (typeof value !== 'string') {
-        console.error('Can not parse JSON from non-string value.', { text: value });
-        throw new Error(spaceTrim$1(`
-                    Can not parse JSON from non-string value.
-
-                    The value type: ${typeof value}
-                    See more in console.
-                `));
-    }
-    try {
-        return JSON.parse(value);
-    }
-    catch (error) {
-        if (!(error instanceof Error)) {
-            throw error;
-        }
-        throw new Error(spaceTrim$1((block) => `
-                    ${block(error.message)}
-            
-                    The JSON text:
-                    ${block(value)}
-                `));
-    }
-}
-/**
- * TODO: !!!! Use in Promptbook.studio
- */
-
-/**
- * Recursively converts JSON strings to JSON objects
-
- * @public exported from `@promptbook/utils`
- */
-function jsonStringsToJsons(object) {
-    if (object === null) {
-        return object;
-    }
-    if (Array.isArray(object)) {
-        return object.map(jsonStringsToJsons);
-    }
-    if (typeof object !== 'object') {
-        return object;
-    }
-    const newObject = { ...object };
-    for (const [key, value] of Object.entries(object)) {
-        if (typeof value === 'string' && isValidJsonString(value)) {
-            newObject[key] = jsonParse(value);
-        }
-        else {
-            newObject[key] = jsonStringsToJsons(value);
-        }
-    }
-    return newObject;
-}
-/**
- * TODO: Type the return type correctly
- */
-
 /**
  * This error indicates problems parsing the format value
  *
@@ -2411,7 +2364,7 @@ class AuthenticationError extends Error {
 }
 
 /**
- * This error indicates that the pipeline collection cannot be propperly loaded
+ * This error indicates that the pipeline collection cannot be properly loaded
  *
  * @public exported from `@promptbook/core`
  */
@@ -2550,6 +2503,101 @@ const ALL_ERRORS = {
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Serializes an error into a [🚉] JSON-serializable object
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function serializeError(error) {
+    const { name, message, stack } = error;
+    const { id } = error;
+    if (!Object.keys(ALL_ERRORS).includes(name)) {
+        console.error(spaceTrim$1((block) => `
+
+                    Cannot serialize error with name "${name}"
+
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
+                    ${block(stack || message)}
+
+                `));
+    }
+    return {
+        name: name,
+        message,
+        stack,
+        id, // Include id in the serialized object
+    };
+}
+
+/**
+ * Converts a JavaScript Object Notation (JSON) string into an object.
+ *
+ * Note: This is wrapper around `JSON.parse()` with better error and type handling
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function jsonParse(value) {
+    if (value === undefined) {
+        throw new Error(`Can not parse JSON from undefined value.`);
+    }
+    else if (typeof value !== 'string') {
+        console.error('Can not parse JSON from non-string value.', { text: value });
+        throw new Error(spaceTrim$1(`
+                    Can not parse JSON from non-string value.
+
+                    The value type: ${typeof value}
+                    See more in console.
+                `));
+    }
+    try {
+        return JSON.parse(value);
+    }
+    catch (error) {
+        if (!(error instanceof Error)) {
+            throw error;
+        }
+        throw new Error(spaceTrim$1((block) => `
+                    ${block(error.message)}
+
+                    The JSON text:
+                    ${block(value)}
+                `));
+    }
+}
+
+/**
+ * Recursively converts JSON strings to JSON objects
+
+ * @public exported from `@promptbook/utils`
+ */
+function jsonStringsToJsons(object) {
+    if (object === null) {
+        return object;
+    }
+    if (Array.isArray(object)) {
+        return object.map(jsonStringsToJsons);
+    }
+    if (typeof object !== 'object') {
+        return object;
+    }
+    const newObject = { ...object };
+    for (const [key, value] of Object.entries(object)) {
+        if (typeof value === 'string' && isValidJsonString(value)) {
+            newObject[key] = jsonParse(value);
+        }
+        else {
+            newObject[key] = jsonStringsToJsons(value);
+        }
+    }
+    return newObject;
+}
+/**
+ * TODO: Type the return type correctly
+ */
+
 /**
  * Deserializes the error object
  *
@@ -2624,8 +2672,9 @@ function assertsTaskSuccessful(executionResult) {
  */
 function createTask(options) {
     const { taskType, taskProcessCallback } = options;
+    let { title } = options;
     // TODO: [🐙] DRY
-    const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
+    const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts   */)}`;
     let status = 'RUNNING';
     const createdAt = new Date();
     let updatedAt = createdAt;
@@ -2635,6 +2684,10 @@ function createTask(options) {
     const partialResultSubject = new Subject();
     // <- Note: Not using `BehaviorSubject` because on error we can't access the last value
     const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
+        if (newOngoingResult.title) {
+            title = newOngoingResult.title;
+        }
+        updatedAt = new Date();
         Object.assign(currentValue, newOngoingResult);
         // <- TODO: assign deep
         partialResultSubject.next(newOngoingResult);
@@ -2680,17 +2733,24 @@ function createTask(options) {
     return {
         taskType,
         taskId,
+        get promptbookVersion() {
+            return PROMPTBOOK_ENGINE_VERSION;
+        },
+        get title() {
+            return title;
+            // <- Note: [1] These must be getters to allow changing the value in the future
+        },
         get status() {
             return status;
-            // <- Note: [1] Theese must be getters to allow changing the value in the future
+            // <- Note: [1] --||--
         },
         get createdAt() {
             return createdAt;
-            // <- Note: [1]
+            // <- Note: [1] --||--
         },
         get updatedAt() {
             return updatedAt;
-            // <- Note: [1]
+            // <- Note: [1] --||--
         },
         asPromise,
         asObservable() {
@@ -2698,15 +2758,15 @@ function createTask(options) {
         },
         get errors() {
             return errors;
-            // <- Note: [1]
+            // <- Note: [1] --||--
         },
         get warnings() {
             return warnings;
-            // <- Note: [1]
+            // <- Note: [1] --||--
         },
         get currentValue() {
             return currentValue;
-            // <- Note: [1]
+            // <- Note: [1] --||--
         },
     };
 }
@@ -2715,64 +2775,6 @@ function createTask(options) {
  * TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
  */
 
-/**
- * Serializes an error into a [🚉] JSON-serializable object
- *
- * @public exported from `@promptbook/utils`
- */
-function serializeError(error) {
-    const { name, message, stack } = error;
-    const { id } = error;
-    if (!Object.keys(ALL_ERRORS).includes(name)) {
-        console.error(spaceTrim$1((block) => `
-
-                    Cannot serialize error with name "${name}"
-
-                    Authors of Promptbook probably forgot to add this error into the list of errors:
-                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
-
-
-                    ${block(stack || message)}
-
-                `));
-    }
-    return {
-        name: name,
-        message,
-        stack,
-        id, // Include id in the serialized object
-    };
-}
-
-/**
- * Async version of Array.forEach
- *
- * @param array - Array to iterate over
- * @param options - Options for the function
- * @param callbackfunction - Function to call for each item
- * @public exported from `@promptbook/utils`
- * @deprecated [🪂] Use queues instead
- */
-async function forEachAsync(array, options, callbackfunction) {
-    const { maxParallelCount = Infinity } = options;
-    let index = 0;
-    let runningTasks = [];
-    const tasks = [];
-    for (const item of array) {
-        const currentIndex = index++;
-        const task = callbackfunction(item, currentIndex, array);
-        tasks.push(task);
-        runningTasks.push(task);
-        /* not await */ Promise.resolve(task).then(() => {
-            runningTasks = runningTasks.filter((t) => t !== task);
-        });
-        if (maxParallelCount < runningTasks.length) {
-            await Promise.race(runningTasks);
-        }
-    }
-    await Promise.all(tasks);
-}
-
 /**
  * Represents the uncertain value
  *
@@ -2816,7 +2818,7 @@ const ZERO_USAGE = $deepFreeze({
  *
  * @public exported from `@promptbook/core`
  */
-$deepFreeze({
+const UNCERTAIN_USAGE = $deepFreeze({
     price: UNCERTAIN_ZERO_VALUE,
     input: {
         tokensCount: UNCERTAIN_ZERO_VALUE,
@@ -2841,6 +2843,35 @@ $deepFreeze({
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Async version of Array.forEach
+ *
+ * @param array - Array to iterate over
+ * @param options - Options for the function
+ * @param callbackfunction - Function to call for each item
+ * @public exported from `@promptbook/utils`
+ * @deprecated [🪂] Use queues instead
+ */
+async function forEachAsync(array, options, callbackfunction) {
+    const { maxParallelCount = Infinity } = options;
+    let index = 0;
+    let runningTasks = [];
+    const tasks = [];
+    for (const item of array) {
+        const currentIndex = index++;
+        const task = callbackfunction(item, currentIndex, array);
+        tasks.push(task);
+        runningTasks.push(task);
+        /* not await */ Promise.resolve(task).then(() => {
+            runningTasks = runningTasks.filter((t) => t !== task);
+        });
+        if (maxParallelCount < runningTasks.length) {
+            await Promise.race(runningTasks);
+        }
+    }
+    await Promise.all(tasks);
+}
+
 /**
  * Function `addUsage` will add multiple usages into one
  *
@@ -2900,12 +2931,14 @@ function countUsage(llmTools) {
     const spending = new Subject();
     const proxyTools = {
         get title() {
-            // TODO: [🧠] Maybe put here some suffix
-            return llmTools.title;
+            return `${llmTools.title} (+usage)`;
+            // <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
+            // <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
         },
         get description() {
-            // TODO: [🧠] Maybe put here some suffix
-            return llmTools.description;
+            return `${llmTools.description} (+usage)`;
+            // <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
+            // <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
         },
         checkConfiguration() {
             return /* not await */ llmTools.checkConfiguration();
@@ -2976,29 +3009,40 @@ class MultipleLlmExecutionTools {
         return 'Multiple LLM Providers';
     }
     get description() {
-        return this.llmExecutionTools.map(({ title }, index) => `${index + 1}) \`${title}\``).join('\n');
+        const innerModelsTitlesAndDescriptions = this.llmExecutionTools
+            .map(({ title, description }, index) => {
+            const headLine = `${index + 1}) \`${title}\``;
+            if (description === undefined) {
+                return headLine;
+            }
+            return spaceTrim$1((block) => `
+                        ${headLine}
+                        
+                          ${ /* <- Note: Indenting the description: */block(description)}
+                    `);
+        })
+            .join('\n\n');
+        return spaceTrim$1((block) => `
+                Multiple LLM Providers:
+
+                ${block(innerModelsTitlesAndDescriptions)}
+            `);
     }
     /**
      * Check the configuration of all execution tools
      */
     async checkConfiguration() {
-        // TODO: Maybe do it in parallel
-        for (const llmExecutionTools of this.llmExecutionTools) {
-            await llmExecutionTools.checkConfiguration();
-        }
+        // Note: Run checks in parallel
+        await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
     }
     /**
      * List all available models that can be used
      * This lists is a combination of all available models from all execution tools
      */
     async listModels() {
-        const availableModels = [];
-        for (const llmExecutionTools of this.llmExecutionTools) {
-            // TODO: [🪂] Obtain models in parallel
-            const models = await llmExecutionTools.listModels();
-            availableModels.push(...models);
-        }
-        return availableModels;
+        // Obtain all models in parallel and flatten
+        const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
+        return modelArrays.flat();
     }
     /**
      * Calls the best available chat model
@@ -3239,7 +3283,8 @@ async function preparePersona(personaDescription, tools, options) {
  */
 
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *
@@ -3250,10 +3295,10 @@ function $getGlobalScope() {
 }
 
 /**
- * @@@
+ * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to SCREAMING_CASE format.
+ * @returns The normalized text in SCREAMING_CASE format.
  * @example 'HELLO_WORLD'
  * @example 'I_LOVE_PROMPTBOOK'
  * @public exported from `@promptbook/utils`
@@ -3305,10 +3350,10 @@ function normalizeTo_SCREAMING_CASE(text) {
  */
 
 /**
- * @@@
+ * Normalizes a text string to snake_case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to snake_case format.
+ * @returns The normalized text in snake_case format.
  * @example 'hello_world'
  * @example 'i_love_promptbook'
  * @public exported from `@promptbook/utils`
@@ -3318,11 +3363,11 @@ function normalizeTo_snake_case(text) {
 }
 
 /**
- * Register is @@@
+ * Global registry for storing and managing registered entities of a given type.
  *
  * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
  *
- * @private internal utility, exported are only signleton instances of this class
+ * @private internal utility, exported are only singleton instances of this class
  */
 class $Register {
     constructor(registerName) {
@@ -3366,10 +3411,10 @@ class $Register {
 }
 
 /**
- * @@@
+ * Global registry for storing metadata about all available scrapers and converters.
  *
- * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * Note: `$` is used to indicate that this interacts with the global scope.
+ * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
  * @public exported from `@promptbook/core`
  */
 const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -3378,10 +3423,11 @@ const $scrapersMetadataRegister = new $Register('scrapers_metadata');
  */
 
 /**
- * @@@
+ * Registry for all available scrapers in the system.
+ * Central point for registering and accessing different types of content scrapers.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
  * @public exported from `@promptbook/core`
  */
 const $scrapersRegister = new $Register('scraper_constructors');
@@ -3549,7 +3595,9 @@ const promptbookFetch = async (urlOrRequest, init) => {
  */
 
 /**
- * @@@
+ * Factory function that creates a handler for processing knowledge sources.
+ * Provides standardized processing of different types of knowledge sources
+ * across various scraper implementations.
  *
  * @public exported from `@promptbook/core`
  */
@@ -3688,7 +3736,7 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
 }
 
 /**
- * Prepares the knowle
+ * Prepares the knowledge pieces
  *
  * @see https://github.com/webgptorg/promptbook/discussions/41
  * @public exported from `@promptbook/core`
@@ -3784,15 +3832,18 @@ TODO: [🧊] This is how it can look in future
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
  *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
  * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
- * TODO: [🧠][❎] Do here propper M:N mapping
+ * TODO: [🧠][❎] Do here proper M:N mapping
  *       [x] One source can make multiple pieces
  *       [ ] One piece can have multiple sources
  */
 
 /**
- * @@@
+ * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
  *
- * @public exported from `@promptbook/core`
+ * @param tasks Sequence of tasks that are chained together to form a pipeline
+ * @returns A promise that resolves to the prepared tasks.
+ *
+ * @private internal utility of `preparePipeline`
  */
 async function prepareTasks(pipeline, tools, options) {
     const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -3830,7 +3881,7 @@ async function prepareTasks(pipeline, tools, options) {
     return { tasksPrepared };
 }
 /**
- * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, simmilar thing with expectations to sync high-level abstractions
+ * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
  * TODO: [🧠] Add context to each task (if missing)
  * TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
  * TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
@@ -3966,7 +4017,7 @@ async function preparePipeline(pipeline, tools, options) {
         order: ORDER_OF_PIPELINE_JSON,
         value: {
             ...pipeline,
-            // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects
+            // <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
             title,
             knowledgeSources: knowledgeSourcesPrepared,
             knowledgePieces: knowledgePiecesPrepared,
@@ -4229,7 +4280,7 @@ function union(...sets) {
 }
 
 /**
- * @@@
+ * Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
  *
  * @public exported from `@promptbook/core`
  */
@@ -4238,11 +4289,29 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
     // encoding: 'utf-8',
 });
 
+/**
+ * Converts a CSV string into an object
+ *
+ * Note: This is wrapper around `papaparse.parse()` with better autohealing
+ *
+ * @private - for now until `@promptbook/csv` is released
+ */
+function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
+    settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
+    // Note: Autoheal invalid '\n' characters
+    if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
+        console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
+        value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    }
+    const csv = parse(value, settings);
+    return csv;
+}
+
 /**
  * Function to check if a string is valid CSV
  *
  * @param value The string to check
- * @returns True if the string is a valid CSV string, false otherwise
+ * @returns `true` if the string is a valid CSV string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4266,7 +4335,7 @@ function isValidCsvString(value) {
  * @public exported from `@promptbook/core`
  *                          <- TODO: [🏢] Export from package `@promptbook/csv`
  */
-const CsvFormatDefinition = {
+const CsvFormatParser = {
     formatName: 'CSV',
     aliases: ['SPREADSHEET', 'TABLE'],
     isValid(value, settings, schema) {
@@ -4278,12 +4347,12 @@ const CsvFormatDefinition = {
     heal(value, settings, schema) {
         throw new Error('Not implemented');
     },
-    subvalueDefinitions: [
+    subvalueParsers: [
         {
             subvalueName: 'ROW',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
-                // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
-                const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
+            async mapValues(options) {
+                const { value, outputParameterName, settings, mapCallback, onProgress } = options;
+                const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim$1((block) => `
                                 CSV parsing error
@@ -4298,23 +4367,37 @@ const CsvFormatDefinition = {
                                 ${block(value)}
                             `));
                 }
-                const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                const mappedData = [];
+                const length = csv.data.length;
+                for (let index = 0; index < length; index++) {
+                    const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
-                    return {
+                    const mappedRow = {
                         ...row,
-                        [outputParameterName]: await mapCallback(row, index),
+                        [outputParameterName]: await mapCallback(row, index, length),
                     };
-                }));
+                    mappedData.push(mappedRow);
+                    if (onProgress) {
+                        // Note: Report the CSV with all rows mapped so far
+                        /*
+                        // TODO: [🛕] Report progress with all the rows including the pending ones
+                          const progressData = mappedData.map((row, i) =>
+                            i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                        );
+                        */
+                        await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                    }
+                }
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
             },
         },
         {
             subvalueName: 'CELL',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
-                // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
-                const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
+            async mapValues(options) {
+                const { value, settings, mapCallback, onProgress } = options;
+                const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim$1((block) => `
                                 CSV parsing error
@@ -4330,9 +4413,9 @@ const CsvFormatDefinition = {
                             `));
                 }
                 const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                         const index = rowIndex * Object.keys(row).length + columnIndex;
-                        return /* not await */ mapCallback({ [key]: value }, index);
+                        return /* not await */ mapCallback({ [key]: value }, index, array.length);
                     }));
                 }));
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -4341,10 +4424,10 @@ const CsvFormatDefinition = {
     ],
 };
 /**
- * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `CsvFormatDefinition` implement `heal
- * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `CsvFormatParser` implement `heal
+ * TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside CSV objects and other formats
  */
 
@@ -4353,7 +4436,7 @@ const CsvFormatDefinition = {
  *
  * @private still in development [🏢]
  */
-const JsonFormatDefinition = {
+const JsonFormatParser = {
     formatName: 'JSON',
     mimeType: 'application/json',
     isValid(value, settings, schema) {
@@ -4365,28 +4448,28 @@ const JsonFormatDefinition = {
     heal(value, settings, schema) {
         throw new Error('Not implemented');
     },
-    subvalueDefinitions: [],
+    subvalueParsers: [],
 };
 /**
- * TODO: [🧠] Maybe propper instance of object
+ * TODO: [🧠] Maybe proper instance of object
  * TODO: [0] Make string_serialized_json
  * TODO: [1] Make type for JSON Settings and Schema
  * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
- * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `JsonFormatDefinition` implement `heal
- * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `JsonFormatParser` implement `heal
+ * TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside JSON objects and other formats
  */
 
 /**
  * Definition for any text - this will be always valid
  *
- * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
  *
  * @public exported from `@promptbook/core`
  */
-const TextFormatDefinition = {
+const TextFormatParser = {
     formatName: 'TEXT',
     isValid(value) {
         return typeof value === 'string';
@@ -4395,19 +4478,20 @@ const TextFormatDefinition = {
         return typeof partialValue === 'string';
     },
     heal() {
-        throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
+        throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
     },
-    subvalueDefinitions: [
+    subvalueParsers: [
         {
             subvalueName: 'LINE',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
-                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                 // TODO: [🧠] Maybe option to skip empty line
                 /* not await */ mapCallback({
                     lineContent,
                     // TODO: [🧠] Maybe also put here `lineNumber`
-                }, lineNumber)));
+                }, lineNumber, array.length)));
                 return mappedLines.join('\n');
             },
         },
@@ -4417,10 +4501,10 @@ const TextFormatDefinition = {
 /**
  * TODO: [1] Make type for XML Text and Schema
  * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
- * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `TextFormatDefinition` implement `heal
- * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `TextFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `TextFormatParser` implement `heal
+ * TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside each item of list and other formats
  */
 
@@ -4428,7 +4512,7 @@ const TextFormatDefinition = {
  * Function to check if a string is valid XML
  *
  * @param value
- * @returns True if the string is a valid XML string, false otherwise
+ * @returns `true` if the string is a valid XML string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4453,7 +4537,7 @@ function isValidXmlString(value) {
  *
  * @private still in development [🏢]
  */
-const XmlFormatDefinition = {
+const XmlFormatParser = {
     formatName: 'XML',
     mimeType: 'application/xml',
     isValid(value, settings, schema) {
@@ -4465,17 +4549,17 @@ const XmlFormatDefinition = {
     heal(value, settings, schema) {
         throw new Error('Not implemented');
     },
-    subvalueDefinitions: [],
+    subvalueParsers: [],
 };
 /**
- * TODO: [🧠] Maybe propper instance of object
+ * TODO: [🧠] Maybe proper instance of object
  * TODO: [0] Make string_serialized_xml
  * TODO: [1] Make type for XML Settings and Schema
  * TODO: [🧠] What to use for validating XMLs - XSD,...
- * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `XmlFormatDefinition` implement `heal
- * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `XmlFormatParser` implement `heal
+ * TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside XML and other formats
  */
 
@@ -4484,24 +4568,19 @@ const XmlFormatDefinition = {
  *
  * @private internal index of `...` <- TODO [🏢]
  */
-const FORMAT_DEFINITIONS = [
-    JsonFormatDefinition,
-    XmlFormatDefinition,
-    TextFormatDefinition,
-    CsvFormatDefinition,
-];
+const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
 /**
- * Maps available parameters to expected parameters
+ * Maps available parameters to expected parameters for a pipeline task.
  *
  * The strategy is:
- * 1) @@@
- * 2) @@@
+ * 1) First, match parameters by name where both available and expected.
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
  *
- * @throws {PipelineExecutionError} @@@
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
  * @private within the repository used in `createPipelineExecutor`
  */
 function mapAvailableToExpectedParameters(options) {
@@ -4524,7 +4603,7 @@ function mapAvailableToExpectedParameters(options) {
         else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
     }
     if (expectedParameterNames.size === 0) {
-        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
         Object.freeze(mappedParameters);
         return mappedParameters;
     }
@@ -4555,7 +4634,7 @@ function mapAvailableToExpectedParameters(options) {
     for (let i = 0; i < expectedParameterNames.size; i++) {
         mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
     }
-    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
     Object.freeze(mappedParameters);
     return mappedParameters;
 }
@@ -4563,8 +4642,8 @@ function mapAvailableToExpectedParameters(options) {
 /**
  * Extracts all code blocks from markdown.
  *
- * Note: There are multiple simmilar function:
- * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
+ * Note: There are multiple similar functions:
+ * - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
  * - `extractJsonBlock` extracts exactly one valid JSON code block
  * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
  * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
@@ -4614,7 +4693,7 @@ function extractAllBlocksFromMarkdown(markdown) {
             if (currentCodeBlock.content !== '') {
                 currentCodeBlock.content += '\n';
             }
-            currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
+            currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
         }
     }
     if (currentCodeBlock !== null) {
@@ -4634,7 +4713,7 @@ function extractAllBlocksFromMarkdown(markdown) {
  * - When there are multiple JSON code blocks the function throws a `ParseError`
  *
  * Note: It is not important if marked as ```json BUT if it is VALID JSON
- * Note: There are multiple simmilar function:
+ * Note: There are multiple similar function:
  * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
  * - `extractJsonBlock` extracts exactly one valid JSON code block
  * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
@@ -4659,7 +4738,7 @@ function extractJsonBlock(markdown) {
 }
 /**
  * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
- * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
+ * TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
  */
 
 /**
@@ -4861,14 +4940,14 @@ const CountUtils = {
     PAGES: countPages,
 };
 /**
- * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
+ * TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
 /**
  * Function checkExpectations will check if the expectations on given value are met
  *
- * Note: There are two simmilar functions:
+ * Note: There are two similar functions:
  * - `checkExpectations` which throws an error if the expectations are not met
  * - `isPassingExpectations` which returns a boolean
  *
@@ -4889,13 +4968,17 @@ function checkExpectations(expectations, value) {
 }
 /**
  * TODO: [💝] Unite object for expecting amount and format
- * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
+ * TODO: [🧠][🤠] This should be part of `TextFormatParser`
  * Note: [💝] and [🤠] are interconnected together
  */
 
 /**
- * @@@
+ * Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
+ * (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
+ * Throws errors if execution fails after all attempts.
  *
+ * @param options - The options for execution, including task, parameters, pipeline, and configuration.
+ * @returns The result string of the executed task.
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeAttempts(options) {
@@ -5117,7 +5200,7 @@ async function executeAttempts(options) {
             if (task.format) {
                 if (task.format === 'JSON') {
                     if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
-                        // TODO: [🏢] Do more universally via `FormatDefinition`
+                        // TODO: [🏢] Do more universally via `FormatParser`
                         try {
                             $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
                         }
@@ -5219,12 +5302,16 @@ async function executeAttempts(options) {
  */
 
 /**
- * @@@
+ * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+ * Handles format and subformat resolution, error handling, and progress reporting.
+ *
+ * @param options - Options for execution, including task details and progress callback.
+ * @returns The result of the subvalue mapping or execution attempts.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeFormatSubvalues(options) {
-    const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+    const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
     if (task.foreach === undefined) {
         return /* not await */ executeAttempts(options);
     }
@@ -5255,16 +5342,16 @@ async function executeFormatSubvalues(options) {
                     ${block(pipelineIdentification)}
                 `));
     }
-    const subvalueDefinition = formatDefinition.subvalueDefinitions.find((subvalueDefinition) => [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(task.foreach.subformatName));
-    if (subvalueDefinition === undefined) {
+    const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
+    if (subvalueParser === undefined) {
         throw new UnexpectedError(
         // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
         spaceTrim$1((block) => `
                     Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
 
                     Available subformat names for format "${formatDefinition.formatName}":
-                    ${block(formatDefinition.subvalueDefinitions
-            .map((subvalueDefinition) => subvalueDefinition.subvalueName)
+                    ${block(formatDefinition.subvalueParsers
+            .map((subvalueParser) => subvalueParser.subvalueName)
             .map((subvalueName) => `- ${subvalueName}`)
             .join('\n'))}
 
@@ -5276,55 +5363,85 @@ async function executeFormatSubvalues(options) {
     let formatSettings;
     if (formatDefinition.formatName === 'CSV') {
         formatSettings = csvSettings;
-        // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
-    }
-    const resultString = await subvalueDefinition.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-        let mappedParameters;
-        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-        // TODO: When done [🐚] Report progress also for each subvalue here
-        try {
-            mappedParameters = mapAvailableToExpectedParameters({
-                expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                availableParameters: subparameters,
-            });
-        }
-        catch (error) {
-            if (!(error instanceof PipelineExecutionError)) {
-                throw error;
+        // <- TODO: [🤹‍♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
+    }
+    const resultString = await subvalueParser.mapValues({
+        value: parameterValue,
+        outputParameterName: task.foreach.outputSubparameterName,
+        settings: formatSettings,
+        onProgress(partialResultString) {
+            return onProgress(Object.freeze({
+                [task.resultingParameterName]: partialResultString,
+            }));
+        },
+        async mapCallback(subparameters, index, length) {
+            let mappedParameters;
+            try {
+                mappedParameters = mapAvailableToExpectedParameters({
+                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                    availableParameters: subparameters,
+                });
             }
-            throw new PipelineExecutionError(spaceTrim$1((block) => `
-                        ${error.message}
+            catch (error) {
+                if (!(error instanceof PipelineExecutionError)) {
+                    throw error;
+                }
+                const highLevelError = new PipelineExecutionError(spaceTrim$1((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
+                            You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
-        }
-        const allSubparameters = {
-            ...parameters,
-            ...mappedParameters,
-        };
-        // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-        Object.freeze(allSubparameters);
-        const subresultString = await executeAttempts({
-            ...options,
-            priority: priority + index,
-            parameters: allSubparameters,
-            pipelineIdentification: spaceTrim$1((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-        });
-        return subresultString;
+                            ${block(pipelineIdentification)}
+                        `));
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(highLevelError);
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw highLevelError;
+            }
+            const allSubparameters = {
+                ...parameters,
+                ...mappedParameters,
+            };
+            Object.freeze(allSubparameters);
+            try {
+                const subresultString = await executeAttempts({
+                    ...options,
+                    priority: priority + index,
+                    parameters: allSubparameters,
+                    pipelineIdentification: spaceTrim$1((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                });
+                return subresultString;
+            }
+            catch (error) {
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(spaceTrim$1((block) => `
+                              ${error.message}
+
+                              This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
+
+                              ${block(pipelineIdentification)}
+                          `));
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw error;
+            }
+        },
     });
     return resultString;
 }
 
 /**
- * @@@
+ * Returns the context for a given task, typically used to provide additional information or variables
+ * required for the execution of the task within a pipeline. The context is returned as a string value
+ * that may include markdown formatting.
  *
+ * @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
+ * @returns The context as a string, formatted as markdown and parameter value.
  * @private internal utility of `createPipelineExecutor`
  */
 async function getContextForTask(task) {
@@ -5332,7 +5449,7 @@ async function getContextForTask(task) {
 }
 
 /**
- * @@@
+ * Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5341,91 +5458,128 @@ async function getExamplesForTask(task) {
 }
 
 /**
- * @@@
+ * Computes the cosine similarity between two embedding vectors
+ *
+ * Note: This is helping function for RAG (retrieval-augmented generation)
  *
- * Here is the place where RAG (retrieval-augmented generation) happens
+ * @param embeddingVector1
+ * @param embeddingVector2
+ * @returns Cosine similarity between the two vectors
+ *
+ * @public exported from `@promptbook/core`
+ */
+function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
+    if (embeddingVector1.length !== embeddingVector2.length) {
+        throw new TypeError('Embedding vectors must have the same length');
+    }
+    const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
+    const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
+    const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
+    return 1 - dotProduct / (magnitude1 * magnitude2);
+}
+
+/**
+ *
+ * @param knowledgePieces
+ * @returns
+ *
+ * @private internal utility of `createPipelineExecutor`
+ */
+function knowledgePiecesToString(knowledgePieces) {
+    return knowledgePieces
+        .map((knowledgePiece) => {
+        const { content } = knowledgePiece;
+        return `- ${content}`;
+    })
+        .join('\n');
+    // <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
+}
+
+/**
+ * Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
+ * This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function getKnowledgeForTask(options) {
-    const { tools, preparedPipeline, task } = options;
+    const { tools, preparedPipeline, task, parameters } = options;
     const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
     const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
     // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
     if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
-        return 'No knowledge pieces found';
+        return ''; // <- Note: Np knowledge present, return empty string
     }
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
-    const _llms = arrayableToArray(tools.llm);
-    const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
-    const taskEmbeddingPrompt = {
-        title: 'Knowledge Search',
-        modelRequirements: {
-            modelVariant: 'EMBEDDING',
-            modelName: firstKnowlegeIndex.modelName,
-        },
-        content: task.content,
-        parameters: {
-        /* !!!!!!!! */
-        },
-    };
-    const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
-    const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
-        const { index } = knowledgePiece;
-        const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
-        // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
-        if (knowledgePieceIndex === undefined) {
+    try {
+        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
+        const _llms = arrayableToArray(tools.llm);
+        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const taskEmbeddingPrompt = {
+            title: 'Knowledge Search',
+            modelRequirements: {
+                modelVariant: 'EMBEDDING',
+                modelName: firstKnowlegeIndex.modelName,
+            },
+            content: task.content,
+            parameters,
+        };
+        const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
+        const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
+            const { index } = knowledgePiece;
+            const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
+            // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
+            if (knowledgePieceIndex === undefined) {
+                return {
+                    content: knowledgePiece.content,
+                    relevance: 0,
+                };
+            }
+            const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
             return {
                 content: knowledgePiece.content,
-                relevance: 0,
+                relevance,
             };
-        }
-        const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
-        return {
-            content: knowledgePiece.content,
-            relevance,
-        };
-    });
-    const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
-    const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
-    console.log('!!! Embedding', {
-        task,
-        taskEmbeddingPrompt,
-        taskEmbeddingResult,
-        firstKnowlegePiece,
-        firstKnowlegeIndex,
-        knowledgePiecesWithRelevance,
-        knowledgePiecesSorted,
-        knowledgePiecesLimited,
-    });
-    return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
-    //                                                      <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
-}
-// TODO: !!!!!! Annotate + to new file
-function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
-    if (embeddingVector1.length !== embeddingVector2.length) {
-        throw new TypeError('Embedding vectors must have the same length');
+        });
+        const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
+        const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
+        /*
+        console.log('`getKnowledgeForTask` Embedding', {
+            task,
+            taskEmbeddingPrompt,
+            taskEmbeddingResult,
+            firstKnowlegePiece,
+            firstKnowlegeIndex,
+            knowledgePiecesWithRelevance,
+            knowledgePiecesSorted,
+            knowledgePiecesLimited,
+        });
+        */
+        return knowledgePiecesToString(knowledgePiecesLimited);
+    }
+    catch (error) {
+        assertsError(error);
+        console.error('Error in `getKnowledgeForTask`', error);
+        // Note: If the LLM fails, just return all knowledge pieces
+        return knowledgePiecesToString(preparedPipeline.knowledgePieces);
     }
-    const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
-    const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
-    const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
-    return 1 - dotProduct / (magnitude1 * magnitude2);
 }
 /**
- * TODO: !!!! Verify if this is working
  * TODO: [♨] Implement Better - use keyword search
  * TODO: [♨] Examples of values
  */
 
 /**
- * @@@
+ * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+ * Ensures all reserved parameters are defined and throws if any are missing.
+ *
+ * @param options - Options including tools, pipeline, task, and context.
+ * @returns An object containing all reserved parameters for the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function getReservedParametersForTask(options) {
-    const { tools, preparedPipeline, task, pipelineIdentification } = options;
+    const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
     const context = await getContextForTask(); // <- [🏍]
-    const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
+    const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
     const examples = await getExamplesForTask();
     const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
     const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5437,6 +5591,9 @@ async function getReservedParametersForTask(options) {
         currentDate,
         modelName,
     };
+    if (isVerbose) {
+        console.info('Reserved parameters for task:', { options, reservedParameters });
+    }
     // Note: Doublecheck that ALL reserved parameters are defined:
     for (const parameterName of RESERVED_PARAMETER_NAMES) {
         if (reservedParameters[parameterName] === undefined) {
@@ -5451,23 +5608,21 @@ async function getReservedParametersForTask(options) {
 }
 
 /**
- * @@@
+ * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+ *
+ * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+ * @returns The output parameters produced by the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeTask(options) {
     const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
     const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-    await onProgress({
-        outputParameters: {
-            [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-        },
-    });
     // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
     const usedParameterNames = extractParameterNamesFromTask(currentTask);
     const dependentParameterNames = new Set(currentTask.dependentParameterNames);
     // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
-    if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
+    if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
         throw new UnexpectedError(spaceTrim((block) => `
                     Dependent parameters are not consistent with used parameters:
 
@@ -5485,13 +5640,16 @@ async function executeTask(options) {
 
                 `));
     }
+    const reservedParameters = await getReservedParametersForTask({
+        tools,
+        preparedPipeline,
+        task: currentTask,
+        pipelineIdentification,
+        parameters: parametersToPass,
+        isVerbose,
+    });
     const definedParameters = Object.freeze({
-        ...(await getReservedParametersForTask({
-            tools,
-            preparedPipeline,
-            task: currentTask,
-            pipelineIdentification,
-        })),
+        ...reservedParameters,
         ...parametersToPass,
     });
     const definedParameterNames = new Set(Object.keys(definedParameters));
@@ -5536,6 +5694,7 @@ async function executeTask(options) {
         preparedPipeline,
         tools,
         $executionReport,
+        onProgress,
         pipelineIdentification,
         maxExecutionAttempts,
         maxParallelCount,
@@ -5563,7 +5722,8 @@ async function executeTask(options) {
  */
 
 /**
- * @@@
+ * Filters and returns only the output parameters from the provided pipeline execution options.
+ * Adds warnings for any expected output parameters that are missing.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5588,9 +5748,12 @@ function filterJustOutputParameters(options) {
 }
 
 /**
- * @@@
+ * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
  *
- * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+ * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
+ *
+ * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+ * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5913,10 +6076,27 @@ function createPipelineExecutor(options) {
             cacheDirname,
             intermediateFilesStrategy,
             isAutoInstalled,
+        }).catch((error) => {
+            assertsError(error);
+            return exportJson({
+                name: 'pipelineExecutorResult',
+                message: `Unuccessful PipelineExecutorResult, last catch`,
+                order: [],
+                value: {
+                    isSuccessful: false,
+                    errors: [serializeError(error)],
+                    warnings: [],
+                    usage: UNCERTAIN_USAGE,
+                    executionReport: null,
+                    outputParameters: {},
+                    preparedPipeline,
+                },
+            });
         });
     };
     const pipelineExecutor = (inputParameters) => createTask({
         taskType: 'EXECUTION',
+        title: pipeline.title,
         taskProcessCallback(updateOngoingResult) {
             return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
                 updateOngoingResult(newOngoingResult);
@@ -6019,7 +6199,7 @@ class MarkdownScraper {
         const knowledge = await Promise.all(
         // TODO: [🪂] Do not send all at once but in chunks
         knowledgeTextPieces.map(async (knowledgeTextPiece, i) => {
-            // Note: Theese are just default values, they will be overwritten by the actual values:
+            // Note: These are just default values, they will be overwritten by the actual values:
             let name = `piece-${i}`;
             let title = spaceTrim$1(knowledgeTextPiece.substring(0, 100));
             const knowledgePieceContent = spaceTrim$1(knowledgeTextPiece);
@@ -6397,7 +6577,8 @@ class LegacyDocumentScraper {
  */
 
 /**
- * @@@
+ * Creates a scraper for legacy document formats (.doc, .rtf, etc).
+ * Uses LibreOffice for conversion to extract content from older document formats.
  *
  * @public exported from `@promptbook/legacy-documents`
  */