@promptbook/markitdown 0.92.0-3 → 0.92.0-30

package/esm/index.es.js

@@ -26,7 +26,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-3';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-30';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -102,6 +102,21 @@ const DEFAULT_BOOK_TITLE = `✨ Untitled Book`;
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+/**
+ * Threshold value that determines when a dataset is considered "big"
+ * and may require special handling or optimizations
+ *
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
+ *
+ * @public exported from `@promptbook/core`
+ */
+const BIG_DATASET_TRESHOLD = 50;
+/**
+ * Placeholder text used to represent a placeholder value of failed operation
+ *
+ * @public exported from `@promptbook/core`
+ */
+const FAILED_VALUE_PLACEHOLDER = '!?';
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -181,7 +196,7 @@ const DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
 const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
 // <- TODO: [🧜‍♂️]
 /**
- * @@@
+ * Default settings for parsing and generating CSV files in Promptbook.
  *
  * @public exported from `@promptbook/core`
  */
@@ -192,19 +207,19 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
     skipEmptyLines: true,
 });
 /**
- * @@@
+ * Controls whether verbose logging is enabled by default throughout the application.
  *
  * @public exported from `@promptbook/core`
  */
 let DEFAULT_IS_VERBOSE = false;
 /**
- * @@@
+ * Controls whether auto-installation of dependencies is enabled by default.
  *
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_IS_AUTO_INSTALLED = false;
 /**
- * @@@
+ * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
  *
  * @private within the repository
  */
@@ -355,7 +370,8 @@ async function isFileExisting(filename, fs) {
  */
 
 /**
- * @@@
+ * Converts a name to a properly formatted subfolder path for cache storage.
+ * Handles normalization and path formatting to create consistent cache directory structures.
  *
  * @private for `FileCacheStorage`
  */
@@ -608,10 +624,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
 */
 
 /**
- * @@@
+ * Removes diacritic marks (accents) from characters in a string.
  *
- * @param input @@@
- * @returns @@@
+ * @param input The string containing diacritics to be normalized.
+ * @returns The string with diacritics removed or normalized.
  * @public exported from `@promptbook/utils`
  */
 function removeDiacritics(input) {
@@ -625,10 +641,10 @@ function removeDiacritics(input) {
  */
 
 /**
- * @@@
+ * Converts a given text to kebab-case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text to be converted.
+ * @returns The kebab-case formatted string.
  * @example 'hello-world'
  * @example 'i-love-promptbook'
  * @public exported from `@promptbook/utils`
@@ -770,11 +786,11 @@ function isValidUrl(url) {
 }
 
 /**
- * @@@
+ * Converts a title string into a normalized name.
  *
- * @param value @@@
- * @returns @@@
- * @example @@@
+ * @param value The title string to be converted to a name.
+ * @returns A normalized name derived from the input title.
+ * @example 'Hello World!' -> 'hello-world'
  * @public exported from `@promptbook/utils`
  */
 function titleToName(value) {
@@ -794,9 +810,8 @@ function titleToName(value) {
 }
 
 /**
- * Create a filename for intermediate cache for scrapers
- *
- * Note: It also checks if directory exists and creates it if not
+ * Retrieves an intermediate source for a scraper based on the knowledge source.
+ * Manages the caching and retrieval of intermediate scraper results for optimized performance.
  *
  * @private as internal utility for scrapers
  */
@@ -847,7 +862,7 @@ async function getScraperIntermediateSource(source, options) {
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */
 
-var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n-   INPUT PARAMETER `{knowledgeContent}` Markdown document content\n-   OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n-   INPUT PARAMETER `{knowledgePieceContent}` The content\n-   OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n-   INPUT PARAMETER `{knowledgePieceContent}` The content\n-   OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n-   EXPECT MIN 1 WORD\n-   EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n-   Your output format is JSON object\n-   Write just the JSON object, no other text should be present\n-   It contains the following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n-   INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n-   INPUT PARAMETER `{personaDescription}` Description of the persona\n-   OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n-   FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n-   Your output format is JSON object\n-   Write just the JSON object, no other text should be present\n-   It contains the following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n-   Title starts with emoticon\n-   Title should not mention the input and output of the workflow but the main purpose of the workflow\n    _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n-   INPUT PARAMETER `{book}` The book to prepare the title for\n-   OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n-   EXPECT MIN 1 Word\n-   EXPECT MAX 8 Words\n-   EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n-   Title starts with emoticon\n-   Title should not mention the input and output of the workflow but the main purpose of the workflow\n    _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
+var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n-   INPUT PARAMETER `{knowledgeContent}` Markdown document content\n-   OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n-   INPUT PARAMETER `{knowledgePieceContent}` The content\n-   OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n-   INPUT PARAMETER `{knowledgePieceContent}` The content\n-   OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n-   EXPECT MIN 1 WORD\n-   EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModels",description:"List of available model names together with their descriptions as JSON",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelsRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n```json\n[\n  {\n  \"modelName\": \"gpt-4o\",\n  \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n  \"temperature\": 0.7\n  },\n  {\n    \"modelName\": \"claude-3-5-sonnet\",\n    \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n    \"temperature\": 0.5\n  }\n]\n```\n\n## Instructions\n\n-   Your output format is JSON array\n-   Sort best-fitting models first\n-   Omit any models that are not suitable\n-   Write just the JSON, no other text should be present\n-   Array contain items with following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n```json\n{availableModels}\n```\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelsRequirements",format:"JSON",dependentParameterNames:["availableModels","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n-   INPUT PARAMETER `{availableModels}` List of available model names together with their descriptions as JSON\n-   INPUT PARAMETER `{personaDescription}` Description of the persona\n-   OUTPUT PARAMETER `{modelsRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n-   FORMAT JSON\n\n```markdown\nYou are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n\\`\\`\\`json\n[\n  {\n  \"modelName\": \"gpt-4o\",\n  \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n  \"temperature\": 0.7\n  },\n  {\n    \"modelName\": \"claude-3-5-sonnet\",\n    \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n    \"temperature\": 0.5\n  }\n]\n\\`\\`\\`\n\n## Instructions\n\n-   Your output format is JSON array\n-   Sort best-fitting models first\n-   Omit any models that are not suitable\n-   Write just the JSON, no other text should be present\n-   Array contain items with following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n\\`\\`\\`json\n{availableModels}\n\\`\\`\\`\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelsRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n-   Title starts with emoticon\n-   Title should not mention the input and output of the workflow but the main purpose of the workflow\n    _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n-   PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n-   INPUT PARAMETER `{book}` The book to prepare the title for\n-   OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n-   EXPECT MIN 1 Word\n-   EXPECT MAX 8 Words\n-   EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n-   Title starts with emoticon\n-   Title should not mention the input and output of the workflow but the main purpose of the workflow\n    _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
 
 /**
  * Checks if value is valid email
@@ -932,7 +947,7 @@ function assertsError(whatWasThrown) {
  * Function isValidJsonString will tell you if the string is valid JSON or not
  *
  * @param value The string to check
- * @returns True if the string is a valid JSON string, false otherwise
+ * @returns `true` if the string is a valid JSON string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -1343,8 +1358,12 @@ function checkSerializableAsJson(options) {
  */
 
 /**
- * @@@
+ * Creates a deep clone of the given object
+ *
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
  *
+ * @param objectValue The object to clone.
+ * @returns A deep, writable clone of the input object.
  * @public exported from `@promptbook/utils`
  */
 function deepClone(objectValue) {
@@ -1426,13 +1445,13 @@ const ORDER_OF_PIPELINE_JSON = [
  */
 const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
- * @@@
+ * Placeholder value indicating a parameter is missing its value.
  *
  * @private within the repository
  */
 const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
 /**
- * @@@
+ * Placeholder value indicating a parameter is restricted and cannot be used directly.
  *
  * @private within the repository
  */
@@ -1890,7 +1909,7 @@ function extractParameterNames(template) {
  */
 function unpreparePipeline(pipeline) {
     let { personas, knowledgeSources, tasks } = pipeline;
-    personas = personas.map((persona) => ({ ...persona, modelRequirements: undefined, preparationIds: undefined }));
+    personas = personas.map((persona) => ({ ...persona, modelsRequirements: undefined, preparationIds: undefined }));
     knowledgeSources = knowledgeSources.map((knowledgeSource) => ({ ...knowledgeSource, preparationIds: undefined }));
     tasks = tasks.map((task) => {
         let { dependentParameterNames } = task;
@@ -1931,7 +1950,7 @@ class SimplePipelineCollection {
     /**
      * Constructs a pipeline collection from pipelines
      *
-     * @param pipelines @@@
+     * @param pipelines Array of pipeline JSON objects to include in the collection
      *
      * Note: During the construction logic of all pipelines are validated
      * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -2095,15 +2114,21 @@ class PipelineExecutionError extends Error {
  * @public exported from `@promptbook/core`
  */
 function isPipelinePrepared(pipeline) {
-    // Note: Ignoring `pipeline.preparations` @@@
-    // Note: Ignoring `pipeline.knowledgePieces` @@@
+    // Note: Ignoring `pipeline.preparations`
+    // Note: Ignoring `pipeline.knowledgePieces`
     if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
+        // TODO: !!! Comment this out
+        console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
         return false;
     }
-    if (!pipeline.personas.every((persona) => persona.modelRequirements !== undefined)) {
+    if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
+        // TODO: !!! Comment this out
+        console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
         return false;
     }
     if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
+        // TODO: !!! Comment this out
+        console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
         return false;
     }
     /*
@@ -2124,36 +2149,6 @@ function isPipelinePrepared(pipeline) {
  *     - [♨] Are tasks prepared
  */
 
-/**
- * Recursively converts JSON strings to JSON objects
-
- * @public exported from `@promptbook/utils`
- */
-function jsonStringsToJsons(object) {
-    if (object === null) {
-        return object;
-    }
-    if (Array.isArray(object)) {
-        return object.map(jsonStringsToJsons);
-    }
-    if (typeof object !== 'object') {
-        return object;
-    }
-    const newObject = { ...object };
-    for (const [key, value] of Object.entries(object)) {
-        if (typeof value === 'string' && isValidJsonString(value)) {
-            newObject[key] = JSON.parse(value);
-        }
-        else {
-            newObject[key] = jsonStringsToJsons(value);
-        }
-    }
-    return newObject;
-}
-/**
- * TODO: Type the return type correctly
- */
-
 /**
  * This error indicates problems parsing the format value
  *
@@ -2337,6 +2332,101 @@ const ALL_ERRORS = {
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Serializes an error into a [🚉] JSON-serializable object
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function serializeError(error) {
+    const { name, message, stack } = error;
+    const { id } = error;
+    if (!Object.keys(ALL_ERRORS).includes(name)) {
+        console.error(spaceTrim((block) => `
+
+                    Cannot serialize error with name "${name}"
+
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
+                    ${block(stack || message)}
+
+                `));
+    }
+    return {
+        name: name,
+        message,
+        stack,
+        id, // Include id in the serialized object
+    };
+}
+
+/**
+ * Converts a JavaScript Object Notation (JSON) string into an object.
+ *
+ * Note: This is wrapper around `JSON.parse()` with better error and type handling
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function jsonParse(value) {
+    if (value === undefined) {
+        throw new Error(`Can not parse JSON from undefined value.`);
+    }
+    else if (typeof value !== 'string') {
+        console.error('Can not parse JSON from non-string value.', { text: value });
+        throw new Error(spaceTrim(`
+                    Can not parse JSON from non-string value.
+
+                    The value type: ${typeof value}
+                    See more in console.
+                `));
+    }
+    try {
+        return JSON.parse(value);
+    }
+    catch (error) {
+        if (!(error instanceof Error)) {
+            throw error;
+        }
+        throw new Error(spaceTrim((block) => `
+                    ${block(error.message)}
+
+                    The JSON text:
+                    ${block(value)}
+                `));
+    }
+}
+
+/**
+ * Recursively converts JSON strings to JSON objects
+
+ * @public exported from `@promptbook/utils`
+ */
+function jsonStringsToJsons(object) {
+    if (object === null) {
+        return object;
+    }
+    if (Array.isArray(object)) {
+        return object.map(jsonStringsToJsons);
+    }
+    if (typeof object !== 'object') {
+        return object;
+    }
+    const newObject = { ...object };
+    for (const [key, value] of Object.entries(object)) {
+        if (typeof value === 'string' && isValidJsonString(value)) {
+            newObject[key] = jsonParse(value);
+        }
+        else {
+            newObject[key] = jsonStringsToJsons(value);
+        }
+    }
+    return newObject;
+}
+/**
+ * TODO: Type the return type correctly
+ */
+
 /**
  * Deserializes the error object
  *
@@ -2502,64 +2592,6 @@ function createTask(options) {
  * TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
  */
 
-/**
- * Serializes an error into a [🚉] JSON-serializable object
- *
- * @public exported from `@promptbook/utils`
- */
-function serializeError(error) {
-    const { name, message, stack } = error;
-    const { id } = error;
-    if (!Object.keys(ALL_ERRORS).includes(name)) {
-        console.error(spaceTrim((block) => `
-
-                    Cannot serialize error with name "${name}"
-
-                    Authors of Promptbook probably forgot to add this error into the list of errors:
-                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
-
-
-                    ${block(stack || message)}
-
-                `));
-    }
-    return {
-        name: name,
-        message,
-        stack,
-        id, // Include id in the serialized object
-    };
-}
-
-/**
- * Async version of Array.forEach
- *
- * @param array - Array to iterate over
- * @param options - Options for the function
- * @param callbackfunction - Function to call for each item
- * @public exported from `@promptbook/utils`
- * @deprecated [🪂] Use queues instead
- */
-async function forEachAsync(array, options, callbackfunction) {
-    const { maxParallelCount = Infinity } = options;
-    let index = 0;
-    let runningTasks = [];
-    const tasks = [];
-    for (const item of array) {
-        const currentIndex = index++;
-        const task = callbackfunction(item, currentIndex, array);
-        tasks.push(task);
-        runningTasks.push(task);
-        /* not await */ Promise.resolve(task).then(() => {
-            runningTasks = runningTasks.filter((t) => t !== task);
-        });
-        if (maxParallelCount < runningTasks.length) {
-            await Promise.race(runningTasks);
-        }
-    }
-    await Promise.all(tasks);
-}
-
 /**
  * Represents the uncertain value
  *
@@ -2603,7 +2635,7 @@ const ZERO_USAGE = $deepFreeze({
  *
  * @public exported from `@promptbook/core`
  */
-$deepFreeze({
+const UNCERTAIN_USAGE = $deepFreeze({
     price: UNCERTAIN_ZERO_VALUE,
     input: {
         tokensCount: UNCERTAIN_ZERO_VALUE,
@@ -2628,6 +2660,35 @@ $deepFreeze({
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Async version of Array.forEach
+ *
+ * @param array - Array to iterate over
+ * @param options - Options for the function
+ * @param callbackfunction - Function to call for each item
+ * @public exported from `@promptbook/utils`
+ * @deprecated [🪂] Use queues instead
+ */
+async function forEachAsync(array, options, callbackfunction) {
+    const { maxParallelCount = Infinity } = options;
+    let index = 0;
+    let runningTasks = [];
+    const tasks = [];
+    for (const item of array) {
+        const currentIndex = index++;
+        const task = callbackfunction(item, currentIndex, array);
+        tasks.push(task);
+        runningTasks.push(task);
+        /* not await */ Promise.resolve(task).then(() => {
+            runningTasks = runningTasks.filter((t) => t !== task);
+        });
+        if (maxParallelCount < runningTasks.length) {
+            await Promise.race(runningTasks);
+        }
+    }
+    await Promise.all(tasks);
+}
+
 /**
  * Function `addUsage` will add multiple usages into one
  *
@@ -2974,27 +3035,48 @@ async function preparePersona(personaDescription, tools, options) {
         pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
         tools,
     });
-    // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
     const _llms = arrayableToArray(tools.llm);
     const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
-    const availableModels = await llmTools.listModels();
-    const availableModelNames = availableModels
+    const availableModels = (await llmTools.listModels())
         .filter(({ modelVariant }) => modelVariant === 'CHAT')
-        .map(({ modelName }) => modelName)
-        .join(',');
-    const result = await preparePersonaExecutor({ availableModelNames, personaDescription }).asPromise();
+        .map(({ modelName, modelDescription }) => ({
+        modelName,
+        modelDescription,
+        // <- Note: `modelTitle` and `modelVariant` is not relevant for this task
+    }));
+    const result = await preparePersonaExecutor({
+        availableModels /* <- Note: Passing as JSON */,
+        personaDescription,
+    }).asPromise();
     const { outputParameters } = result;
-    const { modelRequirements: modelRequirementsRaw } = outputParameters;
-    const modelRequirements = JSON.parse(modelRequirementsRaw);
+    const { modelsRequirements: modelsRequirementsJson } = outputParameters;
+    let modelsRequirementsUnchecked = jsonParse(modelsRequirementsJson);
     if (isVerbose) {
-        console.info(`PERSONA ${personaDescription}`, modelRequirements);
+        console.info(`PERSONA ${personaDescription}`, modelsRequirementsUnchecked);
     }
-    const { modelName, systemMessage, temperature } = modelRequirements;
-    return {
+    if (!Array.isArray(modelsRequirementsUnchecked)) {
+        // <- TODO: Book should have syntax and system to enforce shape of JSON
+        modelsRequirementsUnchecked = [modelsRequirementsUnchecked];
+        /*
+        throw new UnexpectedError(
+            spaceTrim(
+                (block) => `
+                    Invalid \`modelsRequirements\`:
+
+                    \`\`\`json
+                    ${block(JSON.stringify(modelsRequirementsUnchecked, null, 4))}
+                    \`\`\`
+                `,
+            ),
+        );
+        */
+    }
+    const modelsRequirements = modelsRequirementsUnchecked.map((modelRequirements) => ({
         modelVariant: 'CHAT',
-        modelName,
-        systemMessage,
-        temperature,
+        ...modelRequirements,
+    }));
+    return {
+        modelsRequirements,
     };
 }
 /**
@@ -3005,7 +3087,8 @@ async function preparePersona(personaDescription, tools, options) {
  */
 
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *
@@ -3016,10 +3099,10 @@ function $getGlobalScope() {
 }
 
 /**
- * @@@
+ * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to SCREAMING_CASE format.
+ * @returns The normalized text in SCREAMING_CASE format.
  * @example 'HELLO_WORLD'
  * @example 'I_LOVE_PROMPTBOOK'
  * @public exported from `@promptbook/utils`
@@ -3071,10 +3154,10 @@ function normalizeTo_SCREAMING_CASE(text) {
  */
 
 /**
- * @@@
+ * Normalizes a text string to snake_case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to snake_case format.
+ * @returns The normalized text in snake_case format.
  * @example 'hello_world'
  * @example 'i_love_promptbook'
  * @public exported from `@promptbook/utils`
@@ -3084,11 +3167,11 @@ function normalizeTo_snake_case(text) {
 }
 
 /**
- * Register is @@@
+ * Global registry for storing and managing registered entities of a given type.
  *
  * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
  *
- * @private internal utility, exported are only signleton instances of this class
+ * @private internal utility, exported are only singleton instances of this class
  */
 class $Register {
     constructor(registerName) {
@@ -3132,10 +3215,10 @@ class $Register {
 }
 
 /**
- * @@@
+ * Global registry for storing metadata about all available scrapers and converters.
  *
- * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * Note: `$` is used to indicate that this interacts with the global scope.
+ * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
  * @public exported from `@promptbook/core`
  */
 const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -3144,10 +3227,11 @@ const $scrapersMetadataRegister = new $Register('scrapers_metadata');
  */
 
 /**
- * @@@
+ * Registry for all available scrapers in the system.
+ * Central point for registering and accessing different types of content scrapers.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
  * @public exported from `@promptbook/core`
  */
 const $scrapersRegister = new $Register('scraper_constructors');
@@ -3325,7 +3409,9 @@ const promptbookFetch = async (urlOrRequest, init) => {
  */
 
 /**
- * @@@
+ * Factory function that creates a handler for processing knowledge sources.
+ * Provides standardized processing of different types of knowledge sources
+ * across various scraper implementations.
  *
  * @public exported from `@promptbook/core`
  */
@@ -3432,7 +3518,7 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                 > },
             */
             async asJson() {
-                return JSON.parse(await tools.fs.readFile(filename, 'utf-8'));
+                return jsonParse(await tools.fs.readFile(filename, 'utf-8'));
             },
             async asText() {
                 return await tools.fs.readFile(filename, 'utf-8');
@@ -3566,9 +3652,12 @@ TODO: [🧊] This is how it can look in future
  */
 
 /**
- * @@@
+ * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
  *
- * @public exported from `@promptbook/core`
+ * @param tasks Sequence of tasks that are chained together to form a pipeline
+ * @returns A promise that resolves to the prepared tasks.
+ *
+ * @private internal utility of `preparePipeline`
  */
 async function prepareTasks(pipeline, tools, options) {
     const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -3690,14 +3779,14 @@ async function preparePipeline(pipeline, tools, options) {
     // TODO: [🖌][🧠] Implement some `mapAsync` function
     const preparedPersonas = new Array(personas.length);
     await forEachAsync(personas, { maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, async (persona, index) => {
-        const modelRequirements = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
+        const { modelsRequirements } = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
             rootDirname,
             maxParallelCount /* <- TODO:  [🪂] */,
             isVerbose,
         });
         const preparedPersona = {
             ...persona,
-            modelRequirements,
+            modelsRequirements,
             preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id],
             // <- TODO: [🍙] Make some standard order of json properties
         };
@@ -4005,7 +4094,7 @@ function union(...sets) {
 }
 
 /**
- * @@@
+ * Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
  *
  * @public exported from `@promptbook/core`
  */
@@ -4014,11 +4103,29 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
     // encoding: 'utf-8',
 });
 
+/**
+ * Converts a CSV string into an object
+ *
+ * Note: This is wrapper around `papaparse.parse()` with better autohealing
+ *
+ * @private - for now until `@promptbook/csv` is released
+ */
+function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
+    settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
+    // Note: Autoheal invalid '\n' characters
+    if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
+        console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
+        value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+    }
+    const csv = parse(value, settings);
+    return csv;
+}
+
 /**
  * Function to check if a string is valid CSV
  *
  * @param value The string to check
- * @returns True if the string is a valid CSV string, false otherwise
+ * @returns `true` if the string is a valid CSV string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4042,7 +4149,7 @@ function isValidCsvString(value) {
  * @public exported from `@promptbook/core`
  *                          <- TODO: [🏢] Export from package `@promptbook/csv`
  */
-const CsvFormatDefinition = {
+const CsvFormatParser = {
     formatName: 'CSV',
     aliases: ['SPREADSHEET', 'TABLE'],
     isValid(value, settings, schema) {
@@ -4054,12 +4161,12 @@ const CsvFormatDefinition = {
     heal(value, settings, schema) {
         throw new Error('Not implemented');
     },
-    subvalueDefinitions: [
+    subvalueParsers: [
         {
             subvalueName: 'ROW',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
-                // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
-                const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
+            async mapValues(options) {
+                const { value, outputParameterName, settings, mapCallback, onProgress } = options;
+                const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim((block) => `
                                 CSV parsing error
@@ -4074,23 +4181,37 @@ const CsvFormatDefinition = {
                                 ${block(value)}
                             `));
                 }
-                const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                const mappedData = [];
+                const length = csv.data.length;
+                for (let index = 0; index < length; index++) {
+                    const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
-                    return {
+                    const mappedRow = {
                         ...row,
-                        [outputParameterName]: await mapCallback(row, index),
+                        [outputParameterName]: await mapCallback(row, index, length),
                     };
-                }));
+                    mappedData.push(mappedRow);
+                    if (onProgress) {
+                        // Note: Report the CSV with all rows mapped so far
+                        /*
+                        // TODO: [🛕] Report progress with all the rows including the pending ones
+                          const progressData = mappedData.map((row, i) =>
+                            i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                        );
+                        */
+                        await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                    }
+                }
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
             },
         },
         {
             subvalueName: 'CELL',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
-                // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
-                const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
+            async mapValues(options) {
+                const { value, settings, mapCallback, onProgress } = options;
+                const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim((block) => `
                                 CSV parsing error
@@ -4106,9 +4227,9 @@ const CsvFormatDefinition = {
                             `));
                 }
                 const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                         const index = rowIndex * Object.keys(row).length + columnIndex;
-                        return /* not await */ mapCallback({ [key]: value }, index);
+                        return /* not await */ mapCallback({ [key]: value }, index, array.length);
                     }));
                 }));
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -4117,10 +4238,10 @@ const CsvFormatDefinition = {
     ],
 };
 /**
- * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `CsvFormatDefinition` implement `heal
- * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `CsvFormatParser` implement `heal
+ * TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside CSV objects and other formats
  */
 
@@ -4129,7 +4250,7 @@ const CsvFormatDefinition = {
  *
  * @private still in development [🏢]
  */
-const JsonFormatDefinition = {
+const JsonFormatParser = {
     formatName: 'JSON',
     mimeType: 'application/json',
     isValid(value, settings, schema) {
@@ -4141,28 +4262,28 @@ const JsonFormatDefinition = {
     heal(value, settings, schema) {
         throw new Error('Not implemented');
     },
-    subvalueDefinitions: [],
+    subvalueParsers: [],
 };
 /**
  * TODO: [🧠] Maybe propper instance of object
  * TODO: [0] Make string_serialized_json
  * TODO: [1] Make type for JSON Settings and Schema
  * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
- * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `JsonFormatDefinition` implement `heal
- * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `JsonFormatParser` implement `heal
+ * TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside JSON objects and other formats
  */
 
 /**
  * Definition for any text - this will be always valid
  *
- * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
  *
  * @public exported from `@promptbook/core`
  */
-const TextFormatDefinition = {
+const TextFormatParser = {
     formatName: 'TEXT',
     isValid(value) {
         return typeof value === 'string';
@@ -4171,19 +4292,20 @@ const TextFormatDefinition = {
         return typeof partialValue === 'string';
     },
     heal() {
-        throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
+        throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
     },
-    subvalueDefinitions: [
+    subvalueParsers: [
         {
             subvalueName: 'LINE',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
-                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                 // TODO: [🧠] Maybe option to skip empty line
                 /* not await */ mapCallback({
                     lineContent,
                     // TODO: [🧠] Maybe also put here `lineNumber`
-                }, lineNumber)));
+                }, lineNumber, array.length)));
                 return mappedLines.join('\n');
             },
         },
@@ -4193,10 +4315,10 @@ const TextFormatDefinition = {
 /**
  * TODO: [1] Make type for XML Text and Schema
  * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
- * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `TextFormatDefinition` implement `heal
- * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `TextFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `TextFormatParser` implement `heal
+ * TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside each item of list and other formats
  */
 
@@ -4204,7 +4326,7 @@ const TextFormatDefinition = {
  * Function to check if a string is valid XML
  *
  * @param value
- * @returns True if the string is a valid XML string, false otherwise
+ * @returns `true` if the string is a valid XML string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4229,7 +4351,7 @@ function isValidXmlString(value) {
  *
  * @private still in development [🏢]
  */
-const XmlFormatDefinition = {
+const XmlFormatParser = {
     formatName: 'XML',
     mimeType: 'application/xml',
     isValid(value, settings, schema) {
@@ -4241,17 +4363,17 @@ const XmlFormatDefinition = {
     heal(value, settings, schema) {
         throw new Error('Not implemented');
     },
-    subvalueDefinitions: [],
+    subvalueParsers: [],
 };
 /**
  * TODO: [🧠] Maybe propper instance of object
  * TODO: [0] Make string_serialized_xml
  * TODO: [1] Make type for XML Settings and Schema
  * TODO: [🧠] What to use for validating XMLs - XSD,...
- * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
- * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
- * TODO: [🍓] In `XmlFormatDefinition` implement `heal
- * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
+ * TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
+ * TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
+ * TODO: [🍓] In `XmlFormatParser` implement `heal
+ * TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
  * TODO: [🏢] Allow to expect something inside XML and other formats
  */
 
@@ -4260,24 +4382,19 @@ const XmlFormatDefinition = {
  *
  * @private internal index of `...` <- TODO [🏢]
  */
-const FORMAT_DEFINITIONS = [
-    JsonFormatDefinition,
-    XmlFormatDefinition,
-    TextFormatDefinition,
-    CsvFormatDefinition,
-];
+const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
 /**
- * Maps available parameters to expected parameters
+ * Maps available parameters to expected parameters for a pipeline task.
  *
  * The strategy is:
- * 1) @@@
- * 2) @@@
+ * 1) First, match parameters by name where both available and expected.
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
  *
- * @throws {PipelineExecutionError} @@@
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
  * @private within the repository used in `createPipelineExecutor`
  */
 function mapAvailableToExpectedParameters(options) {
@@ -4300,7 +4417,7 @@ function mapAvailableToExpectedParameters(options) {
         else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
     }
     if (expectedParameterNames.size === 0) {
-        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
         Object.freeze(mappedParameters);
         return mappedParameters;
     }
@@ -4331,7 +4448,7 @@ function mapAvailableToExpectedParameters(options) {
     for (let i = 0; i < expectedParameterNames.size; i++) {
         mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
     }
-    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
     Object.freeze(mappedParameters);
     return mappedParameters;
 }
@@ -4435,7 +4552,7 @@ function extractJsonBlock(markdown) {
 }
 /**
  * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
- * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
+ * TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
  */
 
 /**
@@ -4478,10 +4595,12 @@ function templateParameters(template, parameters) {
             throw new PipelineExecutionError('Parameter is already opened or not closed');
         }
         if (parameters[parameterName] === undefined) {
+            console.log('!!! templateParameters 1', { parameterName, template, parameters });
             throw new PipelineExecutionError(`Parameter \`{${parameterName}}\` is not defined`);
         }
         let parameterValue = parameters[parameterName];
         if (parameterValue === undefined) {
+            console.log('!!! templateParameters 2', { parameterName, template, parameters });
             throw new PipelineExecutionError(`Parameter \`{${parameterName}}\` is not defined`);
         }
         parameterValue = valueToString(parameterValue);
@@ -4637,7 +4756,7 @@ const CountUtils = {
     PAGES: countPages,
 };
 /**
- * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
+ * TODO: [🧠][🤠] This should be probbably as part of `TextFormatParser`
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
@@ -4665,13 +4784,17 @@ function checkExpectations(expectations, value) {
 }
 /**
  * TODO: [💝] Unite object for expecting amount and format
- * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
+ * TODO: [🧠][🤠] This should be part of `TextFormatParser`
  * Note: [💝] and [🤠] are interconnected together
  */
 
 /**
- * @@@
+ * Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
+ * (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
+ * Throws errors if execution fails after all attempts.
  *
+ * @param options - The options for execution, including task, parameters, pipeline, and configuration.
+ * @returns The result string of the executed task.
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeAttempts(options) {
@@ -4893,7 +5016,7 @@ async function executeAttempts(options) {
             if (task.format) {
                 if (task.format === 'JSON') {
                     if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
-                        // TODO: [🏢] Do more universally via `FormatDefinition`
+                        // TODO: [🏢] Do more universally via `FormatParser`
                         try {
                             $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
                         }
@@ -4995,12 +5118,16 @@ async function executeAttempts(options) {
  */
 
 /**
- * @@@
+ * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+ * Handles format and subformat resolution, error handling, and progress reporting.
+ *
+ * @param options - Options for execution, including task details and progress callback.
+ * @returns The result of the subvalue mapping or execution attempts.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeFormatSubvalues(options) {
-    const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+    const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
     if (task.foreach === undefined) {
         return /* not await */ executeAttempts(options);
     }
@@ -5031,16 +5158,16 @@ async function executeFormatSubvalues(options) {
                     ${block(pipelineIdentification)}
                 `));
     }
-    const subvalueDefinition = formatDefinition.subvalueDefinitions.find((subvalueDefinition) => [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(task.foreach.subformatName));
-    if (subvalueDefinition === undefined) {
+    const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
+    if (subvalueParser === undefined) {
         throw new UnexpectedError(
         // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
         spaceTrim((block) => `
                     Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
 
                     Available subformat names for format "${formatDefinition.formatName}":
-                    ${block(formatDefinition.subvalueDefinitions
-            .map((subvalueDefinition) => subvalueDefinition.subvalueName)
+                    ${block(formatDefinition.subvalueParsers
+            .map((subvalueParser) => subvalueParser.subvalueName)
             .map((subvalueName) => `- ${subvalueName}`)
             .join('\n'))}
 
@@ -5054,53 +5181,83 @@ async function executeFormatSubvalues(options) {
         formatSettings = csvSettings;
         // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
     }
-    const resultString = await subvalueDefinition.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-        let mappedParameters;
-        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-        // TODO: When done [🐚] Report progress also for each subvalue here
-        try {
-            mappedParameters = mapAvailableToExpectedParameters({
-                expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                availableParameters: subparameters,
-            });
-        }
-        catch (error) {
-            if (!(error instanceof PipelineExecutionError)) {
-                throw error;
+    const resultString = await subvalueParser.mapValues({
+        value: parameterValue,
+        outputParameterName: task.foreach.outputSubparameterName,
+        settings: formatSettings,
+        onProgress(partialResultString) {
+            return onProgress(Object.freeze({
+                [task.resultingParameterName]: partialResultString,
+            }));
+        },
+        async mapCallback(subparameters, index, length) {
+            let mappedParameters;
+            try {
+                mappedParameters = mapAvailableToExpectedParameters({
+                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                    availableParameters: subparameters,
+                });
             }
-            throw new PipelineExecutionError(spaceTrim((block) => `
-                        ${error.message}
+            catch (error) {
+                if (!(error instanceof PipelineExecutionError)) {
+                    throw error;
+                }
+                const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
-        }
-        const allSubparameters = {
-            ...parameters,
-            ...mappedParameters,
-        };
-        // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-        Object.freeze(allSubparameters);
-        const subresultString = await executeAttempts({
-            ...options,
-            priority: priority + index,
-            parameters: allSubparameters,
-            pipelineIdentification: spaceTrim((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-        });
-        return subresultString;
+                            ${block(pipelineIdentification)}
+                        `));
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(highLevelError);
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw highLevelError;
+            }
+            const allSubparameters = {
+                ...parameters,
+                ...mappedParameters,
+            };
+            Object.freeze(allSubparameters);
+            try {
+                const subresultString = await executeAttempts({
+                    ...options,
+                    priority: priority + index,
+                    parameters: allSubparameters,
+                    pipelineIdentification: spaceTrim((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                });
+                return subresultString;
+            }
+            catch (error) {
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(spaceTrim((block) => `
+                              ${error.message}
+
+                              This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
+
+                              ${block(pipelineIdentification)}
+                          `));
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw error;
+            }
+        },
     });
     return resultString;
 }
 
 /**
- * @@@
+ * Returns the context for a given task, typically used to provide additional information or variables
+ * required for the execution of the task within a pipeline. The context is returned as a string value
+ * that may include markdown formatting.
  *
+ * @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
+ * @returns The context as a string, formatted as markdown and parameter value.
  * @private internal utility of `createPipelineExecutor`
  */
 async function getContextForTask(task) {
@@ -5108,7 +5265,7 @@ async function getContextForTask(task) {
 }
 
 /**
- * @@@
+ * Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5117,25 +5274,127 @@ async function getExamplesForTask(task) {
 }
 
 /**
- * @@@
+ * Computes the cosine similarity between two embedding vectors
+ *
+ * Note: This is helping function for RAG (retrieval-augmented generation)
+ *
+ * @param embeddingVector1
+ * @param embeddingVector2
+ * @returns Cosine similarity between the two vectors
+ *
+ * @public exported from `@promptbook/core`
+ */
+function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
+    if (embeddingVector1.length !== embeddingVector2.length) {
+        throw new TypeError('Embedding vectors must have the same length');
+    }
+    const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
+    const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
+    const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
+    return 1 - dotProduct / (magnitude1 * magnitude2);
+}
+
+/**
+ *
+ * @param knowledgePieces
+ * @returns
+ *
+ * @private internal utility of `createPipelineExecutor`
+ */
+function knowledgePiecesToString(knowledgePieces) {
+    return knowledgePieces
+        .map((knowledgePiece) => {
+        const { content } = knowledgePiece;
+        return `- ${content}`;
+    })
+        .join('\n');
+    // <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
+}
+
+/**
+ * Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
+ * This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function getKnowledgeForTask(options) {
-    const { preparedPipeline, task } = options;
-    return preparedPipeline.knowledgePieces.map(({ content }) => `- ${content}`).join('\n');
-    //                                                      <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
+    const { tools, preparedPipeline, task, parameters } = options;
+    const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
+    const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
+    // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
+    if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
+        return ''; // <- Note: Np knowledge present, return empty string
+    }
+    try {
+        // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
+        const _llms = arrayableToArray(tools.llm);
+        const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
+        const taskEmbeddingPrompt = {
+            title: 'Knowledge Search',
+            modelRequirements: {
+                modelVariant: 'EMBEDDING',
+                modelName: firstKnowlegeIndex.modelName,
+            },
+            content: task.content,
+            parameters,
+        };
+        const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
+        const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
+            const { index } = knowledgePiece;
+            const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
+            // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
+            if (knowledgePieceIndex === undefined) {
+                return {
+                    content: knowledgePiece.content,
+                    relevance: 0,
+                };
+            }
+            const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
+            return {
+                content: knowledgePiece.content,
+                relevance,
+            };
+        });
+        const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
+        const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
+        console.log('!!! Embedding', {
+            task,
+            taskEmbeddingPrompt,
+            taskEmbeddingResult,
+            firstKnowlegePiece,
+            firstKnowlegeIndex,
+            knowledgePiecesWithRelevance,
+            knowledgePiecesSorted,
+            knowledgePiecesLimited,
+        });
+        return knowledgePiecesToString(knowledgePiecesLimited);
+    }
+    catch (error) {
+        assertsError(error);
+        console.error('Error in `getKnowledgeForTask`', error);
+        // Note: If the LLM fails, just return all knowledge pieces
+        return knowledgePiecesToString(preparedPipeline.knowledgePieces);
+    }
 }
+/**
+ * TODO: !!!! Verify if this is working
+ * TODO: [♨] Implement Better - use keyword search
+ * TODO: [♨] Examples of values
+ */
 
 /**
- * @@@
+ * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+ * Ensures all reserved parameters are defined and throws if any are missing.
+ *
+ * @param options - Options including tools, pipeline, task, and context.
+ * @returns An object containing all reserved parameters for the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function getReservedParametersForTask(options) {
-    const { preparedPipeline, task, pipelineIdentification } = options;
+    const { tools, preparedPipeline, task, parameters, pipelineIdentification } = options;
     const context = await getContextForTask(); // <- [🏍]
-    const knowledge = await getKnowledgeForTask({ preparedPipeline, task });
+    const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
     const examples = await getExamplesForTask();
     const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
     const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5161,23 +5420,21 @@ async function getReservedParametersForTask(options) {
 }
 
 /**
- * @@@
+ * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+ *
+ * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+ * @returns The output parameters produced by the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeTask(options) {
     const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
     const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-    await onProgress({
-        outputParameters: {
-            [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-        },
-    });
     // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
     const usedParameterNames = extractParameterNamesFromTask(currentTask);
     const dependentParameterNames = new Set(currentTask.dependentParameterNames);
     // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
-    if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
+    if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
         throw new UnexpectedError(spaceTrim$1((block) => `
                     Dependent parameters are not consistent with used parameters:
 
@@ -5197,9 +5454,11 @@ async function executeTask(options) {
     }
     const definedParameters = Object.freeze({
         ...(await getReservedParametersForTask({
+            tools,
             preparedPipeline,
             task: currentTask,
             pipelineIdentification,
+            parameters: parametersToPass,
         })),
         ...parametersToPass,
     });
@@ -5245,6 +5504,7 @@ async function executeTask(options) {
         preparedPipeline,
         tools,
         $executionReport,
+        onProgress,
         pipelineIdentification,
         maxExecutionAttempts,
         maxParallelCount,
@@ -5272,7 +5532,8 @@ async function executeTask(options) {
  */
 
 /**
- * @@@
+ * Filters and returns only the output parameters from the provided pipeline execution options.
+ * Adds warnings for any expected output parameters that are missing.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5297,9 +5558,12 @@ function filterJustOutputParameters(options) {
 }
 
 /**
- * @@@
+ * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
  *
- * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+ * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
+ *
+ * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+ * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5622,6 +5886,22 @@ function createPipelineExecutor(options) {
             cacheDirname,
             intermediateFilesStrategy,
             isAutoInstalled,
+        }).catch((error) => {
+            assertsError(error);
+            return exportJson({
+                name: 'pipelineExecutorResult',
+                message: `Unuccessful PipelineExecutorResult, last catch`,
+                order: [],
+                value: {
+                    isSuccessful: false,
+                    errors: [serializeError(error)],
+                    warnings: [],
+                    usage: UNCERTAIN_USAGE,
+                    executionReport: null,
+                    outputParameters: {},
+                    preparedPipeline,
+                },
+            });
         });
     };
     const pipelineExecutor = (inputParameters) => createTask({
@@ -5882,8 +6162,8 @@ class MarkitdownScraper {
             extension: 'md',
             isVerbose,
         });
-        // TODO: @@@ Preserve, delete or modify
-        // Note: Running Pandoc ONLY if the file in the cache does not exist
+        // TODO: Determine if Markitdown conversion should run only if the cache file doesn't exist, or always.
+        // Note: Running Markitdown conversion ONLY if the file in the cache does not exist
         if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
             const src = source.filename || source.url || null;
             // console.log('!!', { src, source, cacheFilehandler });
@@ -5905,11 +6185,11 @@ class MarkitdownScraper {
         return cacheFilehandler;
     }
     /**
-     * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+     * Scrapes the source document (PDF, DOCX, etc.) and returns the knowledge pieces or `null` if it can't scrape it.
      */
     async scrape(source) {
         const cacheFilehandler = await this.$convert(source);
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: Ensure this correctly creates the source object for the internal MarkdownScraper using the converted file.
         const markdownSource = {
             source: source.source,
             filename: cacheFilehandler.filename,