@promptbook/cli 0.92.0-23 → 0.92.0-25

package/esm/index.es.js

@@ -47,7 +47,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-23';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-25';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -164,11 +164,20 @@ const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
 /**
- * @@@
+ * Threshold value that determines when a dataset is considered "big"
+ * and may require special handling or optimizations
+ *
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
  *
  * @public exported from `@promptbook/core`
  */
 const BIG_DATASET_TRESHOLD = 50;
+/**
+ * Placeholder text used to represent a placeholder value of failed operation
+ *
+ * @public exported from `@promptbook/core`
+ */
+const FAILED_VALUE_PLACEHOLDER = '!?';
 // <- TODO: !!!! Use
 /**
  * Warning message for the generated sections and files files
@@ -703,7 +712,8 @@ class NotYetImplementedError extends Error {
 }
 
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *
@@ -714,10 +724,10 @@ function $getGlobalScope() {
 }
 
 /**
- * @@@
+ * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to SCREAMING_CASE format.
+ * @returns The normalized text in SCREAMING_CASE format.
  * @example 'HELLO_WORLD'
  * @example 'I_LOVE_PROMPTBOOK'
  * @public exported from `@promptbook/utils`
@@ -769,10 +779,10 @@ function normalizeTo_SCREAMING_CASE(text) {
  */
 
 /**
- * @@@
+ * Normalizes a text string to snake_case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to snake_case format.
+ * @returns The normalized text in snake_case format.
  * @example 'hello_world'
  * @example 'i_love_promptbook'
  * @public exported from `@promptbook/utils`
@@ -830,10 +840,10 @@ class $Register {
 }
 
 /**
- * @@@
+ * Register for LLM tools metadata.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
  * @public exported from `@promptbook/core`
  */
 const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
@@ -842,10 +852,10 @@ const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
  */
 
 /**
- * @@@
+ * Register for LLM tools.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
  * @public exported from `@promptbook/core`
  */
 const $llmToolsRegister = new $Register('llm_execution_tools_constructors');
@@ -1105,7 +1115,8 @@ function TODO_USE(...value) {
 }
 
 /**
- * @@@
+ * Provides filesystem access (for example for Node.js-based scrapers)
+ * Creates a standardized filesystem interface that scrapers can use for file operations.
  *
  * @public exported from `@promptbook/node`
  */
@@ -1567,13 +1578,13 @@ const ORDER_OF_PIPELINE_JSON = [
  */
 const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
- * @@@
+ * Placeholder value indicating a parameter is missing its value.
  *
  * @private within the repository
  */
 const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
 /**
- * @@@
+ * Placeholder value indicating a parameter is restricted and cannot be used directly.
  *
  * @private within the repository
  */
@@ -2008,10 +2019,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
 */
 
 /**
- * @@@
+ * Removes diacritic marks (accents) from characters in a string.
  *
- * @param input @@@
- * @returns @@@
+ * @param input The string containing diacritics to be normalized.
+ * @returns The string with diacritics removed or normalized.
  * @public exported from `@promptbook/utils`
  */
 function removeDiacritics(input) {
@@ -2025,10 +2036,10 @@ function removeDiacritics(input) {
  */
 
 /**
- * @@@
+ * Converts a given text to kebab-case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text to be converted.
+ * @returns The kebab-case formatted string.
  * @example 'hello-world'
  * @example 'i-love-promptbook'
  * @public exported from `@promptbook/utils`
@@ -2076,11 +2087,11 @@ function normalizeToKebabCase(text) {
  */
 
 /**
- * @@@
+ * Converts a title string into a normalized name.
  *
- * @param value @@@
- * @returns @@@
- * @example @@@
+ * @param value The title string to be converted to a name.
+ * @returns A normalized name derived from the input title.
+ * @example 'Hello World!' -> 'hello-world'
  * @public exported from `@promptbook/utils`
  */
 function titleToName(value) {
@@ -2100,7 +2111,8 @@ function titleToName(value) {
 }
 
 /**
- * @@@
+ * Converts a name to a properly formatted subfolder path for cache storage.
+ * Handles normalization and path formatting to create consistent cache directory structures.
  *
  * @private for `FileCacheStorage`
  */
@@ -2109,7 +2121,10 @@ function nameToSubfolderPath(name) {
 }
 
 /**
- * @@@
+ * A storage implementation that caches data in files organized in a directory structure.
+ * Provides methods for retrieving, storing, and managing cached data on the filesystem.
+ *
+ * This class implements the PromptbookStorage interface for filesystem-based caching.
  *
  * @public exported from `@promptbook/node`
  */
@@ -2122,7 +2137,8 @@ class FileCacheStorage {
         }
     }
     /**
-     * @@@
+     * Converts a storage key to a filesystem path where the data should be stored.
+     * Creates a consistent, deterministic file path based on the key string.
      */
     getFilenameForKey(key) {
         // TODO: [👬] DRY
@@ -2134,7 +2150,8 @@ class FileCacheStorage {
         ...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${name.substring(0, MAX_FILENAME_LENGTH)}.json`);
     }
     /**
-     * @@@ Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object.
+     * Returns the current value associated with the given key, or null if the given key does not exist.
+     * Retrieves the cached data from the file system storage.
      */
     async getItem(key) {
         const filename = this.getFilenameForKey(key);
@@ -2147,7 +2164,8 @@ class FileCacheStorage {
         return value;
     }
     /**
-     * @@@ Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
+     * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
+     * Persists data to the file system, creating necessary directory structure if it doesn't exist.
      */
     async setItem(key, value) {
         const filename = this.getFilenameForKey(key);
@@ -2159,7 +2177,8 @@ class FileCacheStorage {
         await writeFile(filename, fileContent, 'utf-8');
     }
     /**
-     * @@@ Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists.
+     * Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
+     * Deletes the corresponding file from the filesystem.
      */
     async removeItem(key) {
         const filename = this.getFilenameForKey(key);
@@ -2782,9 +2801,9 @@ function cacheLlmTools(llmTools, options = {}) {
 /**
  * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
  * TODO: [🧠] Is there some meaningfull way how to test this util
- * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
- *            @@@ write discussion about this and storages
- *            @@@ write how to combine multiple interceptors
+ * TODO: [👷‍♂️] Comprehensive manual about construction of llmTools
+ *            Detailed explanation about caching strategies and appropriate storage selection for different use cases
+ *            Examples of how to combine multiple interceptors for advanced caching, logging, and usage tracking
  */
 
 /**
@@ -2974,9 +2993,8 @@ function countUsage(llmTools) {
  */
 
 /**
- * @@@
+ * Provides LLM tools configuration by reading environment variables.
  *
- * @@@ .env
  * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
  *
  * It looks for environment variables:
@@ -2984,7 +3002,8 @@ function countUsage(llmTools) {
  * - `process.env.ANTHROPIC_CLAUDE_API_KEY`
  * - ...
  *
- * @returns @@@
+ * @see Environment variables documentation or .env file for required variables.
+ * @returns A promise that resolves to the LLM tools configuration, or null if configuration is incomplete or missing.
  * @public exported from `@promptbook/node`
  */
 async function $provideLlmToolsConfigurationFromEnv() {
@@ -3902,10 +3921,11 @@ async function $provideExecutablesForNode(options) {
  */
 
 /**
- * @@@
+ * Registry for all available scrapers in the system.
+ * Central point for registering and accessing different types of content scrapers.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
  * @public exported from `@promptbook/core`
  */
 const $scrapersRegister = new $Register('scraper_constructors');
@@ -3914,11 +3934,9 @@ const $scrapersRegister = new $Register('scraper_constructors');
  */
 
 /**
- * @@@
- *
- * 1) @@@
- * 2) @@@
- *
+ * Provides a collection of scrapers optimized for Node.js environment.
+ * 1) `provideScrapersForNode` use as default
+ * 2) `provideScrapersForBrowser` use in limited browser environment *
  * @public exported from `@promptbook/node`
  */
 async function $provideScrapersForNode(tools, options) {
@@ -4822,7 +4840,7 @@ class SimplePipelineCollection {
     /**
      * Constructs a pipeline collection from pipelines
      *
-     * @param pipelines @@@
+     * @param pipelines Array of pipeline JSON objects to include in the collection
      *
      * Note: During the construction logic of all pipelines are validated
      * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -4934,8 +4952,8 @@ function createCollectionFromJson(...promptbooks) {
  * @public exported from `@promptbook/core`
  */
 function isPipelinePrepared(pipeline) {
-    // Note: Ignoring `pipeline.preparations` @@@
-    // Note: Ignoring `pipeline.knowledgePieces` @@@
+    // Note: Ignoring `pipeline.preparations`
+    // Note: Ignoring `pipeline.knowledgePieces`
     if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
         // TODO: !!! Comment this out
         console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
@@ -5510,6 +5528,15 @@ const CsvFormatParser = {
                     mappedData.push(mappedRow);
                     if (onProgress) {
                         // Note: Report the CSV with all rows mapped so far
+                        /*
+                        !!!!
+                         // Report progress with updated value
+                            const progressData = mappedData.map((row, i) =>
+                              i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                          );
+
+
+                        */
                         await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
                     }
                 }
@@ -5726,7 +5753,7 @@ function mapAvailableToExpectedParameters(options) {
         else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
     }
     if (expectedParameterNames.size === 0) {
-        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
         Object.freeze(mappedParameters);
         return mappedParameters;
     }
@@ -5757,7 +5784,7 @@ function mapAvailableToExpectedParameters(options) {
     for (let i = 0; i < expectedParameterNames.size; i++) {
         mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
     }
-    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
     Object.freeze(mappedParameters);
     return mappedParameters;
 }
@@ -6529,15 +6556,14 @@ async function executeFormatSubvalues(options) {
                 const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
                             ${error.message}
 
-                            This is error in FOREACH command when mapping data
+                            This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
                             You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
                             ${block(pipelineIdentification)}
-                            Subparameter index: ${index}
                         `));
                 if (length > BIG_DATASET_TRESHOLD) {
                     console.error(highLevelError);
-                    return '~';
+                    return FAILED_VALUE_PLACEHOLDER;
                 }
                 throw highLevelError;
             }
@@ -6561,14 +6587,13 @@ async function executeFormatSubvalues(options) {
             catch (error) {
                 if (length > BIG_DATASET_TRESHOLD) {
                     console.error(spaceTrim((block) => `
-                              Error in FOREACH command:
+                              ${error.message}
 
-                              ${block(pipelineIdentification)}
+                              This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
 
                               ${block(pipelineIdentification)}
-                              Subparameter index: ${index}
                           `));
-                    return '~';
+                    return FAILED_VALUE_PLACEHOLDER;
                 }
                 throw error;
             }
@@ -7387,7 +7412,9 @@ function mimeTypeToExtension(value) {
 }
 
 /**
- * @@@
+ * Factory function that creates a handler for processing knowledge sources.
+ * Provides standardized processing of different types of knowledge sources
+ * across various scraper implementations.
  *
  * @public exported from `@promptbook/core`
  */
@@ -7628,9 +7655,12 @@ TODO: [🧊] This is how it can look in future
  */
 
 /**
- * @@@
+ * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
  *
- * @public exported from `@promptbook/core`
+ * @param tasks Sequence of tasks that are chained together to form a pipeline
+ * @returns A promise that resolves to the prepared tasks.
+ *
+ * @private internal utility of `preparePipeline`
  */
 async function prepareTasks(pipeline, tools, options) {
     const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -8536,11 +8566,11 @@ const expectCommandParser = {
 };
 
 /**
- * @@@
+ * Normalizes a given text to camelCase format.
  *
- * @param text @@@
- * @param _isFirstLetterCapital @@@
- * @returns @@@
+ * @param text The text to be normalized.
+ * @param _isFirstLetterCapital Whether the first letter should be capitalized.
+ * @returns The camelCase formatted string.
  * @example 'helloWorld'
  * @example 'iLovePromptbook'
  * @public exported from `@promptbook/utils`
@@ -9028,7 +9058,7 @@ const GeneratorFormfactorDefinition = {
 };
 
 /**
- * @@@
+ * Pipeline interface which is equivalent to `any`
  *
  * @see https://github.com/webgptorg/promptbook/discussions/171
  *
@@ -9109,14 +9139,15 @@ const MatcherFormfactorDefinition = {
 };
 
 /**
- * Sheets is form of app that @@@
+ * Sheets is form of app that processes tabular data in CSV format, allowing transformation
+ * and analysis of structured data through AI-powered operations
  *
  * @public exported from `@promptbook/core`
  */
 const SheetsFormfactorDefinition = {
     name: 'SHEETS',
     aliasNames: ['SHEETS', 'SHEET'],
-    description: `@@@`,
+    description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
     documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
     pipelineInterface: {
         inputParameters: [
@@ -9192,7 +9223,7 @@ const FORMFACTOR_DEFINITIONS = [
 /**
  * Parses the formfactor command
  *
- * Note: @@@ This command is used as formfactor for new commands - it should NOT be used in any `.book` file
+ * Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
  *
  * @see `documentationUrl` for more details
  * @public exported from `@promptbook/editable`
@@ -9214,7 +9245,7 @@ const formfactorCommandParser = {
     /**
      * Description of the FORMFACTOR command
      */
-    description: `@@`,
+    description: `Specifies the application type and interface requirements that this promptbook should conform to`,
     /**
      * Link to documentation
      */
@@ -9357,8 +9388,7 @@ const jokerCommandParser = {
 };
 
 /**
- * @@@
- *
+ * @see {@link ModelVariant}
  * @public exported from `@promptbook/core`
  */
 const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
@@ -10331,7 +10361,7 @@ function parseCommandVariant(input) {
 }
 
 /**
- * @@@
+ * Extracts the interface (input and output parameters) from a pipeline.
  *
  * @deprecated https://github.com/webgptorg/promptbook/pull/186
  * @see https://github.com/webgptorg/promptbook/discussions/171
@@ -10364,7 +10394,7 @@ function getPipelineInterface(pipeline) {
 }
 
 /**
- * @@@
+ * Checks if two pipeline interfaces are structurally identical.
  *
  * @deprecated https://github.com/webgptorg/promptbook/pull/186
  * @see https://github.com/webgptorg/promptbook/discussions/171
@@ -10396,10 +10426,11 @@ function isPipelineInterfacesEqual(pipelineInterface1, pipelineInterface2) {
 }
 
 /**
- * @@@
+ * Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
  *
  * @deprecated https://github.com/webgptorg/promptbook/pull/186
  * @see https://github.com/webgptorg/promptbook/discussions/171
+ * @returns `true` if the pipeline implements the interface, `false` otherwise.
  *
  * @public exported from `@promptbook/core`
  */
@@ -10585,7 +10616,8 @@ function removeMarkdownComments(content) {
 }
 
 /**
- * @@@
+ * Utility to determine if a pipeline string is in flat format.
+ * A flat pipeline is a simple text without proper structure (headers, blocks, etc).
  *
  * @public exported from `@promptbook/editable`
  */
@@ -10606,7 +10638,10 @@ function isFlatPipeline(pipelineString) {
 }
 
 /**
- * @@@
+ * Converts a pipeline structure to its string representation.
+ *
+ * Transforms a flat, simple pipeline into a properly formatted pipeline string
+ * with sections for title, prompt, and return statement.
  *
  * @public exported from `@promptbook/editable`
  */
@@ -10663,7 +10698,7 @@ function deflatePipeline(pipelineString) {
  * Note: It can not work with html syntax and comments
  *
  * @param markdown any valid markdown
- * @returns @@@
+ * @returns An array of strings, each representing an individual list item found in the markdown
  * @public exported from `@promptbook/markdown-utils`
  */
 function extractAllListItemsFromMarkdown(markdown) {
@@ -11427,11 +11462,11 @@ function parseKeywordsFromString(input) {
 }
 
 /**
- * @@@
+ * Converts a name string into a URI-compatible format.
  *
- * @param name @@@
- * @returns @@@
- * @example @@@
+ * @param name The string to be converted to a URI-compatible format.
+ * @returns A URI-compatible string derived from the input name.
+ * @example 'Hello World' -> 'hello-world'
  * @public exported from `@promptbook/utils`
  */
 function nameToUriPart(name) {
@@ -11445,11 +11480,11 @@ function nameToUriPart(name) {
 }
 
 /**
- * @@@
+ * Converts a given name into URI-compatible parts.
  *
- * @param name @@@
- * @returns @@@
- * @example @@@
+ * @param name The name to be converted into URI parts.
+ * @returns An array of URI-compatible parts derived from the name.
+ * @example 'Example Name' -> ['example', 'name']
  * @public exported from `@promptbook/utils`
  */
 function nameToUriParts(name) {
@@ -16946,14 +16981,23 @@ function computeOpenAiUsage(promptContent, // <- Note: Intentionally using [] to
 resultContent, rawResponse) {
     var _a, _b;
     if (rawResponse.usage === undefined) {
+        console.log('!!! computeOpenAiUsage', 'The usage is not defined in the response from OpenAI');
         throw new PipelineExecutionError('The usage is not defined in the response from OpenAI');
     }
     if (((_a = rawResponse.usage) === null || _a === void 0 ? void 0 : _a.prompt_tokens) === undefined) {
+        console.log('!!! computeOpenAiUsage', 'In OpenAI response `usage.prompt_tokens` not defined');
         throw new PipelineExecutionError('In OpenAI response `usage.prompt_tokens` not defined');
     }
     const inputTokens = rawResponse.usage.prompt_tokens;
     const outputTokens = ((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completion_tokens) || 0;
     const modelInfo = OPENAI_MODELS.find((model) => model.modelName === rawResponse.model);
+    console.log('!!! computeOpenAiUsage', {
+        inputTokens,
+        outputTokens,
+        rawResponse,
+        resultContent,
+        modelInfo,
+    });
     let price;
     if (modelInfo === undefined || modelInfo.pricing === undefined) {
         price = uncertainNumber();
@@ -17563,9 +17607,8 @@ const _OpenAiAssistantRegistration = $llmToolsRegister.register(createOpenAiAssi
  */
 
 /**
- * Create a filename for intermediate cache for scrapers
- *
- * Note: It also checks if directory exists and creates it if not
+ * Retrieves an intermediate source for a scraper based on the knowledge source.
+ * Manages the caching and retrieval of intermediate scraper results for optimized performance.
  *
  * @private as internal utility for scrapers
  */
@@ -17792,14 +17835,14 @@ const boilerplateScraperMetadata = $deepFreeze({
     packageName: '@promptbook/boilerplate',
     className: 'BoilerplateScraper',
     mimeTypes: [
-        '@@@/@@@',
-        // <- TODO: @@@ Add compatible mime types with Boilerplate scraper
+        '@@/@@',
+        // <- TODO: @@ Add compatible mime types with Boilerplate scraper
     ],
-    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@@',
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
     isAvilableInBrowser: false,
     // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
     requiredExecutables: [
-    /* @@@ 'Pandoc' */
+    /* @@ 'Pandoc' */
     ],
 }); /* <- Note: [🤛] */
 /**
@@ -17817,7 +17860,7 @@ const _BoilerplateScraperMetadataRegistration = $scrapersMetadataRegister.regist
  */
 
 /**
- * Scraper of @@@ files
+ * Scraper of @@ files
  *
  * @see `documentationUrl` for more details
  * @public exported from `@promptbook/boilerplate`
@@ -17835,30 +17878,30 @@ class BoilerplateScraper {
         this.markdownScraper = new MarkdownScraper(tools, options);
     }
     /**
-     * Convert the `.@@@`  to `.md` file and returns intermediate source
+     * Convert the `.@@`  to `.md` file and returns intermediate source
      *
      * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
      */
     async $convert(source) {
         var _a;
         const { rootDirname = process.cwd(), cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, isVerbose = DEFAULT_IS_VERBOSE, } = this.options;
-        // TODO: @@@ Preserve or delete
+        // TODO: @@ Preserve or delete
         if (!$isRunningInNode()) {
             throw new KnowledgeScrapeError('BoilerplateScraper is only supported in Node environment');
         }
-        // TODO: @@@ Preserve or delete
+        // TODO: @@ Preserve or delete
         if (this.tools.fs === undefined) {
             throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools');
             //          <- TODO: [🧠] What is the best error type here`
         }
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: @@ Preserve, delete or modify
         if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
             throw new MissingToolsError('Pandoc is required for scraping .docx files');
         }
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: @@ Preserve, delete or modify
         if (source.filename === null) {
             // TODO: [🧠] Maybe save file as temporary
-            throw new KnowledgeScrapeError('When parsing .@@@ file, it must be real file in the file system');
+            throw new KnowledgeScrapeError('When parsing .@@ file, it must be real file in the file system');
         }
         const extension = getFileExtension(source.filename);
         const cacheFilehandler = await getScraperIntermediateSource(source, {
@@ -17868,7 +17911,7 @@ class BoilerplateScraper {
             extension: 'md',
             isVerbose,
         });
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: @@ Preserve, delete or modify
         // Note: Running Pandoc ONLY if the file in the cache does not exist
         if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
             const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`;
@@ -17894,7 +17937,7 @@ class BoilerplateScraper {
      */
     async scrape(source) {
         const cacheFilehandler = await this.$convert(source);
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: @@ Preserve, delete or modify
         const markdownSource = {
             source: source.source,
             filename: cacheFilehandler.filename,
@@ -17925,7 +17968,7 @@ class BoilerplateScraper {
  * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
  * TODO: [🪂] Do it in parallel
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
- * @@@ Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ * @@ Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
  */
 
 /**
@@ -18259,7 +18302,8 @@ class LegacyDocumentScraper {
  */
 
 /**
- * @@@
+ * Creates a scraper for legacy document formats (.doc, .rtf, etc).
+ * Uses LibreOffice for conversion to extract content from older document formats.
  *
  * @public exported from `@promptbook/legacy-documents`
  */
@@ -18286,7 +18330,7 @@ const _LegacyDocumentScraperRegistration = $scrapersRegister.register(createLega
  */
 
 /**
- * @@@
+ * Creates a scraper for document content.
  *
  * @public exported from `@promptbook/documents`
  */
@@ -18313,7 +18357,7 @@ const _DocumentScraperRegistration = $scrapersRegister.register(createDocumentSc
  */
 
 /**
- * @@@
+ * Creates a scraper for markdown content.
  *
  * @public exported from `@promptbook/markdown-utils`
  */
@@ -18419,8 +18463,8 @@ class MarkitdownScraper {
             extension: 'md',
             isVerbose,
         });
-        // TODO: @@@ Preserve, delete or modify
-        // Note: Running Pandoc ONLY if the file in the cache does not exist
+        // TODO: Determine if Markitdown conversion should run only if the cache file doesn't exist, or always.
+        // Note: Running Markitdown conversion ONLY if the file in the cache does not exist
         if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
             const src = source.filename || source.url || null;
             // console.log('!!', { src, source, cacheFilehandler });
@@ -18442,11 +18486,11 @@ class MarkitdownScraper {
         return cacheFilehandler;
     }
     /**
-     * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+     * Scrapes the source document (PDF, DOCX, etc.) and returns the knowledge pieces or `null` if it can't scrape it.
      */
     async scrape(source) {
         const cacheFilehandler = await this.$convert(source);
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: Ensure this correctly creates the source object for the internal MarkdownScraper using the converted file.
         const markdownSource = {
             source: source.source,
             filename: cacheFilehandler.filename,
@@ -18590,7 +18634,8 @@ class PdfScraper {
  */
 
 /**
- * @@@
+ * Factory function to create an instance of PdfScraper.
+ * It bundles the scraper class with its metadata.
  *
  * @public exported from `@promptbook/pdf`
  */
@@ -18766,7 +18811,8 @@ class WebsiteScraper {
  */
 
 /**
- * @@@
+ * Factory function to create an instance of WebsiteScraper.
+ * It bundles the scraper class with its metadata.
  *
  * @public exported from `@promptbook/website-crawler`
  */