@promptbook/pdf 0.92.0-23 → 0.92.0-25

package/esm/index.es.js

@@ -26,7 +26,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-23';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-25';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -103,11 +103,20 @@ const DEFAULT_BOOK_TITLE = `✨ Untitled Book`;
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
 /**
- * @@@
+ * Threshold value that determines when a dataset is considered "big"
+ * and may require special handling or optimizations
+ *
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
  *
  * @public exported from `@promptbook/core`
  */
 const BIG_DATASET_TRESHOLD = 50;
+/**
+ * Placeholder text used to represent a placeholder value of failed operation
+ *
+ * @public exported from `@promptbook/core`
+ */
+const FAILED_VALUE_PLACEHOLDER = '!?';
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -361,7 +370,8 @@ async function isFileExisting(filename, fs) {
  */
 
 /**
- * @@@
+ * Converts a name to a properly formatted subfolder path for cache storage.
+ * Handles normalization and path formatting to create consistent cache directory structures.
  *
  * @private for `FileCacheStorage`
  */
@@ -614,10 +624,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
 */
 
 /**
- * @@@
+ * Removes diacritic marks (accents) from characters in a string.
  *
- * @param input @@@
- * @returns @@@
+ * @param input The string containing diacritics to be normalized.
+ * @returns The string with diacritics removed or normalized.
  * @public exported from `@promptbook/utils`
  */
 function removeDiacritics(input) {
@@ -631,10 +641,10 @@ function removeDiacritics(input) {
  */
 
 /**
- * @@@
+ * Converts a given text to kebab-case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text to be converted.
+ * @returns The kebab-case formatted string.
  * @example 'hello-world'
  * @example 'i-love-promptbook'
  * @public exported from `@promptbook/utils`
@@ -776,11 +786,11 @@ function isValidUrl(url) {
 }
 
 /**
- * @@@
+ * Converts a title string into a normalized name.
  *
- * @param value @@@
- * @returns @@@
- * @example @@@
+ * @param value The title string to be converted to a name.
+ * @returns A normalized name derived from the input title.
+ * @example 'Hello World!' -> 'hello-world'
  * @public exported from `@promptbook/utils`
  */
 function titleToName(value) {
@@ -813,9 +823,8 @@ function TODO_USE(...value) {
 }
 
 /**
- * Create a filename for intermediate cache for scrapers
- *
- * Note: It also checks if directory exists and creates it if not
+ * Retrieves an intermediate source for a scraper based on the knowledge source.
+ * Manages the caching and retrieval of intermediate scraper results for optimized performance.
  *
  * @private as internal utility for scrapers
  */
@@ -1449,13 +1458,13 @@ const ORDER_OF_PIPELINE_JSON = [
  */
 const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
- * @@@
+ * Placeholder value indicating a parameter is missing its value.
  *
  * @private within the repository
  */
 const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
 /**
- * @@@
+ * Placeholder value indicating a parameter is restricted and cannot be used directly.
  *
  * @private within the repository
  */
@@ -1954,7 +1963,7 @@ class SimplePipelineCollection {
     /**
      * Constructs a pipeline collection from pipelines
      *
-     * @param pipelines @@@
+     * @param pipelines Array of pipeline JSON objects to include in the collection
      *
      * Note: During the construction logic of all pipelines are validated
      * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -2118,8 +2127,8 @@ class PipelineExecutionError extends Error {
  * @public exported from `@promptbook/core`
  */
 function isPipelinePrepared(pipeline) {
-    // Note: Ignoring `pipeline.preparations` @@@
-    // Note: Ignoring `pipeline.knowledgePieces` @@@
+    // Note: Ignoring `pipeline.preparations`
+    // Note: Ignoring `pipeline.knowledgePieces`
     if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
         // TODO: !!! Comment this out
         console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
@@ -3091,7 +3100,8 @@ async function preparePersona(personaDescription, tools, options) {
  */
 
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *
@@ -3102,10 +3112,10 @@ function $getGlobalScope() {
 }
 
 /**
- * @@@
+ * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to SCREAMING_CASE format.
+ * @returns The normalized text in SCREAMING_CASE format.
  * @example 'HELLO_WORLD'
  * @example 'I_LOVE_PROMPTBOOK'
  * @public exported from `@promptbook/utils`
@@ -3157,10 +3167,10 @@ function normalizeTo_SCREAMING_CASE(text) {
  */
 
 /**
- * @@@
+ * Normalizes a text string to snake_case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to snake_case format.
+ * @returns The normalized text in snake_case format.
  * @example 'hello_world'
  * @example 'i_love_promptbook'
  * @public exported from `@promptbook/utils`
@@ -3230,10 +3240,11 @@ const $scrapersMetadataRegister = new $Register('scrapers_metadata');
  */
 
 /**
- * @@@
+ * Registry for all available scrapers in the system.
+ * Central point for registering and accessing different types of content scrapers.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
  * @public exported from `@promptbook/core`
  */
 const $scrapersRegister = new $Register('scraper_constructors');
@@ -3411,7 +3422,9 @@ const promptbookFetch = async (urlOrRequest, init) => {
  */
 
 /**
- * @@@
+ * Factory function that creates a handler for processing knowledge sources.
+ * Provides standardized processing of different types of knowledge sources
+ * across various scraper implementations.
  *
  * @public exported from `@promptbook/core`
  */
@@ -3652,9 +3665,12 @@ TODO: [🧊] This is how it can look in future
  */
 
 /**
- * @@@
+ * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
  *
- * @public exported from `@promptbook/core`
+ * @param tasks Sequence of tasks that are chained together to form a pipeline
+ * @returns A promise that resolves to the prepared tasks.
+ *
+ * @private internal utility of `preparePipeline`
  */
 async function prepareTasks(pipeline, tools, options) {
     const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -4192,6 +4208,15 @@ const CsvFormatParser = {
                     mappedData.push(mappedRow);
                     if (onProgress) {
                         // Note: Report the CSV with all rows mapped so far
+                        /*
+                        !!!!
+                         // Report progress with updated value
+                            const progressData = mappedData.map((row, i) =>
+                              i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                          );
+
+
+                        */
                         await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
                     }
                 }
@@ -4408,7 +4433,7 @@ function mapAvailableToExpectedParameters(options) {
         else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
     }
     if (expectedParameterNames.size === 0) {
-        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
         Object.freeze(mappedParameters);
         return mappedParameters;
     }
@@ -4439,7 +4464,7 @@ function mapAvailableToExpectedParameters(options) {
     for (let i = 0; i < expectedParameterNames.size; i++) {
         mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
     }
-    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
     Object.freeze(mappedParameters);
     return mappedParameters;
 }
@@ -5192,15 +5217,14 @@ async function executeFormatSubvalues(options) {
                 const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
                             ${error.message}
 
-                            This is error in FOREACH command when mapping data
+                            This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
                             You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
                             ${block(pipelineIdentification)}
-                            Subparameter index: ${index}
                         `));
                 if (length > BIG_DATASET_TRESHOLD) {
                     console.error(highLevelError);
-                    return '~';
+                    return FAILED_VALUE_PLACEHOLDER;
                 }
                 throw highLevelError;
             }
@@ -5224,14 +5248,13 @@ async function executeFormatSubvalues(options) {
             catch (error) {
                 if (length > BIG_DATASET_TRESHOLD) {
                     console.error(spaceTrim((block) => `
-                              Error in FOREACH command:
+                              ${error.message}
 
-                              ${block(pipelineIdentification)}
+                              This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
 
                               ${block(pipelineIdentification)}
-                              Subparameter index: ${index}
                           `));
-                    return '~';
+                    return FAILED_VALUE_PLACEHOLDER;
                 }
                 throw error;
             }
@@ -6147,8 +6170,8 @@ class MarkitdownScraper {
             extension: 'md',
             isVerbose,
         });
-        // TODO: @@@ Preserve, delete or modify
-        // Note: Running Pandoc ONLY if the file in the cache does not exist
+        // TODO: Determine if Markitdown conversion should run only if the cache file doesn't exist, or always.
+        // Note: Running Markitdown conversion ONLY if the file in the cache does not exist
         if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
             const src = source.filename || source.url || null;
             // console.log('!!', { src, source, cacheFilehandler });
@@ -6170,11 +6193,11 @@ class MarkitdownScraper {
         return cacheFilehandler;
     }
     /**
-     * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+     * Scrapes the source document (PDF, DOCX, etc.) and returns the knowledge pieces or `null` if it can't scrape it.
      */
     async scrape(source) {
         const cacheFilehandler = await this.$convert(source);
-        // TODO: @@@ Preserve, delete or modify
+        // TODO: Ensure this correctly creates the source object for the internal MarkdownScraper using the converted file.
         const markdownSource = {
             source: source.source,
             filename: cacheFilehandler.filename,
@@ -6318,7 +6341,8 @@ class PdfScraper {
  */
 
 /**
- * @@@
+ * Factory function to create an instance of PdfScraper.
+ * It bundles the scraper class with its metadata.
  *
  * @public exported from `@promptbook/pdf`
  */