@promptbook/website-crawler 0.92.0-22 → 0.92.0-24

package/esm/index.es.js

@@ -29,7 +29,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-22';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-24';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -132,6 +132,21 @@ const DEFAULT_BOOK_TITLE = `✨ Untitled Book`;
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+/**
+ * Threshold value that determines when a dataset is considered "big"
+ * and may require special handling or optimizations
+ *
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
+ *
+ * @public exported from `@promptbook/core`
+ */
+const BIG_DATASET_TRESHOLD = 50;
+/**
+ * Placeholder text used to represent a placeholder value of failed operation
+ *
+ * @public exported from `@promptbook/core`
+ */
+const FAILED_VALUE_PLACEHOLDER = '!?';
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -211,7 +226,7 @@ const DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
 const DEFAULT_SCRAPE_CACHE_DIRNAME = './.promptbook/scrape-cache';
 // <- TODO: [🧜‍♂️]
 /**
- * @@@
+ * Default settings for parsing and generating CSV files in Promptbook.
  *
  * @public exported from `@promptbook/core`
  */
@@ -222,19 +237,19 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
     skipEmptyLines: true,
 });
 /**
- * @@@
+ * Controls whether verbose logging is enabled by default throughout the application.
  *
  * @public exported from `@promptbook/core`
  */
 let DEFAULT_IS_VERBOSE = false;
 /**
- * @@@
+ * Controls whether auto-installation of dependencies is enabled by default.
  *
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_IS_AUTO_INSTALLED = false;
 /**
- * @@@
+ * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
  *
  * @private within the repository
  */
@@ -339,7 +354,8 @@ class UnexpectedError extends Error {
 }
 
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *
@@ -418,11 +434,11 @@ function normalizeTo_snake_case(text) {
 }
 
 /**
- * Register is @@@
+ * Global registry for storing and managing registered entities of a given type.
  *
  * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
  *
- * @private internal utility, exported are only signleton instances of this class
+ * @private internal utility, exported are only singleton instances of this class
  */
 class $Register {
     constructor(registerName) {
@@ -466,10 +482,10 @@ class $Register {
 }
 
 /**
- * @@@
+ * Global registry for storing metadata about all available scrapers and converters.
  *
- * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * Note: `$` is used to indicate that this interacts with the global scope.
+ * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
  * @public exported from `@promptbook/core`
  */
 const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -1110,7 +1126,7 @@ function assertsError(whatWasThrown) {
  * Function isValidJsonString will tell you if the string is valid JSON or not
  *
  * @param value The string to check
- * @returns True if the string is a valid JSON string, false otherwise
+ * @returns `true` if the string is a valid JSON string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -1494,8 +1510,12 @@ function checkSerializableAsJson(options) {
  */
 
 /**
- * @@@
+ * Creates a deep clone of the given object
+ *
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
  *
+ * @param objectValue The object to clone.
+ * @returns A deep, writable clone of the input object.
  * @public exported from `@promptbook/utils`
  */
 function deepClone(objectValue) {
@@ -2082,7 +2102,7 @@ class SimplePipelineCollection {
     /**
      * Constructs a pipeline collection from pipelines
      *
-     * @param pipelines @@@
+     * @param pipelines Array of pipeline JSON objects to include in the collection
      *
      * Note: During the construction logic of all pipelines are validated
      * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -4113,7 +4133,7 @@ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO:
  * Function to check if a string is valid CSV
  *
  * @param value The string to check
- * @returns True if the string is a valid CSV string, false otherwise
+ * @returns `true` if the string is a valid CSV string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4170,18 +4190,28 @@ const CsvFormatParser = {
                             `));
                 }
                 const mappedData = [];
-                for (let index = 0; index < csv.data.length; index++) {
+                const length = csv.data.length;
+                for (let index = 0; index < length; index++) {
                     const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
                     const mappedRow = {
                         ...row,
-                        [outputParameterName]: await mapCallback(row, index),
+                        [outputParameterName]: await mapCallback(row, index, length),
                     };
                     mappedData.push(mappedRow);
                     if (onProgress) {
                         // Note: Report the CSV with all rows mapped so far
+                        /*
+                        !!!!
+                         // Report progress with updated value
+                            const progressData = mappedData.map((row, i) =>
+                              i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                          );
+
+
+                        */
                         await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
                     }
                 }
@@ -4208,9 +4238,9 @@ const CsvFormatParser = {
                             `));
                 }
                 const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                         const index = rowIndex * Object.keys(row).length + columnIndex;
-                        return /* not await */ mapCallback({ [key]: value }, index);
+                        return /* not await */ mapCallback({ [key]: value }, index, array.length);
                     }));
                 }));
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -4281,12 +4311,12 @@ const TextFormatParser = {
             async mapValues(options) {
                 const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
-                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                 // TODO: [🧠] Maybe option to skip empty line
                 /* not await */ mapCallback({
                     lineContent,
                     // TODO: [🧠] Maybe also put here `lineNumber`
-                }, lineNumber)));
+                }, lineNumber, array.length)));
                 return mappedLines.join('\n');
             },
         },
@@ -4307,7 +4337,7 @@ const TextFormatParser = {
  * Function to check if a string is valid XML
  *
  * @param value
- * @returns True if the string is a valid XML string, false otherwise
+ * @returns `true` if the string is a valid XML string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4369,13 +4399,13 @@ const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser,
  */
 
 /**
- * Maps available parameters to expected parameters
+ * Maps available parameters to expected parameters for a pipeline task.
  *
  * The strategy is:
- * 1) @@@
- * 2) @@@
+ * 1) First, match parameters by name where both available and expected.
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
  *
- * @throws {PipelineExecutionError} @@@
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
  * @private within the repository used in `createPipelineExecutor`
  */
 function mapAvailableToExpectedParameters(options) {
@@ -5095,7 +5125,11 @@ async function executeAttempts(options) {
  */
 
 /**
- * @@@
+ * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+ * Handles format and subformat resolution, error handling, and progress reporting.
+ *
+ * @param options - Options for execution, including task details and progress callback.
+ * @returns The result of the subvalue mapping or execution attempts.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5160,15 +5194,11 @@ async function executeFormatSubvalues(options) {
         settings: formatSettings,
         onProgress(partialResultString) {
             return onProgress(Object.freeze({
-                [task.resultingParameterName]: 
-                // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
-                partialResultString,
+                [task.resultingParameterName]: partialResultString,
             }));
         },
-        async mapCallback(subparameters, index) {
+        async mapCallback(subparameters, index, length) {
             let mappedParameters;
-            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-            // TODO: When done [🐚] Report progress also for each subvalue here
             try {
                 mappedParameters = mapAvailableToExpectedParameters({
                     expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
@@ -5179,32 +5209,52 @@ async function executeFormatSubvalues(options) {
                 if (!(error instanceof PipelineExecutionError)) {
                     throw error;
                 }
-                throw new PipelineExecutionError(spaceTrim$1((block) => `
-                        ${error.message}
+                const highLevelError = new PipelineExecutionError(spaceTrim$1((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping data
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `));
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(highLevelError);
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw highLevelError;
             }
             const allSubparameters = {
                 ...parameters,
                 ...mappedParameters,
             };
-            // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
             Object.freeze(allSubparameters);
-            const subresultString = await executeAttempts({
-                ...options,
-                priority: priority + index,
-                parameters: allSubparameters,
-                pipelineIdentification: spaceTrim$1((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-            });
-            return subresultString;
+            try {
+                const subresultString = await executeAttempts({
+                    ...options,
+                    priority: priority + index,
+                    parameters: allSubparameters,
+                    pipelineIdentification: spaceTrim$1((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                });
+                return subresultString;
+            }
+            catch (error) {
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(spaceTrim$1((block) => `
+                              Error in FOREACH command:
+
+                              ${block(pipelineIdentification)}
+
+                              ${block(pipelineIdentification)}
+                              Subparameter index: ${index}
+                          `));
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw error;
+            }
         },
     });
     return resultString;
@@ -5339,7 +5389,11 @@ async function getKnowledgeForTask(options) {
  */
 
 /**
- * @@@
+ * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+ * Ensures all reserved parameters are defined and throws if any are missing.
+ *
+ * @param options - Options including tools, pipeline, task, and context.
+ * @returns An object containing all reserved parameters for the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5372,7 +5426,10 @@ async function getReservedParametersForTask(options) {
 }
 
 /**
- * @@@
+ * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+ *
+ * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+ * @returns The output parameters produced by the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -5506,9 +5563,12 @@ function filterJustOutputParameters(options) {
 }
 
 /**
- * @@@
+ * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
+ *
+ * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
  *
- * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+ * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+ * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
  *
  * @private internal utility of `createPipelineExecutor`
  */