@promptbook/cli 0.92.0-22 → 0.92.0-24

package/esm/index.es.js

@@ -47,7 +47,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-22';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-24';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -163,6 +163,22 @@ const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+/**
+ * Threshold value that determines when a dataset is considered "big"
+ * and may require special handling or optimizations
+ *
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
+ *
+ * @public exported from `@promptbook/core`
+ */
+const BIG_DATASET_TRESHOLD = 50;
+/**
+ * Placeholder text used to represent a placeholder value of failed operation
+ *
+ * @public exported from `@promptbook/core`
+ */
+const FAILED_VALUE_PLACEHOLDER = '!?';
+// <- TODO: !!!! Use
 /**
  * Warning message for the generated sections and files files
  *
@@ -247,6 +263,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
  * @public exported from `@promptbook/core`
  */
 const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
+// <- TODO: [🐝]
 /**
  * Where to store your books
  * This is kind of a "src" for your books
@@ -318,7 +335,7 @@ const MOMENT_ARG_THRESHOLDS = {
 const DEFAULT_REMOTE_SERVER_URL = REMOTE_SERVER_URLS[0].urls[0];
 // <- TODO: [🧜‍♂️]
 /**
- * @@@
+ * Default settings for parsing and generating CSV files in Promptbook.
  *
  * @public exported from `@promptbook/core`
  */
@@ -329,13 +346,13 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
     skipEmptyLines: true,
 });
 /**
- * @@@
+ * Controls whether verbose logging is enabled by default throughout the application.
  *
  * @public exported from `@promptbook/core`
  */
 let DEFAULT_IS_VERBOSE = false;
 /**
- * @@@
+ * Controls whether auto-installation of dependencies is enabled by default.
  *
  * @public exported from `@promptbook/core`
  */
@@ -355,7 +372,7 @@ const DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = `getPipelineCollection`;
  */
 const DEFAULT_RPM$1 = 60;
 /**
- * @@@
+ * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
  *
  * @private within the repository
  */
@@ -695,7 +712,8 @@ class NotYetImplementedError extends Error {
 }
 
 /**
- * @@@
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
+ * regardless of the JavaScript environment in which the code is running
  *
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
  *
@@ -774,11 +792,11 @@ function normalizeTo_snake_case(text) {
 }
 
 /**
- * Register is @@@
+ * Global registry for storing and managing registered entities of a given type.
  *
  * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
  *
- * @private internal utility, exported are only signleton instances of this class
+ * @private internal utility, exported are only singleton instances of this class
  */
 class $Register {
     constructor(registerName) {
@@ -1472,8 +1490,12 @@ function checkSerializableAsJson(options) {
  */
 
 /**
- * @@@
+ * Creates a deep clone of the given object
  *
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
+ *
+ * @param objectValue The object to clone.
+ * @returns A deep, writable clone of the input object.
  * @public exported from `@promptbook/utils`
  */
 function deepClone(objectValue) {
@@ -2720,8 +2742,10 @@ function cacheLlmTools(llmTools, options = {}) {
             sha256(hexEncoder.parse(JSON.stringify({ parameters, content, modelRequirements }))).toString( /* hex */));
         const cacheItem = !isCacheReloaded ? await storage.getItem(key) : null;
         if (cacheItem) {
+            console.log('!!! Cache hit for key:', key);
             return cacheItem.promptResult;
         }
+        console.log('!!! Cache miss for key:', key);
         let promptResult;
         variant: switch (prompt.modelRequirements.modelVariant) {
             case 'CHAT':
@@ -2768,9 +2792,9 @@ function cacheLlmTools(llmTools, options = {}) {
 /**
  * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
  * TODO: [🧠] Is there some meaningfull way how to test this util
- * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
- *            @@@ write discussion about this and storages
- *            @@@ write how to combine multiple interceptors
+ * TODO: [👷‍♂️] Comprehensive manual about construction of llmTools
+ *            Detailed explanation about caching strategies and appropriate storage selection for different use cases
+ *            Examples of how to combine multiple interceptors for advanced caching, logging, and usage tracking
  */
 
 /**
@@ -3929,10 +3953,10 @@ async function $provideScrapersForNode(tools, options) {
  */
 
 /**
- * @@@
+ * Global registry for storing metadata about all available scrapers and converters.
  *
- * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * Note: `$` is used to indicate that this interacts with the global scope.
+ * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
  * @public exported from `@promptbook/core`
  */
 const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -4512,7 +4536,7 @@ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"
  * Function isValidJsonString will tell you if the string is valid JSON or not
  *
  * @param value The string to check
- * @returns True if the string is a valid JSON string, false otherwise
+ * @returns `true` if the string is a valid JSON string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -4808,7 +4832,7 @@ class SimplePipelineCollection {
     /**
      * Constructs a pipeline collection from pipelines
      *
-     * @param pipelines @@@
+     * @param pipelines Array of pipeline JSON objects to include in the collection
      *
      * Note: During the construction logic of all pipelines are validated
      * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -5426,7 +5450,7 @@ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO:
  * Function to check if a string is valid CSV
  *
  * @param value The string to check
- * @returns True if the string is a valid CSV string, false otherwise
+ * @returns `true` if the string is a valid CSV string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -5483,18 +5507,28 @@ const CsvFormatParser = {
                             `));
                 }
                 const mappedData = [];
-                for (let index = 0; index < csv.data.length; index++) {
+                const length = csv.data.length;
+                for (let index = 0; index < length; index++) {
                     const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
                     const mappedRow = {
                         ...row,
-                        [outputParameterName]: await mapCallback(row, index),
+                        [outputParameterName]: await mapCallback(row, index, length),
                     };
                     mappedData.push(mappedRow);
                     if (onProgress) {
                         // Note: Report the CSV with all rows mapped so far
+                        /*
+                        !!!!
+                         // Report progress with updated value
+                            const progressData = mappedData.map((row, i) =>
+                              i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
+                          );
+
+
+                        */
                         await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
                     }
                 }
@@ -5521,9 +5555,9 @@ const CsvFormatParser = {
                             `));
                 }
                 const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
-                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
+                    return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
                         const index = rowIndex * Object.keys(row).length + columnIndex;
-                        return /* not await */ mapCallback({ [key]: value }, index);
+                        return /* not await */ mapCallback({ [key]: value }, index, array.length);
                     }));
                 }));
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -5594,12 +5628,12 @@ const TextFormatParser = {
             async mapValues(options) {
                 const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
-                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
+                const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) => 
                 // TODO: [🧠] Maybe option to skip empty line
                 /* not await */ mapCallback({
                     lineContent,
                     // TODO: [🧠] Maybe also put here `lineNumber`
-                }, lineNumber)));
+                }, lineNumber, array.length)));
                 return mappedLines.join('\n');
             },
         },
@@ -5620,7 +5654,7 @@ const TextFormatParser = {
  * Function to check if a string is valid XML
  *
  * @param value
- * @returns True if the string is a valid XML string, false otherwise
+ * @returns `true` if the string is a valid XML string, false otherwise
  *
  * @public exported from `@promptbook/utils`
  */
@@ -5682,13 +5716,13 @@ const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser,
  */
 
 /**
- * Maps available parameters to expected parameters
+ * Maps available parameters to expected parameters for a pipeline task.
  *
  * The strategy is:
- * 1) @@@
- * 2) @@@
+ * 1) First, match parameters by name where both available and expected.
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
  *
- * @throws {PipelineExecutionError} @@@
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
  * @private within the repository used in `createPipelineExecutor`
  */
 function mapAvailableToExpectedParameters(options) {
@@ -6427,7 +6461,11 @@ async function executeAttempts(options) {
  */
 
 /**
- * @@@
+ * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
+ * Handles format and subformat resolution, error handling, and progress reporting.
+ *
+ * @param options - Options for execution, including task details and progress callback.
+ * @returns The result of the subvalue mapping or execution attempts.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -6492,15 +6530,11 @@ async function executeFormatSubvalues(options) {
         settings: formatSettings,
         onProgress(partialResultString) {
             return onProgress(Object.freeze({
-                [task.resultingParameterName]: 
-                // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
-                partialResultString,
+                [task.resultingParameterName]: partialResultString,
             }));
         },
-        async mapCallback(subparameters, index) {
+        async mapCallback(subparameters, index, length) {
             let mappedParameters;
-            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-            // TODO: When done [🐚] Report progress also for each subvalue here
             try {
                 mappedParameters = mapAvailableToExpectedParameters({
                     expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
@@ -6511,32 +6545,52 @@ async function executeFormatSubvalues(options) {
                 if (!(error instanceof PipelineExecutionError)) {
                     throw error;
                 }
-                throw new PipelineExecutionError(spaceTrim((block) => `
-                        ${error.message}
+                const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
+                            ${error.message}
 
-                        This is error in FOREACH command
-                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
+                            This is error in FOREACH command when mapping data
+                            You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `));
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `));
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(highLevelError);
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw highLevelError;
             }
             const allSubparameters = {
                 ...parameters,
                 ...mappedParameters,
             };
-            // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
             Object.freeze(allSubparameters);
-            const subresultString = await executeAttempts({
-                ...options,
-                priority: priority + index,
-                parameters: allSubparameters,
-                pipelineIdentification: spaceTrim((block) => `
-                        ${block(pipelineIdentification)}
-                        Subparameter index: ${index}
-                    `),
-            });
-            return subresultString;
+            try {
+                const subresultString = await executeAttempts({
+                    ...options,
+                    priority: priority + index,
+                    parameters: allSubparameters,
+                    pipelineIdentification: spaceTrim((block) => `
+                            ${block(pipelineIdentification)}
+                            Subparameter index: ${index}
+                        `),
+                });
+                return subresultString;
+            }
+            catch (error) {
+                if (length > BIG_DATASET_TRESHOLD) {
+                    console.error(spaceTrim((block) => `
+                              Error in FOREACH command:
+
+                              ${block(pipelineIdentification)}
+
+                              ${block(pipelineIdentification)}
+                              Subparameter index: ${index}
+                          `));
+                    return FAILED_VALUE_PLACEHOLDER;
+                }
+                throw error;
+            }
         },
     });
     return resultString;
@@ -6671,7 +6725,11 @@ async function getKnowledgeForTask(options) {
  */
 
 /**
- * @@@
+ * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
+ * Ensures all reserved parameters are defined and throws if any are missing.
+ *
+ * @param options - Options including tools, pipeline, task, and context.
+ * @returns An object containing all reserved parameters for the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -6704,7 +6762,10 @@ async function getReservedParametersForTask(options) {
 }
 
 /**
- * @@@
+ * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
+ *
+ * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
+ * @returns The output parameters produced by the task.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -6838,9 +6899,12 @@ function filterJustOutputParameters(options) {
 }
 
 /**
- * @@@
+ * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
  *
- * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
+ * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
+ *
+ * @param options - Options for execution, including input parameters, pipeline, and callbacks.
+ * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
  *
  * @private internal utility of `createPipelineExecutor`
  */
@@ -8566,11 +8630,12 @@ function removeQuotes(text) {
 }
 
 /**
- * Function `validateParameterName` will @@@
+ * Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
+ * It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
  *
- * @param parameterName @@@
- * @returns @@@
- * @throws {ParseError} @@@
+ * @param parameterName The parameter name to validate and normalize.
+ * @returns The validated and normalized parameter name.
+ * @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
  * @private within the repository
  */
 function validateParameterName(parameterName) {
@@ -9063,14 +9128,15 @@ const MatcherFormfactorDefinition = {
 };
 
 /**
- * Sheets is form of app that @@@
+ * Sheets is form of app that processes tabular data in CSV format, allowing transformation
+ * and analysis of structured data through AI-powered operations
  *
  * @public exported from `@promptbook/core`
  */
 const SheetsFormfactorDefinition = {
     name: 'SHEETS',
     aliasNames: ['SHEETS', 'SHEET'],
-    description: `@@@`,
+    description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
     documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
     pipelineInterface: {
         inputParameters: [
@@ -9146,7 +9212,7 @@ const FORMFACTOR_DEFINITIONS = [
 /**
  * Parses the formfactor command
  *
- * Note: @@@ This command is used as formfactor for new commands - it should NOT be used in any `.book` file
+ * Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
  *
  * @see `documentationUrl` for more details
  * @public exported from `@promptbook/editable`
@@ -9168,7 +9234,7 @@ const formfactorCommandParser = {
     /**
      * Description of the FORMFACTOR command
      */
-    description: `@@`,
+    description: `Specifies the application type and interface requirements that this promptbook should conform to`,
     /**
      * Link to documentation
      */
@@ -9744,10 +9810,10 @@ function $applyToTaskJson(command, $taskJson, $pipelineJson) {
 }
 
 /**
- * @@@
+ * Checks if the given value is a valid JavaScript identifier name.
  *
- * @param javascriptName @@@
- * @returns @@@
+ * @param javascriptName The value to check for JavaScript identifier validity.
+ * @returns `true` if the value is a valid JavaScript name, false otherwise.
  * @public exported from `@promptbook/utils`
  */
 function isValidJavascriptName(javascriptName) {
@@ -16900,14 +16966,23 @@ function computeOpenAiUsage(promptContent, // <- Note: Intentionally using [] to
 resultContent, rawResponse) {
     var _a, _b;
     if (rawResponse.usage === undefined) {
+        console.log('!!! computeOpenAiUsage', 'The usage is not defined in the response from OpenAI');
         throw new PipelineExecutionError('The usage is not defined in the response from OpenAI');
     }
     if (((_a = rawResponse.usage) === null || _a === void 0 ? void 0 : _a.prompt_tokens) === undefined) {
+        console.log('!!! computeOpenAiUsage', 'In OpenAI response `usage.prompt_tokens` not defined');
         throw new PipelineExecutionError('In OpenAI response `usage.prompt_tokens` not defined');
     }
     const inputTokens = rawResponse.usage.prompt_tokens;
     const outputTokens = ((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completion_tokens) || 0;
     const modelInfo = OPENAI_MODELS.find((model) => model.modelName === rawResponse.model);
+    console.log('!!! computeOpenAiUsage', {
+        inputTokens,
+        outputTokens,
+        rawResponse,
+        resultContent,
+        modelInfo,
+    });
     let price;
     if (modelInfo === undefined || modelInfo.pricing === undefined) {
         price = uncertainNumber();