@promptbook/website-crawler 0.92.0-20 → 0.92.0-22

package/esm/index.es.js

@@ -29,7 +29,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-20';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-22';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2499,15 +2499,12 @@ function jsonParse(value) {
         }
         throw new Error(spaceTrim$1((block) => `
                     ${block(error.message)}
-            
+
                     The JSON text:
                     ${block(value)}
                 `));
     }
 }
-/**
- * TODO: !!!! Use in Promptbook.studio
- */
 
 /**
  * Recursively converts JSON strings to JSON objects
@@ -4155,7 +4152,8 @@ const CsvFormatParser = {
     subvalueParsers: [
         {
             subvalueName: 'ROW',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, outputParameterName, settings, mapCallback, onProgress } = options;
                 const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim$1((block) => `
@@ -4171,21 +4169,29 @@ const CsvFormatParser = {
                                 ${block(value)}
                             `));
                 }
-                const mappedData = await Promise.all(csv.data.map(async (row, index) => {
+                const mappedData = [];
+                for (let index = 0; index < csv.data.length; index++) {
+                    const row = csv.data[index];
                     if (row[outputParameterName]) {
                         throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
                     }
-                    return {
+                    const mappedRow = {
                         ...row,
                         [outputParameterName]: await mapCallback(row, index),
                     };
-                }));
+                    mappedData.push(mappedRow);
+                    if (onProgress) {
+                        // Note: Report the CSV with all rows mapped so far
+                        await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
+                    }
+                }
                 return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
             },
         },
         {
             subvalueName: 'CELL',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, settings, mapCallback, onProgress } = options;
                 const csv = csvParse(value, settings);
                 if (csv.errors.length !== 0) {
                     throw new CsvFormatError(spaceTrim$1((block) => `
@@ -4272,7 +4278,8 @@ const TextFormatParser = {
     subvalueParsers: [
         {
             subvalueName: 'LINE',
-            async mapValues(value, outputParameterName, settings, mapCallback) {
+            async mapValues(options) {
+                const { value, mapCallback, onProgress } = options;
                 const lines = value.split('\n');
                 const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) => 
                 // TODO: [🧠] Maybe option to skip empty line
@@ -5093,7 +5100,7 @@ async function executeAttempts(options) {
  * @private internal utility of `createPipelineExecutor`
  */
 async function executeFormatSubvalues(options) {
-    const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
+    const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
     if (task.foreach === undefined) {
         return /* not await */ executeAttempts(options);
     }
@@ -5147,21 +5154,32 @@ async function executeFormatSubvalues(options) {
         formatSettings = csvSettings;
         // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
     }
-    const resultString = await subvalueParser.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
-        let mappedParameters;
-        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
-        // TODO: When done [🐚] Report progress also for each subvalue here
-        try {
-            mappedParameters = mapAvailableToExpectedParameters({
-                expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
-                availableParameters: subparameters,
-            });
-        }
-        catch (error) {
-            if (!(error instanceof PipelineExecutionError)) {
-                throw error;
+    const resultString = await subvalueParser.mapValues({
+        value: parameterValue,
+        outputParameterName: task.foreach.outputSubparameterName,
+        settings: formatSettings,
+        onProgress(partialResultString) {
+            return onProgress(Object.freeze({
+                [task.resultingParameterName]: 
+                // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
+                partialResultString,
+            }));
+        },
+        async mapCallback(subparameters, index) {
+            let mappedParameters;
+            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
+            // TODO: When done [🐚] Report progress also for each subvalue here
+            try {
+                mappedParameters = mapAvailableToExpectedParameters({
+                    expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
+                    availableParameters: subparameters,
+                });
             }
-            throw new PipelineExecutionError(spaceTrim$1((block) => `
+            catch (error) {
+                if (!(error instanceof PipelineExecutionError)) {
+                    throw error;
+                }
+                throw new PipelineExecutionError(spaceTrim$1((block) => `
                         ${error.message}
 
                         This is error in FOREACH command
@@ -5170,23 +5188,24 @@ async function executeFormatSubvalues(options) {
                         ${block(pipelineIdentification)}
                         Subparameter index: ${index}
                     `));
-        }
-        const allSubparameters = {
-            ...parameters,
-            ...mappedParameters,
-        };
-        // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
-        Object.freeze(allSubparameters);
-        const subresultString = await executeAttempts({
-            ...options,
-            priority: priority + index,
-            parameters: allSubparameters,
-            pipelineIdentification: spaceTrim$1((block) => `
+            }
+            const allSubparameters = {
+                ...parameters,
+                ...mappedParameters,
+            };
+            // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
+            Object.freeze(allSubparameters);
+            const subresultString = await executeAttempts({
+                ...options,
+                priority: priority + index,
+                parameters: allSubparameters,
+                pipelineIdentification: spaceTrim$1((block) => `
                         ${block(pipelineIdentification)}
                         Subparameter index: ${index}
                     `),
-        });
-        return subresultString;
+            });
+            return subresultString;
+        },
     });
     return resultString;
 }
@@ -5360,11 +5379,6 @@ async function getReservedParametersForTask(options) {
 async function executeTask(options) {
     const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
     const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
-    await onProgress({
-        outputParameters: {
-            [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
-        },
-    });
     // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
     const usedParameterNames = extractParameterNamesFromTask(currentTask);
     const dependentParameterNames = new Set(currentTask.dependentParameterNames);
@@ -5439,6 +5453,7 @@ async function executeTask(options) {
         preparedPipeline,
         tools,
         $executionReport,
+        onProgress,
         pipelineIdentification,
         maxExecutionAttempts,
         maxParallelCount,