@promptbook/node 0.69.0-20 → 0.69.0-21

package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts

@@ -19,5 +19,9 @@ export type ForeachJson = {
     /**
      * @@@
      */
-    readonly subparameterNames: Array<string_parameter_name>;
+    readonly inputSubparameterNames: Array<string_parameter_name>;
+    /**
+     * @@@
+     */
+    readonly outputSubparameterName: string_parameter_name;
 };

package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts

@@ -22,7 +22,7 @@ type ExecutePipelineOptions = {
     /**
      * @@@
      */
-    readonly onProgress?: (taskProgress: TaskProgress) => Promisable<void>;
+    onProgress?(taskProgress: TaskProgress): Promisable<void>;
     /**
      * @@@
      */

package/esm/typings/src/execution/embeddingVectorToString.d.ts

@@ -4,4 +4,4 @@ import type { EmbeddingVector } from './EmbeddingVector';
  *
  * @public exported from `@promptbook/core`
  */
-export declare function embeddingVectorToString(embeddingVector: EmbeddingVector): string;
+export declare function embeddingVectorToString(embeddingVector: Readonly<EmbeddingVector>): string;

package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts

@@ -1,5 +1,6 @@
 import type { Promisable } from 'type-fest';
 import type { Parameters } from '../../types/typeAliases';
+import type { string_parameter_name } from '../../types/typeAliases';
 import type { string_name } from '../../types/typeAliases';
 import type { string_SCREAMING_CASE } from '../../utils/normalization/normalizeTo_SCREAMING_CASE';
 import type { empty_object } from '../../utils/organization/empty_object';
@@ -23,7 +24,7 @@ export type FormatSubvalueDefinition<TValue extends string, TSettings extends em
      * For example, if you have a JSON object and you want to map all values to uppercase
      * Or iterate over all CSV cells @@@
      */
-    mapValues(value: TValue, settings: TSettings, mapCallback: (subvalues: Parameters, index: number) => Promisable<string>): Promise<string>;
+    mapValues(value: TValue, outputParameterName: string_parameter_name, settings: TSettings, mapCallback: (subvalues: Parameters, index: number) => Promisable<string>): Promise<string>;
 };
 /**
  * Note: [👩🏾‍🤝‍🧑🏽]

package/esm/typings/src/types/PipelineJson/ParameterJson.d.ts

@@ -33,7 +33,7 @@ export type ParameterJson = {
     readonly sampleValues?: Array<string_parameter_value>;
 };
 /**
- * TODO: [🧠] !!!!!! Should be here registered subparameter from foreach or not?
+ * TODO: [🧠] Should be here registered subparameters from foreach or not?
  * TODO: [♈] Probbably move expectations from templates to parameters
  * TODO: [🍙] Make some standard order of json properties
  */

package/esm/typings/src/types/execution-report/executionReportJsonToString.d.ts

@@ -1,3 +1,4 @@
+import type { ReadonlyDeep } from 'type-fest';
 import type { ExecutionReportJson } from './ExecutionReportJson';
 import type { ExecutionReportString } from './ExecutionReportString';
 import type { ExecutionReportStringOptions } from './ExecutionReportStringOptions';
@@ -6,7 +7,7 @@ import type { ExecutionReportStringOptions } from './ExecutionReportStringOption
  *
  * @public exported from `@promptbook/core`
  */
-export declare function executionReportJsonToString(executionReportJson: ExecutionReportJson, options?: Partial<ExecutionReportStringOptions>): ExecutionReportString;
+export declare function executionReportJsonToString(executionReportJson: ReadonlyDeep<ExecutionReportJson>, options?: Partial<ExecutionReportStringOptions>): ExecutionReportString;
 /**
  * TODO: Add mermaid chart for every report
  * TODO: [🧠] Allow to filter out some parts of the report by options

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/node",
-    "version": "0.69.0-20",
+    "version": "0.69.0-21",
     "description": "Supercharge your use of large language models",
     "private": false,
     "sideEffects": false,
@@ -51,7 +51,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/node.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.69.0-20"
+        "@promptbook/core": "0.69.0-21"
     },
     "dependencies": {
         "colors": "1.4.0",

package/umd/index.umd.js

@@ -35,7 +35,7 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.69.0-19';
+    var PROMPTBOOK_VERSION = '0.69.0-20';
     // TODO:[main] !!!! List here all the versions and annotate + put into script
 
     /*! *****************************************************************************
@@ -2547,7 +2547,7 @@
         // Note: [🍭] Fixing dependent subparameterName from FOREACH command
         if (foreach !== undefined) {
             try {
-                for (var _l = __values(foreach.subparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
+                for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
                     var subparameterName = _m.value;
                     if (parameterNames.has(subparameterName)) {
                         parameterNames.delete(subparameterName);
@@ -2703,7 +2703,7 @@
         subvalueDefinitions: [
             {
                 subvalueName: 'ROW',
-                mapValues: function (value, settings, mapCallback) {
+                mapValues: function (value, outputParameterName, settings, mapCallback) {
                     return __awaiter(this, void 0, void 0, function () {
                         var csv, mappedData;
                         var _this = this;
@@ -2716,20 +2716,20 @@
                                         spaceTrim__default["default"](function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                     }
                                     return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
-                                            var _a;
-                                            var _b;
-                                            return __generator(this, function (_c) {
-                                                switch (_c.label) {
+                                            var _a, _b;
+                                            var _c;
+                                            return __generator(this, function (_d) {
+                                                switch (_d.label) {
                                                     case 0:
+                                                        if (row[outputParameterName]) {
+                                                            throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
+                                                            "Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
+                                                        }
                                                         _a = [__assign({}, row)];
-                                                        _b = {};
-                                                        // <- TODO: !!!!!! Dynamic new column name and position
-                                                        // <- TODO: !!!!!! Check name collisions
+                                                        _c = {};
+                                                        _b = outputParameterName;
                                                         return [4 /*yield*/, mapCallback(row, index)];
-                                                    case 1: return [2 /*return*/, (__assign.apply(void 0, _a.concat([(_b.newColumn = 
-                                                            // <- TODO: !!!!!! Dynamic new column name and position
-                                                            // <- TODO: !!!!!! Check name collisions
-                                                            _c.sent(), _b)])))];
+                                                    case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
                                                 }
                                             });
                                         }); }))];
@@ -2743,7 +2743,7 @@
             },
             {
                 subvalueName: 'CELL',
-                mapValues: function (value, settings, mapCallback) {
+                mapValues: function (value, outputParameterName, settings, mapCallback) {
                     return __awaiter(this, void 0, void 0, function () {
                         var csv, mappedData;
                         var _this = this;
@@ -2870,7 +2870,7 @@
         subvalueDefinitions: [
             {
                 subvalueName: 'LINE',
-                mapValues: function (value, settings, mapCallback) {
+                mapValues: function (value, outputParameterName, settings, mapCallback) {
                     return __awaiter(this, void 0, void 0, function () {
                         var lines, mappedLines;
                         return __generator(this, function (_a) {
@@ -3782,7 +3782,7 @@
                             formatSettings = settings.csvSettings;
                             // <- TODO: !!!!!! More universal, make simmilar pattern for other formats for example \n vs \r\n in text
                         }
-                        return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
+                        return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
                                 var mappedParameters, allSubparameters, subresultString;
                                 return __generator(this, function (_a) {
                                     switch (_a.label) {
@@ -3791,7 +3791,7 @@
                                             // TODO: !!!!!!! Report progress
                                             try {
                                                 mappedParameters = mapAvailableToExpectedParameters({
-                                                    expectedParameters: Object.fromEntries(template.foreach.subparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
+                                                    expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
                                                     availableParameters: subparameters,
                                                 });
                                             }
@@ -3799,12 +3799,12 @@
                                                 if (!(error instanceof PipelineExecutionError)) {
                                                     throw error;
                                                 }
-                                                throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n                        ".concat(error.message, "\n\n                        This is error in FOREACH command\n                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n                        ").concat(block(pipelineIdentification), "\n                    "); }));
+                                                throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n                        ".concat(error.message, "\n\n                        This is error in FOREACH command\n                        You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n                        ").concat(block(pipelineIdentification), "\n                        Subparameter index: ").concat(index, "\n                    "); }));
                                             }
                                             allSubparameters = __assign(__assign({}, parameters), mappedParameters);
                                             // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
                                             Object.freeze(allSubparameters);
-                                            return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: pipelineIdentification }))];
+                                            return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: spaceTrim__default["default"](function (block) { return "\n                        ".concat(block(pipelineIdentification), "\n                        Subparameter index: ").concat(index, "\n                    "); }) }))];
                                         case 1:
                                             subresultString = _a.sent();
                                             return [2 /*return*/, subresultString];
@@ -4314,7 +4314,7 @@
                                             },
                                             settings: settings,
                                             $executionReport: executionReport,
-                                            pipelineIdentification: pipelineIdentification,
+                                            pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n                            ".concat(block(pipelineIdentification), "\n                            Template name: ").concat(currentTemplate.name, "\n                            Template title: ").concat(currentTemplate.title, "\n                        "); }),
                                         })
                                             .then(function (newParametersToPass) {
                                             parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
@@ -4437,9 +4437,11 @@
         else if (isNotPreparedWarningSupressed !== true) {
             console.warn(spaceTrim.spaceTrim(function (block) { return "\n                    Pipeline is not prepared\n\n                    ".concat(block(pipelineIdentification), "\n\n                    It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n                    But it is recommended to prepare the pipeline during collection preparation\n\n                    @see more at https://ptbk.io/prepare-pipeline\n                "); }));
         }
+        var runCount = 0;
         var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
             return __generator(this, function (_a) {
-                return [2 /*return*/, executePipeline({
+                runCount++;
+                return [2 /*return*/, /* not await */ executePipeline({
                         pipeline: pipeline,
                         preparedPipeline: preparedPipeline,
                         setPreparedPipeline: function (newPreparedPipeline) {
@@ -4448,7 +4450,7 @@
                         inputParameters: inputParameters,
                         tools: tools,
                         onProgress: onProgress,
-                        pipelineIdentification: pipelineIdentification,
+                        pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(pipelineIdentification), "\n                    ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n                "); }),
                         settings: {
                             maxExecutionAttempts: maxExecutionAttempts,
                             maxParallelCount: maxParallelCount,
@@ -5819,8 +5821,10 @@
          */
         examples: [
             'FOREACH Text Line `{customers}` -> `{customer}`',
-            'FOR Csv Row `{customers}` -> `{firstName}`, `{lastName}`',
-            'EACH Csv Cell `{customers}` -> `{subformat}`',
+            'FOREACH Csv Cell `{customers}` -> `{cell}`',
+            'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
+            'FOR Text Line `{customers}` -> `{customer}`',
+            'EACH Text Line `{customers}` -> `{customer}`',
         ],
         /**
          * Parses the FOREACH command
@@ -5854,20 +5858,49 @@
                 throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
             }
             var parameterName = validateParameterName(parameterNameArg);
-            var subparameterNames = args
+            var outputSubparameterName = null;
+            // TODO: [4] DRY
+            var inputSubparameterNames = args
                 .slice(4)
                 .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+                .filter(function (parameterName) { return !parameterName.includes('+'); })
                 .filter(function (parameterName) { return parameterName !== ''; })
                 .map(validateParameterName);
-            if (subparameterNames.length === 0) {
-                throw new ParseError("FOREACH command must have at least one subparameter");
+            // TODO: [4] DRY
+            var outputSubparameterNames = args
+                .slice(4)
+                .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+                .filter(function (parameterName) { return parameterName.includes('+'); })
+                .map(function (parameterName) { return parameterName.split('+').join(''); })
+                .map(validateParameterName);
+            if (outputSubparameterNames.length === 1) {
+                outputSubparameterName = outputSubparameterNames[0];
+            }
+            else if (outputSubparameterNames.length > 1) {
+                throw new ParseError("FOREACH command can not have more than one output subparameter");
+            }
+            if (inputSubparameterNames.length === 0) {
+                throw new ParseError("FOREACH command must have at least one input subparameter");
+            }
+            if (outputSubparameterName === null) {
+                // TODO: Following code should be unhardcoded from here and moved to the format definition
+                if (formatName === 'CSV' && subformatName === 'CELL') {
+                    outputSubparameterName = 'newCell';
+                }
+                else if (formatName === 'TEXT' && subformatName === 'LINE') {
+                    outputSubparameterName = 'newLine';
+                }
+                else {
+                    throw new ParseError(spaceTrim__default["default"]("\n                        FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n                        Correct example:\n                        - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n                    ")));
+                }
             }
             return {
                 type: 'FOREACH',
                 formatName: formatName,
                 subformatName: subformatName,
                 parameterName: parameterName,
-                subparameterNames: subparameterNames,
+                inputSubparameterNames: inputSubparameterNames,
+                outputSubparameterName: outputSubparameterName,
             };
         },
         /**
@@ -5876,10 +5909,16 @@
          * Note: `$` is used to indicate that this function mutates given `templateJson`
          */
         $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
-            var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, subparameterNames = command.subparameterNames;
+            var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
             // TODO: [🍭] Detect double use
             // TODO: [🍭] Detect usage with JOKER and don't allow it
-            $templateJson.foreach = { formatName: formatName, subformatName: subformatName, parameterName: parameterName, subparameterNames: subparameterNames };
+            $templateJson.foreach = {
+                formatName: formatName,
+                subformatName: subformatName,
+                parameterName: parameterName,
+                inputSubparameterNames: inputSubparameterNames,
+                outputSubparameterName: outputSubparameterName,
+            };
             keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
             // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
         },