@promptbook/node 0.72.0-4 → 0.72.0-5

package/umd/index.umd.js

@@ -35,8 +35,8 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.72.0-3';
-    // TODO:[main] !!!! List here all the versions and annotate + put into script
+    var PROMPTBOOK_VERSION = '0.72.0-4';
+    // TODO: [main] !!!! List here all the versions and annotate + put into script
 
     /*! *****************************************************************************
     Copyright (c) Microsoft Corporation.
@@ -536,7 +536,7 @@
             commands.push("PIPELINE URL ".concat(pipelineUrl));
         }
         commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
-        // TODO:[main] !!! This increase size of the bundle and is probbably not necessary
+        // TODO: [main] !!! This increase size of the bundle and is probbably not necessary
         pipelineString = prettifyMarkdown(pipelineString);
         try {
             for (var _g = __values(parameters.filter(function (_a) {
@@ -684,12 +684,12 @@
                 pipelineString += '```' + contentLanguage;
                 pipelineString += '\n';
                 pipelineString += spaceTrim__default["default"](content);
-                //                   <- TODO:[main] !!! Escape
+                //                   <- TODO: [main] !!! Escape
                 //                   <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
                 pipelineString += '\n';
                 pipelineString += '```';
                 pipelineString += '\n\n';
-                pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO:[main] !!! If the parameter here has description, add it and use templateParameterJsonToString
+                pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!! If the parameter here has description, add it and use templateParameterJsonToString
             }
         }
         catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -923,7 +923,7 @@
         if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
             return false;
         }
-        // <- TODO:[main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
+        // <- TODO: [main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
         return true;
     }
 
@@ -2574,7 +2574,7 @@
         // Note: [🍭] Fixing dependent subparameterName from FOREACH command
         if (foreach !== undefined) {
             try {
-                for (var _l = __values(foreach.subparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
+                for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
                     var subparameterName = _m.value;
                     if (parameterNames.has(subparameterName)) {
                         parameterNames.delete(subparameterName);
@@ -2689,6 +2689,42 @@
         }
     }
 
+    /**
+     * This error indicates problems parsing the format value
+     *
+     * For example, when the format value is not a valid JSON or CSV
+     * This is not thrown directly but in extended classes
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var AbstractFormatError = /** @class */ (function (_super) {
+        __extends(AbstractFormatError, _super);
+        // Note: To allow instanceof do not put here error `name`
+        // public readonly name = 'AbstractFormatError';
+        function AbstractFormatError(message) {
+            var _this = _super.call(this, message) || this;
+            Object.setPrototypeOf(_this, AbstractFormatError.prototype);
+            return _this;
+        }
+        return AbstractFormatError;
+    }(Error));
+
+    /**
+     * This error indicates problem with parsing of CSV
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var CsvFormatError = /** @class */ (function (_super) {
+        __extends(CsvFormatError, _super);
+        function CsvFormatError(message) {
+            var _this = _super.call(this, message) || this;
+            _this.name = 'CsvFormatError';
+            Object.setPrototypeOf(_this, CsvFormatError.prototype);
+            return _this;
+        }
+        return CsvFormatError;
+    }(AbstractFormatError));
+
     /**
      * @@@
      *
@@ -2709,7 +2745,7 @@
         formatName: 'CSV',
         aliases: ['SPREADSHEET', 'TABLE'],
         isValid: function (value, settings, schema) {
-            // TODO: !!!!!! Implement CSV validation
+            // TODO: Implement CSV validation
             TODO_USE(value /* <- TODO: Use value here */);
             TODO_USE(settings /* <- TODO: Use settings here */);
             TODO_USE(schema /* <- TODO: Use schema here */);
@@ -2730,7 +2766,7 @@
         subvalueDefinitions: [
             {
                 subvalueName: 'ROW',
-                mapValues: function (value, settings, mapCallback) {
+                mapValues: function (value, outputParameterName, settings, mapCallback) {
                     return __awaiter(this, void 0, void 0, function () {
                         var csv, mappedData;
                         var _this = this;
@@ -2739,24 +2775,22 @@
                                 case 0:
                                     csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                     if (csv.errors.length !== 0) {
-                                        throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
-                                        spaceTrim__default["default"](function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                        throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                     }
                                     return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
-                                            var _a;
-                                            var _b;
-                                            return __generator(this, function (_c) {
-                                                switch (_c.label) {
+                                            var _a, _b;
+                                            var _c;
+                                            return __generator(this, function (_d) {
+                                                switch (_d.label) {
                                                     case 0:
+                                                        if (row[outputParameterName]) {
+                                                            throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
+                                                        }
                                                         _a = [__assign({}, row)];
-                                                        _b = {};
-                                                        // <- TODO: !!!!!! Dynamic new column name and position
-                                                        // <- TODO: !!!!!! Check name collisions
+                                                        _c = {};
+                                                        _b = outputParameterName;
                                                         return [4 /*yield*/, mapCallback(row, index)];
-                                                    case 1: return [2 /*return*/, (__assign.apply(void 0, _a.concat([(_b.newColumn = 
-                                                            // <- TODO: !!!!!! Dynamic new column name and position
-                                                            // <- TODO: !!!!!! Check name collisions
-                                                            _c.sent(), _b)])))];
+                                                    case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
                                                 }
                                             });
                                         }); }))];
@@ -2770,7 +2804,7 @@
             },
             {
                 subvalueName: 'CELL',
-                mapValues: function (value, settings, mapCallback) {
+                mapValues: function (value, outputParameterName, settings, mapCallback) {
                     return __awaiter(this, void 0, void 0, function () {
                         var csv, mappedData;
                         var _this = this;
@@ -2779,8 +2813,7 @@
                                 case 0:
                                     csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                     if (csv.errors.length !== 0) {
-                                        throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
-                                        spaceTrim__default["default"](function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                        throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                     }
                                     return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
                                             var _this = this;
@@ -2897,7 +2930,7 @@
         subvalueDefinitions: [
             {
                 subvalueName: 'LINE',
-                mapValues: function (value, settings, mapCallback) {
+                mapValues: function (value, outputParameterName, settings, mapCallback) {
                     return __awaiter(this, void 0, void 0, function () {
                         var lines, mappedLines;
                         return __generator(this, function (_a) {
@@ -3465,7 +3498,9 @@
                                             title: template.title,
                                             pipelineUrl: "".concat(preparedPipeline.pipelineUrl
                                                 ? preparedPipeline.pipelineUrl
-                                                : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name),
+                                                : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name
+                                            // <- TODO: Here should be maybe also subformat index to distinguish between same template with different subformat values
+                                            ),
                                             parameters: parameters,
                                             content: preparedContent,
                                             modelRequirements: modelRequirements,
@@ -3769,7 +3804,7 @@
      *
      * @private internal utility of `createPipelineExecutor`
      */
-    function executeFormatCells(options) {
+    function executeFormatSubvalues(options) {
         return __awaiter(this, void 0, void 0, function () {
             var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
             var _this = this;
@@ -3807,18 +3842,18 @@
                         }
                         if (formatDefinition.formatName === 'CSV') {
                             formatSettings = settings.csvSettings;
-                            // <- TODO: !!!!!! More universal, make simmilar pattern for other formats for example \n vs \r\n in text
+                            // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
                         }
-                        return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
+                        return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
                                 var mappedParameters, allSubparameters, subresultString;
                                 return __generator(this, function (_a) {
                                     switch (_a.label) {
                                         case 0:
-                                            // TODO: !!!!!!! Limit to N concurrent executions
-                                            // TODO: !!!!!!! Report progress
+                                            // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
+                                            // TODO: When done [🐚] Report progress also for each subvalue here
                                             try {
                                                 mappedParameters = mapAvailableToExpectedParameters({
-                                                    expectedParameters: Object.fromEntries(template.foreach.subparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
+                                                    expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
                                                     availableParameters: subparameters,
                                                 });
                                             }
@@ -3845,10 +3880,6 @@
             });
         });
     }
-    /**
-     * TODO: !!!!!! Make pipelineIdentification more precise
-     * TODO: !!!!!! How FOREACH execution looks in the report
-     */
 
     /**
      * @@@
@@ -4042,7 +4073,7 @@
                         preparedContent = (currentTemplate.preparedContent || '{content}')
                             .split('{content}')
                             .join(currentTemplate.content);
-                        return [4 /*yield*/, executeFormatCells({
+                        return [4 /*yield*/, executeFormatSubvalues({
                                 jokerParameterNames: jokerParameterNames,
                                 priority: priority,
                                 maxAttempts: maxAttempts,
@@ -4082,6 +4113,9 @@
     /**
      * TODO: [🤹‍♂️]
      */
+    /**
+     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     */
 
     /**
      * @@@
@@ -4433,6 +4467,9 @@
             });
         });
     }
+    /**
+     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     */
 
     /**
      * Creates executor function from pipeline and execution tools.
@@ -4490,6 +4527,9 @@
         }); };
         return pipelineExecutor;
     }
+    /**
+     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     */
 
     /**
      * @@@
@@ -4541,7 +4581,7 @@
                         outputParameters = result.outputParameters;
                         knowledgePiecesRaw = outputParameters.knowledgePieces;
                         knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
-                        //                                                               <- TODO:[main] !!!!! Smarter split and filter out empty pieces
+                        //                                                               <- TODO: [main] !!!!! Smarter split and filter out empty pieces
                         if (isVerbose) {
                             console.info('knowledgeTextPieces:', knowledgeTextPieces);
                         }
@@ -4891,7 +4931,7 @@
                     case 0:
                         _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
                         templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
-                        // TODO:[main] !!!!! Apply samples to each template (if missing and is for the template defined)
+                        // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
                         TODO_USE(parameters);
                         templatesPrepared = new Array(
                         //            <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -5095,7 +5135,7 @@
             if (sourceContent === '') {
                 throw new ParseError("Source is not defined");
             }
-            // TODO:[main] !!!! Following checks should be applied every link in the `sourceContent`
+            // TODO: [main] !!!! Following checks should be applied every link in the `sourceContent`
             if (sourceContent.startsWith('http://')) {
                 throw new ParseError("Source is not secure");
             }
@@ -5848,8 +5888,10 @@
          */
         examples: [
             'FOREACH Text Line `{customers}` -> `{customer}`',
-            'FOR Csv Row `{customers}` -> `{firstName}`, `{lastName}`',
-            'EACH Csv Cell `{customers}` -> `{subformat}`',
+            'FOREACH Csv Cell `{customers}` -> `{cell}`',
+            'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
+            'FOR Text Line `{customers}` -> `{customer}`',
+            'EACH Text Line `{customers}` -> `{customer}`',
         ],
         /**
          * Parses the FOREACH command
@@ -5883,20 +5925,49 @@
                 throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
             }
             var parameterName = validateParameterName(parameterNameArg);
-            var subparameterNames = args
+            var outputSubparameterName = null;
+            // TODO: [4] DRY
+            var inputSubparameterNames = args
                 .slice(4)
                 .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+                .filter(function (parameterName) { return !parameterName.includes('+'); })
                 .filter(function (parameterName) { return parameterName !== ''; })
                 .map(validateParameterName);
-            if (subparameterNames.length === 0) {
-                throw new ParseError("FOREACH command must have at least one subparameter");
+            // TODO: [4] DRY
+            var outputSubparameterNames = args
+                .slice(4)
+                .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+                .filter(function (parameterName) { return parameterName.includes('+'); })
+                .map(function (parameterName) { return parameterName.split('+').join(''); })
+                .map(validateParameterName);
+            if (outputSubparameterNames.length === 1) {
+                outputSubparameterName = outputSubparameterNames[0];
+            }
+            else if (outputSubparameterNames.length > 1) {
+                throw new ParseError("FOREACH command can not have more than one output subparameter");
+            }
+            if (inputSubparameterNames.length === 0) {
+                throw new ParseError("FOREACH command must have at least one input subparameter");
+            }
+            if (outputSubparameterName === null) {
+                // TODO: Following code should be unhardcoded from here and moved to the format definition
+                if (formatName === 'CSV' && subformatName === 'CELL') {
+                    outputSubparameterName = 'newCell';
+                }
+                else if (formatName === 'TEXT' && subformatName === 'LINE') {
+                    outputSubparameterName = 'newLine';
+                }
+                else {
+                    throw new ParseError(spaceTrim__default["default"]("\n                        FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n                        Correct example:\n                        - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n                    ")));
+                }
             }
             return {
                 type: 'FOREACH',
                 formatName: formatName,
                 subformatName: subformatName,
                 parameterName: parameterName,
-                subparameterNames: subparameterNames,
+                inputSubparameterNames: inputSubparameterNames,
+                outputSubparameterName: outputSubparameterName,
             };
         },
         /**
@@ -5905,10 +5976,16 @@
          * Note: `$` is used to indicate that this function mutates given `templateJson`
          */
         $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
-            var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, subparameterNames = command.subparameterNames;
+            var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
             // TODO: [🍭] Detect double use
             // TODO: [🍭] Detect usage with JOKER and don't allow it
-            $templateJson.foreach = { formatName: formatName, subformatName: subformatName, parameterName: parameterName, subparameterNames: subparameterNames };
+            $templateJson.foreach = {
+                formatName: formatName,
+                subformatName: subformatName,
+                parameterName: parameterName,
+                inputSubparameterNames: inputSubparameterNames,
+                outputSubparameterName: outputSubparameterName,
+            };
             keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
             // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
         },
@@ -7730,7 +7807,7 @@
         return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
     }
     /**
-     * TODO:[main] !!!! Warn if used only sync version
+     * TODO: [main] !!!! Warn if used only sync version
      * TODO: [🚞] Report here line/column of error
      * TODO: Use spaceTrim more effectively
      * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal