@promptbook/node 0.72.0-4 → 0.72.0-5

package/esm/index.es.js

@@ -15,8 +15,8 @@ import * as dotenv from 'dotenv';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-3';
-// TODO:[main] !!!! List here all the versions and annotate + put into script
+var PROMPTBOOK_VERSION = '0.72.0-4';
+// TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
 Copyright (c) Microsoft Corporation.
@@ -516,7 +516,7 @@ function pipelineJsonToString(pipelineJson) {
         commands.push("PIPELINE URL ".concat(pipelineUrl));
     }
     commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
-    // TODO:[main] !!! This increase size of the bundle and is probbably not necessary
+    // TODO: [main] !!! This increase size of the bundle and is probbably not necessary
     pipelineString = prettifyMarkdown(pipelineString);
     try {
         for (var _g = __values(parameters.filter(function (_a) {
@@ -664,12 +664,12 @@ function pipelineJsonToString(pipelineJson) {
             pipelineString += '```' + contentLanguage;
             pipelineString += '\n';
             pipelineString += spaceTrim$1(content);
-            //                   <- TODO:[main] !!! Escape
+            //                   <- TODO: [main] !!! Escape
             //                   <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
             pipelineString += '\n';
             pipelineString += '```';
             pipelineString += '\n\n';
-            pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO:[main] !!! If the parameter here has description, add it and use templateParameterJsonToString
+            pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!! If the parameter here has description, add it and use templateParameterJsonToString
         }
     }
     catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -903,7 +903,7 @@ function isValidPromptbookVersion(version) {
     if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
         return false;
     }
-    // <- TODO:[main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
+    // <- TODO: [main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
     return true;
 }
 
@@ -2554,7 +2554,7 @@ function extractParameterNamesFromTemplate(template) {
     // Note: [🍭] Fixing dependent subparameterName from FOREACH command
     if (foreach !== undefined) {
         try {
-            for (var _l = __values(foreach.subparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
+            for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
                 var subparameterName = _m.value;
                 if (parameterNames.has(subparameterName)) {
                     parameterNames.delete(subparameterName);
@@ -2669,6 +2669,42 @@ function TODO_USE() {
     }
 }
 
+/**
+ * This error indicates problems parsing the format value
+ *
+ * For example, when the format value is not a valid JSON or CSV
+ * This is not thrown directly but in extended classes
+ *
+ * @public exported from `@promptbook/core`
+ */
+var AbstractFormatError = /** @class */ (function (_super) {
+    __extends(AbstractFormatError, _super);
+    // Note: To allow instanceof do not put here error `name`
+    // public readonly name = 'AbstractFormatError';
+    function AbstractFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        Object.setPrototypeOf(_this, AbstractFormatError.prototype);
+        return _this;
+    }
+    return AbstractFormatError;
+}(Error));
+
+/**
+ * This error indicates problem with parsing of CSV
+ *
+ * @public exported from `@promptbook/core`
+ */
+var CsvFormatError = /** @class */ (function (_super) {
+    __extends(CsvFormatError, _super);
+    function CsvFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'CsvFormatError';
+        Object.setPrototypeOf(_this, CsvFormatError.prototype);
+        return _this;
+    }
+    return CsvFormatError;
+}(AbstractFormatError));
+
 /**
  * @@@
  *
@@ -2689,7 +2725,7 @@ var CsvFormatDefinition = {
     formatName: 'CSV',
     aliases: ['SPREADSHEET', 'TABLE'],
     isValid: function (value, settings, schema) {
-        // TODO: !!!!!! Implement CSV validation
+        // TODO: Implement CSV validation
         TODO_USE(value /* <- TODO: Use value here */);
         TODO_USE(settings /* <- TODO: Use settings here */);
         TODO_USE(schema /* <- TODO: Use schema here */);
@@ -2710,7 +2746,7 @@ var CsvFormatDefinition = {
     subvalueDefinitions: [
         {
             subvalueName: 'ROW',
-            mapValues: function (value, settings, mapCallback) {
+            mapValues: function (value, outputParameterName, settings, mapCallback) {
                 return __awaiter(this, void 0, void 0, function () {
                     var csv, mappedData;
                     var _this = this;
@@ -2719,24 +2755,22 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
-                                    spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
-                                        var _a;
-                                        var _b;
-                                        return __generator(this, function (_c) {
-                                            switch (_c.label) {
+                                        var _a, _b;
+                                        var _c;
+                                        return __generator(this, function (_d) {
+                                            switch (_d.label) {
                                                 case 0:
+                                                    if (row[outputParameterName]) {
+                                                        throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
+                                                    }
                                                     _a = [__assign({}, row)];
-                                                    _b = {};
-                                                    // <- TODO: !!!!!! Dynamic new column name and position
-                                                    // <- TODO: !!!!!! Check name collisions
+                                                    _c = {};
+                                                    _b = outputParameterName;
                                                     return [4 /*yield*/, mapCallback(row, index)];
-                                                case 1: return [2 /*return*/, (__assign.apply(void 0, _a.concat([(_b.newColumn = 
-                                                        // <- TODO: !!!!!! Dynamic new column name and position
-                                                        // <- TODO: !!!!!! Check name collisions
-                                                        _c.sent(), _b)])))];
+                                                case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
                                             }
                                         });
                                     }); }))];
@@ -2750,7 +2784,7 @@ var CsvFormatDefinition = {
         },
         {
             subvalueName: 'CELL',
-            mapValues: function (value, settings, mapCallback) {
+            mapValues: function (value, outputParameterName, settings, mapCallback) {
                 return __awaiter(this, void 0, void 0, function () {
                     var csv, mappedData;
                     var _this = this;
@@ -2759,8 +2793,7 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
-                                    spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
                                         var _this = this;
@@ -2877,7 +2910,7 @@ var TextFormatDefinition = {
     subvalueDefinitions: [
         {
             subvalueName: 'LINE',
-            mapValues: function (value, settings, mapCallback) {
+            mapValues: function (value, outputParameterName, settings, mapCallback) {
                 return __awaiter(this, void 0, void 0, function () {
                     var lines, mappedLines;
                     return __generator(this, function (_a) {
@@ -3445,7 +3478,9 @@ function executeAttempts(options) {
                                         title: template.title,
                                         pipelineUrl: "".concat(preparedPipeline.pipelineUrl
                                             ? preparedPipeline.pipelineUrl
-                                            : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name),
+                                            : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name
+                                        // <- TODO: Here should be maybe also subformat index to distinguish between same template with different subformat values
+                                        ),
                                         parameters: parameters,
                                         content: preparedContent,
                                         modelRequirements: modelRequirements,
@@ -3749,7 +3784,7 @@ function executeAttempts(options) {
  *
  * @private internal utility of `createPipelineExecutor`
  */
-function executeFormatCells(options) {
+function executeFormatSubvalues(options) {
     return __awaiter(this, void 0, void 0, function () {
         var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
         var _this = this;
@@ -3787,18 +3822,18 @@ function executeFormatCells(options) {
                     }
                     if (formatDefinition.formatName === 'CSV') {
                         formatSettings = settings.csvSettings;
-                        // <- TODO: !!!!!! More universal, make simmilar pattern for other formats for example \n vs \r\n in text
+                        // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
                     }
-                    return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
+                    return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
                             var mappedParameters, allSubparameters, subresultString;
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
                                     case 0:
-                                        // TODO: !!!!!!! Limit to N concurrent executions
-                                        // TODO: !!!!!!! Report progress
+                                        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
+                                        // TODO: When done [🐚] Report progress also for each subvalue here
                                         try {
                                             mappedParameters = mapAvailableToExpectedParameters({
-                                                expectedParameters: Object.fromEntries(template.foreach.subparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
+                                                expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
                                                 availableParameters: subparameters,
                                             });
                                         }
@@ -3825,10 +3860,6 @@ function executeFormatCells(options) {
         });
     });
 }
-/**
- * TODO: !!!!!! Make pipelineIdentification more precise
- * TODO: !!!!!! How FOREACH execution looks in the report
- */
 
 /**
  * @@@
@@ -4022,7 +4053,7 @@ function executeTemplate(options) {
                     preparedContent = (currentTemplate.preparedContent || '{content}')
                         .split('{content}')
                         .join(currentTemplate.content);
-                    return [4 /*yield*/, executeFormatCells({
+                    return [4 /*yield*/, executeFormatSubvalues({
                             jokerParameterNames: jokerParameterNames,
                             priority: priority,
                             maxAttempts: maxAttempts,
@@ -4062,6 +4093,9 @@ function executeTemplate(options) {
 /**
  * TODO: [🤹‍♂️]
  */
+/**
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+ */
 
 /**
  * @@@
@@ -4413,6 +4447,9 @@ function executePipeline(options) {
         });
     });
 }
+/**
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+ */
 
 /**
  * Creates executor function from pipeline and execution tools.
@@ -4470,6 +4507,9 @@ function createPipelineExecutor(options) {
     }); };
     return pipelineExecutor;
 }
+/**
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+ */
 
 /**
  * @@@
@@ -4521,7 +4561,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
                     outputParameters = result.outputParameters;
                     knowledgePiecesRaw = outputParameters.knowledgePieces;
                     knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
-                    //                                                               <- TODO:[main] !!!!! Smarter split and filter out empty pieces
+                    //                                                               <- TODO: [main] !!!!! Smarter split and filter out empty pieces
                     if (isVerbose) {
                         console.info('knowledgeTextPieces:', knowledgeTextPieces);
                     }
@@ -4871,7 +4911,7 @@ function prepareTemplates(pipeline, options) {
                 case 0:
                     _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
                     templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
-                    // TODO:[main] !!!!! Apply samples to each template (if missing and is for the template defined)
+                    // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
                     TODO_USE(parameters);
                     templatesPrepared = new Array(
                     //            <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -5075,7 +5115,7 @@ var knowledgeCommandParser = {
         if (sourceContent === '') {
             throw new ParseError("Source is not defined");
         }
-        // TODO:[main] !!!! Following checks should be applied every link in the `sourceContent`
+        // TODO: [main] !!!! Following checks should be applied every link in the `sourceContent`
         if (sourceContent.startsWith('http://')) {
             throw new ParseError("Source is not secure");
         }
@@ -5828,8 +5868,10 @@ var foreachCommandParser = {
      */
     examples: [
         'FOREACH Text Line `{customers}` -> `{customer}`',
-        'FOR Csv Row `{customers}` -> `{firstName}`, `{lastName}`',
-        'EACH Csv Cell `{customers}` -> `{subformat}`',
+        'FOREACH Csv Cell `{customers}` -> `{cell}`',
+        'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
+        'FOR Text Line `{customers}` -> `{customer}`',
+        'EACH Text Line `{customers}` -> `{customer}`',
     ],
     /**
      * Parses the FOREACH command
@@ -5863,20 +5905,49 @@ var foreachCommandParser = {
             throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
         }
         var parameterName = validateParameterName(parameterNameArg);
-        var subparameterNames = args
+        var outputSubparameterName = null;
+        // TODO: [4] DRY
+        var inputSubparameterNames = args
             .slice(4)
             .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+            .filter(function (parameterName) { return !parameterName.includes('+'); })
             .filter(function (parameterName) { return parameterName !== ''; })
             .map(validateParameterName);
-        if (subparameterNames.length === 0) {
-            throw new ParseError("FOREACH command must have at least one subparameter");
+        // TODO: [4] DRY
+        var outputSubparameterNames = args
+            .slice(4)
+            .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+            .filter(function (parameterName) { return parameterName.includes('+'); })
+            .map(function (parameterName) { return parameterName.split('+').join(''); })
+            .map(validateParameterName);
+        if (outputSubparameterNames.length === 1) {
+            outputSubparameterName = outputSubparameterNames[0];
+        }
+        else if (outputSubparameterNames.length > 1) {
+            throw new ParseError("FOREACH command can not have more than one output subparameter");
+        }
+        if (inputSubparameterNames.length === 0) {
+            throw new ParseError("FOREACH command must have at least one input subparameter");
+        }
+        if (outputSubparameterName === null) {
+            // TODO: Following code should be unhardcoded from here and moved to the format definition
+            if (formatName === 'CSV' && subformatName === 'CELL') {
+                outputSubparameterName = 'newCell';
+            }
+            else if (formatName === 'TEXT' && subformatName === 'LINE') {
+                outputSubparameterName = 'newLine';
+            }
+            else {
+                throw new ParseError(spaceTrim$1("\n                        FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n                        Correct example:\n                        - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n                    ")));
+            }
         }
         return {
             type: 'FOREACH',
             formatName: formatName,
             subformatName: subformatName,
             parameterName: parameterName,
-            subparameterNames: subparameterNames,
+            inputSubparameterNames: inputSubparameterNames,
+            outputSubparameterName: outputSubparameterName,
         };
     },
     /**
@@ -5885,10 +5956,16 @@ var foreachCommandParser = {
      * Note: `$` is used to indicate that this function mutates given `templateJson`
      */
     $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
-        var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, subparameterNames = command.subparameterNames;
+        var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
         // TODO: [🍭] Detect double use
         // TODO: [🍭] Detect usage with JOKER and don't allow it
-        $templateJson.foreach = { formatName: formatName, subformatName: subformatName, parameterName: parameterName, subparameterNames: subparameterNames };
+        $templateJson.foreach = {
+            formatName: formatName,
+            subformatName: subformatName,
+            parameterName: parameterName,
+            inputSubparameterNames: inputSubparameterNames,
+            outputSubparameterName: outputSubparameterName,
+        };
         keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
         // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
     },
@@ -7710,7 +7787,7 @@ function pipelineStringToJsonSync(pipelineString) {
     return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
 }
 /**
- * TODO:[main] !!!! Warn if used only sync version
+ * TODO: [main] !!!! Warn if used only sync version
  * TODO: [🚞] Report here line/column of error
  * TODO: Use spaceTrim more effectively
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal