@promptbook/cli 0.72.0-3 → 0.72.0-5

package/esm/index.es.js

@@ -21,8 +21,8 @@ import OpenAI from 'openai';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-2';
-// TODO:[main] !!!! List here all the versions and annotate + put into script
+var PROMPTBOOK_VERSION = '0.72.0-4';
+// TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
 Copyright (c) Microsoft Corporation.
@@ -682,7 +682,7 @@ function pipelineJsonToString(pipelineJson) {
         commands.push("PIPELINE URL ".concat(pipelineUrl));
     }
     commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
-    // TODO:[main] !!! This increase size of the bundle and is probbably not necessary
+    // TODO: [main] !!! This increase size of the bundle and is probbably not necessary
     pipelineString = prettifyMarkdown(pipelineString);
     try {
         for (var _g = __values(parameters.filter(function (_a) {
@@ -830,12 +830,12 @@ function pipelineJsonToString(pipelineJson) {
             pipelineString += '```' + contentLanguage;
             pipelineString += '\n';
             pipelineString += spaceTrim$1(content);
-            //                   <- TODO:[main] !!! Escape
+            //                   <- TODO: [main] !!! Escape
             //                   <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
             pipelineString += '\n';
             pipelineString += '```';
             pipelineString += '\n\n';
-            pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO:[main] !!! If the parameter here has description, add it and use templateParameterJsonToString
+            pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!! If the parameter here has description, add it and use templateParameterJsonToString
         }
     }
     catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -1069,7 +1069,7 @@ function isValidPromptbookVersion(version) {
     if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
         return false;
     }
-    // <- TODO:[main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
+    // <- TODO: [main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
     return true;
 }
 
@@ -2704,7 +2704,7 @@ function extractParameterNamesFromTemplate(template) {
     // Note: [🍭] Fixing dependent subparameterName from FOREACH command
     if (foreach !== undefined) {
         try {
-            for (var _l = __values(foreach.subparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
+            for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
                 var subparameterName = _m.value;
                 if (parameterNames.has(subparameterName)) {
                     parameterNames.delete(subparameterName);
@@ -2819,6 +2819,42 @@ function TODO_USE() {
     }
 }
 
+/**
+ * This error indicates problems parsing the format value
+ *
+ * For example, when the format value is not a valid JSON or CSV
+ * This is not thrown directly but in extended classes
+ *
+ * @public exported from `@promptbook/core`
+ */
+var AbstractFormatError = /** @class */ (function (_super) {
+    __extends(AbstractFormatError, _super);
+    // Note: To allow instanceof do not put here error `name`
+    // public readonly name = 'AbstractFormatError';
+    function AbstractFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        Object.setPrototypeOf(_this, AbstractFormatError.prototype);
+        return _this;
+    }
+    return AbstractFormatError;
+}(Error));
+
+/**
+ * This error indicates problem with parsing of CSV
+ *
+ * @public exported from `@promptbook/core`
+ */
+var CsvFormatError = /** @class */ (function (_super) {
+    __extends(CsvFormatError, _super);
+    function CsvFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'CsvFormatError';
+        Object.setPrototypeOf(_this, CsvFormatError.prototype);
+        return _this;
+    }
+    return CsvFormatError;
+}(AbstractFormatError));
+
 /**
  * @@@
  *
@@ -2839,7 +2875,7 @@ var CsvFormatDefinition = {
     formatName: 'CSV',
     aliases: ['SPREADSHEET', 'TABLE'],
     isValid: function (value, settings, schema) {
-        // TODO: !!!!!! Implement CSV validation
+        // TODO: Implement CSV validation
         TODO_USE(value /* <- TODO: Use value here */);
         TODO_USE(settings /* <- TODO: Use settings here */);
         TODO_USE(schema /* <- TODO: Use schema here */);
@@ -2860,7 +2896,7 @@ var CsvFormatDefinition = {
     subvalueDefinitions: [
         {
             subvalueName: 'ROW',
-            mapValues: function (value, settings, mapCallback) {
+            mapValues: function (value, outputParameterName, settings, mapCallback) {
                 return __awaiter(this, void 0, void 0, function () {
                     var csv, mappedData;
                     var _this = this;
@@ -2869,24 +2905,22 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
-                                    spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
-                                        var _a;
-                                        var _b;
-                                        return __generator(this, function (_c) {
-                                            switch (_c.label) {
+                                        var _a, _b;
+                                        var _c;
+                                        return __generator(this, function (_d) {
+                                            switch (_d.label) {
                                                 case 0:
+                                                    if (row[outputParameterName]) {
+                                                        throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
+                                                    }
                                                     _a = [__assign({}, row)];
-                                                    _b = {};
-                                                    // <- TODO: !!!!!! Dynamic new column name and position
-                                                    // <- TODO: !!!!!! Check name collisions
+                                                    _c = {};
+                                                    _b = outputParameterName;
                                                     return [4 /*yield*/, mapCallback(row, index)];
-                                                case 1: return [2 /*return*/, (__assign.apply(void 0, _a.concat([(_b.newColumn = 
-                                                        // <- TODO: !!!!!! Dynamic new column name and position
-                                                        // <- TODO: !!!!!! Check name collisions
-                                                        _c.sent(), _b)])))];
+                                                case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
                                             }
                                         });
                                     }); }))];
@@ -2900,7 +2934,7 @@ var CsvFormatDefinition = {
         },
         {
             subvalueName: 'CELL',
-            mapValues: function (value, settings, mapCallback) {
+            mapValues: function (value, outputParameterName, settings, mapCallback) {
                 return __awaiter(this, void 0, void 0, function () {
                     var csv, mappedData;
                     var _this = this;
@@ -2909,8 +2943,7 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
-                                    spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
                                         var _this = this;
@@ -3027,7 +3060,7 @@ var TextFormatDefinition = {
     subvalueDefinitions: [
         {
             subvalueName: 'LINE',
-            mapValues: function (value, settings, mapCallback) {
+            mapValues: function (value, outputParameterName, settings, mapCallback) {
                 return __awaiter(this, void 0, void 0, function () {
                     var lines, mappedLines;
                     return __generator(this, function (_a) {
@@ -3595,7 +3628,9 @@ function executeAttempts(options) {
                                         title: template.title,
                                         pipelineUrl: "".concat(preparedPipeline.pipelineUrl
                                             ? preparedPipeline.pipelineUrl
-                                            : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name),
+                                            : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name
+                                        // <- TODO: Here should be maybe also subformat index to distinguish between same template with different subformat values
+                                        ),
                                         parameters: parameters,
                                         content: preparedContent,
                                         modelRequirements: modelRequirements,
@@ -3899,7 +3934,7 @@ function executeAttempts(options) {
  *
  * @private internal utility of `createPipelineExecutor`
  */
-function executeFormatCells(options) {
+function executeFormatSubvalues(options) {
     return __awaiter(this, void 0, void 0, function () {
         var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
         var _this = this;
@@ -3937,18 +3972,18 @@ function executeFormatCells(options) {
                     }
                     if (formatDefinition.formatName === 'CSV') {
                         formatSettings = settings.csvSettings;
-                        // <- TODO: !!!!!! More universal, make simmilar pattern for other formats for example \n vs \r\n in text
+                        // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
                     }
-                    return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
+                    return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
                             var mappedParameters, allSubparameters, subresultString;
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
                                     case 0:
-                                        // TODO: !!!!!!! Limit to N concurrent executions
-                                        // TODO: !!!!!!! Report progress
+                                        // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
+                                        // TODO: When done [🐚] Report progress also for each subvalue here
                                         try {
                                             mappedParameters = mapAvailableToExpectedParameters({
-                                                expectedParameters: Object.fromEntries(template.foreach.subparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
+                                                expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
                                                 availableParameters: subparameters,
                                             });
                                         }
@@ -3975,10 +4010,6 @@ function executeFormatCells(options) {
         });
     });
 }
-/**
- * TODO: !!!!!! Make pipelineIdentification more precise
- * TODO: !!!!!! How FOREACH execution looks in the report
- */
 
 /**
  * @@@
@@ -4172,7 +4203,7 @@ function executeTemplate(options) {
                     preparedContent = (currentTemplate.preparedContent || '{content}')
                         .split('{content}')
                         .join(currentTemplate.content);
-                    return [4 /*yield*/, executeFormatCells({
+                    return [4 /*yield*/, executeFormatSubvalues({
                             jokerParameterNames: jokerParameterNames,
                             priority: priority,
                             maxAttempts: maxAttempts,
@@ -4212,6 +4243,9 @@ function executeTemplate(options) {
 /**
  * TODO: [🤹‍♂️]
  */
+/**
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+ */
 
 /**
  * @@@
@@ -4563,6 +4597,9 @@ function executePipeline(options) {
         });
     });
 }
+/**
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+ */
 
 /**
  * Creates executor function from pipeline and execution tools.
@@ -4620,6 +4657,9 @@ function createPipelineExecutor(options) {
     }); };
     return pipelineExecutor;
 }
+/**
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+ */
 
 /**
  * @@@
@@ -4671,7 +4711,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
                     outputParameters = result.outputParameters;
                     knowledgePiecesRaw = outputParameters.knowledgePieces;
                     knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
-                    //                                                               <- TODO:[main] !!!!! Smarter split and filter out empty pieces
+                    //                                                               <- TODO: [main] !!!!! Smarter split and filter out empty pieces
                     if (isVerbose) {
                         console.info('knowledgeTextPieces:', knowledgeTextPieces);
                     }
@@ -5021,7 +5061,7 @@ function prepareTemplates(pipeline, options) {
                 case 0:
                     _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
                     templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
-                    // TODO:[main] !!!!! Apply samples to each template (if missing and is for the template defined)
+                    // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
                     TODO_USE(parameters);
                     templatesPrepared = new Array(
                     //            <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -5225,7 +5265,7 @@ var knowledgeCommandParser = {
         if (sourceContent === '') {
             throw new ParseError("Source is not defined");
         }
-        // TODO:[main] !!!! Following checks should be applied every link in the `sourceContent`
+        // TODO: [main] !!!! Following checks should be applied every link in the `sourceContent`
         if (sourceContent.startsWith('http://')) {
             throw new ParseError("Source is not secure");
         }
@@ -5978,8 +6018,10 @@ var foreachCommandParser = {
      */
     examples: [
         'FOREACH Text Line `{customers}` -> `{customer}`',
-        'FOR Csv Row `{customers}` -> `{firstName}`, `{lastName}`',
-        'EACH Csv Cell `{customers}` -> `{subformat}`',
+        'FOREACH Csv Cell `{customers}` -> `{cell}`',
+        'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
+        'FOR Text Line `{customers}` -> `{customer}`',
+        'EACH Text Line `{customers}` -> `{customer}`',
     ],
     /**
      * Parses the FOREACH command
@@ -6013,20 +6055,49 @@ var foreachCommandParser = {
             throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
         }
         var parameterName = validateParameterName(parameterNameArg);
-        var subparameterNames = args
+        var outputSubparameterName = null;
+        // TODO: [4] DRY
+        var inputSubparameterNames = args
             .slice(4)
             .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+            .filter(function (parameterName) { return !parameterName.includes('+'); })
             .filter(function (parameterName) { return parameterName !== ''; })
             .map(validateParameterName);
-        if (subparameterNames.length === 0) {
-            throw new ParseError("FOREACH command must have at least one subparameter");
+        // TODO: [4] DRY
+        var outputSubparameterNames = args
+            .slice(4)
+            .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
+            .filter(function (parameterName) { return parameterName.includes('+'); })
+            .map(function (parameterName) { return parameterName.split('+').join(''); })
+            .map(validateParameterName);
+        if (outputSubparameterNames.length === 1) {
+            outputSubparameterName = outputSubparameterNames[0];
+        }
+        else if (outputSubparameterNames.length > 1) {
+            throw new ParseError("FOREACH command can not have more than one output subparameter");
+        }
+        if (inputSubparameterNames.length === 0) {
+            throw new ParseError("FOREACH command must have at least one input subparameter");
+        }
+        if (outputSubparameterName === null) {
+            // TODO: Following code should be unhardcoded from here and moved to the format definition
+            if (formatName === 'CSV' && subformatName === 'CELL') {
+                outputSubparameterName = 'newCell';
+            }
+            else if (formatName === 'TEXT' && subformatName === 'LINE') {
+                outputSubparameterName = 'newLine';
+            }
+            else {
+                throw new ParseError(spaceTrim$1("\n                        FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n                        Correct example:\n                        - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n                    ")));
+            }
         }
         return {
             type: 'FOREACH',
             formatName: formatName,
             subformatName: subformatName,
             parameterName: parameterName,
-            subparameterNames: subparameterNames,
+            inputSubparameterNames: inputSubparameterNames,
+            outputSubparameterName: outputSubparameterName,
         };
     },
     /**
@@ -6035,10 +6106,16 @@ var foreachCommandParser = {
      * Note: `$` is used to indicate that this function mutates given `templateJson`
      */
     $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
-        var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, subparameterNames = command.subparameterNames;
+        var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
         // TODO: [🍭] Detect double use
         // TODO: [🍭] Detect usage with JOKER and don't allow it
-        $templateJson.foreach = { formatName: formatName, subformatName: subformatName, parameterName: parameterName, subparameterNames: subparameterNames };
+        $templateJson.foreach = {
+            formatName: formatName,
+            subformatName: subformatName,
+            parameterName: parameterName,
+            inputSubparameterNames: inputSubparameterNames,
+            outputSubparameterName: outputSubparameterName,
+        };
         keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
         // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
     },
@@ -7860,7 +7937,7 @@ function pipelineStringToJsonSync(pipelineString) {
     return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
 }
 /**
- * TODO:[main] !!!! Warn if used only sync version
+ * TODO: [main] !!!! Warn if used only sync version
  * TODO: [🚞] Report here line/column of error
  * TODO: Use spaceTrim more effectively
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
@@ -9645,7 +9722,7 @@ var RemoteLlmExecutionTools = /** @class */ (function () {
             socket.on('connect', function () {
                 resolve(socket);
             });
-            // TODO:[main] !!!! Better timeout handling
+            // TODO: [main] !!!! Better timeout handling
             setTimeout(function () {
                 reject(new Error("Timeout while connecting to ".concat(_this.options.remoteUrl)));
             }, CONNECTION_TIMEOUT_MS);
@@ -9825,7 +9902,7 @@ var ANTHROPIC_CLAUDE_MODELS = $asDeeplyFrozenSerializableJson('ANTHROPIC_CLAUDE_
             output: computeUsage("$2.40 / 1M tokens"),
         },
     },
-    // TODO:[main] !!! Claude 1 and 2 has also completion versions - ask Hoagy
+    // TODO: [main] !!! Claude 1 and 2 has also completion versions - ask Hoagy
 ]);
 /**
  * Note: [🤖] Add models of new variant
@@ -10565,7 +10642,7 @@ var OPENAI_MODELS = $asDeeplyFrozenSerializableJson('OPENAI_MODELS', [
             prompt: computeUsage("$5.00 / 1M tokens"),
             output: computeUsage("$15.00 / 1M tokens"),
         },
-        //TODO:[main] !!!!!! Add gpt-4o-mini-2024-07-18 and all others to be up to date
+        //TODO: [main] !!! Add gpt-4o-mini-2024-07-18 and all others to be up to date
     },
     /**/
     /**/
@@ -10595,7 +10672,7 @@ var OPENAI_MODELS = $asDeeplyFrozenSerializableJson('OPENAI_MODELS', [
         modelVariant: 'CHAT',
         modelTitle: 'o1-preview-2024-09-12',
         modelName: 'o1-preview-2024-09-12',
-        //             <- TODO:[main] !!!!!! Some better system to organize theese date suffixes and versions
+        //             <- TODO: [main] !!! Some better system to organize theese date suffixes and versions
         pricing: {
             prompt: computeUsage("$15.00 / 1M tokens"),
             output: computeUsage("$60.00 / 1M tokens"),
@@ -10713,7 +10790,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
     AzureOpenAiExecutionTools.prototype.listModels = function () {
         return __awaiter(this, void 0, void 0, function () {
             return __generator(this, function (_a) {
-                // TODO:[main] !!! Do here some filtering which models are really available as deployment
+                // TODO: [main] !!! Do here some filtering which models are really available as deployment
                 //       @see https://management.azure.com/subscriptions/subscriptionId/resourceGroups/resourceGroupName/providers/Microsoft.CognitiveServices/accounts/accountName/deployments?api-version=2023-05-01
                 return [2 /*return*/, OPENAI_MODELS.map(function (_a) {
                         var modelTitle = _a.modelTitle, modelName = _a.modelName, modelVariant = _a.modelVariant;
@@ -11527,8 +11604,7 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
                         rawPromptContent = replaceParameters(content, __assign(__assign({}, parameters), { modelName: 'assistant' }));
                         rawRequest = {
                             // [👨‍👨‍👧‍👧] ...modelSettings,
-                            assistant_id: 'asst_CJCZzFCbBL0f2D4OWMXVTdBB',
-                            //             <- Note: This is not a private information, just ID of the assistant which is accessible only with correct API key
+                            assistant_id: this.assistantId,
                             thread: {
                                 messages: [
                                     // TODO: !!!!!! Unhardcode
@@ -11542,6 +11618,7 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
                         if (this.options.isVerbose) {
                             console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
                         }
+                        console.log('!!!!!! OpenAI client', client);
                         return [4 /*yield*/, client.beta.threads.createAndRunStream(rawRequest)];
                     case 2:
                         stream = _g.sent();