@promptbook/node 0.69.0-8 → 0.69.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +4 -1
  2. package/esm/index.es.js +985 -310
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/promptbook-collection/index.d.ts +0 -3
  5. package/esm/typings/src/_packages/core.index.d.ts +12 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +6 -0
  7. package/esm/typings/src/_packages/utils.index.d.ts +4 -4
  8. package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
  9. package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +1 -1
  10. package/esm/typings/src/commands/FOREACH/ForeachCommand.d.ts +1 -1
  11. package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -5
  12. package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -2
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +1 -1
  14. package/esm/typings/src/config.d.ts +11 -4
  15. package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +1 -1
  16. package/esm/typings/src/conversion/prettify/renderPipelineMermaidOptions.d.ts +3 -3
  17. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +6 -5
  18. package/esm/typings/src/errors/AbstractFormatError.d.ts +11 -0
  19. package/esm/typings/src/execution/PipelineExecutor.d.ts +1 -0
  20. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +5 -6
  21. package/esm/typings/src/execution/createPipelineExecutor/$OngoingTemplateResult.d.ts +24 -0
  22. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +7 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +3 -0
  24. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +4 -1
  25. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +3 -0
  26. package/esm/typings/src/execution/createPipelineExecutor/{30-executeFormatCells.d.ts → 30-executeFormatSubvalues.d.ts} +2 -2
  27. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +2 -8
  28. package/esm/typings/src/execution/embeddingVectorToString.d.ts +1 -1
  29. package/esm/typings/src/execution/utils/checkExpectations.d.ts +2 -0
  30. package/esm/typings/src/execution/utils/usageToHuman.d.ts +3 -4
  31. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +14 -15
  32. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +31 -0
  33. package/esm/typings/src/formats/csv/{ListFormatDefinition.d.ts → CsvFormatDefinition.d.ts} +6 -3
  34. package/esm/typings/src/formats/csv/CsvFormatError.d.ts +10 -0
  35. package/esm/typings/src/formats/csv/CsvSettings.d.ts +13 -0
  36. package/esm/typings/src/formats/index.d.ts +1 -1
  37. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +4 -3
  38. package/esm/typings/src/formats/text/TextFormatDefinition.d.ts +19 -0
  39. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +4 -3
  40. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +1 -1
  41. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.d.ts +2 -2
  45. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +2 -2
  46. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +1 -1
  47. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  48. package/esm/typings/src/prepare/isPipelinePrepared.d.ts +1 -1
  49. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  50. package/esm/typings/src/types/PipelineJson/ParameterJson.d.ts +1 -1
  51. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +1 -1
  52. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +0 -3
  53. package/esm/typings/src/types/execution-report/executionReportJsonToString.d.ts +2 -1
  54. package/esm/typings/src/types/typeAliases.d.ts +1 -1
  55. package/esm/typings/src/utils/expectation-counters/index.d.ts +3 -0
  56. package/esm/typings/src/utils/organization/{f.d.ts → empty_object.d.ts} +5 -1
  57. package/esm/typings/src/utils/organization/just_empty_object.d.ts +12 -0
  58. package/esm/typings/src/utils/{extractParameterNames.d.ts → parameters/extractParameterNames.d.ts} +2 -2
  59. package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +27 -0
  60. package/esm/typings/src/utils/{replaceParameters.d.ts → parameters/replaceParameters.d.ts} +2 -2
  61. package/esm/typings/src/utils/serialization/checkSerializableAsJson.d.ts +1 -1
  62. package/esm/typings/src/utils/serialization/isSerializableAsJson.d.ts +1 -1
  63. package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +10 -0
  64. package/package.json +18 -13
  65. package/umd/index.umd.js +988 -314
  66. package/umd/index.umd.js.map +1 -1
  67. package/esm/typings/src/formats/list/ListFormatDefinition.d.ts +0 -16
  68. /package/esm/typings/src/utils/{extractParameterNames.test.d.ts → parameters/extractParameterNames.test.d.ts} +0 -0
  69. /package/esm/typings/src/{execution/utils/usageToHuman.test.d.ts → utils/parameters/mapAvailableToExpectedParameters.test.d.ts} +0 -0
  70. /package/esm/typings/src/utils/{replaceParameters.test.d.ts → parameters/replaceParameters.test.d.ts} +0 -0
  71. /package/esm/typings/src/{personas/preparePersona.test.d.ts → utils/validators/parameterName/validateParameterName.test.d.ts} +0 -0
package/umd/index.umd.js CHANGED
@@ -1,8 +1,8 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('path/posix'), require('dotenv')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'crypto-js/enc-hex', 'crypto-js/sha256', 'path/posix', 'dotenv'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.hexEncoder, global.sha256, global.posix, global.dotenv));
5
- })(this, (function (exports, colors, promises, path, spaceTrim, prettier, parserHtml, waitasecond, hexEncoder, sha256, posix, dotenv) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('path/posix'), require('dotenv')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'path/posix', 'dotenv'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.posix, global.dotenv));
5
+ })(this, (function (exports, colors, promises, path, spaceTrim, prettier, parserHtml, waitasecond, papaparse, hexEncoder, sha256, posix, dotenv) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
@@ -35,8 +35,8 @@
35
35
  /**
36
36
  * The version of the Promptbook library
37
37
  */
38
- var PROMPTBOOK_VERSION = '0.69.0-7';
39
- // TODO: !!!! List here all the versions and annotate + put into script
38
+ var PROMPTBOOK_VERSION = '0.69.0-21';
39
+ // TODO: [main] !!!! List here all the versions and annotate + put into script
40
40
 
41
41
  /*! *****************************************************************************
42
42
  Copyright (c) Microsoft Corporation.
@@ -346,7 +346,7 @@
346
346
  }
347
347
  /**
348
348
  * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
349
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
349
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
350
350
  * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
351
351
  */
352
352
 
@@ -442,6 +442,17 @@
442
442
  */
443
443
  var RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE;
444
444
  // <- TODO: [🧜‍♂️]
445
+ /**
446
+ * @@@
447
+ *
448
+ * @public exported from `@promptbook/core`
449
+ */
450
+ var DEFAULT_CSV_SETTINGS = Object.freeze({
451
+ delimiter: ',',
452
+ quoteChar: '"',
453
+ newline: '\n',
454
+ skipEmptyLines: true,
455
+ });
445
456
  /**
446
457
  * @@@
447
458
  *
@@ -525,7 +536,7 @@
525
536
  commands.push("PIPELINE URL ".concat(pipelineUrl));
526
537
  }
527
538
  commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
528
- // TODO: !!! This increase size of the bundle and is probbably not necessary
539
+ // TODO: [main] !!! This increase size of the bundle and is probbably not necessary
529
540
  pipelineString = prettifyMarkdown(pipelineString);
530
541
  try {
531
542
  for (var _g = __values(parameters.filter(function (_a) {
@@ -673,12 +684,12 @@
673
684
  pipelineString += '```' + contentLanguage;
674
685
  pipelineString += '\n';
675
686
  pipelineString += spaceTrim__default["default"](content);
676
- // <- TODO: !!! Escape
687
+ // <- TODO: [main] !!! Escape
677
688
  // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
678
689
  pipelineString += '\n';
679
690
  pipelineString += '```';
680
691
  pipelineString += '\n\n';
681
- pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: !!! If the parameter here has description, add it and use templateParameterJsonToString
692
+ pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!! If the parameter here has description, add it and use templateParameterJsonToString
682
693
  }
683
694
  }
684
695
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -905,7 +916,7 @@
905
916
  });
906
917
  }
907
918
 
908
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
919
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
909
920
 
910
921
  /**
911
922
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -980,7 +991,7 @@
980
991
  if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
981
992
  return false;
982
993
  }
983
- // <- TODO: !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
994
+ // <- TODO: [main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
984
995
  return true;
985
996
  }
986
997
 
@@ -1149,7 +1160,7 @@
1149
1160
  // <- Note: [🚲]
1150
1161
  throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1151
1162
  }
1152
- if (!isValidPromptbookVersion(pipeline.promptbookVersion)) {
1163
+ if (pipeline.promptbookVersion !== undefined && !isValidPromptbookVersion(pipeline.promptbookVersion)) {
1153
1164
  // <- Note: [🚲]
1154
1165
  throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n Invalid Promptbook Version \"".concat(pipeline.promptbookVersion, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1155
1166
  }
@@ -1332,6 +1343,7 @@
1332
1343
  }
1333
1344
  }
1334
1345
  /**
1346
+ * TODO: !!!!! [🧞‍♀️] Do not allow joker + foreach
1335
1347
  * TODO: [🧠] Work with promptbookVersion
1336
1348
  * TODO: Use here some json-schema, Zod or something similar and change it to:
1337
1349
  * > /**
@@ -1343,11 +1355,11 @@
1343
1355
  * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
1344
1356
  */
1345
1357
  /**
1346
- * TODO: [🐣] !!!! Validate that all samples match expectations
1347
- * TODO: [🐣][🐝] !!!! Validate that knowledge is valid (non-void)
1348
- * TODO: [🐣] !!!! Validate that persona can be used only with CHAT variant
1349
- * TODO: [🐣] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1350
- * TODO: [🐣] !!!! Validate that reserved parameter is not used as joker
1358
+ * TODO: [🐣][main] !!!! Validate that all samples match expectations
1359
+ * TODO: [🐣][🐝][main] !!!! Validate that knowledge is valid (non-void)
1360
+ * TODO: [🐣][main] !!!! Validate that persona can be used only with CHAT variant
1361
+ * TODO: [🐣][main] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1362
+ * TODO: [🐣][main] !!!! Validate that reserved parameter is not used as joker
1351
1363
  * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists
1352
1364
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
1353
1365
  */
@@ -2091,7 +2103,7 @@
2091
2103
  return true;
2092
2104
  }
2093
2105
  /**
2094
- * TODO: [🔃] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2106
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2095
2107
  * TODO: [🐠] Maybe base this on `makeValidator`
2096
2108
  * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2097
2109
  * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
@@ -2143,9 +2155,10 @@
2143
2155
  });
2144
2156
  Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2145
2157
  get: function () {
2146
- return this.llmExecutionTools
2147
- .map(function (tools, index) { return "".concat(index + 1, ") ").concat(tools.title, " ").concat(tools.description || ''); })
2148
- .join('\n');
2158
+ return this.llmExecutionTools.map(function (_a, index) {
2159
+ var title = _a.title;
2160
+ return "".concat(index + 1, ") `").concat(title, "`");
2161
+ }).join('\n');
2149
2162
  },
2150
2163
  enumerable: false,
2151
2164
  configurable: true
@@ -2343,9 +2356,7 @@
2343
2356
  throw new PipelineExecutionError("You have not provided any `LlmExecutionTools`");
2344
2357
  }
2345
2358
  else {
2346
- throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.llmExecutionTools
2347
- .map(function (tools) { return "- ".concat(tools.title, " ").concat(tools.description || ''); })
2348
- .join('\n')), "\n\n "); }));
2359
+ throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\"\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.description), "\n\n "); }));
2349
2360
  }
2350
2361
  }
2351
2362
  });
@@ -2487,47 +2498,47 @@
2487
2498
  * @public exported from `@promptbook/utils`
2488
2499
  */
2489
2500
  function extractParameterNamesFromTemplate(template) {
2490
- var e_1, _a, e_2, _b, e_3, _c;
2501
+ var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
2491
2502
  var title = template.title, description = template.description, templateType = template.templateType, content = template.content, preparedContent = template.preparedContent, jokerParameterNames = template.jokerParameterNames, foreach = template.foreach;
2492
2503
  var parameterNames = new Set();
2493
2504
  try {
2494
- for (var _d = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _e = _d.next(); !_e.done; _e = _d.next()) {
2495
- var parameterName = _e.value;
2505
+ for (var _e = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _f = _e.next(); !_f.done; _f = _e.next()) {
2506
+ var parameterName = _f.value;
2496
2507
  parameterNames.add(parameterName);
2497
2508
  }
2498
2509
  }
2499
2510
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
2500
2511
  finally {
2501
2512
  try {
2502
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2513
+ if (_f && !_f.done && (_a = _e.return)) _a.call(_e);
2503
2514
  }
2504
2515
  finally { if (e_1) throw e_1.error; }
2505
2516
  }
2506
2517
  if (templateType === 'SCRIPT_TEMPLATE') {
2507
2518
  try {
2508
- for (var _f = __values(extractVariables(content)), _g = _f.next(); !_g.done; _g = _f.next()) {
2509
- var parameterName = _g.value;
2519
+ for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
2520
+ var parameterName = _h.value;
2510
2521
  parameterNames.add(parameterName);
2511
2522
  }
2512
2523
  }
2513
2524
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
2514
2525
  finally {
2515
2526
  try {
2516
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
2527
+ if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
2517
2528
  }
2518
2529
  finally { if (e_2) throw e_2.error; }
2519
2530
  }
2520
2531
  }
2521
2532
  try {
2522
- for (var _h = __values(jokerParameterNames || []), _j = _h.next(); !_j.done; _j = _h.next()) {
2523
- var jokerName = _j.value;
2533
+ for (var _j = __values(jokerParameterNames || []), _k = _j.next(); !_k.done; _k = _j.next()) {
2534
+ var jokerName = _k.value;
2524
2535
  parameterNames.add(jokerName);
2525
2536
  }
2526
2537
  }
2527
2538
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
2528
2539
  finally {
2529
2540
  try {
2530
- if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
2541
+ if (_k && !_k.done && (_c = _j.return)) _c.call(_j);
2531
2542
  }
2532
2543
  finally { if (e_3) throw e_3.error; }
2533
2544
  }
@@ -2535,10 +2546,22 @@
2535
2546
  // <- Note {websiteContent} is used in `preparedContent`
2536
2547
  // Note: [🍭] Fixing dependent subparameterName from FOREACH command
2537
2548
  if (foreach !== undefined) {
2538
- if (parameterNames.has(foreach.subparameterName)) {
2539
- parameterNames.delete(foreach.subparameterName);
2540
- parameterNames.add(foreach.parameterName);
2541
- // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2549
+ try {
2550
+ for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
2551
+ var subparameterName = _m.value;
2552
+ if (parameterNames.has(subparameterName)) {
2553
+ parameterNames.delete(subparameterName);
2554
+ parameterNames.add(foreach.parameterName);
2555
+ // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2556
+ }
2557
+ }
2558
+ }
2559
+ catch (e_4_1) { e_4 = { error: e_4_1 }; }
2560
+ finally {
2561
+ try {
2562
+ if (_m && !_m.done && (_d = _l.return)) _d.call(_l);
2563
+ }
2564
+ finally { if (e_4) throw e_4.error; }
2542
2565
  }
2543
2566
  }
2544
2567
  return parameterNames;
@@ -2622,6 +2645,183 @@
2622
2645
  return union;
2623
2646
  }
2624
2647
 
2648
+ /**
2649
+ * Just marks a place of place where should be something implemented
2650
+ * No side effects.
2651
+ *
2652
+ * Note: It can be usefull suppressing eslint errors of unused variables
2653
+ *
2654
+ * @param value any values
2655
+ * @returns void
2656
+ * @private within the repository
2657
+ */
2658
+ function TODO_USE() {
2659
+ var value = [];
2660
+ for (var _i = 0; _i < arguments.length; _i++) {
2661
+ value[_i] = arguments[_i];
2662
+ }
2663
+ }
2664
+
2665
+ /**
2666
+ * This error indicates problems parsing the format value
2667
+ *
2668
+ * For example, when the format value is not a valid JSON or CSV
2669
+ * This is not thrown directly but in extended classes
2670
+ *
2671
+ * @public exported from `@promptbook/core`
2672
+ */
2673
+ var AbstractFormatError = /** @class */ (function (_super) {
2674
+ __extends(AbstractFormatError, _super);
2675
+ // Note: To allow instanceof do not put here error `name`
2676
+ // public readonly name = 'AbstractFormatError';
2677
+ function AbstractFormatError(message) {
2678
+ var _this = _super.call(this, message) || this;
2679
+ Object.setPrototypeOf(_this, AbstractFormatError.prototype);
2680
+ return _this;
2681
+ }
2682
+ return AbstractFormatError;
2683
+ }(Error));
2684
+
2685
+ /**
2686
+ * This error indicates problem with parsing of CSV
2687
+ *
2688
+ * @public exported from `@promptbook/core`
2689
+ */
2690
+ var CsvFormatError = /** @class */ (function (_super) {
2691
+ __extends(CsvFormatError, _super);
2692
+ function CsvFormatError(message) {
2693
+ var _this = _super.call(this, message) || this;
2694
+ _this.name = 'CsvFormatError';
2695
+ Object.setPrototypeOf(_this, CsvFormatError.prototype);
2696
+ return _this;
2697
+ }
2698
+ return CsvFormatError;
2699
+ }(AbstractFormatError));
2700
+
2701
+ /**
2702
+ * @@@
2703
+ *
2704
+ * @public exported from `@promptbook/core`
2705
+ */
2706
+ var MANDATORY_CSV_SETTINGS = Object.freeze({
2707
+ header: true,
2708
+ // encoding: 'utf8',
2709
+ });
2710
+
2711
+ /**
2712
+ * Definition for CSV spreadsheet
2713
+ *
2714
+ * @public exported from `@promptbook/core`
2715
+ * <- TODO: [🏢] Export from package `@promptbook/csv`
2716
+ */
2717
+ var CsvFormatDefinition = {
2718
+ formatName: 'CSV',
2719
+ aliases: ['SPREADSHEET', 'TABLE'],
2720
+ isValid: function (value, settings, schema) {
2721
+ // TODO: Implement CSV validation
2722
+ TODO_USE(value /* <- TODO: Use value here */);
2723
+ TODO_USE(settings /* <- TODO: Use settings here */);
2724
+ TODO_USE(schema /* <- TODO: Use schema here */);
2725
+ return true;
2726
+ },
2727
+ canBeValid: function (partialValue, settings, schema) {
2728
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2729
+ TODO_USE(settings /* <- TODO: Use settings here */);
2730
+ TODO_USE(schema /* <- TODO: Use schema here */);
2731
+ return true;
2732
+ },
2733
+ heal: function (value, settings, schema) {
2734
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2735
+ TODO_USE(settings /* <- TODO: Use settings here */);
2736
+ TODO_USE(schema /* <- TODO: Use schema here */);
2737
+ throw new Error('Not implemented');
2738
+ },
2739
+ subvalueDefinitions: [
2740
+ {
2741
+ subvalueName: 'ROW',
2742
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2743
+ return __awaiter(this, void 0, void 0, function () {
2744
+ var csv, mappedData;
2745
+ var _this = this;
2746
+ return __generator(this, function (_a) {
2747
+ switch (_a.label) {
2748
+ case 0:
2749
+ csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2750
+ if (csv.errors.length !== 0) {
2751
+ throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2752
+ }
2753
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
2754
+ var _a, _b;
2755
+ var _c;
2756
+ return __generator(this, function (_d) {
2757
+ switch (_d.label) {
2758
+ case 0:
2759
+ if (row[outputParameterName]) {
2760
+ throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
2761
+ }
2762
+ _a = [__assign({}, row)];
2763
+ _c = {};
2764
+ _b = outputParameterName;
2765
+ return [4 /*yield*/, mapCallback(row, index)];
2766
+ case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
2767
+ }
2768
+ });
2769
+ }); }))];
2770
+ case 1:
2771
+ mappedData = _a.sent();
2772
+ return [2 /*return*/, papaparse.unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2773
+ }
2774
+ });
2775
+ });
2776
+ },
2777
+ },
2778
+ {
2779
+ subvalueName: 'CELL',
2780
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2781
+ return __awaiter(this, void 0, void 0, function () {
2782
+ var csv, mappedData;
2783
+ var _this = this;
2784
+ return __generator(this, function (_a) {
2785
+ switch (_a.label) {
2786
+ case 0:
2787
+ csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2788
+ if (csv.errors.length !== 0) {
2789
+ throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2790
+ }
2791
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
2792
+ var _this = this;
2793
+ return __generator(this, function (_a) {
2794
+ return [2 /*return*/, /* not await */ Promise.all(Object.entries(row).map(function (_a, columnIndex) {
2795
+ var _b = __read(_a, 2), key = _b[0], value = _b[1];
2796
+ return __awaiter(_this, void 0, void 0, function () {
2797
+ var index;
2798
+ var _c;
2799
+ return __generator(this, function (_d) {
2800
+ index = rowIndex * Object.keys(row).length + columnIndex;
2801
+ return [2 /*return*/, /* not await */ mapCallback((_c = {}, _c[key] = value, _c), index)];
2802
+ });
2803
+ });
2804
+ }))];
2805
+ });
2806
+ }); }))];
2807
+ case 1:
2808
+ mappedData = _a.sent();
2809
+ return [2 /*return*/, papaparse.unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2810
+ }
2811
+ });
2812
+ });
2813
+ },
2814
+ },
2815
+ ],
2816
+ };
2817
+ /**
2818
+ * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
2819
+ * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
2820
+ * TODO: [🍓] In `CsvFormatDefinition` implement `heal
2821
+ * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
2822
+ * TODO: [🏢] Allow to expect something inside CSV objects and other formats
2823
+ */
2824
+
2625
2825
  /**
2626
2826
  * Function isValidJsonString will tell you if the string is valid JSON or not
2627
2827
  *
@@ -2643,6 +2843,222 @@
2643
2843
  }
2644
2844
  }
2645
2845
 
2846
+ /**
2847
+ * Definition for JSON format
2848
+ *
2849
+ * @private still in development [🏢]
2850
+ */
2851
+ var JsonFormatDefinition = {
2852
+ formatName: 'JSON',
2853
+ mimeType: 'application/json',
2854
+ isValid: function (value, settings, schema) {
2855
+ TODO_USE(schema /* <- TODO: Use schema here */);
2856
+ TODO_USE(settings /* <- TODO: Use settings here */);
2857
+ return isValidJsonString(value);
2858
+ },
2859
+ canBeValid: function (partialValue, settings, schema) {
2860
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2861
+ TODO_USE(settings /* <- TODO: Use settings here */);
2862
+ TODO_USE(schema /* <- TODO: Use schema here */);
2863
+ return true;
2864
+ },
2865
+ heal: function (value, settings, schema) {
2866
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2867
+ TODO_USE(settings /* <- TODO: Use settings here */);
2868
+ TODO_USE(schema /* <- TODO: Use schema here */);
2869
+ throw new Error('Not implemented');
2870
+ },
2871
+ subvalueDefinitions: [],
2872
+ };
2873
+ /**
2874
+ * TODO: [🧠] Maybe propper instance of object
2875
+ * TODO: [0] Make string_serialized_json
2876
+ * TODO: [1] Make type for JSON Settings and Schema
2877
+ * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
2878
+ * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
2879
+ * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
2880
+ * TODO: [🍓] In `JsonFormatDefinition` implement `heal
2881
+ * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
2882
+ * TODO: [🏢] Allow to expect something inside JSON objects and other formats
2883
+ */
2884
+
2885
+ /**
2886
+ * Definition for any text - this will be always valid
2887
+ *
2888
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
2889
+ *
2890
+ * @public exported from `@promptbook/core`
2891
+ */
2892
+ var TextFormatDefinition = {
2893
+ formatName: 'TEXT',
2894
+ isValid: function (value) {
2895
+ return typeof value === 'string';
2896
+ },
2897
+ canBeValid: function (partialValue) {
2898
+ return typeof partialValue === 'string';
2899
+ },
2900
+ heal: function () {
2901
+ throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
2902
+ },
2903
+ subvalueDefinitions: [
2904
+ {
2905
+ subvalueName: 'LINE',
2906
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2907
+ return __awaiter(this, void 0, void 0, function () {
2908
+ var lines, mappedLines;
2909
+ return __generator(this, function (_a) {
2910
+ switch (_a.label) {
2911
+ case 0:
2912
+ lines = value.split('\n');
2913
+ return [4 /*yield*/, Promise.all(lines.map(function (lineContent, lineNumber) {
2914
+ // TODO: [🧠] Maybe option to skip empty line
2915
+ /* not await */ return mapCallback({
2916
+ lineContent: lineContent,
2917
+ // TODO: [🧠] Maybe also put here `lineNumber`
2918
+ }, lineNumber);
2919
+ }))];
2920
+ case 1:
2921
+ mappedLines = _a.sent();
2922
+ return [2 /*return*/, mappedLines.join('\n')];
2923
+ }
2924
+ });
2925
+ });
2926
+ },
2927
+ },
2928
+ // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2929
+ ],
2930
+ };
2931
+ /**
2932
+ * TODO: [1] Make type for XML Text and Schema
2933
+ * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2934
+ * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
2935
+ * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
2936
+ * TODO: [🍓] In `TextFormatDefinition` implement `heal
2937
+ * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
2938
+ * TODO: [🏢] Allow to expect something inside each item of list and other formats
2939
+ */
2940
+
2941
+ /**
2942
+ * Definition for XML format
2943
+ *
2944
+ * @private still in development [🏢]
2945
+ */
2946
+ var XmlFormatDefinition = {
2947
+ formatName: 'XML',
2948
+ mimeType: 'application/xml',
2949
+ isValid: function (value, settings, schema) {
2950
+ TODO_USE(value /* <- TODO: Use value here */);
2951
+ TODO_USE(settings /* <- TODO: Use settings here */);
2952
+ TODO_USE(schema /* <- TODO: Use schema here */);
2953
+ return true;
2954
+ },
2955
+ canBeValid: function (partialValue, settings, schema) {
2956
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2957
+ TODO_USE(settings /* <- TODO: Use settings here */);
2958
+ TODO_USE(schema /* <- TODO: Use schema here */);
2959
+ return true;
2960
+ },
2961
+ heal: function (value, settings, schema) {
2962
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2963
+ TODO_USE(settings /* <- TODO: Use settings here */);
2964
+ TODO_USE(schema /* <- TODO: Use schema here */);
2965
+ throw new Error('Not implemented');
2966
+ },
2967
+ subvalueDefinitions: [],
2968
+ };
2969
+ /**
2970
+ * TODO: [🧠] Maybe propper instance of object
2971
+ * TODO: [0] Make string_serialized_xml
2972
+ * TODO: [1] Make type for XML Settings and Schema
2973
+ * TODO: [🧠] What to use for validating XMLs - XSD,...
2974
+ * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
2975
+ * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
2976
+ * TODO: [🍓] In `XmlFormatDefinition` implement `heal
2977
+ * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
2978
+ * TODO: [🏢] Allow to expect something inside XML and other formats
2979
+ */
2980
+
2981
+ /**
2982
+ * Definitions for all formats supported by Promptbook
2983
+ *
2984
+ * @private internal index of `...` <- TODO [🏢]
2985
+ */
2986
+ var FORMAT_DEFINITIONS = [
2987
+ JsonFormatDefinition,
2988
+ XmlFormatDefinition,
2989
+ TextFormatDefinition,
2990
+ CsvFormatDefinition,
2991
+ ];
2992
+
2993
+ /**
2994
+ * Maps available parameters to expected parameters
2995
+ *
2996
+ * The strategy is:
2997
+ * 1) @@@
2998
+ * 2) @@@
2999
+ *
3000
+ * @throws {PipelineExecutionError} @@@
3001
+ * @private within the repository used in `createPipelineExecutor`
3002
+ */
3003
+ function mapAvailableToExpectedParameters(options) {
3004
+ var e_1, _a;
3005
+ var expectedParameters = options.expectedParameters, availableParameters = options.availableParameters;
3006
+ var availableParametersNames = new Set(Object.keys(availableParameters));
3007
+ var expectedParameterNames = new Set(Object.keys(expectedParameters));
3008
+ var mappedParameters = {};
3009
+ try {
3010
+ // Phase 1️⃣: Matching mapping
3011
+ for (var _b = __values(Array.from(union(availableParametersNames, expectedParameterNames))), _c = _b.next(); !_c.done; _c = _b.next()) {
3012
+ var parameterName = _c.value;
3013
+ // Situation: Parameter is available and expected
3014
+ if (availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
3015
+ mappedParameters[parameterName] = availableParameters[parameterName];
3016
+ // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here?
3017
+ availableParametersNames.delete(parameterName);
3018
+ expectedParameterNames.delete(parameterName);
3019
+ }
3020
+ // Situation: Parameter is available but NOT expected
3021
+ else if (availableParametersNames.has(parameterName) && !expectedParameterNames.has(parameterName)) {
3022
+ // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping
3023
+ }
3024
+ // Situation: Parameter is NOT available BUT expected
3025
+ else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
3026
+ // Do nothing here - this will be maybe fixed in the non-matching mapping
3027
+ }
3028
+ }
3029
+ }
3030
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3031
+ finally {
3032
+ try {
3033
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3034
+ }
3035
+ finally { if (e_1) throw e_1.error; }
3036
+ }
3037
+ if (expectedParameterNames.size === 0) {
3038
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3039
+ Object.freeze(mappedParameters);
3040
+ return mappedParameters;
3041
+ }
3042
+ // Phase 2️⃣: Non-matching mapping
3043
+ if (expectedParameterNames.size !== availableParametersNames.size) {
3044
+ throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n Can not map available parameters to expected parameters\n\n Mapped parameters:\n ".concat(block(Object.keys(mappedParameters)
3045
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3046
+ .join('\n')), "\n\n Expected parameters which can not be mapped:\n ").concat(block(Array.from(expectedParameterNames)
3047
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3048
+ .join('\n')), "\n\n Remaining available parameters:\n ").concat(block(Array.from(availableParametersNames)
3049
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3050
+ .join('\n')), "\n\n "); }));
3051
+ }
3052
+ var expectedParameterNamesArray = Array.from(expectedParameterNames);
3053
+ var availableParametersNamesArray = Array.from(availableParametersNames);
3054
+ for (var i = 0; i < expectedParameterNames.size; i++) {
3055
+ mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
3056
+ }
3057
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3058
+ Object.freeze(mappedParameters);
3059
+ return mappedParameters;
3060
+ }
3061
+
2646
3062
  /**
2647
3063
  * Extracts all code blocks from markdown.
2648
3064
  *
@@ -2948,6 +3364,9 @@
2948
3364
  LINES: countLines,
2949
3365
  PAGES: countPages,
2950
3366
  };
3367
+ /**
3368
+ * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
3369
+ */
2951
3370
 
2952
3371
  /**
2953
3372
  * Function checkExpectations will check if the expectations on given value are met
@@ -2984,6 +3403,8 @@
2984
3403
  }
2985
3404
  /**
2986
3405
  * TODO: [💝] Unite object for expecting amount and format
3406
+ * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
3407
+ * Note: [💝] and [🤠] are interconnected together
2987
3408
  */
2988
3409
 
2989
3410
  /**
@@ -2993,58 +3414,66 @@
2993
3414
  */
2994
3415
  function executeAttempts(options) {
2995
3416
  return __awaiter(this, void 0, void 0, function () {
2996
- var $ongoingTemplateResult, jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, _loop_1, attempt, state_1;
3417
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
2997
3418
  return __generator(this, function (_a) {
2998
3419
  switch (_a.label) {
2999
3420
  case 0:
3000
- $ongoingTemplateResult = options.$ongoingTemplateResult, jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3421
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3001
3422
  maxExecutionAttempts = settings.maxExecutionAttempts;
3423
+ $ongoingTemplateResult = {
3424
+ $result: null,
3425
+ $resultString: null,
3426
+ $expectError: null,
3427
+ $scriptPipelineExecutionErrors: [],
3428
+ };
3002
3429
  _loop_1 = function (attempt) {
3003
- var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, _h, scriptTools, _j, error_1, e_1_1, _k, _l, _m, functionName, postprocessingError, _o, _p, scriptTools, _q, error_2, e_2_1, e_3_1, error_3;
3004
- var e_1, _r, e_3, _s, e_2, _t;
3005
- return __generator(this, function (_u) {
3006
- switch (_u.label) {
3430
+ var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
3431
+ var e_1, _q, e_3, _r, e_2, _s;
3432
+ return __generator(this, function (_t) {
3433
+ switch (_t.label) {
3007
3434
  case 0:
3008
3435
  isJokerAttempt = attempt < 0;
3009
3436
  jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
3010
- // TODO: [🧠] !!!!!! JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3437
+ // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3011
3438
  if (isJokerAttempt && !jokerParameterName) {
3012
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
3439
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
3013
3440
  }
3014
3441
  $ongoingTemplateResult.$result = null;
3015
3442
  $ongoingTemplateResult.$resultString = null;
3016
3443
  $ongoingTemplateResult.$expectError = null;
3017
3444
  if (isJokerAttempt) {
3018
3445
  if (parameters[jokerParameterName] === undefined) {
3019
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3446
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3020
3447
  // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
3021
3448
  }
3022
3449
  else {
3023
3450
  $ongoingTemplateResult.$resultString = parameters[jokerParameterName];
3024
3451
  }
3025
3452
  }
3026
- _u.label = 1;
3453
+ _t.label = 1;
3027
3454
  case 1:
3028
- _u.trys.push([1, 44, 45, 46]);
3029
- if (!!isJokerAttempt) return [3 /*break*/, 26];
3455
+ _t.trys.push([1, 43, 44, 45]);
3456
+ if (!!isJokerAttempt) return [3 /*break*/, 25];
3030
3457
  _b = template.templateType;
3031
3458
  switch (_b) {
3032
3459
  case 'SIMPLE_TEMPLATE': return [3 /*break*/, 2];
3033
3460
  case 'PROMPT_TEMPLATE': return [3 /*break*/, 3];
3034
- case 'SCRIPT_TEMPLATE': return [3 /*break*/, 12];
3035
- case 'DIALOG_TEMPLATE': return [3 /*break*/, 23];
3461
+ case 'SCRIPT_TEMPLATE': return [3 /*break*/, 11];
3462
+ case 'DIALOG_TEMPLATE': return [3 /*break*/, 22];
3036
3463
  }
3037
- return [3 /*break*/, 25];
3464
+ return [3 /*break*/, 24];
3038
3465
  case 2:
3039
3466
  $ongoingTemplateResult.$resultString = replaceParameters(preparedContent, parameters);
3040
- return [3 /*break*/, 26];
3467
+ return [3 /*break*/, 25];
3041
3468
  case 3:
3042
3469
  modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (preparedPipeline.defaultModelRequirements || {})), (template.modelRequirements || {}));
3043
3470
  $ongoingTemplateResult.$prompt = {
3044
3471
  title: template.title,
3045
3472
  pipelineUrl: "".concat(preparedPipeline.pipelineUrl
3046
3473
  ? preparedPipeline.pipelineUrl
3047
- : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name),
3474
+ : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name
3475
+ // <- TODO: Here should be maybe also subformat index to distinguish between same template with different subformat values
3476
+ ),
3048
3477
  parameters: parameters,
3049
3478
  content: preparedContent,
3050
3479
  modelRequirements: modelRequirements,
@@ -3062,67 +3491,57 @@
3062
3491
  case 'COMPLETION': return [3 /*break*/, 6];
3063
3492
  case 'EMBEDDING': return [3 /*break*/, 8];
3064
3493
  }
3065
- return [3 /*break*/, 10];
3494
+ return [3 /*break*/, 9];
3066
3495
  case 4:
3067
3496
  _d = $ongoingTemplateResult;
3068
3497
  return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3069
3498
  case 5:
3070
- _d.$chatResult = _u.sent();
3499
+ _d.$chatResult = _t.sent();
3071
3500
  // TODO: [🍬] Destroy chatThread
3072
3501
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$chatResult;
3073
3502
  $ongoingTemplateResult.$resultString = $ongoingTemplateResult.$chatResult.content;
3074
- return [3 /*break*/, 11];
3503
+ return [3 /*break*/, 10];
3075
3504
  case 6:
3076
3505
  _e = $ongoingTemplateResult;
3077
3506
  return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3078
3507
  case 7:
3079
- _e.$completionResult = _u.sent();
3508
+ _e.$completionResult = _t.sent();
3080
3509
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
3081
3510
  $ongoingTemplateResult.$resultString =
3082
3511
  $ongoingTemplateResult.$completionResult.content;
3083
- return [3 /*break*/, 11];
3084
- case 8:
3085
- // TODO: [🧠] This is weird, embedding model can not be used such a way in the pipeline
3086
- _f = $ongoingTemplateResult;
3087
- return [4 /*yield*/, llmTools.callEmbeddingModel($deepFreeze($ongoingTemplateResult.$prompt))];
3088
- case 9:
3089
- // TODO: [🧠] This is weird, embedding model can not be used such a way in the pipeline
3090
- _f.$embeddingResult = _u.sent();
3091
- $ongoingTemplateResult.$result = $ongoingTemplateResult.$embeddingResult;
3092
- $ongoingTemplateResult.$resultString =
3093
- $ongoingTemplateResult.$embeddingResult.content.join(',');
3094
- return [3 /*break*/, 11];
3095
- case 10: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Unknown model variant \"".concat(template.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3096
- case 11: return [3 /*break*/, 26];
3097
- case 12:
3512
+ return [3 /*break*/, 10];
3513
+ case 8: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Embedding model can not be used in pipeline\n\n This should be catched during parsing\n\n ".concat(block(pipelineIdentification), "\n\n "); }));
3514
+ case 9: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Unknown model variant \"".concat(template.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3515
+ case 10: return [3 /*break*/, 25];
3516
+ case 11:
3098
3517
  if (arrayableToArray(tools.script).length === 0) {
3099
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3518
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3100
3519
  }
3101
3520
  if (!template.contentLanguage) {
3102
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(template.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3521
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(template.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3103
3522
  }
3104
- _u.label = 13;
3523
+ _t.label = 12;
3524
+ case 12:
3525
+ _t.trys.push([12, 19, 20, 21]);
3526
+ _f = (e_1 = void 0, __values(arrayableToArray(tools.script))), _g = _f.next();
3527
+ _t.label = 13;
3105
3528
  case 13:
3106
- _u.trys.push([13, 20, 21, 22]);
3107
- _g = (e_1 = void 0, __values(arrayableToArray(tools.script))), _h = _g.next();
3108
- _u.label = 14;
3529
+ if (!!_g.done) return [3 /*break*/, 18];
3530
+ scriptTools = _g.value;
3531
+ _t.label = 14;
3109
3532
  case 14:
3110
- if (!!_h.done) return [3 /*break*/, 19];
3111
- scriptTools = _h.value;
3112
- _u.label = 15;
3113
- case 15:
3114
- _u.trys.push([15, 17, , 18]);
3115
- _j = $ongoingTemplateResult;
3533
+ _t.trys.push([14, 16, , 17]);
3534
+ _h = $ongoingTemplateResult;
3116
3535
  return [4 /*yield*/, scriptTools.execute($deepFreeze({
3117
3536
  scriptLanguage: template.contentLanguage,
3118
3537
  script: preparedContent,
3119
3538
  parameters: parameters,
3120
3539
  }))];
3540
+ case 15:
3541
+ _h.$resultString = _t.sent();
3542
+ return [3 /*break*/, 18];
3121
3543
  case 16:
3122
- _j.$resultString = _u.sent();
3123
- return [3 /*break*/, 19];
3124
- case 17:
3125
- error_1 = _u.sent();
3544
+ error_1 = _t.sent();
3126
3545
  if (!(error_1 instanceof Error)) {
3127
3546
  throw error_1;
3128
3547
  }
@@ -3130,39 +3549,39 @@
3130
3549
  throw error_1;
3131
3550
  }
3132
3551
  $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_1);
3133
- return [3 /*break*/, 18];
3134
- case 18:
3135
- _h = _g.next();
3136
- return [3 /*break*/, 14];
3137
- case 19: return [3 /*break*/, 22];
3138
- case 20:
3139
- e_1_1 = _u.sent();
3552
+ return [3 /*break*/, 17];
3553
+ case 17:
3554
+ _g = _f.next();
3555
+ return [3 /*break*/, 13];
3556
+ case 18: return [3 /*break*/, 21];
3557
+ case 19:
3558
+ e_1_1 = _t.sent();
3140
3559
  e_1 = { error: e_1_1 };
3141
- return [3 /*break*/, 22];
3142
- case 21:
3560
+ return [3 /*break*/, 21];
3561
+ case 20:
3143
3562
  try {
3144
- if (_h && !_h.done && (_r = _g.return)) _r.call(_g);
3563
+ if (_g && !_g.done && (_q = _f.return)) _q.call(_f);
3145
3564
  }
3146
3565
  finally { if (e_1) throw e_1.error; }
3147
3566
  return [7 /*endfinally*/];
3148
- case 22:
3567
+ case 21:
3149
3568
  if ($ongoingTemplateResult.$resultString !== null) {
3150
- return [3 /*break*/, 26];
3569
+ return [3 /*break*/, 25];
3151
3570
  }
3152
3571
  if ($ongoingTemplateResult.$scriptPipelineExecutionErrors.length === 1) {
3153
3572
  throw $ongoingTemplateResult.$scriptPipelineExecutionErrors[0];
3154
3573
  }
3155
3574
  else {
3156
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTemplateResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTemplateResult.$scriptPipelineExecutionErrors
3575
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTemplateResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTemplateResult.$scriptPipelineExecutionErrors
3157
3576
  .map(function (error) { return '- ' + error.message; })
3158
- .join('\n\n')), "\n "); }));
3577
+ .join('\n\n')), "\n "); }));
3159
3578
  }
3160
- case 23:
3579
+ case 22:
3161
3580
  if (tools.userInterface === undefined) {
3162
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3581
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3163
3582
  }
3164
3583
  // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3165
- _k = $ongoingTemplateResult;
3584
+ _j = $ongoingTemplateResult;
3166
3585
  return [4 /*yield*/, tools.userInterface.promptDialog($deepFreeze({
3167
3586
  promptTitle: template.title,
3168
3587
  promptMessage: replaceParameters(template.description || '', parameters),
@@ -3171,34 +3590,34 @@
3171
3590
  placeholder: undefined,
3172
3591
  priority: priority,
3173
3592
  }))];
3174
- case 24:
3593
+ case 23:
3175
3594
  // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3176
- _k.$resultString = _u.sent();
3177
- return [3 /*break*/, 26];
3178
- case 25: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Unknown execution type \"".concat(template.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3595
+ _j.$resultString = _t.sent();
3596
+ return [3 /*break*/, 25];
3597
+ case 24: throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Unknown execution type \"".concat(template.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3598
+ case 25:
3599
+ if (!(!isJokerAttempt && template.postprocessingFunctionNames)) return [3 /*break*/, 42];
3600
+ _t.label = 26;
3179
3601
  case 26:
3180
- if (!(!isJokerAttempt && template.postprocessingFunctionNames)) return [3 /*break*/, 43];
3181
- _u.label = 27;
3602
+ _t.trys.push([26, 40, 41, 42]);
3603
+ _k = (e_3 = void 0, __values(template.postprocessingFunctionNames)), _l = _k.next();
3604
+ _t.label = 27;
3182
3605
  case 27:
3183
- _u.trys.push([27, 41, 42, 43]);
3184
- _l = (e_3 = void 0, __values(template.postprocessingFunctionNames)), _m = _l.next();
3185
- _u.label = 28;
3186
- case 28:
3187
- if (!!_m.done) return [3 /*break*/, 40];
3188
- functionName = _m.value;
3606
+ if (!!_l.done) return [3 /*break*/, 39];
3607
+ functionName = _l.value;
3189
3608
  postprocessingError = null;
3190
- _u.label = 29;
3609
+ _t.label = 28;
3610
+ case 28:
3611
+ _t.trys.push([28, 35, 36, 37]);
3612
+ _m = (e_2 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next();
3613
+ _t.label = 29;
3191
3614
  case 29:
3192
- _u.trys.push([29, 36, 37, 38]);
3193
- _o = (e_2 = void 0, __values(arrayableToArray(tools.script))), _p = _o.next();
3194
- _u.label = 30;
3615
+ if (!!_o.done) return [3 /*break*/, 34];
3616
+ scriptTools = _o.value;
3617
+ _t.label = 30;
3195
3618
  case 30:
3196
- if (!!_p.done) return [3 /*break*/, 35];
3197
- scriptTools = _p.value;
3198
- _u.label = 31;
3199
- case 31:
3200
- _u.trys.push([31, 33, , 34]);
3201
- _q = $ongoingTemplateResult;
3619
+ _t.trys.push([30, 32, , 33]);
3620
+ _p = $ongoingTemplateResult;
3202
3621
  return [4 /*yield*/, scriptTools.execute({
3203
3622
  scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
3204
3623
  script: "".concat(functionName, "(resultString)"),
@@ -3207,12 +3626,12 @@
3207
3626
  // Note: No ...parametersForTemplate, because working with result only
3208
3627
  },
3209
3628
  })];
3210
- case 32:
3211
- _q.$resultString = _u.sent();
3629
+ case 31:
3630
+ _p.$resultString = _t.sent();
3212
3631
  postprocessingError = null;
3213
- return [3 /*break*/, 35];
3214
- case 33:
3215
- error_2 = _u.sent();
3632
+ return [3 /*break*/, 34];
3633
+ case 32:
3634
+ error_2 = _t.sent();
3216
3635
  if (!(error_2 instanceof Error)) {
3217
3636
  throw error_2;
3218
3637
  }
@@ -3221,41 +3640,41 @@
3221
3640
  }
3222
3641
  postprocessingError = error_2;
3223
3642
  $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_2);
3224
- return [3 /*break*/, 34];
3225
- case 34:
3226
- _p = _o.next();
3227
- return [3 /*break*/, 30];
3228
- case 35: return [3 /*break*/, 38];
3229
- case 36:
3230
- e_2_1 = _u.sent();
3643
+ return [3 /*break*/, 33];
3644
+ case 33:
3645
+ _o = _m.next();
3646
+ return [3 /*break*/, 29];
3647
+ case 34: return [3 /*break*/, 37];
3648
+ case 35:
3649
+ e_2_1 = _t.sent();
3231
3650
  e_2 = { error: e_2_1 };
3232
- return [3 /*break*/, 38];
3233
- case 37:
3651
+ return [3 /*break*/, 37];
3652
+ case 36:
3234
3653
  try {
3235
- if (_p && !_p.done && (_t = _o.return)) _t.call(_o);
3654
+ if (_o && !_o.done && (_s = _m.return)) _s.call(_m);
3236
3655
  }
3237
3656
  finally { if (e_2) throw e_2.error; }
3238
3657
  return [7 /*endfinally*/];
3239
- case 38:
3658
+ case 37:
3240
3659
  if (postprocessingError) {
3241
3660
  throw postprocessingError;
3242
3661
  }
3243
- _u.label = 39;
3244
- case 39:
3245
- _m = _l.next();
3246
- return [3 /*break*/, 28];
3247
- case 40: return [3 /*break*/, 43];
3248
- case 41:
3249
- e_3_1 = _u.sent();
3662
+ _t.label = 38;
3663
+ case 38:
3664
+ _l = _k.next();
3665
+ return [3 /*break*/, 27];
3666
+ case 39: return [3 /*break*/, 42];
3667
+ case 40:
3668
+ e_3_1 = _t.sent();
3250
3669
  e_3 = { error: e_3_1 };
3251
- return [3 /*break*/, 43];
3252
- case 42:
3670
+ return [3 /*break*/, 42];
3671
+ case 41:
3253
3672
  try {
3254
- if (_m && !_m.done && (_s = _l.return)) _s.call(_l);
3673
+ if (_l && !_l.done && (_r = _k.return)) _r.call(_k);
3255
3674
  }
3256
3675
  finally { if (e_3) throw e_3.error; }
3257
3676
  return [7 /*endfinally*/];
3258
- case 43:
3677
+ case 42:
3259
3678
  // TODO: [💝] Unite object for expecting amount and format
3260
3679
  if (template.format) {
3261
3680
  if (template.format === 'JSON') {
@@ -3266,13 +3685,13 @@
3266
3685
  }
3267
3686
  catch (error) {
3268
3687
  keepUnused(error);
3269
- throw new ExpectError(spaceTrim.spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block(
3270
- /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); }));
3688
+ throw new ExpectError(spaceTrim.spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block(
3689
+ /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); }));
3271
3690
  }
3272
3691
  }
3273
3692
  }
3274
3693
  else {
3275
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Unknown format \"".concat(template.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3694
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Unknown format \"".concat(template.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3276
3695
  }
3277
3696
  }
3278
3697
  // TODO: [💝] Unite object for expecting amount and format
@@ -3280,14 +3699,14 @@
3280
3699
  checkExpectations(template.expectations, $ongoingTemplateResult.$resultString || '');
3281
3700
  }
3282
3701
  return [2 /*return*/, "break-attempts"];
3283
- case 44:
3284
- error_3 = _u.sent();
3702
+ case 43:
3703
+ error_3 = _t.sent();
3285
3704
  if (!(error_3 instanceof ExpectError)) {
3286
3705
  throw error_3;
3287
3706
  }
3288
3707
  $ongoingTemplateResult.$expectError = error_3;
3289
- return [3 /*break*/, 46];
3290
- case 45:
3708
+ return [3 /*break*/, 45];
3709
+ case 44:
3291
3710
  if (!isJokerAttempt &&
3292
3711
  template.templateType === 'PROMPT_TEMPLATE' &&
3293
3712
  $ongoingTemplateResult.$prompt
@@ -3304,22 +3723,22 @@
3304
3723
  });
3305
3724
  }
3306
3725
  return [7 /*endfinally*/];
3307
- case 46:
3726
+ case 45:
3308
3727
  if ($ongoingTemplateResult.$expectError !== null && attempt === maxAttempts - 1) {
3309
3728
  throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) {
3310
3729
  var _a, _b, _c;
3311
- return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTemplateResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3730
+ return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTemplateResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3312
3731
  .split('\n')
3313
3732
  .map(function (line) { return "> ".concat(line); })
3314
- .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTemplateResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTemplateResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3733
+ .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTemplateResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTemplateResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3315
3734
  .split('\n')
3316
3735
  .map(function (line) { return "> ".concat(line); })
3317
- .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTemplateResult.$resultString === null
3736
+ .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTemplateResult.$resultString === null
3318
3737
  ? 'null'
3319
3738
  : $ongoingTemplateResult.$resultString
3320
3739
  .split('\n')
3321
3740
  .map(function (line) { return "> ".concat(line); })
3322
- .join('\n')), "\n ---\n ");
3741
+ .join('\n')), "\n ---\n ");
3323
3742
  }));
3324
3743
  }
3325
3744
  return [2 /*return*/];
@@ -3340,7 +3759,11 @@
3340
3759
  case 3:
3341
3760
  attempt++;
3342
3761
  return [3 /*break*/, 1];
3343
- case 4: return [2 /*return*/];
3762
+ case 4:
3763
+ if ($ongoingTemplateResult.$resultString === null) {
3764
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3765
+ }
3766
+ return [2 /*return*/, $ongoingTemplateResult.$resultString];
3344
3767
  }
3345
3768
  });
3346
3769
  });
@@ -3354,36 +3777,83 @@
3354
3777
  *
3355
3778
  * @private internal utility of `createPipelineExecutor`
3356
3779
  */
3357
- function executeFormatCells(options) {
3780
+ function executeFormatSubvalues(options) {
3358
3781
  return __awaiter(this, void 0, void 0, function () {
3359
- var template;
3782
+ var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3783
+ var _this = this;
3360
3784
  return __generator(this, function (_a) {
3361
- template = options.template;
3362
- if (template.foreach === undefined) {
3363
- return [2 /*return*/, /* not await */ executeAttempts(options)];
3785
+ switch (_a.label) {
3786
+ case 0:
3787
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3788
+ if (template.foreach === undefined) {
3789
+ return [2 /*return*/, /* not await */ executeAttempts(options)];
3790
+ }
3791
+ if (jokerParameterNames.length !== 0) {
3792
+ throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n JOKER parameters are not supported together with FOREACH command\n\n [\uD83E\uDDDE\u200D\u2640\uFE0F] This should be prevented in `validatePipeline`\n\n ".concat(block(pipelineIdentification), "\n "); }));
3793
+ }
3794
+ parameterValue = parameters[template.foreach.parameterName] || '';
3795
+ formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
3796
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(template.foreach.formatName);
3797
+ });
3798
+ if (formatDefinition === undefined) {
3799
+ throw new UnexpectedError(
3800
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3801
+ spaceTrim__default["default"](function (block) { return "\n Unsupported format \"".concat(template.foreach.formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
3802
+ .map(function (formatName) { return "- ".concat(formatName); })
3803
+ .join('\n')), "\n\n [\u26F7] This should never happen because format name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3804
+ }
3805
+ subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
3806
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(template.foreach.subformatName);
3807
+ });
3808
+ if (subvalueDefinition === undefined) {
3809
+ throw new UnexpectedError(
3810
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3811
+ spaceTrim__default["default"](function (block) { return "\n Unsupported subformat name \"".concat(template.foreach.subformatName, "\" for format \"").concat(template.foreach.formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
3812
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
3813
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
3814
+ .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3815
+ }
3816
+ if (formatDefinition.formatName === 'CSV') {
3817
+ formatSettings = settings.csvSettings;
3818
+ // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
3819
+ }
3820
+ return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
3821
+ var mappedParameters, allSubparameters, subresultString;
3822
+ return __generator(this, function (_a) {
3823
+ switch (_a.label) {
3824
+ case 0:
3825
+ // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
3826
+ // TODO: When done [🐚] Report progress also for each subvalue here
3827
+ try {
3828
+ mappedParameters = mapAvailableToExpectedParameters({
3829
+ expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
3830
+ availableParameters: subparameters,
3831
+ });
3832
+ }
3833
+ catch (error) {
3834
+ if (!(error instanceof PipelineExecutionError)) {
3835
+ throw error;
3836
+ }
3837
+ throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n ".concat(error.message, "\n\n This is error in FOREACH command\n You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n ").concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }));
3838
+ }
3839
+ allSubparameters = __assign(__assign({}, parameters), mappedParameters);
3840
+ // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
3841
+ Object.freeze(allSubparameters);
3842
+ return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: spaceTrim__default["default"](function (block) { return "\n ".concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }) }))];
3843
+ case 1:
3844
+ subresultString = _a.sent();
3845
+ return [2 /*return*/, subresultString];
3846
+ }
3847
+ });
3848
+ }); })];
3849
+ case 1:
3850
+ resultString = _a.sent();
3851
+ return [2 /*return*/, resultString];
3364
3852
  }
3365
- throw new NotYetImplementedError('FOREACH execution not implemented yet');
3366
3853
  });
3367
3854
  });
3368
3855
  }
3369
3856
 
3370
- /**
3371
- * Just marks a place of place where should be something implemented
3372
- * No side effects.
3373
- *
3374
- * Note: It can be usefull suppressing eslint errors of unused variables
3375
- *
3376
- * @param value any values
3377
- * @returns void
3378
- * @private within the repository
3379
- */
3380
- function TODO_USE() {
3381
- var value = [];
3382
- for (var _i = 0; _i < arguments.length; _i++) {
3383
- value[_i] = arguments[_i];
3384
- }
3385
- }
3386
-
3387
3857
  /**
3388
3858
  * @@@
3389
3859
  *
@@ -3497,7 +3967,7 @@
3497
3967
  */
3498
3968
  function executeTemplate(options) {
3499
3969
  return __awaiter(this, void 0, void 0, function () {
3500
- var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, $ongoingTemplateResult, maxAttempts, jokerParameterNames, preparedContent;
3970
+ var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
3501
3971
  var e_1, _f, _g;
3502
3972
  return __generator(this, function (_h) {
3503
3973
  switch (_h.label) {
@@ -3521,12 +3991,13 @@
3521
3991
  _h.sent();
3522
3992
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
3523
3993
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
3994
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3524
3995
  if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
3525
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Dependent parameters:\n ").concat(Array.from(dependentParameterNames)
3996
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames)
3526
3997
  .map(function (name) { return "{".concat(name, "}"); })
3527
3998
  .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames)
3528
3999
  .map(function (name) { return "{".concat(name, "}"); })
3529
- .join(', '), "\n\n "); }));
4000
+ .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3530
4001
  }
3531
4002
  _b = (_a = Object).freeze;
3532
4003
  _c = [{}];
@@ -3555,6 +4026,7 @@
3555
4026
  };
3556
4027
  try {
3557
4028
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4029
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3558
4030
  for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
3559
4031
  parameterName = _e.value;
3560
4032
  _loop_1(parameterName);
@@ -3567,22 +4039,14 @@
3567
4039
  }
3568
4040
  finally { if (e_1) throw e_1.error; }
3569
4041
  }
3570
- // Note: Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
4042
+ // Note: [👨‍👨‍👧] Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
3571
4043
  Object.freeze(parameters);
3572
- $ongoingTemplateResult = {
3573
- $result: null,
3574
- $resultString: null,
3575
- $expectError: null,
3576
- $scriptPipelineExecutionErrors: [],
3577
- };
3578
4044
  maxAttempts = currentTemplate.templateType === 'DIALOG_TEMPLATE' ? Infinity : maxExecutionAttempts;
3579
4045
  jokerParameterNames = currentTemplate.jokerParameterNames || [];
3580
4046
  preparedContent = (currentTemplate.preparedContent || '{content}')
3581
4047
  .split('{content}')
3582
4048
  .join(currentTemplate.content);
3583
- // <- TODO: [🍵] Use here `replaceParameters` to replace {websiteContent} with option to ignore missing parameters
3584
- return [4 /*yield*/, executeFormatCells({
3585
- $ongoingTemplateResult: $ongoingTemplateResult,
4049
+ return [4 /*yield*/, executeFormatSubvalues({
3586
4050
  jokerParameterNames: jokerParameterNames,
3587
4051
  priority: priority,
3588
4052
  maxAttempts: maxAttempts,
@@ -3597,11 +4061,7 @@
3597
4061
  pipelineIdentification: pipelineIdentification,
3598
4062
  })];
3599
4063
  case 3:
3600
- // <- TODO: [🍵] Use here `replaceParameters` to replace {websiteContent} with option to ignore missing parameters
3601
- _h.sent();
3602
- if ($ongoingTemplateResult.$resultString === null) {
3603
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3604
- }
4064
+ resultString = _h.sent();
3605
4065
  return [4 /*yield*/, onProgress({
3606
4066
  name: name,
3607
4067
  title: title,
@@ -3609,13 +4069,15 @@
3609
4069
  isDone: true,
3610
4070
  templateType: currentTemplate.templateType,
3611
4071
  parameterName: currentTemplate.resultingParameterName,
3612
- parameterValue: $ongoingTemplateResult.$resultString,
4072
+ parameterValue: resultString,
3613
4073
  // <- [🍸]
3614
4074
  })];
3615
4075
  case 4:
3616
4076
  _h.sent();
3617
4077
  return [2 /*return*/, Object.freeze((_g = {},
3618
- _g[currentTemplate.resultingParameterName] = $ongoingTemplateResult.$resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */,
4078
+ _g[currentTemplate.resultingParameterName] =
4079
+ // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4080
+ resultString,
3619
4081
  _g))];
3620
4082
  }
3621
4083
  });
@@ -3624,6 +4086,9 @@
3624
4086
  /**
3625
4087
  * TODO: [🤹‍♂️]
3626
4088
  */
4089
+ /**
4090
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4091
+ */
3627
4092
 
3628
4093
  /**
3629
4094
  * @@@
@@ -3644,6 +4109,7 @@
3644
4109
  };
3645
4110
  try {
3646
4111
  // Note: Filter ONLY output parameters
4112
+ // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters`
3647
4113
  for (var _b = __values(preparedPipeline.parameters.filter(function (_a) {
3648
4114
  var isOutput = _a.isOutput;
3649
4115
  return isOutput;
@@ -3758,7 +4224,7 @@
3758
4224
  return name === parameterName;
3759
4225
  });
3760
4226
  if (!(parameter === undefined)) return [3 /*break*/, 1];
3761
- warnings.push(new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
4227
+ warnings.push(new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
3762
4228
  return [3 /*break*/, 4];
3763
4229
  case 1:
3764
4230
  if (!(parameter.isInput === false)) return [3 /*break*/, 4];
@@ -3770,10 +4236,10 @@
3770
4236
  // Note: Wait a short time to prevent race conditions
3771
4237
  _h.sent();
3772
4238
  _h.label = 3;
3773
- case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim.spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
4239
+ case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim.spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
3774
4240
  isSuccessful: false,
3775
4241
  errors: __spreadArray([
3776
- new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
4242
+ new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
3777
4243
  ], __read(errors), false).map(serializeError),
3778
4244
  warnings: warnings.map(serializeError),
3779
4245
  executionReport: executionReport,
@@ -3837,7 +4303,7 @@
3837
4303
  case 0:
3838
4304
  if (loopLimit-- < 0) {
3839
4305
  // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
3840
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
4306
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
3841
4307
  }
3842
4308
  currentTemplate = unresovedTemplates_1.find(function (template) {
3843
4309
  return template.dependentParameterNames.every(function (name) {
@@ -3847,14 +4313,14 @@
3847
4313
  if (!(!currentTemplate && resolving_1.length === 0)) return [3 /*break*/, 1];
3848
4314
  throw new UnexpectedError(
3849
4315
  // TODO: [🐎] DRY
3850
- spaceTrim.spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
4316
+ spaceTrim.spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
3851
4317
  .map(function (_a) {
3852
4318
  var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
3853
4319
  return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
3854
4320
  .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
3855
4321
  .join(' and '));
3856
4322
  })
3857
- .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
4323
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
3858
4324
  case 1:
3859
4325
  if (!!currentTemplate) return [3 /*break*/, 3];
3860
4326
  /* [🤹‍♂️] */ return [4 /*yield*/, Promise.race(resolving_1)];
@@ -3871,10 +4337,10 @@
3871
4337
  llmTools: llmTools,
3872
4338
  onProgress: function (progress) {
3873
4339
  if (isReturned) {
3874
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4340
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
3875
4341
  .split('\n')
3876
4342
  .map(function (line) { return "> ".concat(line); })
3877
- .join('\n')), "\n "); }));
4343
+ .join('\n')), "\n "); }));
3878
4344
  }
3879
4345
  if (onProgress) {
3880
4346
  onProgress(progress);
@@ -3882,7 +4348,7 @@
3882
4348
  },
3883
4349
  settings: settings,
3884
4350
  $executionReport: executionReport,
3885
- pipelineIdentification: pipelineIdentification,
4351
+ pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
3886
4352
  })
3887
4353
  .then(function (newParametersToPass) {
3888
4354
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
@@ -3974,6 +4440,9 @@
3974
4440
  });
3975
4441
  });
3976
4442
  }
4443
+ /**
4444
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4445
+ */
3977
4446
 
3978
4447
  /**
3979
4448
  * Creates executor function from pipeline and execution tools.
@@ -3985,7 +4454,7 @@
3985
4454
  function createPipelineExecutor(options) {
3986
4455
  var _this = this;
3987
4456
  var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
3988
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d, _e = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e;
4457
+ var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f;
3989
4458
  validatePipeline(pipeline);
3990
4459
  var pipelineIdentification = (function () {
3991
4460
  // Note: This is a 😐 implementation of [🚞]
@@ -4005,9 +4474,11 @@
4005
4474
  else if (isNotPreparedWarningSupressed !== true) {
4006
4475
  console.warn(spaceTrim.spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
4007
4476
  }
4477
+ var runCount = 0;
4008
4478
  var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
4009
4479
  return __generator(this, function (_a) {
4010
- return [2 /*return*/, executePipeline({
4480
+ runCount++;
4481
+ return [2 /*return*/, /* not await */ executePipeline({
4011
4482
  pipeline: pipeline,
4012
4483
  preparedPipeline: preparedPipeline,
4013
4484
  setPreparedPipeline: function (newPreparedPipeline) {
@@ -4016,10 +4487,11 @@
4016
4487
  inputParameters: inputParameters,
4017
4488
  tools: tools,
4018
4489
  onProgress: onProgress,
4019
- pipelineIdentification: pipelineIdentification,
4490
+ pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
4020
4491
  settings: {
4021
4492
  maxExecutionAttempts: maxExecutionAttempts,
4022
4493
  maxParallelCount: maxParallelCount,
4494
+ csvSettings: csvSettings,
4023
4495
  isVerbose: isVerbose,
4024
4496
  isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4025
4497
  },
@@ -4028,6 +4500,9 @@
4028
4500
  }); };
4029
4501
  return pipelineExecutor;
4030
4502
  }
4503
+ /**
4504
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4505
+ */
4031
4506
 
4032
4507
  /**
4033
4508
  * @@@
@@ -4079,7 +4554,7 @@
4079
4554
  outputParameters = result.outputParameters;
4080
4555
  knowledgePiecesRaw = outputParameters.knowledgePieces;
4081
4556
  knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
4082
- // <- TODO: !!!!! Smarter split and filter out empty pieces
4557
+ // <- TODO: [main] !!!!! Smarter split and filter out empty pieces
4083
4558
  if (isVerbose) {
4084
4559
  console.info('knowledgeTextPieces:', knowledgeTextPieces);
4085
4560
  }
@@ -4137,8 +4612,13 @@
4137
4612
  case 6: return [3 /*break*/, 8];
4138
4613
  case 7:
4139
4614
  error_1 = _c.sent();
4615
+ // Note: Here is expected error:
4616
+ // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
4617
+ if (!(error_1 instanceof PipelineExecutionError)) {
4618
+ throw error_1;
4619
+ }
4140
4620
  // TODO: [🟥] Detect browser / node and make it colorfull
4141
- console.error(error_1);
4621
+ console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
4142
4622
  return [3 /*break*/, 8];
4143
4623
  case 8: return [2 /*return*/, {
4144
4624
  name: name,
@@ -4159,7 +4639,7 @@
4159
4639
  });
4160
4640
  }
4161
4641
  /**
4162
- * TODO: [🐝][🔼] !!! Export via `@promptbook/markdown`
4642
+ * TODO: [🐝][🔼][main] !!! Export via `@promptbook/markdown`
4163
4643
  * TODO: [🪂] Do it in parallel 11:11
4164
4644
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4165
4645
  */
@@ -4183,7 +4663,7 @@
4183
4663
  var partialPieces, pieces;
4184
4664
  return __generator(this, function (_a) {
4185
4665
  switch (_a.label) {
4186
- case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4666
+ case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝][main] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4187
4667
  options)];
4188
4668
  case 1:
4189
4669
  partialPieces = _a.sent();
@@ -4375,7 +4855,7 @@
4375
4855
  });
4376
4856
  }
4377
4857
  /**
4378
- * TODO: [🔃] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4858
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4379
4859
  * TODO: [🏢] !! Check validity of `modelName` in pipeline
4380
4860
  * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4381
4861
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
@@ -4424,7 +4904,7 @@
4424
4904
  case 0:
4425
4905
  _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
4426
4906
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
4427
- // TODO: !!!!! Apply samples to each template (if missing and is for the template defined)
4907
+ // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
4428
4908
  TODO_USE(parameters);
4429
4909
  templatesPrepared = new Array(
4430
4910
  // <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -4456,7 +4936,7 @@
4456
4936
  /**
4457
4937
  * TODO: [🧠] Add context to each template (if missing)
4458
4938
  * TODO: [🧠] What is better name `prepareTemplate` or `prepareTemplateAndParameters`
4459
- * TODO: [♨] !!! Prepare index the samples and maybe templates
4939
+ * TODO: [♨][main] !!! Prepare index the samples and maybe templates
4460
4940
  * TODO: Write tests for `preparePipeline`
4461
4941
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
4462
4942
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
@@ -4628,7 +5108,7 @@
4628
5108
  if (sourceContent === '') {
4629
5109
  throw new ParseError("Source is not defined");
4630
5110
  }
4631
- // TODO: !!!! Following checks should be applied every link in the `sourceContent`
5111
+ // TODO: [main] !!!! Following checks should be applied every link in the `sourceContent`
4632
5112
  if (sourceContent.startsWith('http://')) {
4633
5113
  throw new ParseError("Source is not secure");
4634
5114
  }
@@ -4831,7 +5311,7 @@
4831
5311
  if (command.templateType === 'KNOWLEDGE') {
4832
5312
  knowledgeCommandParser.$applyToPipelineJson({
4833
5313
  type: 'KNOWLEDGE',
4834
- sourceContent: $templateJson.content, // <- TODO: [🐝] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
5314
+ sourceContent: $templateJson.content, // <- TODO: [🐝][main] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
4835
5315
  }, $pipelineJson);
4836
5316
  $templateJson.isTemplate = false;
4837
5317
  return;
@@ -5181,6 +5661,171 @@
5181
5661
  * TODO: [🌺] Use some intermediate util splitWords
5182
5662
  */
5183
5663
 
5664
+ /**
5665
+ * @@@
5666
+ *
5667
+ * @param text @@@
5668
+ * @param _isFirstLetterCapital @@@
5669
+ * @returns @@@
5670
+ * @example 'helloWorld'
5671
+ * @example 'iLovePromptbook'
5672
+ * @public exported from `@promptbook/utils`
5673
+ */
5674
+ function normalizeTo_camelCase(text, _isFirstLetterCapital) {
5675
+ var e_1, _a;
5676
+ if (_isFirstLetterCapital === void 0) { _isFirstLetterCapital = false; }
5677
+ var charType;
5678
+ var lastCharType = null;
5679
+ var normalizedName = '';
5680
+ try {
5681
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5682
+ var char = text_1_1.value;
5683
+ var normalizedChar = void 0;
5684
+ if (/^[a-z]$/.test(char)) {
5685
+ charType = 'LOWERCASE';
5686
+ normalizedChar = char;
5687
+ }
5688
+ else if (/^[A-Z]$/.test(char)) {
5689
+ charType = 'UPPERCASE';
5690
+ normalizedChar = char.toLowerCase();
5691
+ }
5692
+ else if (/^[0-9]$/.test(char)) {
5693
+ charType = 'NUMBER';
5694
+ normalizedChar = char;
5695
+ }
5696
+ else {
5697
+ charType = 'OTHER';
5698
+ normalizedChar = '';
5699
+ }
5700
+ if (!lastCharType) {
5701
+ if (_isFirstLetterCapital) {
5702
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: DRY
5703
+ }
5704
+ }
5705
+ else if (charType !== lastCharType &&
5706
+ !(charType === 'LOWERCASE' && lastCharType === 'UPPERCASE') &&
5707
+ !(lastCharType === 'NUMBER') &&
5708
+ !(charType === 'NUMBER')) {
5709
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: [🌺] DRY
5710
+ }
5711
+ normalizedName += normalizedChar;
5712
+ lastCharType = charType;
5713
+ }
5714
+ }
5715
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5716
+ finally {
5717
+ try {
5718
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5719
+ }
5720
+ finally { if (e_1) throw e_1.error; }
5721
+ }
5722
+ return normalizedName;
5723
+ }
5724
+ /**
5725
+ * TODO: [🌺] Use some intermediate util splitWords
5726
+ */
5727
+
5728
+ /**
5729
+ * Removes quotes from a string
5730
+ *
5731
+ * Tip: This is very usefull for post-processing of the result of the LLM model
5732
+ * Note: This function removes only the same quotes from the beginning and the end of the string
5733
+ * Note: There are two simmilar functions:
5734
+ * - `removeQuotes` which removes only bounding quotes
5735
+ * - `unwrapResult` which removes whole introduce sentence
5736
+ *
5737
+ * @param text optionally quoted text
5738
+ * @returns text without quotes
5739
+ * @public exported from `@promptbook/utils`
5740
+ */
5741
+ function removeQuotes(text) {
5742
+ if (text.startsWith('"') && text.endsWith('"')) {
5743
+ return text.slice(1, -1);
5744
+ }
5745
+ if (text.startsWith('\'') && text.endsWith('\'')) {
5746
+ return text.slice(1, -1);
5747
+ }
5748
+ return text;
5749
+ }
5750
+
5751
+ /**
5752
+ * Function `validateParameterName` will @@@
5753
+ *
5754
+ * @param parameterName @@@
5755
+ * @returns @@@
5756
+ * @throws {ParseError} @@@
5757
+ * @private within the repository
5758
+ */
5759
+ function validateParameterName(parameterName) {
5760
+ var e_1, _a;
5761
+ var rawParameterName = parameterName;
5762
+ try {
5763
+ for (var _b = __values([
5764
+ ['`', '`'],
5765
+ ['{', '}'],
5766
+ ['[', ']'],
5767
+ ['(', ')'],
5768
+ ['<', '>'],
5769
+ ]), _c = _b.next(); !_c.done; _c = _b.next()) {
5770
+ var _d = __read(_c.value, 2), start = _d[0], end = _d[1];
5771
+ if (parameterName.substring(0, 1) === start &&
5772
+ parameterName.substring(parameterName.length - 1, parameterName.length) === end
5773
+ // <- TODO: More universal that 1 character
5774
+ ) {
5775
+ parameterName = parameterName.substring(1, parameterName.length - 1);
5776
+ // <- TODO: More universal that 1 character
5777
+ }
5778
+ }
5779
+ }
5780
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5781
+ finally {
5782
+ try {
5783
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
5784
+ }
5785
+ finally { if (e_1) throw e_1.error; }
5786
+ }
5787
+ // TODO: [🐠] Following try-catch block should be part of common validators logic
5788
+ try {
5789
+ /*
5790
+ Note: We don't need to check for spaces because we are going to normalize the parameter name to camelCase
5791
+ if (parameterName.includes(' ')) {
5792
+ throw new ParseError(`Parameter name cannot contain spaces`);
5793
+ }
5794
+ */
5795
+ if (parameterName.includes('.')) {
5796
+ throw new ParseError("Parameter name cannot contain dots");
5797
+ }
5798
+ if (parameterName.includes('/') || parameterName.includes('\\')) {
5799
+ throw new ParseError("Parameter name cannot contain slashes");
5800
+ }
5801
+ if (parameterName.includes('(') ||
5802
+ parameterName.includes(')') ||
5803
+ parameterName.includes('{') ||
5804
+ parameterName.includes('}') ||
5805
+ parameterName.includes('[') ||
5806
+ parameterName.includes(']')) {
5807
+ throw new ParseError("Parameter name cannot contain braces");
5808
+ }
5809
+ parameterName = removeDiacritics(parameterName);
5810
+ parameterName = removeEmojis(parameterName);
5811
+ parameterName = removeQuotes(parameterName);
5812
+ parameterName = normalizeTo_camelCase(parameterName);
5813
+ if (parameterName === '') {
5814
+ throw new ParseError("Parameter name cannot be empty");
5815
+ }
5816
+ if (RESERVED_PARAMETER_NAMES.includes(parameterName)) {
5817
+ throw new ParseError("{".concat(parameterName, "} is a reserved parameter name"));
5818
+ }
5819
+ }
5820
+ catch (error) {
5821
+ if (!(error instanceof ParseError)) {
5822
+ throw error;
5823
+ }
5824
+ throw new ParseError(spaceTrim__default["default"](function (block) { return "\n ".concat(block(error.message), "\n\n Tried to validate parameter name:\n ").concat(block(rawParameterName), "\n "); }));
5825
+ }
5826
+ return parameterName;
5827
+ }
5828
+
5184
5829
  /**
5185
5830
  * Parses the foreach command
5186
5831
  *
@@ -5210,15 +5855,16 @@
5210
5855
  /**
5211
5856
  * Link to discussion
5212
5857
  */
5213
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5858
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/148',
5214
5859
  /**
5215
5860
  * Example usages of the FOREACH command
5216
5861
  */
5217
5862
  examples: [
5218
- 'FOREACH List Line `{customers}` -> `{customer}`',
5219
- 'FOR List Line `{customers}` -> `{customer}`',
5220
- 'EACH List Line `{customers}` -> `{customer}`',
5221
- // <- TODO: [🍭] !!!!!! More
5863
+ 'FOREACH Text Line `{customers}` -> `{customer}`',
5864
+ 'FOREACH Csv Cell `{customers}` -> `{cell}`',
5865
+ 'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
5866
+ 'FOR Text Line `{customers}` -> `{customer}`',
5867
+ 'EACH Text Line `{customers}` -> `{customer}`',
5222
5868
  ],
5223
5869
  /**
5224
5870
  * Parses the FOREACH command
@@ -5226,55 +5872,75 @@
5226
5872
  parse: function (input) {
5227
5873
  var args = input.args;
5228
5874
  var formatName = normalizeTo_SCREAMING_CASE(args[0] || '');
5229
- var cellName = normalizeTo_SCREAMING_CASE(args[1] || '');
5230
- var parameterNameWrapped = args[2];
5875
+ var subformatName = normalizeTo_SCREAMING_CASE(args[1] || '');
5876
+ var parameterNameArg = args[2] || '';
5231
5877
  var assignSign = args[3];
5232
- var subparameterNameWrapped = args[4];
5233
- if (![
5234
- 'LIST',
5235
- 'CSV',
5236
- // <- TODO: [🏢] Unhardcode formats
5237
- ].includes(formatName)) {
5238
- console.info({ args: args, formatName: formatName });
5239
- throw new Error("Unsupported format \"".concat(formatName, "\""));
5878
+ var formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
5879
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(formatName);
5880
+ });
5881
+ if (formatDefinition === undefined) {
5882
+ throw new ParseError(spaceTrim__default["default"](function (block) { return "\n Unsupported format \"".concat(formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
5883
+ .map(function (formatName) { return "- ".concat(formatName); })
5884
+ .join('\n')), "\n "); }));
5240
5885
  // <- TODO: [🏢] List all supported format names
5241
5886
  }
5242
- if (![
5243
- 'LINE',
5244
- 'ROW',
5245
- 'COLUMN',
5246
- 'CELL',
5247
- // <- TODO: [🏢] Unhardcode format cells
5248
- ].includes(cellName)) {
5249
- console.info({ args: args, cellName: cellName });
5250
- throw new Error("Format ".concat(formatName, " does not support cell \"").concat(cellName, "\""));
5251
- // <- TODO: [🏢] List all supported cell names for the format
5887
+ var subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
5888
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(subformatName);
5889
+ });
5890
+ if (subvalueDefinition === undefined) {
5891
+ throw new ParseError(spaceTrim__default["default"](function (block) { return "\n Unsupported subformat name \"".concat(subformatName, "\" for format \"").concat(formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
5892
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
5893
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
5894
+ .join('\n')), "\n "); }));
5895
+ // <- TODO: [🏢] List all supported subformat names for the format
5252
5896
  }
5253
5897
  if (assignSign !== '->') {
5254
- console.info({ args: args, assignSign: assignSign });
5255
- throw new Error("FOREACH command must have '->' to assign the value to the parameter");
5256
- }
5257
- // TODO: !!!!!! Replace with propper parameter name validation
5258
- if ((parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1)) !== '{' ||
5259
- (parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length)) !== '}') {
5260
- console.info({ args: args, parameterNameWrapped: parameterNameWrapped }, parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1), parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length));
5261
- throw new Error("!!!!!! 1 Here will be error (with rules and precise error) from validateParameterName");
5262
- }
5263
- var parameterName = parameterNameWrapped.substring(1, parameterNameWrapped.length - 1);
5264
- // TODO: !!!!!! Replace with propper parameter name validation
5265
- if ((subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(0, 1)) !== '{' ||
5266
- (subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(subparameterNameWrapped.length - 1, subparameterNameWrapped.length)) !==
5267
- '}') {
5268
- console.info({ args: args, subparameterNameWrapped: subparameterNameWrapped });
5269
- throw new Error("!!!!!! 2 Here will be error (with rules and precise error) from validateParameterName");
5270
- }
5271
- var subparameterName = subparameterNameWrapped.substring(1, subparameterNameWrapped.length - 1);
5898
+ throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
5899
+ }
5900
+ var parameterName = validateParameterName(parameterNameArg);
5901
+ var outputSubparameterName = null;
5902
+ // TODO: [4] DRY
5903
+ var inputSubparameterNames = args
5904
+ .slice(4)
5905
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5906
+ .filter(function (parameterName) { return !parameterName.includes('+'); })
5907
+ .filter(function (parameterName) { return parameterName !== ''; })
5908
+ .map(validateParameterName);
5909
+ // TODO: [4] DRY
5910
+ var outputSubparameterNames = args
5911
+ .slice(4)
5912
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5913
+ .filter(function (parameterName) { return parameterName.includes('+'); })
5914
+ .map(function (parameterName) { return parameterName.split('+').join(''); })
5915
+ .map(validateParameterName);
5916
+ if (outputSubparameterNames.length === 1) {
5917
+ outputSubparameterName = outputSubparameterNames[0];
5918
+ }
5919
+ else if (outputSubparameterNames.length > 1) {
5920
+ throw new ParseError("FOREACH command can not have more than one output subparameter");
5921
+ }
5922
+ if (inputSubparameterNames.length === 0) {
5923
+ throw new ParseError("FOREACH command must have at least one input subparameter");
5924
+ }
5925
+ if (outputSubparameterName === null) {
5926
+ // TODO: Following code should be unhardcoded from here and moved to the format definition
5927
+ if (formatName === 'CSV' && subformatName === 'CELL') {
5928
+ outputSubparameterName = 'newCell';
5929
+ }
5930
+ else if (formatName === 'TEXT' && subformatName === 'LINE') {
5931
+ outputSubparameterName = 'newLine';
5932
+ }
5933
+ else {
5934
+ throw new ParseError(spaceTrim__default["default"]("\n FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n Correct example:\n - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n ")));
5935
+ }
5936
+ }
5272
5937
  return {
5273
5938
  type: 'FOREACH',
5274
5939
  formatName: formatName,
5275
- cellName: cellName,
5940
+ subformatName: subformatName,
5276
5941
  parameterName: parameterName,
5277
- subparameterName: subparameterName,
5942
+ inputSubparameterNames: inputSubparameterNames,
5943
+ outputSubparameterName: outputSubparameterName,
5278
5944
  };
5279
5945
  },
5280
5946
  /**
@@ -5283,11 +5949,17 @@
5283
5949
  * Note: `$` is used to indicate that this function mutates given `templateJson`
5284
5950
  */
5285
5951
  $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
5286
- var formatName = command.formatName, cellName = command.cellName, parameterName = command.parameterName, subparameterName = command.subparameterName;
5287
- // TODO: !!!!!! Detect double use
5288
- // TODO: !!!!!! Detect usage with JOKER and don't allow it
5289
- $templateJson.foreach = { formatName: formatName, cellName: cellName, parameterName: parameterName, subparameterName: subparameterName };
5290
- keepUnused($pipelineJson); // <- TODO: !!!!!! BUT Maybe register subparameter from foreach into parameters of the pipeline
5952
+ var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
5953
+ // TODO: [🍭] Detect double use
5954
+ // TODO: [🍭] Detect usage with JOKER and don't allow it
5955
+ $templateJson.foreach = {
5956
+ formatName: formatName,
5957
+ subformatName: subformatName,
5958
+ parameterName: parameterName,
5959
+ inputSubparameterNames: inputSubparameterNames,
5960
+ outputSubparameterName: outputSubparameterName,
5961
+ };
5962
+ keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
5291
5963
  // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
5292
5964
  },
5293
5965
  /**
@@ -5310,8 +5982,7 @@
5310
5982
  },
5311
5983
  };
5312
5984
  /**
5313
- * TODO: !!!!!! Comment console logs
5314
- * TODO: [🍭] !!!!!! Make .ptbk.md file with examples of the FOREACH command and also with wrong parsing and logic
5985
+ * TODO: [🍭] Make .ptbk.md file with examples of the FOREACH with wrong parsing and logic
5315
5986
  */
5316
5987
 
5317
5988
  /**
@@ -5421,12 +6092,11 @@
5421
6092
  */
5422
6093
  parse: function (input) {
5423
6094
  var args = input.args;
5424
- // TODO: !!!!!! Replace with propper parameter name validation
5425
- var parametersMatch = (args.pop() || '').match(/^\{(?<parameterName>[a-z0-9_]+)\}$/im);
5426
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5427
- throw new ParseError("Invalid joker");
6095
+ if (args.length !== 1) {
6096
+ throw new ParseError("JOKE command expects exactly one parameter name");
5428
6097
  }
5429
- var parameterName = parametersMatch.groups.parameterName;
6098
+ var parameterNameArg = args[0] || '';
6099
+ var parameterName = validateParameterName(parameterNameArg);
5430
6100
  return {
5431
6101
  type: 'JOKER',
5432
6102
  parameterName: parameterName,
@@ -5501,6 +6171,9 @@
5501
6171
  */
5502
6172
  parse: function (input) {
5503
6173
  var args = input.args, normalized = input.normalized;
6174
+ var availableVariantsMessage = spaceTrim__default["default"](function (block) { return "\n Available variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) {
6175
+ return "- ".concat(variantName).concat(variantName !== 'EMBEDDING' ? '' : ' (Not available in pipeline)');
6176
+ }).join('\n')), "\n "); });
5504
6177
  // TODO: Make this more elegant and dynamically
5505
6178
  if (normalized.startsWith('MODEL_VARIANT')) {
5506
6179
  if (normalized === 'MODEL_VARIANT_CHAT') {
@@ -5516,17 +6189,13 @@
5516
6189
  key: 'modelVariant',
5517
6190
  value: 'COMPLETION',
5518
6191
  };
6192
+ // <- Note: [🤖]
5519
6193
  }
5520
6194
  else if (normalized.startsWith('MODEL_VARIANT_EMBED')) {
5521
- return {
5522
- type: 'MODEL',
5523
- key: 'modelVariant',
5524
- value: 'EMBEDDING',
5525
- };
5526
- // <- Note: [🤖]
6195
+ spaceTrim__default["default"](function (block) { return "\n Embedding model can not be used in pipeline\n\n ".concat(block(availableVariantsMessage), "\n "); });
5527
6196
  }
5528
6197
  else {
5529
- throw new ParseError(spaceTrim__default["default"](function (block) { return "\n Unknown model variant in command:\n\n Supported variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) { return "- ".concat(variantName); }).join('\n')), "\n "); }));
6198
+ throw new ParseError(spaceTrim__default["default"](function (block) { return "\n Unknown model variant in command:\n\n ".concat(block(availableVariantsMessage), "\n "); }));
5530
6199
  }
5531
6200
  }
5532
6201
  if (normalized.startsWith('MODEL_NAME')) {
@@ -5651,14 +6320,13 @@
5651
6320
  * Parses the PARAMETER command
5652
6321
  */
5653
6322
  parse: function (input) {
5654
- var normalized = input.normalized, raw = input.raw;
5655
- var parametersMatch = raw.match(/\{(?<parameterName>[a-z0-9_]+)\}[^\S\r\n]*(?<parameterDescription>.*)$/im);
5656
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5657
- throw new ParseError("Invalid parameter");
5658
- }
5659
- var _a = parametersMatch.groups, parameterName = _a.parameterName, parameterDescription = _a.parameterDescription;
5660
- if (parameterDescription && parameterDescription.match(/\{(?<parameterName>[a-z0-9_]+)\}/im)) {
5661
- throw new ParseError("Parameter {".concat(parameterName, "} can not contain another parameter in description"));
6323
+ var normalized = input.normalized, args = input.args, raw = input.raw;
6324
+ var parameterNameRaw = args.shift() || '';
6325
+ var parameterDescriptionRaw = args.join(' ');
6326
+ // <- TODO: When [🥶] fixed, change to:
6327
+ // > const parameterDescriptionRaw = rawArgs.split(parameterNameRaw).join('').trim();
6328
+ if (parameterDescriptionRaw && parameterDescriptionRaw.match(/\{(?<embeddedParameterName>[a-z0-9_]+)\}/im)) {
6329
+ throw new ParseError(spaceTrim__default["default"](function (block) { return "\n Parameter {".concat(parameterNameRaw, "} can not contain another parameter in description\n\n The description:\n ").concat(block(parameterDescriptionRaw), "\n "); }));
5662
6330
  }
5663
6331
  var isInput = normalized.startsWith('INPUT');
5664
6332
  var isOutput = normalized.startsWith('OUTPUT');
@@ -5666,11 +6334,12 @@
5666
6334
  isInput = false;
5667
6335
  isOutput = false;
5668
6336
  }
5669
- // TODO: !!!!!! Add parameter name validation
6337
+ var parameterName = validateParameterName(parameterNameRaw);
6338
+ var parameterDescription = parameterDescriptionRaw.trim() || null;
5670
6339
  return {
5671
6340
  type: 'PARAMETER',
5672
6341
  parameterName: parameterName,
5673
- parameterDescription: parameterDescription.trim() || null,
6342
+ parameterDescription: parameterDescription,
5674
6343
  isInput: isInput,
5675
6344
  isOutput: isOutput,
5676
6345
  };
@@ -5995,6 +6664,7 @@
5995
6664
  * Note: `$` is used to indicate that this function mutates given `pipelineJson`
5996
6665
  */
5997
6666
  $applyToPipelineJson: function (command, $pipelineJson) {
6667
+ // TODO: Warn if the version is overridden
5998
6668
  $pipelineJson.promptbookVersion = command.promptbookVersion;
5999
6669
  },
6000
6670
  /**
@@ -6477,7 +7147,9 @@
6477
7147
  for (var commandNameSegmentsCount = 0; commandNameSegmentsCount < Math.min(items.length, 3); commandNameSegmentsCount++) {
6478
7148
  var commandNameRaw = items.slice(0, commandNameSegmentsCount + 1).join('_');
6479
7149
  var args = items.slice(commandNameSegmentsCount + 1);
6480
- var rawArgs = raw.substring(commandNameRaw.length).trim();
7150
+ var rawArgs = raw
7151
+ .substring(commandNameRaw.length)
7152
+ .trim();
6481
7153
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6482
7154
  if (command !== null) {
6483
7155
  return command;
@@ -6488,7 +7160,9 @@
6488
7160
  {
6489
7161
  var commandNameRaw = items.slice(-1).join('_');
6490
7162
  var args = items.slice(0, -1); // <- Note: This is probbably not correct
6491
- var rawArgs = raw.substring(0, raw.length - commandNameRaw.length).trim();
7163
+ var rawArgs = raw
7164
+ .substring(0, raw.length - commandNameRaw.length)
7165
+ .trim();
6492
7166
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6493
7167
  if (command !== null) {
6494
7168
  return command;
@@ -6628,7 +7302,7 @@
6628
7302
  function extractOneBlockFromMarkdown(markdown) {
6629
7303
  var codeBlocks = extractAllBlocksFromMarkdown(markdown);
6630
7304
  if (codeBlocks.length !== 1) {
6631
- throw new ParseError(spaceTrim__default["default"](function (block) { return "\n There should be exactly 1 code block, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
7305
+ throw new ParseError(spaceTrim__default["default"](function (block) { return "\n There should be exactly 1 code block in template, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
6632
7306
  }
6633
7307
  return codeBlocks[0];
6634
7308
  }
@@ -6815,7 +7489,7 @@
6815
7489
  var $pipelineJson = {
6816
7490
  title: undefined /* <- Note: [🍙] Putting here placeholder to keep `title` on top at final JSON */,
6817
7491
  pipelineUrl: undefined /* <- Note: Putting here placeholder to keep `pipelineUrl` on top at final JSON */,
6818
- promptbookVersion: PROMPTBOOK_VERSION,
7492
+ promptbookVersion: undefined /* <- Note: By default no explicit version */,
6819
7493
  description: undefined /* <- Note: [🍙] Putting here placeholder to keep `description` on top at final JSON */,
6820
7494
  parameters: [],
6821
7495
  templates: [],
@@ -7106,7 +7780,7 @@
7106
7780
  return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
7107
7781
  }
7108
7782
  /**
7109
- * TODO: !!!! Warn if used only sync version
7783
+ * TODO: [main] !!!! Warn if used only sync version
7110
7784
  * TODO: [🚞] Report here line/column of error
7111
7785
  * TODO: Use spaceTrim more effectively
7112
7786
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
@@ -7889,7 +8563,7 @@
7889
8563
  }
7890
8564
  }
7891
8565
  /**
7892
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
8566
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
7893
8567
  * TODO: [🧠][💺] Can be done this on type-level?
7894
8568
  */
7895
8569