@promptbook/node 0.69.0-8 → 0.69.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +4 -1
  2. package/esm/index.es.js +985 -310
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/promptbook-collection/index.d.ts +0 -3
  5. package/esm/typings/src/_packages/core.index.d.ts +12 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +6 -0
  7. package/esm/typings/src/_packages/utils.index.d.ts +4 -4
  8. package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
  9. package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +1 -1
  10. package/esm/typings/src/commands/FOREACH/ForeachCommand.d.ts +1 -1
  11. package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -5
  12. package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -2
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +1 -1
  14. package/esm/typings/src/config.d.ts +11 -4
  15. package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +1 -1
  16. package/esm/typings/src/conversion/prettify/renderPipelineMermaidOptions.d.ts +3 -3
  17. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +6 -5
  18. package/esm/typings/src/errors/AbstractFormatError.d.ts +11 -0
  19. package/esm/typings/src/execution/PipelineExecutor.d.ts +1 -0
  20. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +5 -6
  21. package/esm/typings/src/execution/createPipelineExecutor/$OngoingTemplateResult.d.ts +24 -0
  22. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +7 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +3 -0
  24. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +4 -1
  25. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +3 -0
  26. package/esm/typings/src/execution/createPipelineExecutor/{30-executeFormatCells.d.ts → 30-executeFormatSubvalues.d.ts} +2 -2
  27. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +2 -8
  28. package/esm/typings/src/execution/embeddingVectorToString.d.ts +1 -1
  29. package/esm/typings/src/execution/utils/checkExpectations.d.ts +2 -0
  30. package/esm/typings/src/execution/utils/usageToHuman.d.ts +3 -4
  31. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +14 -15
  32. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +31 -0
  33. package/esm/typings/src/formats/csv/{ListFormatDefinition.d.ts → CsvFormatDefinition.d.ts} +6 -3
  34. package/esm/typings/src/formats/csv/CsvFormatError.d.ts +10 -0
  35. package/esm/typings/src/formats/csv/CsvSettings.d.ts +13 -0
  36. package/esm/typings/src/formats/index.d.ts +1 -1
  37. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +4 -3
  38. package/esm/typings/src/formats/text/TextFormatDefinition.d.ts +19 -0
  39. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +4 -3
  40. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +1 -1
  41. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.d.ts +2 -2
  45. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +2 -2
  46. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +1 -1
  47. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  48. package/esm/typings/src/prepare/isPipelinePrepared.d.ts +1 -1
  49. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  50. package/esm/typings/src/types/PipelineJson/ParameterJson.d.ts +1 -1
  51. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +1 -1
  52. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +0 -3
  53. package/esm/typings/src/types/execution-report/executionReportJsonToString.d.ts +2 -1
  54. package/esm/typings/src/types/typeAliases.d.ts +1 -1
  55. package/esm/typings/src/utils/expectation-counters/index.d.ts +3 -0
  56. package/esm/typings/src/utils/organization/{f.d.ts → empty_object.d.ts} +5 -1
  57. package/esm/typings/src/utils/organization/just_empty_object.d.ts +12 -0
  58. package/esm/typings/src/utils/{extractParameterNames.d.ts → parameters/extractParameterNames.d.ts} +2 -2
  59. package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +27 -0
  60. package/esm/typings/src/utils/{replaceParameters.d.ts → parameters/replaceParameters.d.ts} +2 -2
  61. package/esm/typings/src/utils/serialization/checkSerializableAsJson.d.ts +1 -1
  62. package/esm/typings/src/utils/serialization/isSerializableAsJson.d.ts +1 -1
  63. package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +10 -0
  64. package/package.json +18 -13
  65. package/umd/index.umd.js +988 -314
  66. package/umd/index.umd.js.map +1 -1
  67. package/esm/typings/src/formats/list/ListFormatDefinition.d.ts +0 -16
  68. /package/esm/typings/src/utils/{extractParameterNames.test.d.ts → parameters/extractParameterNames.test.d.ts} +0 -0
  69. /package/esm/typings/src/{execution/utils/usageToHuman.test.d.ts → utils/parameters/mapAvailableToExpectedParameters.test.d.ts} +0 -0
  70. /package/esm/typings/src/utils/{replaceParameters.test.d.ts → parameters/replaceParameters.test.d.ts} +0 -0
  71. /package/esm/typings/src/{personas/preparePersona.test.d.ts → utils/validators/parameterName/validateParameterName.test.d.ts} +0 -0
package/esm/index.es.js CHANGED
@@ -5,6 +5,7 @@ import spaceTrim$1, { spaceTrim } from 'spacetrim';
5
5
  import { format } from 'prettier';
6
6
  import parserHtml from 'prettier/parser-html';
7
7
  import { forTime } from 'waitasecond';
8
+ import { unparse, parse } from 'papaparse';
8
9
  import hexEncoder from 'crypto-js/enc-hex';
9
10
  import sha256 from 'crypto-js/sha256';
10
11
  import { join } from 'path/posix';
@@ -14,8 +15,8 @@ import * as dotenv from 'dotenv';
14
15
  /**
15
16
  * The version of the Promptbook library
16
17
  */
17
- var PROMPTBOOK_VERSION = '0.69.0-7';
18
- // TODO: !!!! List here all the versions and annotate + put into script
18
+ var PROMPTBOOK_VERSION = '0.69.0-21';
19
+ // TODO: [main] !!!! List here all the versions and annotate + put into script
19
20
 
20
21
  /*! *****************************************************************************
21
22
  Copyright (c) Microsoft Corporation.
@@ -325,7 +326,7 @@ function checkSerializableAsJson(name, value) {
325
326
  }
326
327
  /**
327
328
  * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
328
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
329
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
329
330
  * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
330
331
  */
331
332
 
@@ -421,6 +422,17 @@ var RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
421
422
  */
422
423
  var RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE;
423
424
  // <- TODO: [🧜‍♂️]
425
+ /**
426
+ * @@@
427
+ *
428
+ * @public exported from `@promptbook/core`
429
+ */
430
+ var DEFAULT_CSV_SETTINGS = Object.freeze({
431
+ delimiter: ',',
432
+ quoteChar: '"',
433
+ newline: '\n',
434
+ skipEmptyLines: true,
435
+ });
424
436
  /**
425
437
  * @@@
426
438
  *
@@ -504,7 +516,7 @@ function pipelineJsonToString(pipelineJson) {
504
516
  commands.push("PIPELINE URL ".concat(pipelineUrl));
505
517
  }
506
518
  commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
507
- // TODO: !!! This increase size of the bundle and is probbably not necessary
519
+ // TODO: [main] !!! This increase size of the bundle and is probbably not necessary
508
520
  pipelineString = prettifyMarkdown(pipelineString);
509
521
  try {
510
522
  for (var _g = __values(parameters.filter(function (_a) {
@@ -652,12 +664,12 @@ function pipelineJsonToString(pipelineJson) {
652
664
  pipelineString += '```' + contentLanguage;
653
665
  pipelineString += '\n';
654
666
  pipelineString += spaceTrim$1(content);
655
- // <- TODO: !!! Escape
667
+ // <- TODO: [main] !!! Escape
656
668
  // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
657
669
  pipelineString += '\n';
658
670
  pipelineString += '```';
659
671
  pipelineString += '\n\n';
660
- pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: !!! If the parameter here has description, add it and use templateParameterJsonToString
672
+ pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!! If the parameter here has description, add it and use templateParameterJsonToString
661
673
  }
662
674
  }
663
675
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -884,7 +896,7 @@ function forEachAsync(array, options, callbackfunction) {
884
896
  });
885
897
  }
886
898
 
887
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.69.0-7",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
899
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
888
900
 
889
901
  /**
890
902
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -959,7 +971,7 @@ function isValidPromptbookVersion(version) {
959
971
  if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
960
972
  return false;
961
973
  }
962
- // <- TODO: !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
974
+ // <- TODO: [main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
963
975
  return true;
964
976
  }
965
977
 
@@ -1128,7 +1140,7 @@ function validatePipelineCore(pipeline) {
1128
1140
  // <- Note: [🚲]
1129
1141
  throw new PipelineLogicError(spaceTrim(function (block) { return "\n Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1130
1142
  }
1131
- if (!isValidPromptbookVersion(pipeline.promptbookVersion)) {
1143
+ if (pipeline.promptbookVersion !== undefined && !isValidPromptbookVersion(pipeline.promptbookVersion)) {
1132
1144
  // <- Note: [🚲]
1133
1145
  throw new PipelineLogicError(spaceTrim(function (block) { return "\n Invalid Promptbook Version \"".concat(pipeline.promptbookVersion, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1134
1146
  }
@@ -1311,6 +1323,7 @@ function validatePipelineCore(pipeline) {
1311
1323
  }
1312
1324
  }
1313
1325
  /**
1326
+ * TODO: !!!!! [🧞‍♀️] Do not allow joker + foreach
1314
1327
  * TODO: [🧠] Work with promptbookVersion
1315
1328
  * TODO: Use here some json-schema, Zod or something similar and change it to:
1316
1329
  * > /**
@@ -1322,11 +1335,11 @@ function validatePipelineCore(pipeline) {
1322
1335
  * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
1323
1336
  */
1324
1337
  /**
1325
- * TODO: [🐣] !!!! Validate that all samples match expectations
1326
- * TODO: [🐣][🐝] !!!! Validate that knowledge is valid (non-void)
1327
- * TODO: [🐣] !!!! Validate that persona can be used only with CHAT variant
1328
- * TODO: [🐣] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1329
- * TODO: [🐣] !!!! Validate that reserved parameter is not used as joker
1338
+ * TODO: [🐣][main] !!!! Validate that all samples match expectations
1339
+ * TODO: [🐣][🐝][main] !!!! Validate that knowledge is valid (non-void)
1340
+ * TODO: [🐣][main] !!!! Validate that persona can be used only with CHAT variant
1341
+ * TODO: [🐣][main] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1342
+ * TODO: [🐣][main] !!!! Validate that reserved parameter is not used as joker
1330
1343
  * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists
1331
1344
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
1332
1345
  */
@@ -2070,7 +2083,7 @@ function isPipelinePrepared(pipeline) {
2070
2083
  return true;
2071
2084
  }
2072
2085
  /**
2073
- * TODO: [🔃] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2086
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2074
2087
  * TODO: [🐠] Maybe base this on `makeValidator`
2075
2088
  * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2076
2089
  * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
@@ -2122,9 +2135,10 @@ var MultipleLlmExecutionTools = /** @class */ (function () {
2122
2135
  });
2123
2136
  Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2124
2137
  get: function () {
2125
- return this.llmExecutionTools
2126
- .map(function (tools, index) { return "".concat(index + 1, ") ").concat(tools.title, " ").concat(tools.description || ''); })
2127
- .join('\n');
2138
+ return this.llmExecutionTools.map(function (_a, index) {
2139
+ var title = _a.title;
2140
+ return "".concat(index + 1, ") `").concat(title, "`");
2141
+ }).join('\n');
2128
2142
  },
2129
2143
  enumerable: false,
2130
2144
  configurable: true
@@ -2322,9 +2336,7 @@ var MultipleLlmExecutionTools = /** @class */ (function () {
2322
2336
  throw new PipelineExecutionError("You have not provided any `LlmExecutionTools`");
2323
2337
  }
2324
2338
  else {
2325
- throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.llmExecutionTools
2326
- .map(function (tools) { return "- ".concat(tools.title, " ").concat(tools.description || ''); })
2327
- .join('\n')), "\n\n "); }));
2339
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\"\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.description), "\n\n "); }));
2328
2340
  }
2329
2341
  }
2330
2342
  });
@@ -2466,47 +2478,47 @@ function extractVariables(script) {
2466
2478
  * @public exported from `@promptbook/utils`
2467
2479
  */
2468
2480
  function extractParameterNamesFromTemplate(template) {
2469
- var e_1, _a, e_2, _b, e_3, _c;
2481
+ var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
2470
2482
  var title = template.title, description = template.description, templateType = template.templateType, content = template.content, preparedContent = template.preparedContent, jokerParameterNames = template.jokerParameterNames, foreach = template.foreach;
2471
2483
  var parameterNames = new Set();
2472
2484
  try {
2473
- for (var _d = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _e = _d.next(); !_e.done; _e = _d.next()) {
2474
- var parameterName = _e.value;
2485
+ for (var _e = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _f = _e.next(); !_f.done; _f = _e.next()) {
2486
+ var parameterName = _f.value;
2475
2487
  parameterNames.add(parameterName);
2476
2488
  }
2477
2489
  }
2478
2490
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
2479
2491
  finally {
2480
2492
  try {
2481
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2493
+ if (_f && !_f.done && (_a = _e.return)) _a.call(_e);
2482
2494
  }
2483
2495
  finally { if (e_1) throw e_1.error; }
2484
2496
  }
2485
2497
  if (templateType === 'SCRIPT_TEMPLATE') {
2486
2498
  try {
2487
- for (var _f = __values(extractVariables(content)), _g = _f.next(); !_g.done; _g = _f.next()) {
2488
- var parameterName = _g.value;
2499
+ for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
2500
+ var parameterName = _h.value;
2489
2501
  parameterNames.add(parameterName);
2490
2502
  }
2491
2503
  }
2492
2504
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
2493
2505
  finally {
2494
2506
  try {
2495
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
2507
+ if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
2496
2508
  }
2497
2509
  finally { if (e_2) throw e_2.error; }
2498
2510
  }
2499
2511
  }
2500
2512
  try {
2501
- for (var _h = __values(jokerParameterNames || []), _j = _h.next(); !_j.done; _j = _h.next()) {
2502
- var jokerName = _j.value;
2513
+ for (var _j = __values(jokerParameterNames || []), _k = _j.next(); !_k.done; _k = _j.next()) {
2514
+ var jokerName = _k.value;
2503
2515
  parameterNames.add(jokerName);
2504
2516
  }
2505
2517
  }
2506
2518
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
2507
2519
  finally {
2508
2520
  try {
2509
- if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
2521
+ if (_k && !_k.done && (_c = _j.return)) _c.call(_j);
2510
2522
  }
2511
2523
  finally { if (e_3) throw e_3.error; }
2512
2524
  }
@@ -2514,10 +2526,22 @@ function extractParameterNamesFromTemplate(template) {
2514
2526
  // <- Note {websiteContent} is used in `preparedContent`
2515
2527
  // Note: [🍭] Fixing dependent subparameterName from FOREACH command
2516
2528
  if (foreach !== undefined) {
2517
- if (parameterNames.has(foreach.subparameterName)) {
2518
- parameterNames.delete(foreach.subparameterName);
2519
- parameterNames.add(foreach.parameterName);
2520
- // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2529
+ try {
2530
+ for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
2531
+ var subparameterName = _m.value;
2532
+ if (parameterNames.has(subparameterName)) {
2533
+ parameterNames.delete(subparameterName);
2534
+ parameterNames.add(foreach.parameterName);
2535
+ // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2536
+ }
2537
+ }
2538
+ }
2539
+ catch (e_4_1) { e_4 = { error: e_4_1 }; }
2540
+ finally {
2541
+ try {
2542
+ if (_m && !_m.done && (_d = _l.return)) _d.call(_l);
2543
+ }
2544
+ finally { if (e_4) throw e_4.error; }
2521
2545
  }
2522
2546
  }
2523
2547
  return parameterNames;
@@ -2601,6 +2625,183 @@ function union() {
2601
2625
  return union;
2602
2626
  }
2603
2627
 
2628
+ /**
2629
+ * Just marks a place of place where should be something implemented
2630
+ * No side effects.
2631
+ *
2632
+ * Note: It can be usefull suppressing eslint errors of unused variables
2633
+ *
2634
+ * @param value any values
2635
+ * @returns void
2636
+ * @private within the repository
2637
+ */
2638
+ function TODO_USE() {
2639
+ var value = [];
2640
+ for (var _i = 0; _i < arguments.length; _i++) {
2641
+ value[_i] = arguments[_i];
2642
+ }
2643
+ }
2644
+
2645
+ /**
2646
+ * This error indicates problems parsing the format value
2647
+ *
2648
+ * For example, when the format value is not a valid JSON or CSV
2649
+ * This is not thrown directly but in extended classes
2650
+ *
2651
+ * @public exported from `@promptbook/core`
2652
+ */
2653
+ var AbstractFormatError = /** @class */ (function (_super) {
2654
+ __extends(AbstractFormatError, _super);
2655
+ // Note: To allow instanceof do not put here error `name`
2656
+ // public readonly name = 'AbstractFormatError';
2657
+ function AbstractFormatError(message) {
2658
+ var _this = _super.call(this, message) || this;
2659
+ Object.setPrototypeOf(_this, AbstractFormatError.prototype);
2660
+ return _this;
2661
+ }
2662
+ return AbstractFormatError;
2663
+ }(Error));
2664
+
2665
+ /**
2666
+ * This error indicates problem with parsing of CSV
2667
+ *
2668
+ * @public exported from `@promptbook/core`
2669
+ */
2670
+ var CsvFormatError = /** @class */ (function (_super) {
2671
+ __extends(CsvFormatError, _super);
2672
+ function CsvFormatError(message) {
2673
+ var _this = _super.call(this, message) || this;
2674
+ _this.name = 'CsvFormatError';
2675
+ Object.setPrototypeOf(_this, CsvFormatError.prototype);
2676
+ return _this;
2677
+ }
2678
+ return CsvFormatError;
2679
+ }(AbstractFormatError));
2680
+
2681
+ /**
2682
+ * @@@
2683
+ *
2684
+ * @public exported from `@promptbook/core`
2685
+ */
2686
+ var MANDATORY_CSV_SETTINGS = Object.freeze({
2687
+ header: true,
2688
+ // encoding: 'utf8',
2689
+ });
2690
+
2691
+ /**
2692
+ * Definition for CSV spreadsheet
2693
+ *
2694
+ * @public exported from `@promptbook/core`
2695
+ * <- TODO: [🏢] Export from package `@promptbook/csv`
2696
+ */
2697
+ var CsvFormatDefinition = {
2698
+ formatName: 'CSV',
2699
+ aliases: ['SPREADSHEET', 'TABLE'],
2700
+ isValid: function (value, settings, schema) {
2701
+ // TODO: Implement CSV validation
2702
+ TODO_USE(value /* <- TODO: Use value here */);
2703
+ TODO_USE(settings /* <- TODO: Use settings here */);
2704
+ TODO_USE(schema /* <- TODO: Use schema here */);
2705
+ return true;
2706
+ },
2707
+ canBeValid: function (partialValue, settings, schema) {
2708
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2709
+ TODO_USE(settings /* <- TODO: Use settings here */);
2710
+ TODO_USE(schema /* <- TODO: Use schema here */);
2711
+ return true;
2712
+ },
2713
+ heal: function (value, settings, schema) {
2714
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2715
+ TODO_USE(settings /* <- TODO: Use settings here */);
2716
+ TODO_USE(schema /* <- TODO: Use schema here */);
2717
+ throw new Error('Not implemented');
2718
+ },
2719
+ subvalueDefinitions: [
2720
+ {
2721
+ subvalueName: 'ROW',
2722
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2723
+ return __awaiter(this, void 0, void 0, function () {
2724
+ var csv, mappedData;
2725
+ var _this = this;
2726
+ return __generator(this, function (_a) {
2727
+ switch (_a.label) {
2728
+ case 0:
2729
+ csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2730
+ if (csv.errors.length !== 0) {
2731
+ throw new CsvFormatError(spaceTrim$1(function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2732
+ }
2733
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
2734
+ var _a, _b;
2735
+ var _c;
2736
+ return __generator(this, function (_d) {
2737
+ switch (_d.label) {
2738
+ case 0:
2739
+ if (row[outputParameterName]) {
2740
+ throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
2741
+ }
2742
+ _a = [__assign({}, row)];
2743
+ _c = {};
2744
+ _b = outputParameterName;
2745
+ return [4 /*yield*/, mapCallback(row, index)];
2746
+ case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
2747
+ }
2748
+ });
2749
+ }); }))];
2750
+ case 1:
2751
+ mappedData = _a.sent();
2752
+ return [2 /*return*/, unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2753
+ }
2754
+ });
2755
+ });
2756
+ },
2757
+ },
2758
+ {
2759
+ subvalueName: 'CELL',
2760
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2761
+ return __awaiter(this, void 0, void 0, function () {
2762
+ var csv, mappedData;
2763
+ var _this = this;
2764
+ return __generator(this, function (_a) {
2765
+ switch (_a.label) {
2766
+ case 0:
2767
+ csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2768
+ if (csv.errors.length !== 0) {
2769
+ throw new CsvFormatError(spaceTrim$1(function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2770
+ }
2771
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
2772
+ var _this = this;
2773
+ return __generator(this, function (_a) {
2774
+ return [2 /*return*/, /* not await */ Promise.all(Object.entries(row).map(function (_a, columnIndex) {
2775
+ var _b = __read(_a, 2), key = _b[0], value = _b[1];
2776
+ return __awaiter(_this, void 0, void 0, function () {
2777
+ var index;
2778
+ var _c;
2779
+ return __generator(this, function (_d) {
2780
+ index = rowIndex * Object.keys(row).length + columnIndex;
2781
+ return [2 /*return*/, /* not await */ mapCallback((_c = {}, _c[key] = value, _c), index)];
2782
+ });
2783
+ });
2784
+ }))];
2785
+ });
2786
+ }); }))];
2787
+ case 1:
2788
+ mappedData = _a.sent();
2789
+ return [2 /*return*/, unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2790
+ }
2791
+ });
2792
+ });
2793
+ },
2794
+ },
2795
+ ],
2796
+ };
2797
+ /**
2798
+ * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
2799
+ * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
2800
+ * TODO: [🍓] In `CsvFormatDefinition` implement `heal
2801
+ * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
2802
+ * TODO: [🏢] Allow to expect something inside CSV objects and other formats
2803
+ */
2804
+
2604
2805
  /**
2605
2806
  * Function isValidJsonString will tell you if the string is valid JSON or not
2606
2807
  *
@@ -2622,6 +2823,222 @@ function isValidJsonString(value /* <- [👨‍⚖️] */) {
2622
2823
  }
2623
2824
  }
2624
2825
 
2826
+ /**
2827
+ * Definition for JSON format
2828
+ *
2829
+ * @private still in development [🏢]
2830
+ */
2831
+ var JsonFormatDefinition = {
2832
+ formatName: 'JSON',
2833
+ mimeType: 'application/json',
2834
+ isValid: function (value, settings, schema) {
2835
+ TODO_USE(schema /* <- TODO: Use schema here */);
2836
+ TODO_USE(settings /* <- TODO: Use settings here */);
2837
+ return isValidJsonString(value);
2838
+ },
2839
+ canBeValid: function (partialValue, settings, schema) {
2840
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2841
+ TODO_USE(settings /* <- TODO: Use settings here */);
2842
+ TODO_USE(schema /* <- TODO: Use schema here */);
2843
+ return true;
2844
+ },
2845
+ heal: function (value, settings, schema) {
2846
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2847
+ TODO_USE(settings /* <- TODO: Use settings here */);
2848
+ TODO_USE(schema /* <- TODO: Use schema here */);
2849
+ throw new Error('Not implemented');
2850
+ },
2851
+ subvalueDefinitions: [],
2852
+ };
2853
+ /**
2854
+ * TODO: [🧠] Maybe propper instance of object
2855
+ * TODO: [0] Make string_serialized_json
2856
+ * TODO: [1] Make type for JSON Settings and Schema
2857
+ * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
2858
+ * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
2859
+ * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
2860
+ * TODO: [🍓] In `JsonFormatDefinition` implement `heal
2861
+ * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
2862
+ * TODO: [🏢] Allow to expect something inside JSON objects and other formats
2863
+ */
2864
+
2865
+ /**
2866
+ * Definition for any text - this will be always valid
2867
+ *
2868
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
2869
+ *
2870
+ * @public exported from `@promptbook/core`
2871
+ */
2872
+ var TextFormatDefinition = {
2873
+ formatName: 'TEXT',
2874
+ isValid: function (value) {
2875
+ return typeof value === 'string';
2876
+ },
2877
+ canBeValid: function (partialValue) {
2878
+ return typeof partialValue === 'string';
2879
+ },
2880
+ heal: function () {
2881
+ throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
2882
+ },
2883
+ subvalueDefinitions: [
2884
+ {
2885
+ subvalueName: 'LINE',
2886
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2887
+ return __awaiter(this, void 0, void 0, function () {
2888
+ var lines, mappedLines;
2889
+ return __generator(this, function (_a) {
2890
+ switch (_a.label) {
2891
+ case 0:
2892
+ lines = value.split('\n');
2893
+ return [4 /*yield*/, Promise.all(lines.map(function (lineContent, lineNumber) {
2894
+ // TODO: [🧠] Maybe option to skip empty line
2895
+ /* not await */ return mapCallback({
2896
+ lineContent: lineContent,
2897
+ // TODO: [🧠] Maybe also put here `lineNumber`
2898
+ }, lineNumber);
2899
+ }))];
2900
+ case 1:
2901
+ mappedLines = _a.sent();
2902
+ return [2 /*return*/, mappedLines.join('\n')];
2903
+ }
2904
+ });
2905
+ });
2906
+ },
2907
+ },
2908
+ // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2909
+ ],
2910
+ };
2911
+ /**
2912
+ * TODO: [1] Make type for XML Text and Schema
2913
+ * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2914
+ * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
2915
+ * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
2916
+ * TODO: [🍓] In `TextFormatDefinition` implement `heal
2917
+ * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
2918
+ * TODO: [🏢] Allow to expect something inside each item of list and other formats
2919
+ */
2920
+
2921
+ /**
2922
+ * Definition for XML format
2923
+ *
2924
+ * @private still in development [🏢]
2925
+ */
2926
+ var XmlFormatDefinition = {
2927
+ formatName: 'XML',
2928
+ mimeType: 'application/xml',
2929
+ isValid: function (value, settings, schema) {
2930
+ TODO_USE(value /* <- TODO: Use value here */);
2931
+ TODO_USE(settings /* <- TODO: Use settings here */);
2932
+ TODO_USE(schema /* <- TODO: Use schema here */);
2933
+ return true;
2934
+ },
2935
+ canBeValid: function (partialValue, settings, schema) {
2936
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2937
+ TODO_USE(settings /* <- TODO: Use settings here */);
2938
+ TODO_USE(schema /* <- TODO: Use schema here */);
2939
+ return true;
2940
+ },
2941
+ heal: function (value, settings, schema) {
2942
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2943
+ TODO_USE(settings /* <- TODO: Use settings here */);
2944
+ TODO_USE(schema /* <- TODO: Use schema here */);
2945
+ throw new Error('Not implemented');
2946
+ },
2947
+ subvalueDefinitions: [],
2948
+ };
2949
+ /**
2950
+ * TODO: [🧠] Maybe propper instance of object
2951
+ * TODO: [0] Make string_serialized_xml
2952
+ * TODO: [1] Make type for XML Settings and Schema
2953
+ * TODO: [🧠] What to use for validating XMLs - XSD,...
2954
+ * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
2955
+ * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
2956
+ * TODO: [🍓] In `XmlFormatDefinition` implement `heal
2957
+ * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
2958
+ * TODO: [🏢] Allow to expect something inside XML and other formats
2959
+ */
2960
+
2961
+ /**
2962
+ * Definitions for all formats supported by Promptbook
2963
+ *
2964
+ * @private internal index of `...` <- TODO [🏢]
2965
+ */
2966
+ var FORMAT_DEFINITIONS = [
2967
+ JsonFormatDefinition,
2968
+ XmlFormatDefinition,
2969
+ TextFormatDefinition,
2970
+ CsvFormatDefinition,
2971
+ ];
2972
+
2973
+ /**
2974
+ * Maps available parameters to expected parameters
2975
+ *
2976
+ * The strategy is:
2977
+ * 1) @@@
2978
+ * 2) @@@
2979
+ *
2980
+ * @throws {PipelineExecutionError} @@@
2981
+ * @private within the repository used in `createPipelineExecutor`
2982
+ */
2983
+ function mapAvailableToExpectedParameters(options) {
2984
+ var e_1, _a;
2985
+ var expectedParameters = options.expectedParameters, availableParameters = options.availableParameters;
2986
+ var availableParametersNames = new Set(Object.keys(availableParameters));
2987
+ var expectedParameterNames = new Set(Object.keys(expectedParameters));
2988
+ var mappedParameters = {};
2989
+ try {
2990
+ // Phase 1️⃣: Matching mapping
2991
+ for (var _b = __values(Array.from(union(availableParametersNames, expectedParameterNames))), _c = _b.next(); !_c.done; _c = _b.next()) {
2992
+ var parameterName = _c.value;
2993
+ // Situation: Parameter is available and expected
2994
+ if (availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
2995
+ mappedParameters[parameterName] = availableParameters[parameterName];
2996
+ // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here?
2997
+ availableParametersNames.delete(parameterName);
2998
+ expectedParameterNames.delete(parameterName);
2999
+ }
3000
+ // Situation: Parameter is available but NOT expected
3001
+ else if (availableParametersNames.has(parameterName) && !expectedParameterNames.has(parameterName)) {
3002
+ // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping
3003
+ }
3004
+ // Situation: Parameter is NOT available BUT expected
3005
+ else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
3006
+ // Do nothing here - this will be maybe fixed in the non-matching mapping
3007
+ }
3008
+ }
3009
+ }
3010
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3011
+ finally {
3012
+ try {
3013
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3014
+ }
3015
+ finally { if (e_1) throw e_1.error; }
3016
+ }
3017
+ if (expectedParameterNames.size === 0) {
3018
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3019
+ Object.freeze(mappedParameters);
3020
+ return mappedParameters;
3021
+ }
3022
+ // Phase 2️⃣: Non-matching mapping
3023
+ if (expectedParameterNames.size !== availableParametersNames.size) {
3024
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n Can not map available parameters to expected parameters\n\n Mapped parameters:\n ".concat(block(Object.keys(mappedParameters)
3025
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3026
+ .join('\n')), "\n\n Expected parameters which can not be mapped:\n ").concat(block(Array.from(expectedParameterNames)
3027
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3028
+ .join('\n')), "\n\n Remaining available parameters:\n ").concat(block(Array.from(availableParametersNames)
3029
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3030
+ .join('\n')), "\n\n "); }));
3031
+ }
3032
+ var expectedParameterNamesArray = Array.from(expectedParameterNames);
3033
+ var availableParametersNamesArray = Array.from(availableParametersNames);
3034
+ for (var i = 0; i < expectedParameterNames.size; i++) {
3035
+ mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
3036
+ }
3037
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3038
+ Object.freeze(mappedParameters);
3039
+ return mappedParameters;
3040
+ }
3041
+
2625
3042
  /**
2626
3043
  * Extracts all code blocks from markdown.
2627
3044
  *
@@ -2927,6 +3344,9 @@ var CountUtils = {
2927
3344
  LINES: countLines,
2928
3345
  PAGES: countPages,
2929
3346
  };
3347
+ /**
3348
+ * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
3349
+ */
2930
3350
 
2931
3351
  /**
2932
3352
  * Function checkExpectations will check if the expectations on given value are met
@@ -2963,6 +3383,8 @@ function checkExpectations(expectations, value) {
2963
3383
  }
2964
3384
  /**
2965
3385
  * TODO: [💝] Unite object for expecting amount and format
3386
+ * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
3387
+ * Note: [💝] and [🤠] are interconnected together
2966
3388
  */
2967
3389
 
2968
3390
  /**
@@ -2972,58 +3394,66 @@ function checkExpectations(expectations, value) {
2972
3394
  */
2973
3395
  function executeAttempts(options) {
2974
3396
  return __awaiter(this, void 0, void 0, function () {
2975
- var $ongoingTemplateResult, jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, _loop_1, attempt, state_1;
3397
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
2976
3398
  return __generator(this, function (_a) {
2977
3399
  switch (_a.label) {
2978
3400
  case 0:
2979
- $ongoingTemplateResult = options.$ongoingTemplateResult, jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3401
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
2980
3402
  maxExecutionAttempts = settings.maxExecutionAttempts;
3403
+ $ongoingTemplateResult = {
3404
+ $result: null,
3405
+ $resultString: null,
3406
+ $expectError: null,
3407
+ $scriptPipelineExecutionErrors: [],
3408
+ };
2981
3409
  _loop_1 = function (attempt) {
2982
- var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, _h, scriptTools, _j, error_1, e_1_1, _k, _l, _m, functionName, postprocessingError, _o, _p, scriptTools, _q, error_2, e_2_1, e_3_1, error_3;
2983
- var e_1, _r, e_3, _s, e_2, _t;
2984
- return __generator(this, function (_u) {
2985
- switch (_u.label) {
3410
+ var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
3411
+ var e_1, _q, e_3, _r, e_2, _s;
3412
+ return __generator(this, function (_t) {
3413
+ switch (_t.label) {
2986
3414
  case 0:
2987
3415
  isJokerAttempt = attempt < 0;
2988
3416
  jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
2989
- // TODO: [🧠] !!!!!! JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3417
+ // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
2990
3418
  if (isJokerAttempt && !jokerParameterName) {
2991
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
3419
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
2992
3420
  }
2993
3421
  $ongoingTemplateResult.$result = null;
2994
3422
  $ongoingTemplateResult.$resultString = null;
2995
3423
  $ongoingTemplateResult.$expectError = null;
2996
3424
  if (isJokerAttempt) {
2997
3425
  if (parameters[jokerParameterName] === undefined) {
2998
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3426
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
2999
3427
  // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
3000
3428
  }
3001
3429
  else {
3002
3430
  $ongoingTemplateResult.$resultString = parameters[jokerParameterName];
3003
3431
  }
3004
3432
  }
3005
- _u.label = 1;
3433
+ _t.label = 1;
3006
3434
  case 1:
3007
- _u.trys.push([1, 44, 45, 46]);
3008
- if (!!isJokerAttempt) return [3 /*break*/, 26];
3435
+ _t.trys.push([1, 43, 44, 45]);
3436
+ if (!!isJokerAttempt) return [3 /*break*/, 25];
3009
3437
  _b = template.templateType;
3010
3438
  switch (_b) {
3011
3439
  case 'SIMPLE_TEMPLATE': return [3 /*break*/, 2];
3012
3440
  case 'PROMPT_TEMPLATE': return [3 /*break*/, 3];
3013
- case 'SCRIPT_TEMPLATE': return [3 /*break*/, 12];
3014
- case 'DIALOG_TEMPLATE': return [3 /*break*/, 23];
3441
+ case 'SCRIPT_TEMPLATE': return [3 /*break*/, 11];
3442
+ case 'DIALOG_TEMPLATE': return [3 /*break*/, 22];
3015
3443
  }
3016
- return [3 /*break*/, 25];
3444
+ return [3 /*break*/, 24];
3017
3445
  case 2:
3018
3446
  $ongoingTemplateResult.$resultString = replaceParameters(preparedContent, parameters);
3019
- return [3 /*break*/, 26];
3447
+ return [3 /*break*/, 25];
3020
3448
  case 3:
3021
3449
  modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (preparedPipeline.defaultModelRequirements || {})), (template.modelRequirements || {}));
3022
3450
  $ongoingTemplateResult.$prompt = {
3023
3451
  title: template.title,
3024
3452
  pipelineUrl: "".concat(preparedPipeline.pipelineUrl
3025
3453
  ? preparedPipeline.pipelineUrl
3026
- : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name),
3454
+ : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name
3455
+ // <- TODO: Here should be maybe also subformat index to distinguish between same template with different subformat values
3456
+ ),
3027
3457
  parameters: parameters,
3028
3458
  content: preparedContent,
3029
3459
  modelRequirements: modelRequirements,
@@ -3041,67 +3471,57 @@ function executeAttempts(options) {
3041
3471
  case 'COMPLETION': return [3 /*break*/, 6];
3042
3472
  case 'EMBEDDING': return [3 /*break*/, 8];
3043
3473
  }
3044
- return [3 /*break*/, 10];
3474
+ return [3 /*break*/, 9];
3045
3475
  case 4:
3046
3476
  _d = $ongoingTemplateResult;
3047
3477
  return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3048
3478
  case 5:
3049
- _d.$chatResult = _u.sent();
3479
+ _d.$chatResult = _t.sent();
3050
3480
  // TODO: [🍬] Destroy chatThread
3051
3481
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$chatResult;
3052
3482
  $ongoingTemplateResult.$resultString = $ongoingTemplateResult.$chatResult.content;
3053
- return [3 /*break*/, 11];
3483
+ return [3 /*break*/, 10];
3054
3484
  case 6:
3055
3485
  _e = $ongoingTemplateResult;
3056
3486
  return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3057
3487
  case 7:
3058
- _e.$completionResult = _u.sent();
3488
+ _e.$completionResult = _t.sent();
3059
3489
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
3060
3490
  $ongoingTemplateResult.$resultString =
3061
3491
  $ongoingTemplateResult.$completionResult.content;
3062
- return [3 /*break*/, 11];
3063
- case 8:
3064
- // TODO: [🧠] This is weird, embedding model can not be used such a way in the pipeline
3065
- _f = $ongoingTemplateResult;
3066
- return [4 /*yield*/, llmTools.callEmbeddingModel($deepFreeze($ongoingTemplateResult.$prompt))];
3067
- case 9:
3068
- // TODO: [🧠] This is weird, embedding model can not be used such a way in the pipeline
3069
- _f.$embeddingResult = _u.sent();
3070
- $ongoingTemplateResult.$result = $ongoingTemplateResult.$embeddingResult;
3071
- $ongoingTemplateResult.$resultString =
3072
- $ongoingTemplateResult.$embeddingResult.content.join(',');
3073
- return [3 /*break*/, 11];
3074
- case 10: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown model variant \"".concat(template.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3075
- case 11: return [3 /*break*/, 26];
3076
- case 12:
3492
+ return [3 /*break*/, 10];
3493
+ case 8: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Embedding model can not be used in pipeline\n\n This should be catched during parsing\n\n ".concat(block(pipelineIdentification), "\n\n "); }));
3494
+ case 9: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown model variant \"".concat(template.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3495
+ case 10: return [3 /*break*/, 25];
3496
+ case 11:
3077
3497
  if (arrayableToArray(tools.script).length === 0) {
3078
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3498
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3079
3499
  }
3080
3500
  if (!template.contentLanguage) {
3081
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(template.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3501
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(template.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3082
3502
  }
3083
- _u.label = 13;
3503
+ _t.label = 12;
3504
+ case 12:
3505
+ _t.trys.push([12, 19, 20, 21]);
3506
+ _f = (e_1 = void 0, __values(arrayableToArray(tools.script))), _g = _f.next();
3507
+ _t.label = 13;
3084
3508
  case 13:
3085
- _u.trys.push([13, 20, 21, 22]);
3086
- _g = (e_1 = void 0, __values(arrayableToArray(tools.script))), _h = _g.next();
3087
- _u.label = 14;
3509
+ if (!!_g.done) return [3 /*break*/, 18];
3510
+ scriptTools = _g.value;
3511
+ _t.label = 14;
3088
3512
  case 14:
3089
- if (!!_h.done) return [3 /*break*/, 19];
3090
- scriptTools = _h.value;
3091
- _u.label = 15;
3092
- case 15:
3093
- _u.trys.push([15, 17, , 18]);
3094
- _j = $ongoingTemplateResult;
3513
+ _t.trys.push([14, 16, , 17]);
3514
+ _h = $ongoingTemplateResult;
3095
3515
  return [4 /*yield*/, scriptTools.execute($deepFreeze({
3096
3516
  scriptLanguage: template.contentLanguage,
3097
3517
  script: preparedContent,
3098
3518
  parameters: parameters,
3099
3519
  }))];
3520
+ case 15:
3521
+ _h.$resultString = _t.sent();
3522
+ return [3 /*break*/, 18];
3100
3523
  case 16:
3101
- _j.$resultString = _u.sent();
3102
- return [3 /*break*/, 19];
3103
- case 17:
3104
- error_1 = _u.sent();
3524
+ error_1 = _t.sent();
3105
3525
  if (!(error_1 instanceof Error)) {
3106
3526
  throw error_1;
3107
3527
  }
@@ -3109,39 +3529,39 @@ function executeAttempts(options) {
3109
3529
  throw error_1;
3110
3530
  }
3111
3531
  $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_1);
3112
- return [3 /*break*/, 18];
3113
- case 18:
3114
- _h = _g.next();
3115
- return [3 /*break*/, 14];
3116
- case 19: return [3 /*break*/, 22];
3117
- case 20:
3118
- e_1_1 = _u.sent();
3532
+ return [3 /*break*/, 17];
3533
+ case 17:
3534
+ _g = _f.next();
3535
+ return [3 /*break*/, 13];
3536
+ case 18: return [3 /*break*/, 21];
3537
+ case 19:
3538
+ e_1_1 = _t.sent();
3119
3539
  e_1 = { error: e_1_1 };
3120
- return [3 /*break*/, 22];
3121
- case 21:
3540
+ return [3 /*break*/, 21];
3541
+ case 20:
3122
3542
  try {
3123
- if (_h && !_h.done && (_r = _g.return)) _r.call(_g);
3543
+ if (_g && !_g.done && (_q = _f.return)) _q.call(_f);
3124
3544
  }
3125
3545
  finally { if (e_1) throw e_1.error; }
3126
3546
  return [7 /*endfinally*/];
3127
- case 22:
3547
+ case 21:
3128
3548
  if ($ongoingTemplateResult.$resultString !== null) {
3129
- return [3 /*break*/, 26];
3549
+ return [3 /*break*/, 25];
3130
3550
  }
3131
3551
  if ($ongoingTemplateResult.$scriptPipelineExecutionErrors.length === 1) {
3132
3552
  throw $ongoingTemplateResult.$scriptPipelineExecutionErrors[0];
3133
3553
  }
3134
3554
  else {
3135
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTemplateResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTemplateResult.$scriptPipelineExecutionErrors
3555
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTemplateResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTemplateResult.$scriptPipelineExecutionErrors
3136
3556
  .map(function (error) { return '- ' + error.message; })
3137
- .join('\n\n')), "\n "); }));
3557
+ .join('\n\n')), "\n "); }));
3138
3558
  }
3139
- case 23:
3559
+ case 22:
3140
3560
  if (tools.userInterface === undefined) {
3141
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3561
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3142
3562
  }
3143
3563
  // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3144
- _k = $ongoingTemplateResult;
3564
+ _j = $ongoingTemplateResult;
3145
3565
  return [4 /*yield*/, tools.userInterface.promptDialog($deepFreeze({
3146
3566
  promptTitle: template.title,
3147
3567
  promptMessage: replaceParameters(template.description || '', parameters),
@@ -3150,34 +3570,34 @@ function executeAttempts(options) {
3150
3570
  placeholder: undefined,
3151
3571
  priority: priority,
3152
3572
  }))];
3153
- case 24:
3573
+ case 23:
3154
3574
  // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3155
- _k.$resultString = _u.sent();
3156
- return [3 /*break*/, 26];
3157
- case 25: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown execution type \"".concat(template.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3575
+ _j.$resultString = _t.sent();
3576
+ return [3 /*break*/, 25];
3577
+ case 24: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown execution type \"".concat(template.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3578
+ case 25:
3579
+ if (!(!isJokerAttempt && template.postprocessingFunctionNames)) return [3 /*break*/, 42];
3580
+ _t.label = 26;
3158
3581
  case 26:
3159
- if (!(!isJokerAttempt && template.postprocessingFunctionNames)) return [3 /*break*/, 43];
3160
- _u.label = 27;
3582
+ _t.trys.push([26, 40, 41, 42]);
3583
+ _k = (e_3 = void 0, __values(template.postprocessingFunctionNames)), _l = _k.next();
3584
+ _t.label = 27;
3161
3585
  case 27:
3162
- _u.trys.push([27, 41, 42, 43]);
3163
- _l = (e_3 = void 0, __values(template.postprocessingFunctionNames)), _m = _l.next();
3164
- _u.label = 28;
3165
- case 28:
3166
- if (!!_m.done) return [3 /*break*/, 40];
3167
- functionName = _m.value;
3586
+ if (!!_l.done) return [3 /*break*/, 39];
3587
+ functionName = _l.value;
3168
3588
  postprocessingError = null;
3169
- _u.label = 29;
3589
+ _t.label = 28;
3590
+ case 28:
3591
+ _t.trys.push([28, 35, 36, 37]);
3592
+ _m = (e_2 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next();
3593
+ _t.label = 29;
3170
3594
  case 29:
3171
- _u.trys.push([29, 36, 37, 38]);
3172
- _o = (e_2 = void 0, __values(arrayableToArray(tools.script))), _p = _o.next();
3173
- _u.label = 30;
3595
+ if (!!_o.done) return [3 /*break*/, 34];
3596
+ scriptTools = _o.value;
3597
+ _t.label = 30;
3174
3598
  case 30:
3175
- if (!!_p.done) return [3 /*break*/, 35];
3176
- scriptTools = _p.value;
3177
- _u.label = 31;
3178
- case 31:
3179
- _u.trys.push([31, 33, , 34]);
3180
- _q = $ongoingTemplateResult;
3599
+ _t.trys.push([30, 32, , 33]);
3600
+ _p = $ongoingTemplateResult;
3181
3601
  return [4 /*yield*/, scriptTools.execute({
3182
3602
  scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
3183
3603
  script: "".concat(functionName, "(resultString)"),
@@ -3186,12 +3606,12 @@ function executeAttempts(options) {
3186
3606
  // Note: No ...parametersForTemplate, because working with result only
3187
3607
  },
3188
3608
  })];
3189
- case 32:
3190
- _q.$resultString = _u.sent();
3609
+ case 31:
3610
+ _p.$resultString = _t.sent();
3191
3611
  postprocessingError = null;
3192
- return [3 /*break*/, 35];
3193
- case 33:
3194
- error_2 = _u.sent();
3612
+ return [3 /*break*/, 34];
3613
+ case 32:
3614
+ error_2 = _t.sent();
3195
3615
  if (!(error_2 instanceof Error)) {
3196
3616
  throw error_2;
3197
3617
  }
@@ -3200,41 +3620,41 @@ function executeAttempts(options) {
3200
3620
  }
3201
3621
  postprocessingError = error_2;
3202
3622
  $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_2);
3203
- return [3 /*break*/, 34];
3204
- case 34:
3205
- _p = _o.next();
3206
- return [3 /*break*/, 30];
3207
- case 35: return [3 /*break*/, 38];
3208
- case 36:
3209
- e_2_1 = _u.sent();
3623
+ return [3 /*break*/, 33];
3624
+ case 33:
3625
+ _o = _m.next();
3626
+ return [3 /*break*/, 29];
3627
+ case 34: return [3 /*break*/, 37];
3628
+ case 35:
3629
+ e_2_1 = _t.sent();
3210
3630
  e_2 = { error: e_2_1 };
3211
- return [3 /*break*/, 38];
3212
- case 37:
3631
+ return [3 /*break*/, 37];
3632
+ case 36:
3213
3633
  try {
3214
- if (_p && !_p.done && (_t = _o.return)) _t.call(_o);
3634
+ if (_o && !_o.done && (_s = _m.return)) _s.call(_m);
3215
3635
  }
3216
3636
  finally { if (e_2) throw e_2.error; }
3217
3637
  return [7 /*endfinally*/];
3218
- case 38:
3638
+ case 37:
3219
3639
  if (postprocessingError) {
3220
3640
  throw postprocessingError;
3221
3641
  }
3222
- _u.label = 39;
3223
- case 39:
3224
- _m = _l.next();
3225
- return [3 /*break*/, 28];
3226
- case 40: return [3 /*break*/, 43];
3227
- case 41:
3228
- e_3_1 = _u.sent();
3642
+ _t.label = 38;
3643
+ case 38:
3644
+ _l = _k.next();
3645
+ return [3 /*break*/, 27];
3646
+ case 39: return [3 /*break*/, 42];
3647
+ case 40:
3648
+ e_3_1 = _t.sent();
3229
3649
  e_3 = { error: e_3_1 };
3230
- return [3 /*break*/, 43];
3231
- case 42:
3650
+ return [3 /*break*/, 42];
3651
+ case 41:
3232
3652
  try {
3233
- if (_m && !_m.done && (_s = _l.return)) _s.call(_l);
3653
+ if (_l && !_l.done && (_r = _k.return)) _r.call(_k);
3234
3654
  }
3235
3655
  finally { if (e_3) throw e_3.error; }
3236
3656
  return [7 /*endfinally*/];
3237
- case 43:
3657
+ case 42:
3238
3658
  // TODO: [💝] Unite object for expecting amount and format
3239
3659
  if (template.format) {
3240
3660
  if (template.format === 'JSON') {
@@ -3245,13 +3665,13 @@ function executeAttempts(options) {
3245
3665
  }
3246
3666
  catch (error) {
3247
3667
  keepUnused(error);
3248
- throw new ExpectError(spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block(
3249
- /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); }));
3668
+ throw new ExpectError(spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block(
3669
+ /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); }));
3250
3670
  }
3251
3671
  }
3252
3672
  }
3253
3673
  else {
3254
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Unknown format \"".concat(template.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3674
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Unknown format \"".concat(template.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3255
3675
  }
3256
3676
  }
3257
3677
  // TODO: [💝] Unite object for expecting amount and format
@@ -3259,14 +3679,14 @@ function executeAttempts(options) {
3259
3679
  checkExpectations(template.expectations, $ongoingTemplateResult.$resultString || '');
3260
3680
  }
3261
3681
  return [2 /*return*/, "break-attempts"];
3262
- case 44:
3263
- error_3 = _u.sent();
3682
+ case 43:
3683
+ error_3 = _t.sent();
3264
3684
  if (!(error_3 instanceof ExpectError)) {
3265
3685
  throw error_3;
3266
3686
  }
3267
3687
  $ongoingTemplateResult.$expectError = error_3;
3268
- return [3 /*break*/, 46];
3269
- case 45:
3688
+ return [3 /*break*/, 45];
3689
+ case 44:
3270
3690
  if (!isJokerAttempt &&
3271
3691
  template.templateType === 'PROMPT_TEMPLATE' &&
3272
3692
  $ongoingTemplateResult.$prompt
@@ -3283,22 +3703,22 @@ function executeAttempts(options) {
3283
3703
  });
3284
3704
  }
3285
3705
  return [7 /*endfinally*/];
3286
- case 46:
3706
+ case 45:
3287
3707
  if ($ongoingTemplateResult.$expectError !== null && attempt === maxAttempts - 1) {
3288
3708
  throw new PipelineExecutionError(spaceTrim(function (block) {
3289
3709
  var _a, _b, _c;
3290
- return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTemplateResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3710
+ return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTemplateResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3291
3711
  .split('\n')
3292
3712
  .map(function (line) { return "> ".concat(line); })
3293
- .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTemplateResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTemplateResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3713
+ .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTemplateResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTemplateResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3294
3714
  .split('\n')
3295
3715
  .map(function (line) { return "> ".concat(line); })
3296
- .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTemplateResult.$resultString === null
3716
+ .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTemplateResult.$resultString === null
3297
3717
  ? 'null'
3298
3718
  : $ongoingTemplateResult.$resultString
3299
3719
  .split('\n')
3300
3720
  .map(function (line) { return "> ".concat(line); })
3301
- .join('\n')), "\n ---\n ");
3721
+ .join('\n')), "\n ---\n ");
3302
3722
  }));
3303
3723
  }
3304
3724
  return [2 /*return*/];
@@ -3319,7 +3739,11 @@ function executeAttempts(options) {
3319
3739
  case 3:
3320
3740
  attempt++;
3321
3741
  return [3 /*break*/, 1];
3322
- case 4: return [2 /*return*/];
3742
+ case 4:
3743
+ if ($ongoingTemplateResult.$resultString === null) {
3744
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3745
+ }
3746
+ return [2 /*return*/, $ongoingTemplateResult.$resultString];
3323
3747
  }
3324
3748
  });
3325
3749
  });
@@ -3333,36 +3757,83 @@ function executeAttempts(options) {
3333
3757
  *
3334
3758
  * @private internal utility of `createPipelineExecutor`
3335
3759
  */
3336
- function executeFormatCells(options) {
3760
+ function executeFormatSubvalues(options) {
3337
3761
  return __awaiter(this, void 0, void 0, function () {
3338
- var template;
3762
+ var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3763
+ var _this = this;
3339
3764
  return __generator(this, function (_a) {
3340
- template = options.template;
3341
- if (template.foreach === undefined) {
3342
- return [2 /*return*/, /* not await */ executeAttempts(options)];
3765
+ switch (_a.label) {
3766
+ case 0:
3767
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3768
+ if (template.foreach === undefined) {
3769
+ return [2 /*return*/, /* not await */ executeAttempts(options)];
3770
+ }
3771
+ if (jokerParameterNames.length !== 0) {
3772
+ throw new UnexpectedError(spaceTrim$1(function (block) { return "\n JOKER parameters are not supported together with FOREACH command\n\n [\uD83E\uDDDE\u200D\u2640\uFE0F] This should be prevented in `validatePipeline`\n\n ".concat(block(pipelineIdentification), "\n "); }));
3773
+ }
3774
+ parameterValue = parameters[template.foreach.parameterName] || '';
3775
+ formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
3776
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(template.foreach.formatName);
3777
+ });
3778
+ if (formatDefinition === undefined) {
3779
+ throw new UnexpectedError(
3780
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3781
+ spaceTrim$1(function (block) { return "\n Unsupported format \"".concat(template.foreach.formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
3782
+ .map(function (formatName) { return "- ".concat(formatName); })
3783
+ .join('\n')), "\n\n [\u26F7] This should never happen because format name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3784
+ }
3785
+ subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
3786
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(template.foreach.subformatName);
3787
+ });
3788
+ if (subvalueDefinition === undefined) {
3789
+ throw new UnexpectedError(
3790
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3791
+ spaceTrim$1(function (block) { return "\n Unsupported subformat name \"".concat(template.foreach.subformatName, "\" for format \"").concat(template.foreach.formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
3792
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
3793
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
3794
+ .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3795
+ }
3796
+ if (formatDefinition.formatName === 'CSV') {
3797
+ formatSettings = settings.csvSettings;
3798
+ // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
3799
+ }
3800
+ return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
3801
+ var mappedParameters, allSubparameters, subresultString;
3802
+ return __generator(this, function (_a) {
3803
+ switch (_a.label) {
3804
+ case 0:
3805
+ // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
3806
+ // TODO: When done [🐚] Report progress also for each subvalue here
3807
+ try {
3808
+ mappedParameters = mapAvailableToExpectedParameters({
3809
+ expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
3810
+ availableParameters: subparameters,
3811
+ });
3812
+ }
3813
+ catch (error) {
3814
+ if (!(error instanceof PipelineExecutionError)) {
3815
+ throw error;
3816
+ }
3817
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n ".concat(error.message, "\n\n This is error in FOREACH command\n You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n ").concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }));
3818
+ }
3819
+ allSubparameters = __assign(__assign({}, parameters), mappedParameters);
3820
+ // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
3821
+ Object.freeze(allSubparameters);
3822
+ return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }) }))];
3823
+ case 1:
3824
+ subresultString = _a.sent();
3825
+ return [2 /*return*/, subresultString];
3826
+ }
3827
+ });
3828
+ }); })];
3829
+ case 1:
3830
+ resultString = _a.sent();
3831
+ return [2 /*return*/, resultString];
3343
3832
  }
3344
- throw new NotYetImplementedError('FOREACH execution not implemented yet');
3345
3833
  });
3346
3834
  });
3347
3835
  }
3348
3836
 
3349
- /**
3350
- * Just marks a place of place where should be something implemented
3351
- * No side effects.
3352
- *
3353
- * Note: It can be usefull suppressing eslint errors of unused variables
3354
- *
3355
- * @param value any values
3356
- * @returns void
3357
- * @private within the repository
3358
- */
3359
- function TODO_USE() {
3360
- var value = [];
3361
- for (var _i = 0; _i < arguments.length; _i++) {
3362
- value[_i] = arguments[_i];
3363
- }
3364
- }
3365
-
3366
3837
  /**
3367
3838
  * @@@
3368
3839
  *
@@ -3476,7 +3947,7 @@ function getReservedParametersForTemplate(options) {
3476
3947
  */
3477
3948
  function executeTemplate(options) {
3478
3949
  return __awaiter(this, void 0, void 0, function () {
3479
- var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, $ongoingTemplateResult, maxAttempts, jokerParameterNames, preparedContent;
3950
+ var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
3480
3951
  var e_1, _f, _g;
3481
3952
  return __generator(this, function (_h) {
3482
3953
  switch (_h.label) {
@@ -3500,12 +3971,13 @@ function executeTemplate(options) {
3500
3971
  _h.sent();
3501
3972
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
3502
3973
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
3974
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3503
3975
  if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
3504
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Dependent parameters:\n ").concat(Array.from(dependentParameterNames)
3976
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames)
3505
3977
  .map(function (name) { return "{".concat(name, "}"); })
3506
3978
  .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames)
3507
3979
  .map(function (name) { return "{".concat(name, "}"); })
3508
- .join(', '), "\n\n "); }));
3980
+ .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3509
3981
  }
3510
3982
  _b = (_a = Object).freeze;
3511
3983
  _c = [{}];
@@ -3534,6 +4006,7 @@ function executeTemplate(options) {
3534
4006
  };
3535
4007
  try {
3536
4008
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4009
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3537
4010
  for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
3538
4011
  parameterName = _e.value;
3539
4012
  _loop_1(parameterName);
@@ -3546,22 +4019,14 @@ function executeTemplate(options) {
3546
4019
  }
3547
4020
  finally { if (e_1) throw e_1.error; }
3548
4021
  }
3549
- // Note: Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
4022
+ // Note: [👨‍👨‍👧] Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
3550
4023
  Object.freeze(parameters);
3551
- $ongoingTemplateResult = {
3552
- $result: null,
3553
- $resultString: null,
3554
- $expectError: null,
3555
- $scriptPipelineExecutionErrors: [],
3556
- };
3557
4024
  maxAttempts = currentTemplate.templateType === 'DIALOG_TEMPLATE' ? Infinity : maxExecutionAttempts;
3558
4025
  jokerParameterNames = currentTemplate.jokerParameterNames || [];
3559
4026
  preparedContent = (currentTemplate.preparedContent || '{content}')
3560
4027
  .split('{content}')
3561
4028
  .join(currentTemplate.content);
3562
- // <- TODO: [🍵] Use here `replaceParameters` to replace {websiteContent} with option to ignore missing parameters
3563
- return [4 /*yield*/, executeFormatCells({
3564
- $ongoingTemplateResult: $ongoingTemplateResult,
4029
+ return [4 /*yield*/, executeFormatSubvalues({
3565
4030
  jokerParameterNames: jokerParameterNames,
3566
4031
  priority: priority,
3567
4032
  maxAttempts: maxAttempts,
@@ -3576,11 +4041,7 @@ function executeTemplate(options) {
3576
4041
  pipelineIdentification: pipelineIdentification,
3577
4042
  })];
3578
4043
  case 3:
3579
- // <- TODO: [🍵] Use here `replaceParameters` to replace {websiteContent} with option to ignore missing parameters
3580
- _h.sent();
3581
- if ($ongoingTemplateResult.$resultString === null) {
3582
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3583
- }
4044
+ resultString = _h.sent();
3584
4045
  return [4 /*yield*/, onProgress({
3585
4046
  name: name,
3586
4047
  title: title,
@@ -3588,13 +4049,15 @@ function executeTemplate(options) {
3588
4049
  isDone: true,
3589
4050
  templateType: currentTemplate.templateType,
3590
4051
  parameterName: currentTemplate.resultingParameterName,
3591
- parameterValue: $ongoingTemplateResult.$resultString,
4052
+ parameterValue: resultString,
3592
4053
  // <- [🍸]
3593
4054
  })];
3594
4055
  case 4:
3595
4056
  _h.sent();
3596
4057
  return [2 /*return*/, Object.freeze((_g = {},
3597
- _g[currentTemplate.resultingParameterName] = $ongoingTemplateResult.$resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */,
4058
+ _g[currentTemplate.resultingParameterName] =
4059
+ // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4060
+ resultString,
3598
4061
  _g))];
3599
4062
  }
3600
4063
  });
@@ -3603,6 +4066,9 @@ function executeTemplate(options) {
3603
4066
  /**
3604
4067
  * TODO: [🤹‍♂️]
3605
4068
  */
4069
+ /**
4070
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4071
+ */
3606
4072
 
3607
4073
  /**
3608
4074
  * @@@
@@ -3623,6 +4089,7 @@ function filterJustOutputParameters(options) {
3623
4089
  };
3624
4090
  try {
3625
4091
  // Note: Filter ONLY output parameters
4092
+ // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters`
3626
4093
  for (var _b = __values(preparedPipeline.parameters.filter(function (_a) {
3627
4094
  var isOutput = _a.isOutput;
3628
4095
  return isOutput;
@@ -3737,7 +4204,7 @@ function executePipeline(options) {
3737
4204
  return name === parameterName;
3738
4205
  });
3739
4206
  if (!(parameter === undefined)) return [3 /*break*/, 1];
3740
- warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
4207
+ warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
3741
4208
  return [3 /*break*/, 4];
3742
4209
  case 1:
3743
4210
  if (!(parameter.isInput === false)) return [3 /*break*/, 4];
@@ -3749,10 +4216,10 @@ function executePipeline(options) {
3749
4216
  // Note: Wait a short time to prevent race conditions
3750
4217
  _h.sent();
3751
4218
  _h.label = 3;
3752
- case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
4219
+ case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
3753
4220
  isSuccessful: false,
3754
4221
  errors: __spreadArray([
3755
- new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
4222
+ new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
3756
4223
  ], __read(errors), false).map(serializeError),
3757
4224
  warnings: warnings.map(serializeError),
3758
4225
  executionReport: executionReport,
@@ -3816,7 +4283,7 @@ function executePipeline(options) {
3816
4283
  case 0:
3817
4284
  if (loopLimit-- < 0) {
3818
4285
  // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
3819
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
4286
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
3820
4287
  }
3821
4288
  currentTemplate = unresovedTemplates_1.find(function (template) {
3822
4289
  return template.dependentParameterNames.every(function (name) {
@@ -3826,14 +4293,14 @@ function executePipeline(options) {
3826
4293
  if (!(!currentTemplate && resolving_1.length === 0)) return [3 /*break*/, 1];
3827
4294
  throw new UnexpectedError(
3828
4295
  // TODO: [🐎] DRY
3829
- spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
4296
+ spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
3830
4297
  .map(function (_a) {
3831
4298
  var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
3832
4299
  return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
3833
4300
  .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
3834
4301
  .join(' and '));
3835
4302
  })
3836
- .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
4303
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
3837
4304
  case 1:
3838
4305
  if (!!currentTemplate) return [3 /*break*/, 3];
3839
4306
  /* [🤹‍♂️] */ return [4 /*yield*/, Promise.race(resolving_1)];
@@ -3850,10 +4317,10 @@ function executePipeline(options) {
3850
4317
  llmTools: llmTools,
3851
4318
  onProgress: function (progress) {
3852
4319
  if (isReturned) {
3853
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4320
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
3854
4321
  .split('\n')
3855
4322
  .map(function (line) { return "> ".concat(line); })
3856
- .join('\n')), "\n "); }));
4323
+ .join('\n')), "\n "); }));
3857
4324
  }
3858
4325
  if (onProgress) {
3859
4326
  onProgress(progress);
@@ -3861,7 +4328,7 @@ function executePipeline(options) {
3861
4328
  },
3862
4329
  settings: settings,
3863
4330
  $executionReport: executionReport,
3864
- pipelineIdentification: pipelineIdentification,
4331
+ pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
3865
4332
  })
3866
4333
  .then(function (newParametersToPass) {
3867
4334
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
@@ -3953,6 +4420,9 @@ function executePipeline(options) {
3953
4420
  });
3954
4421
  });
3955
4422
  }
4423
+ /**
4424
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4425
+ */
3956
4426
 
3957
4427
  /**
3958
4428
  * Creates executor function from pipeline and execution tools.
@@ -3964,7 +4434,7 @@ function executePipeline(options) {
3964
4434
  function createPipelineExecutor(options) {
3965
4435
  var _this = this;
3966
4436
  var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
3967
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d, _e = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e;
4437
+ var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f;
3968
4438
  validatePipeline(pipeline);
3969
4439
  var pipelineIdentification = (function () {
3970
4440
  // Note: This is a 😐 implementation of [🚞]
@@ -3984,9 +4454,11 @@ function createPipelineExecutor(options) {
3984
4454
  else if (isNotPreparedWarningSupressed !== true) {
3985
4455
  console.warn(spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
3986
4456
  }
4457
+ var runCount = 0;
3987
4458
  var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
3988
4459
  return __generator(this, function (_a) {
3989
- return [2 /*return*/, executePipeline({
4460
+ runCount++;
4461
+ return [2 /*return*/, /* not await */ executePipeline({
3990
4462
  pipeline: pipeline,
3991
4463
  preparedPipeline: preparedPipeline,
3992
4464
  setPreparedPipeline: function (newPreparedPipeline) {
@@ -3995,10 +4467,11 @@ function createPipelineExecutor(options) {
3995
4467
  inputParameters: inputParameters,
3996
4468
  tools: tools,
3997
4469
  onProgress: onProgress,
3998
- pipelineIdentification: pipelineIdentification,
4470
+ pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
3999
4471
  settings: {
4000
4472
  maxExecutionAttempts: maxExecutionAttempts,
4001
4473
  maxParallelCount: maxParallelCount,
4474
+ csvSettings: csvSettings,
4002
4475
  isVerbose: isVerbose,
4003
4476
  isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4004
4477
  },
@@ -4007,6 +4480,9 @@ function createPipelineExecutor(options) {
4007
4480
  }); };
4008
4481
  return pipelineExecutor;
4009
4482
  }
4483
+ /**
4484
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4485
+ */
4010
4486
 
4011
4487
  /**
4012
4488
  * @@@
@@ -4058,7 +4534,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
4058
4534
  outputParameters = result.outputParameters;
4059
4535
  knowledgePiecesRaw = outputParameters.knowledgePieces;
4060
4536
  knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
4061
- // <- TODO: !!!!! Smarter split and filter out empty pieces
4537
+ // <- TODO: [main] !!!!! Smarter split and filter out empty pieces
4062
4538
  if (isVerbose) {
4063
4539
  console.info('knowledgeTextPieces:', knowledgeTextPieces);
4064
4540
  }
@@ -4116,8 +4592,13 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
4116
4592
  case 6: return [3 /*break*/, 8];
4117
4593
  case 7:
4118
4594
  error_1 = _c.sent();
4595
+ // Note: Here is expected error:
4596
+ // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
4597
+ if (!(error_1 instanceof PipelineExecutionError)) {
4598
+ throw error_1;
4599
+ }
4119
4600
  // TODO: [🟥] Detect browser / node and make it colorfull
4120
- console.error(error_1);
4601
+ console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
4121
4602
  return [3 /*break*/, 8];
4122
4603
  case 8: return [2 /*return*/, {
4123
4604
  name: name,
@@ -4138,7 +4619,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
4138
4619
  });
4139
4620
  }
4140
4621
  /**
4141
- * TODO: [🐝][🔼] !!! Export via `@promptbook/markdown`
4622
+ * TODO: [🐝][🔼][main] !!! Export via `@promptbook/markdown`
4142
4623
  * TODO: [🪂] Do it in parallel 11:11
4143
4624
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4144
4625
  */
@@ -4162,7 +4643,7 @@ function prepareKnowledgePieces(knowledgeSources, options) {
4162
4643
  var partialPieces, pieces;
4163
4644
  return __generator(this, function (_a) {
4164
4645
  switch (_a.label) {
4165
- case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4646
+ case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝][main] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4166
4647
  options)];
4167
4648
  case 1:
4168
4649
  partialPieces = _a.sent();
@@ -4354,7 +4835,7 @@ function preparePersona(personaDescription, options) {
4354
4835
  });
4355
4836
  }
4356
4837
  /**
4357
- * TODO: [🔃] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4838
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4358
4839
  * TODO: [🏢] !! Check validity of `modelName` in pipeline
4359
4840
  * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4360
4841
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
@@ -4403,7 +4884,7 @@ function prepareTemplates(pipeline, options) {
4403
4884
  case 0:
4404
4885
  _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
4405
4886
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
4406
- // TODO: !!!!! Apply samples to each template (if missing and is for the template defined)
4887
+ // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
4407
4888
  TODO_USE(parameters);
4408
4889
  templatesPrepared = new Array(
4409
4890
  // <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -4435,7 +4916,7 @@ function prepareTemplates(pipeline, options) {
4435
4916
  /**
4436
4917
  * TODO: [🧠] Add context to each template (if missing)
4437
4918
  * TODO: [🧠] What is better name `prepareTemplate` or `prepareTemplateAndParameters`
4438
- * TODO: [♨] !!! Prepare index the samples and maybe templates
4919
+ * TODO: [♨][main] !!! Prepare index the samples and maybe templates
4439
4920
  * TODO: Write tests for `preparePipeline`
4440
4921
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
4441
4922
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
@@ -4607,7 +5088,7 @@ var knowledgeCommandParser = {
4607
5088
  if (sourceContent === '') {
4608
5089
  throw new ParseError("Source is not defined");
4609
5090
  }
4610
- // TODO: !!!! Following checks should be applied every link in the `sourceContent`
5091
+ // TODO: [main] !!!! Following checks should be applied every link in the `sourceContent`
4611
5092
  if (sourceContent.startsWith('http://')) {
4612
5093
  throw new ParseError("Source is not secure");
4613
5094
  }
@@ -4810,7 +5291,7 @@ var templateCommandParser = {
4810
5291
  if (command.templateType === 'KNOWLEDGE') {
4811
5292
  knowledgeCommandParser.$applyToPipelineJson({
4812
5293
  type: 'KNOWLEDGE',
4813
- sourceContent: $templateJson.content, // <- TODO: [🐝] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
5294
+ sourceContent: $templateJson.content, // <- TODO: [🐝][main] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
4814
5295
  }, $pipelineJson);
4815
5296
  $templateJson.isTemplate = false;
4816
5297
  return;
@@ -5160,6 +5641,171 @@ function normalizeTo_SCREAMING_CASE(text) {
5160
5641
  * TODO: [🌺] Use some intermediate util splitWords
5161
5642
  */
5162
5643
 
5644
+ /**
5645
+ * @@@
5646
+ *
5647
+ * @param text @@@
5648
+ * @param _isFirstLetterCapital @@@
5649
+ * @returns @@@
5650
+ * @example 'helloWorld'
5651
+ * @example 'iLovePromptbook'
5652
+ * @public exported from `@promptbook/utils`
5653
+ */
5654
+ function normalizeTo_camelCase(text, _isFirstLetterCapital) {
5655
+ var e_1, _a;
5656
+ if (_isFirstLetterCapital === void 0) { _isFirstLetterCapital = false; }
5657
+ var charType;
5658
+ var lastCharType = null;
5659
+ var normalizedName = '';
5660
+ try {
5661
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5662
+ var char = text_1_1.value;
5663
+ var normalizedChar = void 0;
5664
+ if (/^[a-z]$/.test(char)) {
5665
+ charType = 'LOWERCASE';
5666
+ normalizedChar = char;
5667
+ }
5668
+ else if (/^[A-Z]$/.test(char)) {
5669
+ charType = 'UPPERCASE';
5670
+ normalizedChar = char.toLowerCase();
5671
+ }
5672
+ else if (/^[0-9]$/.test(char)) {
5673
+ charType = 'NUMBER';
5674
+ normalizedChar = char;
5675
+ }
5676
+ else {
5677
+ charType = 'OTHER';
5678
+ normalizedChar = '';
5679
+ }
5680
+ if (!lastCharType) {
5681
+ if (_isFirstLetterCapital) {
5682
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: DRY
5683
+ }
5684
+ }
5685
+ else if (charType !== lastCharType &&
5686
+ !(charType === 'LOWERCASE' && lastCharType === 'UPPERCASE') &&
5687
+ !(lastCharType === 'NUMBER') &&
5688
+ !(charType === 'NUMBER')) {
5689
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: [🌺] DRY
5690
+ }
5691
+ normalizedName += normalizedChar;
5692
+ lastCharType = charType;
5693
+ }
5694
+ }
5695
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5696
+ finally {
5697
+ try {
5698
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5699
+ }
5700
+ finally { if (e_1) throw e_1.error; }
5701
+ }
5702
+ return normalizedName;
5703
+ }
5704
+ /**
5705
+ * TODO: [🌺] Use some intermediate util splitWords
5706
+ */
5707
+
5708
+ /**
5709
+ * Removes quotes from a string
5710
+ *
5711
+ * Tip: This is very usefull for post-processing of the result of the LLM model
5712
+ * Note: This function removes only the same quotes from the beginning and the end of the string
5713
+ * Note: There are two simmilar functions:
5714
+ * - `removeQuotes` which removes only bounding quotes
5715
+ * - `unwrapResult` which removes whole introduce sentence
5716
+ *
5717
+ * @param text optionally quoted text
5718
+ * @returns text without quotes
5719
+ * @public exported from `@promptbook/utils`
5720
+ */
5721
+ function removeQuotes(text) {
5722
+ if (text.startsWith('"') && text.endsWith('"')) {
5723
+ return text.slice(1, -1);
5724
+ }
5725
+ if (text.startsWith('\'') && text.endsWith('\'')) {
5726
+ return text.slice(1, -1);
5727
+ }
5728
+ return text;
5729
+ }
5730
+
5731
+ /**
5732
+ * Function `validateParameterName` will @@@
5733
+ *
5734
+ * @param parameterName @@@
5735
+ * @returns @@@
5736
+ * @throws {ParseError} @@@
5737
+ * @private within the repository
5738
+ */
5739
+ function validateParameterName(parameterName) {
5740
+ var e_1, _a;
5741
+ var rawParameterName = parameterName;
5742
+ try {
5743
+ for (var _b = __values([
5744
+ ['`', '`'],
5745
+ ['{', '}'],
5746
+ ['[', ']'],
5747
+ ['(', ')'],
5748
+ ['<', '>'],
5749
+ ]), _c = _b.next(); !_c.done; _c = _b.next()) {
5750
+ var _d = __read(_c.value, 2), start = _d[0], end = _d[1];
5751
+ if (parameterName.substring(0, 1) === start &&
5752
+ parameterName.substring(parameterName.length - 1, parameterName.length) === end
5753
+ // <- TODO: More universal that 1 character
5754
+ ) {
5755
+ parameterName = parameterName.substring(1, parameterName.length - 1);
5756
+ // <- TODO: More universal that 1 character
5757
+ }
5758
+ }
5759
+ }
5760
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5761
+ finally {
5762
+ try {
5763
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
5764
+ }
5765
+ finally { if (e_1) throw e_1.error; }
5766
+ }
5767
+ // TODO: [🐠] Following try-catch block should be part of common validators logic
5768
+ try {
5769
+ /*
5770
+ Note: We don't need to check for spaces because we are going to normalize the parameter name to camelCase
5771
+ if (parameterName.includes(' ')) {
5772
+ throw new ParseError(`Parameter name cannot contain spaces`);
5773
+ }
5774
+ */
5775
+ if (parameterName.includes('.')) {
5776
+ throw new ParseError("Parameter name cannot contain dots");
5777
+ }
5778
+ if (parameterName.includes('/') || parameterName.includes('\\')) {
5779
+ throw new ParseError("Parameter name cannot contain slashes");
5780
+ }
5781
+ if (parameterName.includes('(') ||
5782
+ parameterName.includes(')') ||
5783
+ parameterName.includes('{') ||
5784
+ parameterName.includes('}') ||
5785
+ parameterName.includes('[') ||
5786
+ parameterName.includes(']')) {
5787
+ throw new ParseError("Parameter name cannot contain braces");
5788
+ }
5789
+ parameterName = removeDiacritics(parameterName);
5790
+ parameterName = removeEmojis(parameterName);
5791
+ parameterName = removeQuotes(parameterName);
5792
+ parameterName = normalizeTo_camelCase(parameterName);
5793
+ if (parameterName === '') {
5794
+ throw new ParseError("Parameter name cannot be empty");
5795
+ }
5796
+ if (RESERVED_PARAMETER_NAMES.includes(parameterName)) {
5797
+ throw new ParseError("{".concat(parameterName, "} is a reserved parameter name"));
5798
+ }
5799
+ }
5800
+ catch (error) {
5801
+ if (!(error instanceof ParseError)) {
5802
+ throw error;
5803
+ }
5804
+ throw new ParseError(spaceTrim$1(function (block) { return "\n ".concat(block(error.message), "\n\n Tried to validate parameter name:\n ").concat(block(rawParameterName), "\n "); }));
5805
+ }
5806
+ return parameterName;
5807
+ }
5808
+
5163
5809
  /**
5164
5810
  * Parses the foreach command
5165
5811
  *
@@ -5189,15 +5835,16 @@ var foreachCommandParser = {
5189
5835
  /**
5190
5836
  * Link to discussion
5191
5837
  */
5192
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5838
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/148',
5193
5839
  /**
5194
5840
  * Example usages of the FOREACH command
5195
5841
  */
5196
5842
  examples: [
5197
- 'FOREACH List Line `{customers}` -> `{customer}`',
5198
- 'FOR List Line `{customers}` -> `{customer}`',
5199
- 'EACH List Line `{customers}` -> `{customer}`',
5200
- // <- TODO: [🍭] !!!!!! More
5843
+ 'FOREACH Text Line `{customers}` -> `{customer}`',
5844
+ 'FOREACH Csv Cell `{customers}` -> `{cell}`',
5845
+ 'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
5846
+ 'FOR Text Line `{customers}` -> `{customer}`',
5847
+ 'EACH Text Line `{customers}` -> `{customer}`',
5201
5848
  ],
5202
5849
  /**
5203
5850
  * Parses the FOREACH command
@@ -5205,55 +5852,75 @@ var foreachCommandParser = {
5205
5852
  parse: function (input) {
5206
5853
  var args = input.args;
5207
5854
  var formatName = normalizeTo_SCREAMING_CASE(args[0] || '');
5208
- var cellName = normalizeTo_SCREAMING_CASE(args[1] || '');
5209
- var parameterNameWrapped = args[2];
5855
+ var subformatName = normalizeTo_SCREAMING_CASE(args[1] || '');
5856
+ var parameterNameArg = args[2] || '';
5210
5857
  var assignSign = args[3];
5211
- var subparameterNameWrapped = args[4];
5212
- if (![
5213
- 'LIST',
5214
- 'CSV',
5215
- // <- TODO: [🏢] Unhardcode formats
5216
- ].includes(formatName)) {
5217
- console.info({ args: args, formatName: formatName });
5218
- throw new Error("Unsupported format \"".concat(formatName, "\""));
5858
+ var formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
5859
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(formatName);
5860
+ });
5861
+ if (formatDefinition === undefined) {
5862
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unsupported format \"".concat(formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
5863
+ .map(function (formatName) { return "- ".concat(formatName); })
5864
+ .join('\n')), "\n "); }));
5219
5865
  // <- TODO: [🏢] List all supported format names
5220
5866
  }
5221
- if (![
5222
- 'LINE',
5223
- 'ROW',
5224
- 'COLUMN',
5225
- 'CELL',
5226
- // <- TODO: [🏢] Unhardcode format cells
5227
- ].includes(cellName)) {
5228
- console.info({ args: args, cellName: cellName });
5229
- throw new Error("Format ".concat(formatName, " does not support cell \"").concat(cellName, "\""));
5230
- // <- TODO: [🏢] List all supported cell names for the format
5867
+ var subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
5868
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(subformatName);
5869
+ });
5870
+ if (subvalueDefinition === undefined) {
5871
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unsupported subformat name \"".concat(subformatName, "\" for format \"").concat(formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
5872
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
5873
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
5874
+ .join('\n')), "\n "); }));
5875
+ // <- TODO: [🏢] List all supported subformat names for the format
5231
5876
  }
5232
5877
  if (assignSign !== '->') {
5233
- console.info({ args: args, assignSign: assignSign });
5234
- throw new Error("FOREACH command must have '->' to assign the value to the parameter");
5235
- }
5236
- // TODO: !!!!!! Replace with propper parameter name validation
5237
- if ((parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1)) !== '{' ||
5238
- (parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length)) !== '}') {
5239
- console.info({ args: args, parameterNameWrapped: parameterNameWrapped }, parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1), parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length));
5240
- throw new Error("!!!!!! 1 Here will be error (with rules and precise error) from validateParameterName");
5241
- }
5242
- var parameterName = parameterNameWrapped.substring(1, parameterNameWrapped.length - 1);
5243
- // TODO: !!!!!! Replace with propper parameter name validation
5244
- if ((subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(0, 1)) !== '{' ||
5245
- (subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(subparameterNameWrapped.length - 1, subparameterNameWrapped.length)) !==
5246
- '}') {
5247
- console.info({ args: args, subparameterNameWrapped: subparameterNameWrapped });
5248
- throw new Error("!!!!!! 2 Here will be error (with rules and precise error) from validateParameterName");
5249
- }
5250
- var subparameterName = subparameterNameWrapped.substring(1, subparameterNameWrapped.length - 1);
5878
+ throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
5879
+ }
5880
+ var parameterName = validateParameterName(parameterNameArg);
5881
+ var outputSubparameterName = null;
5882
+ // TODO: [4] DRY
5883
+ var inputSubparameterNames = args
5884
+ .slice(4)
5885
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5886
+ .filter(function (parameterName) { return !parameterName.includes('+'); })
5887
+ .filter(function (parameterName) { return parameterName !== ''; })
5888
+ .map(validateParameterName);
5889
+ // TODO: [4] DRY
5890
+ var outputSubparameterNames = args
5891
+ .slice(4)
5892
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5893
+ .filter(function (parameterName) { return parameterName.includes('+'); })
5894
+ .map(function (parameterName) { return parameterName.split('+').join(''); })
5895
+ .map(validateParameterName);
5896
+ if (outputSubparameterNames.length === 1) {
5897
+ outputSubparameterName = outputSubparameterNames[0];
5898
+ }
5899
+ else if (outputSubparameterNames.length > 1) {
5900
+ throw new ParseError("FOREACH command can not have more than one output subparameter");
5901
+ }
5902
+ if (inputSubparameterNames.length === 0) {
5903
+ throw new ParseError("FOREACH command must have at least one input subparameter");
5904
+ }
5905
+ if (outputSubparameterName === null) {
5906
+ // TODO: Following code should be unhardcoded from here and moved to the format definition
5907
+ if (formatName === 'CSV' && subformatName === 'CELL') {
5908
+ outputSubparameterName = 'newCell';
5909
+ }
5910
+ else if (formatName === 'TEXT' && subformatName === 'LINE') {
5911
+ outputSubparameterName = 'newLine';
5912
+ }
5913
+ else {
5914
+ throw new ParseError(spaceTrim$1("\n FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n Correct example:\n - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n ")));
5915
+ }
5916
+ }
5251
5917
  return {
5252
5918
  type: 'FOREACH',
5253
5919
  formatName: formatName,
5254
- cellName: cellName,
5920
+ subformatName: subformatName,
5255
5921
  parameterName: parameterName,
5256
- subparameterName: subparameterName,
5922
+ inputSubparameterNames: inputSubparameterNames,
5923
+ outputSubparameterName: outputSubparameterName,
5257
5924
  };
5258
5925
  },
5259
5926
  /**
@@ -5262,11 +5929,17 @@ var foreachCommandParser = {
5262
5929
  * Note: `$` is used to indicate that this function mutates given `templateJson`
5263
5930
  */
5264
5931
  $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
5265
- var formatName = command.formatName, cellName = command.cellName, parameterName = command.parameterName, subparameterName = command.subparameterName;
5266
- // TODO: !!!!!! Detect double use
5267
- // TODO: !!!!!! Detect usage with JOKER and don't allow it
5268
- $templateJson.foreach = { formatName: formatName, cellName: cellName, parameterName: parameterName, subparameterName: subparameterName };
5269
- keepUnused($pipelineJson); // <- TODO: !!!!!! BUT Maybe register subparameter from foreach into parameters of the pipeline
5932
+ var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
5933
+ // TODO: [🍭] Detect double use
5934
+ // TODO: [🍭] Detect usage with JOKER and don't allow it
5935
+ $templateJson.foreach = {
5936
+ formatName: formatName,
5937
+ subformatName: subformatName,
5938
+ parameterName: parameterName,
5939
+ inputSubparameterNames: inputSubparameterNames,
5940
+ outputSubparameterName: outputSubparameterName,
5941
+ };
5942
+ keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
5270
5943
  // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
5271
5944
  },
5272
5945
  /**
@@ -5289,8 +5962,7 @@ var foreachCommandParser = {
5289
5962
  },
5290
5963
  };
5291
5964
  /**
5292
- * TODO: !!!!!! Comment console logs
5293
- * TODO: [🍭] !!!!!! Make .ptbk.md file with examples of the FOREACH command and also with wrong parsing and logic
5965
+ * TODO: [🍭] Make .ptbk.md file with examples of the FOREACH with wrong parsing and logic
5294
5966
  */
5295
5967
 
5296
5968
  /**
@@ -5400,12 +6072,11 @@ var jokerCommandParser = {
5400
6072
  */
5401
6073
  parse: function (input) {
5402
6074
  var args = input.args;
5403
- // TODO: !!!!!! Replace with propper parameter name validation
5404
- var parametersMatch = (args.pop() || '').match(/^\{(?<parameterName>[a-z0-9_]+)\}$/im);
5405
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5406
- throw new ParseError("Invalid joker");
6075
+ if (args.length !== 1) {
6076
+ throw new ParseError("JOKE command expects exactly one parameter name");
5407
6077
  }
5408
- var parameterName = parametersMatch.groups.parameterName;
6078
+ var parameterNameArg = args[0] || '';
6079
+ var parameterName = validateParameterName(parameterNameArg);
5409
6080
  return {
5410
6081
  type: 'JOKER',
5411
6082
  parameterName: parameterName,
@@ -5480,6 +6151,9 @@ var modelCommandParser = {
5480
6151
  */
5481
6152
  parse: function (input) {
5482
6153
  var args = input.args, normalized = input.normalized;
6154
+ var availableVariantsMessage = spaceTrim$1(function (block) { return "\n Available variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) {
6155
+ return "- ".concat(variantName).concat(variantName !== 'EMBEDDING' ? '' : ' (Not available in pipeline)');
6156
+ }).join('\n')), "\n "); });
5483
6157
  // TODO: Make this more elegant and dynamically
5484
6158
  if (normalized.startsWith('MODEL_VARIANT')) {
5485
6159
  if (normalized === 'MODEL_VARIANT_CHAT') {
@@ -5495,17 +6169,13 @@ var modelCommandParser = {
5495
6169
  key: 'modelVariant',
5496
6170
  value: 'COMPLETION',
5497
6171
  };
6172
+ // <- Note: [🤖]
5498
6173
  }
5499
6174
  else if (normalized.startsWith('MODEL_VARIANT_EMBED')) {
5500
- return {
5501
- type: 'MODEL',
5502
- key: 'modelVariant',
5503
- value: 'EMBEDDING',
5504
- };
5505
- // <- Note: [🤖]
6175
+ spaceTrim$1(function (block) { return "\n Embedding model can not be used in pipeline\n\n ".concat(block(availableVariantsMessage), "\n "); });
5506
6176
  }
5507
6177
  else {
5508
- throw new ParseError(spaceTrim$1(function (block) { return "\n Unknown model variant in command:\n\n Supported variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) { return "- ".concat(variantName); }).join('\n')), "\n "); }));
6178
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unknown model variant in command:\n\n ".concat(block(availableVariantsMessage), "\n "); }));
5509
6179
  }
5510
6180
  }
5511
6181
  if (normalized.startsWith('MODEL_NAME')) {
@@ -5630,14 +6300,13 @@ var parameterCommandParser = {
5630
6300
  * Parses the PARAMETER command
5631
6301
  */
5632
6302
  parse: function (input) {
5633
- var normalized = input.normalized, raw = input.raw;
5634
- var parametersMatch = raw.match(/\{(?<parameterName>[a-z0-9_]+)\}[^\S\r\n]*(?<parameterDescription>.*)$/im);
5635
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5636
- throw new ParseError("Invalid parameter");
5637
- }
5638
- var _a = parametersMatch.groups, parameterName = _a.parameterName, parameterDescription = _a.parameterDescription;
5639
- if (parameterDescription && parameterDescription.match(/\{(?<parameterName>[a-z0-9_]+)\}/im)) {
5640
- throw new ParseError("Parameter {".concat(parameterName, "} can not contain another parameter in description"));
6303
+ var normalized = input.normalized, args = input.args, raw = input.raw;
6304
+ var parameterNameRaw = args.shift() || '';
6305
+ var parameterDescriptionRaw = args.join(' ');
6306
+ // <- TODO: When [🥶] fixed, change to:
6307
+ // > const parameterDescriptionRaw = rawArgs.split(parameterNameRaw).join('').trim();
6308
+ if (parameterDescriptionRaw && parameterDescriptionRaw.match(/\{(?<embeddedParameterName>[a-z0-9_]+)\}/im)) {
6309
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Parameter {".concat(parameterNameRaw, "} can not contain another parameter in description\n\n The description:\n ").concat(block(parameterDescriptionRaw), "\n "); }));
5641
6310
  }
5642
6311
  var isInput = normalized.startsWith('INPUT');
5643
6312
  var isOutput = normalized.startsWith('OUTPUT');
@@ -5645,11 +6314,12 @@ var parameterCommandParser = {
5645
6314
  isInput = false;
5646
6315
  isOutput = false;
5647
6316
  }
5648
- // TODO: !!!!!! Add parameter name validation
6317
+ var parameterName = validateParameterName(parameterNameRaw);
6318
+ var parameterDescription = parameterDescriptionRaw.trim() || null;
5649
6319
  return {
5650
6320
  type: 'PARAMETER',
5651
6321
  parameterName: parameterName,
5652
- parameterDescription: parameterDescription.trim() || null,
6322
+ parameterDescription: parameterDescription,
5653
6323
  isInput: isInput,
5654
6324
  isOutput: isOutput,
5655
6325
  };
@@ -5974,6 +6644,7 @@ var promptbookVersionCommandParser = {
5974
6644
  * Note: `$` is used to indicate that this function mutates given `pipelineJson`
5975
6645
  */
5976
6646
  $applyToPipelineJson: function (command, $pipelineJson) {
6647
+ // TODO: Warn if the version is overridden
5977
6648
  $pipelineJson.promptbookVersion = command.promptbookVersion;
5978
6649
  },
5979
6650
  /**
@@ -6456,7 +7127,9 @@ function parseCommand(raw, usagePlace) {
6456
7127
  for (var commandNameSegmentsCount = 0; commandNameSegmentsCount < Math.min(items.length, 3); commandNameSegmentsCount++) {
6457
7128
  var commandNameRaw = items.slice(0, commandNameSegmentsCount + 1).join('_');
6458
7129
  var args = items.slice(commandNameSegmentsCount + 1);
6459
- var rawArgs = raw.substring(commandNameRaw.length).trim();
7130
+ var rawArgs = raw
7131
+ .substring(commandNameRaw.length)
7132
+ .trim();
6460
7133
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6461
7134
  if (command !== null) {
6462
7135
  return command;
@@ -6467,7 +7140,9 @@ function parseCommand(raw, usagePlace) {
6467
7140
  {
6468
7141
  var commandNameRaw = items.slice(-1).join('_');
6469
7142
  var args = items.slice(0, -1); // <- Note: This is probbably not correct
6470
- var rawArgs = raw.substring(0, raw.length - commandNameRaw.length).trim();
7143
+ var rawArgs = raw
7144
+ .substring(0, raw.length - commandNameRaw.length)
7145
+ .trim();
6471
7146
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6472
7147
  if (command !== null) {
6473
7148
  return command;
@@ -6607,7 +7282,7 @@ function extractAllListItemsFromMarkdown(markdown) {
6607
7282
  function extractOneBlockFromMarkdown(markdown) {
6608
7283
  var codeBlocks = extractAllBlocksFromMarkdown(markdown);
6609
7284
  if (codeBlocks.length !== 1) {
6610
- throw new ParseError(spaceTrim$1(function (block) { return "\n There should be exactly 1 code block, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
7285
+ throw new ParseError(spaceTrim$1(function (block) { return "\n There should be exactly 1 code block in template, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
6611
7286
  }
6612
7287
  return codeBlocks[0];
6613
7288
  }
@@ -6794,7 +7469,7 @@ function pipelineStringToJsonSync(pipelineString) {
6794
7469
  var $pipelineJson = {
6795
7470
  title: undefined /* <- Note: [🍙] Putting here placeholder to keep `title` on top at final JSON */,
6796
7471
  pipelineUrl: undefined /* <- Note: Putting here placeholder to keep `pipelineUrl` on top at final JSON */,
6797
- promptbookVersion: PROMPTBOOK_VERSION,
7472
+ promptbookVersion: undefined /* <- Note: By default no explicit version */,
6798
7473
  description: undefined /* <- Note: [🍙] Putting here placeholder to keep `description` on top at final JSON */,
6799
7474
  parameters: [],
6800
7475
  templates: [],
@@ -7085,7 +7760,7 @@ function pipelineStringToJsonSync(pipelineString) {
7085
7760
  return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
7086
7761
  }
7087
7762
  /**
7088
- * TODO: !!!! Warn if used only sync version
7763
+ * TODO: [main] !!!! Warn if used only sync version
7089
7764
  * TODO: [🚞] Report here line/column of error
7090
7765
  * TODO: Use spaceTrim more effectively
7091
7766
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
@@ -7868,7 +8543,7 @@ function isSerializableAsJson(value) {
7868
8543
  }
7869
8544
  }
7870
8545
  /**
7871
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
8546
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
7872
8547
  * TODO: [🧠][💺] Can be done this on type-level?
7873
8548
  */
7874
8549