@promptbook/node 0.69.0-7 → 0.69.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +4 -1
  2. package/esm/index.es.js +1510 -783
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/promptbook-collection/index.d.ts +0 -3
  5. package/esm/typings/src/_packages/core.index.d.ts +12 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +6 -0
  7. package/esm/typings/src/_packages/utils.index.d.ts +4 -4
  8. package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
  9. package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +1 -1
  10. package/esm/typings/src/commands/FOREACH/ForeachCommand.d.ts +1 -1
  11. package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -5
  12. package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -2
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +1 -1
  14. package/esm/typings/src/config.d.ts +11 -4
  15. package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +1 -1
  16. package/esm/typings/src/conversion/prettify/renderPipelineMermaidOptions.d.ts +3 -3
  17. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +6 -5
  18. package/esm/typings/src/errors/AbstractFormatError.d.ts +11 -0
  19. package/esm/typings/src/execution/PipelineExecutor.d.ts +1 -0
  20. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +5 -6
  21. package/esm/typings/src/execution/createPipelineExecutor/$OngoingTemplateResult.d.ts +45 -0
  22. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +10 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +3 -0
  24. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +5 -2
  25. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +3 -0
  26. package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatSubvalues.d.ts +15 -0
  27. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +74 -0
  28. package/esm/typings/src/execution/embeddingVectorToString.d.ts +1 -1
  29. package/esm/typings/src/execution/utils/checkExpectations.d.ts +2 -0
  30. package/esm/typings/src/execution/utils/usageToHuman.d.ts +3 -4
  31. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +14 -15
  32. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +31 -0
  33. package/esm/typings/src/formats/csv/{ListFormatDefinition.d.ts → CsvFormatDefinition.d.ts} +6 -3
  34. package/esm/typings/src/formats/csv/CsvFormatError.d.ts +10 -0
  35. package/esm/typings/src/formats/csv/CsvSettings.d.ts +13 -0
  36. package/esm/typings/src/formats/index.d.ts +1 -1
  37. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +4 -3
  38. package/esm/typings/src/formats/text/TextFormatDefinition.d.ts +19 -0
  39. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +4 -3
  40. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +1 -1
  41. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.d.ts +2 -2
  45. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +2 -2
  46. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +1 -1
  47. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  48. package/esm/typings/src/prepare/isPipelinePrepared.d.ts +1 -1
  49. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  50. package/esm/typings/src/types/PipelineJson/ParameterJson.d.ts +1 -1
  51. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +1 -1
  52. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +0 -3
  53. package/esm/typings/src/types/execution-report/executionReportJsonToString.d.ts +2 -1
  54. package/esm/typings/src/types/typeAliases.d.ts +1 -1
  55. package/esm/typings/src/utils/expectation-counters/index.d.ts +3 -0
  56. package/esm/typings/src/utils/organization/{f.d.ts → empty_object.d.ts} +5 -1
  57. package/esm/typings/src/utils/organization/just_empty_object.d.ts +12 -0
  58. package/esm/typings/src/utils/{extractParameterNames.d.ts → parameters/extractParameterNames.d.ts} +2 -2
  59. package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +27 -0
  60. package/esm/typings/src/utils/{replaceParameters.d.ts → parameters/replaceParameters.d.ts} +2 -2
  61. package/esm/typings/src/utils/serialization/checkSerializableAsJson.d.ts +1 -1
  62. package/esm/typings/src/utils/serialization/isSerializableAsJson.d.ts +1 -1
  63. package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +10 -0
  64. package/package.json +18 -13
  65. package/umd/index.umd.js +1513 -787
  66. package/umd/index.umd.js.map +1 -1
  67. package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatCell.d.ts +0 -30
  68. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempt.d.ts +0 -30
  69. package/esm/typings/src/formats/list/ListFormatDefinition.d.ts +0 -16
  70. /package/esm/typings/src/utils/{extractParameterNames.test.d.ts → parameters/extractParameterNames.test.d.ts} +0 -0
  71. /package/esm/typings/src/{execution/utils/usageToHuman.test.d.ts → utils/parameters/mapAvailableToExpectedParameters.test.d.ts} +0 -0
  72. /package/esm/typings/src/utils/{replaceParameters.test.d.ts → parameters/replaceParameters.test.d.ts} +0 -0
  73. /package/esm/typings/src/{personas/preparePersona.test.d.ts → utils/validators/parameterName/validateParameterName.test.d.ts} +0 -0
package/esm/index.es.js CHANGED
@@ -5,6 +5,7 @@ import spaceTrim$1, { spaceTrim } from 'spacetrim';
5
5
  import { format } from 'prettier';
6
6
  import parserHtml from 'prettier/parser-html';
7
7
  import { forTime } from 'waitasecond';
8
+ import { unparse, parse } from 'papaparse';
8
9
  import hexEncoder from 'crypto-js/enc-hex';
9
10
  import sha256 from 'crypto-js/sha256';
10
11
  import { join } from 'path/posix';
@@ -14,8 +15,8 @@ import * as dotenv from 'dotenv';
14
15
  /**
15
16
  * The version of the Promptbook library
16
17
  */
17
- var PROMPTBOOK_VERSION = '0.69.0-6';
18
- // TODO: !!!! List here all the versions and annotate + put into script
18
+ var PROMPTBOOK_VERSION = '0.69.0-21';
19
+ // TODO: [main] !!!! List here all the versions and annotate + put into script
19
20
 
20
21
  /*! *****************************************************************************
21
22
  Copyright (c) Microsoft Corporation.
@@ -325,7 +326,7 @@ function checkSerializableAsJson(name, value) {
325
326
  }
326
327
  /**
327
328
  * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
328
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
329
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
329
330
  * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
330
331
  */
331
332
 
@@ -367,13 +368,13 @@ var IMMEDIATE_TIME = 10;
367
368
  *
368
369
  * @public exported from `@promptbook/core`
369
370
  */
370
- var MAX_PARALLEL_COUNT = 5;
371
+ var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
371
372
  /**
372
373
  * The maximum number of attempts to execute LLM task before giving up
373
374
  *
374
375
  * @public exported from `@promptbook/core`
375
376
  */
376
- var MAX_EXECUTION_ATTEMPTS = 3;
377
+ var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
377
378
  /**
378
379
  * The maximum length of the (generated) filename
379
380
  *
@@ -421,6 +422,17 @@ var RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
421
422
  */
422
423
  var RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE;
423
424
  // <- TODO: [🧜‍♂️]
425
+ /**
426
+ * @@@
427
+ *
428
+ * @public exported from `@promptbook/core`
429
+ */
430
+ var DEFAULT_CSV_SETTINGS = Object.freeze({
431
+ delimiter: ',',
432
+ quoteChar: '"',
433
+ newline: '\n',
434
+ skipEmptyLines: true,
435
+ });
424
436
  /**
425
437
  * @@@
426
438
  *
@@ -504,7 +516,7 @@ function pipelineJsonToString(pipelineJson) {
504
516
  commands.push("PIPELINE URL ".concat(pipelineUrl));
505
517
  }
506
518
  commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
507
- // TODO: !!! This increase size of the bundle and is probbably not necessary
519
+ // TODO: [main] !!! This increase size of the bundle and is probbably not necessary
508
520
  pipelineString = prettifyMarkdown(pipelineString);
509
521
  try {
510
522
  for (var _g = __values(parameters.filter(function (_a) {
@@ -652,12 +664,12 @@ function pipelineJsonToString(pipelineJson) {
652
664
  pipelineString += '```' + contentLanguage;
653
665
  pipelineString += '\n';
654
666
  pipelineString += spaceTrim$1(content);
655
- // <- TODO: !!! Escape
667
+ // <- TODO: [main] !!! Escape
656
668
  // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
657
669
  pipelineString += '\n';
658
670
  pipelineString += '```';
659
671
  pipelineString += '\n\n';
660
- pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: !!! If the parameter here has description, add it and use templateParameterJsonToString
672
+ pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: [main] !!! If the parameter here has description, add it and use templateParameterJsonToString
661
673
  }
662
674
  }
663
675
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -884,7 +896,7 @@ function forEachAsync(array, options, callbackfunction) {
884
896
  });
885
897
  }
886
898
 
887
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.69.0-6",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.69.0-6",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.69.0-6",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.69.0-6",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
899
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
888
900
 
889
901
  /**
890
902
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -959,7 +971,7 @@ function isValidPromptbookVersion(version) {
959
971
  if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
960
972
  return false;
961
973
  }
962
- // <- TODO: !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
974
+ // <- TODO: [main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
963
975
  return true;
964
976
  }
965
977
 
@@ -1128,7 +1140,7 @@ function validatePipelineCore(pipeline) {
1128
1140
  // <- Note: [🚲]
1129
1141
  throw new PipelineLogicError(spaceTrim(function (block) { return "\n Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1130
1142
  }
1131
- if (!isValidPromptbookVersion(pipeline.promptbookVersion)) {
1143
+ if (pipeline.promptbookVersion !== undefined && !isValidPromptbookVersion(pipeline.promptbookVersion)) {
1132
1144
  // <- Note: [🚲]
1133
1145
  throw new PipelineLogicError(spaceTrim(function (block) { return "\n Invalid Promptbook Version \"".concat(pipeline.promptbookVersion, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1134
1146
  }
@@ -1311,6 +1323,7 @@ function validatePipelineCore(pipeline) {
1311
1323
  }
1312
1324
  }
1313
1325
  /**
1326
+ * TODO: !!!!! [🧞‍♀️] Do not allow joker + foreach
1314
1327
  * TODO: [🧠] Work with promptbookVersion
1315
1328
  * TODO: Use here some json-schema, Zod or something similar and change it to:
1316
1329
  * > /**
@@ -1322,11 +1335,11 @@ function validatePipelineCore(pipeline) {
1322
1335
  * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
1323
1336
  */
1324
1337
  /**
1325
- * TODO: [🐣] !!!! Validate that all samples match expectations
1326
- * TODO: [🐣][🐝] !!!! Validate that knowledge is valid (non-void)
1327
- * TODO: [🐣] !!!! Validate that persona can be used only with CHAT variant
1328
- * TODO: [🐣] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1329
- * TODO: [🐣] !!!! Validate that reserved parameter is not used as joker
1338
+ * TODO: [🐣][main] !!!! Validate that all samples match expectations
1339
+ * TODO: [🐣][🐝][main] !!!! Validate that knowledge is valid (non-void)
1340
+ * TODO: [🐣][main] !!!! Validate that persona can be used only with CHAT variant
1341
+ * TODO: [🐣][main] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1342
+ * TODO: [🐣][main] !!!! Validate that reserved parameter is not used as joker
1330
1343
  * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists
1331
1344
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
1332
1345
  */
@@ -2070,7 +2083,7 @@ function isPipelinePrepared(pipeline) {
2070
2083
  return true;
2071
2084
  }
2072
2085
  /**
2073
- * TODO: [🔃] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2086
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2074
2087
  * TODO: [🐠] Maybe base this on `makeValidator`
2075
2088
  * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2076
2089
  * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
@@ -2079,6 +2092,23 @@ function isPipelinePrepared(pipeline) {
2079
2092
  * - [♨] Are templates prepared
2080
2093
  */
2081
2094
 
2095
+ /**
2096
+ * Serializes an error into a [🚉] JSON-serializable object
2097
+ *
2098
+ * @public exported from `@promptbook/utils`
2099
+ */
2100
+ function serializeError(error) {
2101
+ var name = error.name, message = error.message, stack = error.stack;
2102
+ if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2103
+ throw new UnexpectedError(spaceTrim$1(function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2104
+ }
2105
+ return {
2106
+ name: name,
2107
+ message: message,
2108
+ stack: stack,
2109
+ };
2110
+ }
2111
+
2082
2112
  /**
2083
2113
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2084
2114
  *
@@ -2105,9 +2135,10 @@ var MultipleLlmExecutionTools = /** @class */ (function () {
2105
2135
  });
2106
2136
  Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2107
2137
  get: function () {
2108
- return this.llmExecutionTools
2109
- .map(function (tools, index) { return "".concat(index + 1, ") ").concat(tools.title, " ").concat(tools.description || ''); })
2110
- .join('\n');
2138
+ return this.llmExecutionTools.map(function (_a, index) {
2139
+ var title = _a.title;
2140
+ return "".concat(index + 1, ") `").concat(title, "`");
2141
+ }).join('\n');
2111
2142
  },
2112
2143
  enumerable: false,
2113
2144
  configurable: true
@@ -2305,9 +2336,7 @@ var MultipleLlmExecutionTools = /** @class */ (function () {
2305
2336
  throw new PipelineExecutionError("You have not provided any `LlmExecutionTools`");
2306
2337
  }
2307
2338
  else {
2308
- throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.llmExecutionTools
2309
- .map(function (tools) { return "- ".concat(tools.title, " ").concat(tools.description || ''); })
2310
- .join('\n')), "\n\n "); }));
2339
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\"\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.description), "\n\n "); }));
2311
2340
  }
2312
2341
  }
2313
2342
  });
@@ -2372,23 +2401,6 @@ function joinLlmExecutionTools() {
2372
2401
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2373
2402
  */
2374
2403
 
2375
- /**
2376
- * Serializes an error into a [🚉] JSON-serializable object
2377
- *
2378
- * @public exported from `@promptbook/utils`
2379
- */
2380
- function serializeError(error) {
2381
- var name = error.name, message = error.message, stack = error.stack;
2382
- if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2383
- throw new UnexpectedError(spaceTrim$1(function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2384
- }
2385
- return {
2386
- name: name,
2387
- message: message,
2388
- stack: stack,
2389
- };
2390
- }
2391
-
2392
2404
  /**
2393
2405
  * Takes an item or an array of items and returns an array of items
2394
2406
  *
@@ -2466,47 +2478,47 @@ function extractVariables(script) {
2466
2478
  * @public exported from `@promptbook/utils`
2467
2479
  */
2468
2480
  function extractParameterNamesFromTemplate(template) {
2469
- var e_1, _a, e_2, _b, e_3, _c;
2481
+ var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
2470
2482
  var title = template.title, description = template.description, templateType = template.templateType, content = template.content, preparedContent = template.preparedContent, jokerParameterNames = template.jokerParameterNames, foreach = template.foreach;
2471
2483
  var parameterNames = new Set();
2472
2484
  try {
2473
- for (var _d = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _e = _d.next(); !_e.done; _e = _d.next()) {
2474
- var parameterName = _e.value;
2485
+ for (var _e = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _f = _e.next(); !_f.done; _f = _e.next()) {
2486
+ var parameterName = _f.value;
2475
2487
  parameterNames.add(parameterName);
2476
2488
  }
2477
2489
  }
2478
2490
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
2479
2491
  finally {
2480
2492
  try {
2481
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2493
+ if (_f && !_f.done && (_a = _e.return)) _a.call(_e);
2482
2494
  }
2483
2495
  finally { if (e_1) throw e_1.error; }
2484
2496
  }
2485
2497
  if (templateType === 'SCRIPT_TEMPLATE') {
2486
2498
  try {
2487
- for (var _f = __values(extractVariables(content)), _g = _f.next(); !_g.done; _g = _f.next()) {
2488
- var parameterName = _g.value;
2499
+ for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
2500
+ var parameterName = _h.value;
2489
2501
  parameterNames.add(parameterName);
2490
2502
  }
2491
2503
  }
2492
2504
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
2493
2505
  finally {
2494
2506
  try {
2495
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
2507
+ if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
2496
2508
  }
2497
2509
  finally { if (e_2) throw e_2.error; }
2498
2510
  }
2499
2511
  }
2500
2512
  try {
2501
- for (var _h = __values(jokerParameterNames || []), _j = _h.next(); !_j.done; _j = _h.next()) {
2502
- var jokerName = _j.value;
2513
+ for (var _j = __values(jokerParameterNames || []), _k = _j.next(); !_k.done; _k = _j.next()) {
2514
+ var jokerName = _k.value;
2503
2515
  parameterNames.add(jokerName);
2504
2516
  }
2505
2517
  }
2506
2518
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
2507
2519
  finally {
2508
2520
  try {
2509
- if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
2521
+ if (_k && !_k.done && (_c = _j.return)) _c.call(_j);
2510
2522
  }
2511
2523
  finally { if (e_3) throw e_3.error; }
2512
2524
  }
@@ -2514,10 +2526,22 @@ function extractParameterNamesFromTemplate(template) {
2514
2526
  // <- Note {websiteContent} is used in `preparedContent`
2515
2527
  // Note: [🍭] Fixing dependent subparameterName from FOREACH command
2516
2528
  if (foreach !== undefined) {
2517
- if (parameterNames.has(foreach.subparameterName)) {
2518
- parameterNames.delete(foreach.subparameterName);
2519
- parameterNames.add(foreach.parameterName);
2520
- // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2529
+ try {
2530
+ for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
2531
+ var subparameterName = _m.value;
2532
+ if (parameterNames.has(subparameterName)) {
2533
+ parameterNames.delete(subparameterName);
2534
+ parameterNames.add(foreach.parameterName);
2535
+ // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2536
+ }
2537
+ }
2538
+ }
2539
+ catch (e_4_1) { e_4 = { error: e_4_1 }; }
2540
+ finally {
2541
+ try {
2542
+ if (_m && !_m.done && (_d = _l.return)) _d.call(_l);
2543
+ }
2544
+ finally { if (e_4) throw e_4.error; }
2521
2545
  }
2522
2546
  }
2523
2547
  return parameterNames;
@@ -2527,188 +2551,656 @@ function extractParameterNamesFromTemplate(template) {
2527
2551
  */
2528
2552
 
2529
2553
  /**
2530
- * Function isValidJsonString will tell you if the string is valid JSON or not
2554
+ * Create difference set of two sets.
2531
2555
  *
2556
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2532
2557
  * @public exported from `@promptbook/utils`
2533
2558
  */
2534
- function isValidJsonString(value /* <- [👨‍⚖️] */) {
2559
+ function difference(a, b, isEqual) {
2560
+ var e_1, _a;
2561
+ if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
2562
+ var diff = new Set();
2563
+ var _loop_1 = function (itemA) {
2564
+ if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
2565
+ diff.add(itemA);
2566
+ }
2567
+ };
2535
2568
  try {
2536
- JSON.parse(value);
2537
- return true;
2538
- }
2539
- catch (error) {
2540
- if (!(error instanceof Error)) {
2541
- throw error;
2569
+ for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
2570
+ var itemA = _c.value;
2571
+ _loop_1(itemA);
2542
2572
  }
2543
- if (error.message.includes('Unexpected token')) {
2544
- return false;
2573
+ }
2574
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2575
+ finally {
2576
+ try {
2577
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2545
2578
  }
2546
- return false;
2579
+ finally { if (e_1) throw e_1.error; }
2547
2580
  }
2581
+ return diff;
2548
2582
  }
2583
+ /**
2584
+ * TODO: [🧠][💯] Maybe also implement symmetricDifference
2585
+ */
2549
2586
 
2550
2587
  /**
2551
- * Extracts all code blocks from markdown.
2552
- *
2553
- * Note: There are multiple simmilar function:
2554
- * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
2555
- * - `extractJsonBlock` extracts exactly one valid JSON code block
2556
- * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
2557
- * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
2588
+ * Creates a new set with all elements that are present in either set
2558
2589
  *
2559
- * @param markdown any valid markdown
2560
- * @returns code blocks with language and content
2561
- * @throws {ParseError} if block is not closed properly
2562
- * @public exported from `@promptbook/markdown-utils`
2590
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2591
+ * @public exported from `@promptbook/utils`
2563
2592
  */
2564
- function extractAllBlocksFromMarkdown(markdown) {
2565
- var e_1, _a;
2566
- var codeBlocks = [];
2567
- var lines = markdown.split('\n');
2568
- // Note: [0] Ensure that the last block notated by gt > will be closed
2569
- lines.push('');
2570
- var currentCodeBlock = null;
2593
+ function union() {
2594
+ var e_1, _a, e_2, _b;
2595
+ var sets = [];
2596
+ for (var _i = 0; _i < arguments.length; _i++) {
2597
+ sets[_i] = arguments[_i];
2598
+ }
2599
+ var union = new Set();
2571
2600
  try {
2572
- for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) {
2573
- var line = lines_1_1.value;
2574
- if (line.startsWith('> ') || line === '>') {
2575
- if (currentCodeBlock === null) {
2576
- currentCodeBlock = { blockNotation: '>', language: null, content: '' };
2577
- } /* not else */
2578
- if (currentCodeBlock.blockNotation === '>') {
2579
- if (currentCodeBlock.content !== '') {
2580
- currentCodeBlock.content += '\n';
2581
- }
2582
- currentCodeBlock.content += line.slice(2);
2583
- }
2584
- }
2585
- else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) {
2586
- codeBlocks.push(currentCodeBlock);
2587
- currentCodeBlock = null;
2588
- }
2589
- /* not else */
2590
- if (line.startsWith('```')) {
2591
- var language = line.slice(3).trim() || null;
2592
- if (currentCodeBlock === null) {
2593
- currentCodeBlock = { blockNotation: '```', language: language, content: '' };
2594
- }
2595
- else {
2596
- if (language !== null) {
2597
- throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
2598
- }
2599
- codeBlocks.push(currentCodeBlock);
2600
- currentCodeBlock = null;
2601
+ for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
2602
+ var set = sets_1_1.value;
2603
+ try {
2604
+ for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
2605
+ var item = _d.value;
2606
+ union.add(item);
2601
2607
  }
2602
2608
  }
2603
- else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') {
2604
- if (currentCodeBlock.content !== '') {
2605
- currentCodeBlock.content += '\n';
2609
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2610
+ finally {
2611
+ try {
2612
+ if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
2606
2613
  }
2607
- currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
2614
+ finally { if (e_2) throw e_2.error; }
2608
2615
  }
2609
2616
  }
2610
2617
  }
2611
2618
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
2612
2619
  finally {
2613
2620
  try {
2614
- if (lines_1_1 && !lines_1_1.done && (_a = lines_1.return)) _a.call(lines_1);
2621
+ if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
2615
2622
  }
2616
2623
  finally { if (e_1) throw e_1.error; }
2617
2624
  }
2618
- if (currentCodeBlock !== null) {
2619
- throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
2620
- }
2621
- return codeBlocks;
2625
+ return union;
2622
2626
  }
2623
- /**
2624
- * TODO: Maybe name for `blockNotation` instead of '```' and '>'
2625
- */
2626
2627
 
2627
2628
  /**
2628
- * Extracts extracts exactly one valid JSON code block
2629
- *
2630
- * - When given string is a valid JSON as it is, it just returns it
2631
- * - When there is no JSON code block the function throws a `ParseError`
2632
- * - When there are multiple JSON code blocks the function throws a `ParseError`
2629
+ * Just marks a place of place where should be something implemented
2630
+ * No side effects.
2633
2631
  *
2634
- * Note: It is not important if marked as ```json BUT if it is VALID JSON
2635
- * Note: There are multiple simmilar function:
2636
- * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
2637
- * - `extractJsonBlock` extracts exactly one valid JSON code block
2638
- * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
2639
- * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
2632
+ * Note: It can be usefull suppressing eslint errors of unused variables
2640
2633
  *
2641
- * @public exported from `@promptbook/markdown-utils`
2642
- * @throws {ParseError} if there is no valid JSON block in the markdown
2634
+ * @param value any values
2635
+ * @returns void
2636
+ * @private within the repository
2643
2637
  */
2644
- function extractJsonBlock(markdown) {
2645
- if (isValidJsonString(markdown)) {
2646
- return markdown;
2647
- }
2648
- var codeBlocks = extractAllBlocksFromMarkdown(markdown);
2649
- var jsonBlocks = codeBlocks.filter(function (_a) {
2650
- var content = _a.content;
2651
- return isValidJsonString(content);
2652
- });
2653
- if (jsonBlocks.length === 0) {
2654
- throw new Error('There is no valid JSON block in the markdown');
2655
- }
2656
- if (jsonBlocks.length > 1) {
2657
- throw new Error('There are multiple JSON code blocks in the markdown');
2638
+ function TODO_USE() {
2639
+ var value = [];
2640
+ for (var _i = 0; _i < arguments.length; _i++) {
2641
+ value[_i] = arguments[_i];
2658
2642
  }
2659
- return jsonBlocks[0].content;
2660
2643
  }
2661
- /**
2662
- * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
2663
- * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
2664
- */
2665
2644
 
2666
2645
  /**
2667
- * Just says that the variable is not used but should be kept
2668
- * No side effects.
2669
- *
2670
- * Note: It can be usefull for:
2646
+ * This error indicates problems parsing the format value
2671
2647
  *
2672
- * 1) Suppressing eager optimization of unused imports
2673
- * 2) Suppressing eslint errors of unused variables in the tests
2674
- * 3) Keeping the type of the variable for type testing
2648
+ * For example, when the format value is not a valid JSON or CSV
2649
+ * This is not thrown directly but in extended classes
2675
2650
  *
2676
- * @param value any values
2677
- * @returns void
2678
- * @private within the repository
2651
+ * @public exported from `@promptbook/core`
2679
2652
  */
2680
- function keepUnused() {
2681
- var valuesToKeep = [];
2682
- for (var _i = 0; _i < arguments.length; _i++) {
2683
- valuesToKeep[_i] = arguments[_i];
2653
+ var AbstractFormatError = /** @class */ (function (_super) {
2654
+ __extends(AbstractFormatError, _super);
2655
+ // Note: To allow instanceof do not put here error `name`
2656
+ // public readonly name = 'AbstractFormatError';
2657
+ function AbstractFormatError(message) {
2658
+ var _this = _super.call(this, message) || this;
2659
+ Object.setPrototypeOf(_this, AbstractFormatError.prototype);
2660
+ return _this;
2684
2661
  }
2685
- }
2662
+ return AbstractFormatError;
2663
+ }(Error));
2686
2664
 
2687
2665
  /**
2688
- * Replaces parameters in template with values from parameters object
2666
+ * This error indicates problem with parsing of CSV
2689
2667
  *
2690
- * @param template the template with parameters in {curly} braces
2691
- * @param parameters the object with parameters
2692
- * @returns the template with replaced parameters
2693
- * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened
2694
- * @public exported from `@promptbook/utils`
2668
+ * @public exported from `@promptbook/core`
2695
2669
  */
2696
- function replaceParameters(template, parameters) {
2697
- var e_1, _a;
2698
- try {
2699
- for (var _b = __values(Object.entries(parameters)), _c = _b.next(); !_c.done; _c = _b.next()) {
2700
- var _d = __read(_c.value, 2), parameterName = _d[0], parameterValue = _d[1];
2701
- if (parameterValue === RESERVED_PARAMETER_MISSING_VALUE) {
2702
- throw new UnexpectedError("Parameter {".concat(parameterName, "} has missing value"));
2703
- }
2704
- else if (parameterValue === RESERVED_PARAMETER_RESTRICTED) {
2705
- // TODO: [🍵]
2706
- throw new UnexpectedError("Parameter {".concat(parameterName, "} is restricted to use"));
2707
- }
2708
- }
2670
+ var CsvFormatError = /** @class */ (function (_super) {
2671
+ __extends(CsvFormatError, _super);
2672
+ function CsvFormatError(message) {
2673
+ var _this = _super.call(this, message) || this;
2674
+ _this.name = 'CsvFormatError';
2675
+ Object.setPrototypeOf(_this, CsvFormatError.prototype);
2676
+ return _this;
2709
2677
  }
2710
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2711
- finally {
2678
+ return CsvFormatError;
2679
+ }(AbstractFormatError));
2680
+
2681
+ /**
2682
+ * @@@
2683
+ *
2684
+ * @public exported from `@promptbook/core`
2685
+ */
2686
+ var MANDATORY_CSV_SETTINGS = Object.freeze({
2687
+ header: true,
2688
+ // encoding: 'utf8',
2689
+ });
2690
+
2691
+ /**
2692
+ * Definition for CSV spreadsheet
2693
+ *
2694
+ * @public exported from `@promptbook/core`
2695
+ * <- TODO: [🏢] Export from package `@promptbook/csv`
2696
+ */
2697
+ var CsvFormatDefinition = {
2698
+ formatName: 'CSV',
2699
+ aliases: ['SPREADSHEET', 'TABLE'],
2700
+ isValid: function (value, settings, schema) {
2701
+ // TODO: Implement CSV validation
2702
+ TODO_USE(value /* <- TODO: Use value here */);
2703
+ TODO_USE(settings /* <- TODO: Use settings here */);
2704
+ TODO_USE(schema /* <- TODO: Use schema here */);
2705
+ return true;
2706
+ },
2707
+ canBeValid: function (partialValue, settings, schema) {
2708
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2709
+ TODO_USE(settings /* <- TODO: Use settings here */);
2710
+ TODO_USE(schema /* <- TODO: Use schema here */);
2711
+ return true;
2712
+ },
2713
+ heal: function (value, settings, schema) {
2714
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2715
+ TODO_USE(settings /* <- TODO: Use settings here */);
2716
+ TODO_USE(schema /* <- TODO: Use schema here */);
2717
+ throw new Error('Not implemented');
2718
+ },
2719
+ subvalueDefinitions: [
2720
+ {
2721
+ subvalueName: 'ROW',
2722
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2723
+ return __awaiter(this, void 0, void 0, function () {
2724
+ var csv, mappedData;
2725
+ var _this = this;
2726
+ return __generator(this, function (_a) {
2727
+ switch (_a.label) {
2728
+ case 0:
2729
+ csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2730
+ if (csv.errors.length !== 0) {
2731
+ throw new CsvFormatError(spaceTrim$1(function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2732
+ }
2733
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
2734
+ var _a, _b;
2735
+ var _c;
2736
+ return __generator(this, function (_d) {
2737
+ switch (_d.label) {
2738
+ case 0:
2739
+ if (row[outputParameterName]) {
2740
+ throw new CsvFormatError("Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
2741
+ }
2742
+ _a = [__assign({}, row)];
2743
+ _c = {};
2744
+ _b = outputParameterName;
2745
+ return [4 /*yield*/, mapCallback(row, index)];
2746
+ case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
2747
+ }
2748
+ });
2749
+ }); }))];
2750
+ case 1:
2751
+ mappedData = _a.sent();
2752
+ return [2 /*return*/, unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2753
+ }
2754
+ });
2755
+ });
2756
+ },
2757
+ },
2758
+ {
2759
+ subvalueName: 'CELL',
2760
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2761
+ return __awaiter(this, void 0, void 0, function () {
2762
+ var csv, mappedData;
2763
+ var _this = this;
2764
+ return __generator(this, function (_a) {
2765
+ switch (_a.label) {
2766
+ case 0:
2767
+ csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2768
+ if (csv.errors.length !== 0) {
2769
+ throw new CsvFormatError(spaceTrim$1(function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2770
+ }
2771
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
2772
+ var _this = this;
2773
+ return __generator(this, function (_a) {
2774
+ return [2 /*return*/, /* not await */ Promise.all(Object.entries(row).map(function (_a, columnIndex) {
2775
+ var _b = __read(_a, 2), key = _b[0], value = _b[1];
2776
+ return __awaiter(_this, void 0, void 0, function () {
2777
+ var index;
2778
+ var _c;
2779
+ return __generator(this, function (_d) {
2780
+ index = rowIndex * Object.keys(row).length + columnIndex;
2781
+ return [2 /*return*/, /* not await */ mapCallback((_c = {}, _c[key] = value, _c), index)];
2782
+ });
2783
+ });
2784
+ }))];
2785
+ });
2786
+ }); }))];
2787
+ case 1:
2788
+ mappedData = _a.sent();
2789
+ return [2 /*return*/, unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2790
+ }
2791
+ });
2792
+ });
2793
+ },
2794
+ },
2795
+ ],
2796
+ };
2797
+ /**
2798
+ * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
2799
+ * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
2800
+ * TODO: [🍓] In `CsvFormatDefinition` implement `heal
2801
+ * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
2802
+ * TODO: [🏢] Allow to expect something inside CSV objects and other formats
2803
+ */
2804
+
2805
+ /**
2806
+ * Function isValidJsonString will tell you if the string is valid JSON or not
2807
+ *
2808
+ * @public exported from `@promptbook/utils`
2809
+ */
2810
+ function isValidJsonString(value /* <- [👨‍⚖️] */) {
2811
+ try {
2812
+ JSON.parse(value);
2813
+ return true;
2814
+ }
2815
+ catch (error) {
2816
+ if (!(error instanceof Error)) {
2817
+ throw error;
2818
+ }
2819
+ if (error.message.includes('Unexpected token')) {
2820
+ return false;
2821
+ }
2822
+ return false;
2823
+ }
2824
+ }
2825
+
2826
+ /**
2827
+ * Definition for JSON format
2828
+ *
2829
+ * @private still in development [🏢]
2830
+ */
2831
+ var JsonFormatDefinition = {
2832
+ formatName: 'JSON',
2833
+ mimeType: 'application/json',
2834
+ isValid: function (value, settings, schema) {
2835
+ TODO_USE(schema /* <- TODO: Use schema here */);
2836
+ TODO_USE(settings /* <- TODO: Use settings here */);
2837
+ return isValidJsonString(value);
2838
+ },
2839
+ canBeValid: function (partialValue, settings, schema) {
2840
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2841
+ TODO_USE(settings /* <- TODO: Use settings here */);
2842
+ TODO_USE(schema /* <- TODO: Use schema here */);
2843
+ return true;
2844
+ },
2845
+ heal: function (value, settings, schema) {
2846
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2847
+ TODO_USE(settings /* <- TODO: Use settings here */);
2848
+ TODO_USE(schema /* <- TODO: Use schema here */);
2849
+ throw new Error('Not implemented');
2850
+ },
2851
+ subvalueDefinitions: [],
2852
+ };
2853
+ /**
2854
+ * TODO: [🧠] Maybe propper instance of object
2855
+ * TODO: [0] Make string_serialized_json
2856
+ * TODO: [1] Make type for JSON Settings and Schema
2857
+ * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
2858
+ * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
2859
+ * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
2860
+ * TODO: [🍓] In `JsonFormatDefinition` implement `heal
2861
+ * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
2862
+ * TODO: [🏢] Allow to expect something inside JSON objects and other formats
2863
+ */
2864
+
2865
+ /**
2866
+ * Definition for any text - this will be always valid
2867
+ *
2868
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
2869
+ *
2870
+ * @public exported from `@promptbook/core`
2871
+ */
2872
+ var TextFormatDefinition = {
2873
+ formatName: 'TEXT',
2874
+ isValid: function (value) {
2875
+ return typeof value === 'string';
2876
+ },
2877
+ canBeValid: function (partialValue) {
2878
+ return typeof partialValue === 'string';
2879
+ },
2880
+ heal: function () {
2881
+ throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
2882
+ },
2883
+ subvalueDefinitions: [
2884
+ {
2885
+ subvalueName: 'LINE',
2886
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2887
+ return __awaiter(this, void 0, void 0, function () {
2888
+ var lines, mappedLines;
2889
+ return __generator(this, function (_a) {
2890
+ switch (_a.label) {
2891
+ case 0:
2892
+ lines = value.split('\n');
2893
+ return [4 /*yield*/, Promise.all(lines.map(function (lineContent, lineNumber) {
2894
+ // TODO: [🧠] Maybe option to skip empty line
2895
+ /* not await */ return mapCallback({
2896
+ lineContent: lineContent,
2897
+ // TODO: [🧠] Maybe also put here `lineNumber`
2898
+ }, lineNumber);
2899
+ }))];
2900
+ case 1:
2901
+ mappedLines = _a.sent();
2902
+ return [2 /*return*/, mappedLines.join('\n')];
2903
+ }
2904
+ });
2905
+ });
2906
+ },
2907
+ },
2908
+ // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2909
+ ],
2910
+ };
2911
+ /**
2912
+ * TODO: [1] Make type for XML Text and Schema
2913
+ * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2914
+ * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
2915
+ * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
2916
+ * TODO: [🍓] In `TextFormatDefinition` implement `heal
2917
+ * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
2918
+ * TODO: [🏢] Allow to expect something inside each item of list and other formats
2919
+ */
2920
+
2921
+ /**
2922
+ * Definition for XML format
2923
+ *
2924
+ * @private still in development [🏢]
2925
+ */
2926
+ var XmlFormatDefinition = {
2927
+ formatName: 'XML',
2928
+ mimeType: 'application/xml',
2929
+ isValid: function (value, settings, schema) {
2930
+ TODO_USE(value /* <- TODO: Use value here */);
2931
+ TODO_USE(settings /* <- TODO: Use settings here */);
2932
+ TODO_USE(schema /* <- TODO: Use schema here */);
2933
+ return true;
2934
+ },
2935
+ canBeValid: function (partialValue, settings, schema) {
2936
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2937
+ TODO_USE(settings /* <- TODO: Use settings here */);
2938
+ TODO_USE(schema /* <- TODO: Use schema here */);
2939
+ return true;
2940
+ },
2941
+ heal: function (value, settings, schema) {
2942
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2943
+ TODO_USE(settings /* <- TODO: Use settings here */);
2944
+ TODO_USE(schema /* <- TODO: Use schema here */);
2945
+ throw new Error('Not implemented');
2946
+ },
2947
+ subvalueDefinitions: [],
2948
+ };
2949
+ /**
2950
+ * TODO: [🧠] Maybe propper instance of object
2951
+ * TODO: [0] Make string_serialized_xml
2952
+ * TODO: [1] Make type for XML Settings and Schema
2953
+ * TODO: [🧠] What to use for validating XMLs - XSD,...
2954
+ * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
2955
+ * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
2956
+ * TODO: [🍓] In `XmlFormatDefinition` implement `heal
2957
+ * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
2958
+ * TODO: [🏢] Allow to expect something inside XML and other formats
2959
+ */
2960
+
2961
+ /**
2962
+ * Definitions for all formats supported by Promptbook
2963
+ *
2964
+ * @private internal index of `...` <- TODO [🏢]
2965
+ */
2966
+ var FORMAT_DEFINITIONS = [
2967
+ JsonFormatDefinition,
2968
+ XmlFormatDefinition,
2969
+ TextFormatDefinition,
2970
+ CsvFormatDefinition,
2971
+ ];
2972
+
2973
+ /**
2974
+ * Maps available parameters to expected parameters
2975
+ *
2976
+ * The strategy is:
2977
+ * 1) @@@
2978
+ * 2) @@@
2979
+ *
2980
+ * @throws {PipelineExecutionError} @@@
2981
+ * @private within the repository used in `createPipelineExecutor`
2982
+ */
2983
+ function mapAvailableToExpectedParameters(options) {
2984
+ var e_1, _a;
2985
+ var expectedParameters = options.expectedParameters, availableParameters = options.availableParameters;
2986
+ var availableParametersNames = new Set(Object.keys(availableParameters));
2987
+ var expectedParameterNames = new Set(Object.keys(expectedParameters));
2988
+ var mappedParameters = {};
2989
+ try {
2990
+ // Phase 1️⃣: Matching mapping
2991
+ for (var _b = __values(Array.from(union(availableParametersNames, expectedParameterNames))), _c = _b.next(); !_c.done; _c = _b.next()) {
2992
+ var parameterName = _c.value;
2993
+ // Situation: Parameter is available and expected
2994
+ if (availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
2995
+ mappedParameters[parameterName] = availableParameters[parameterName];
2996
+ // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here?
2997
+ availableParametersNames.delete(parameterName);
2998
+ expectedParameterNames.delete(parameterName);
2999
+ }
3000
+ // Situation: Parameter is available but NOT expected
3001
+ else if (availableParametersNames.has(parameterName) && !expectedParameterNames.has(parameterName)) {
3002
+ // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping
3003
+ }
3004
+ // Situation: Parameter is NOT available BUT expected
3005
+ else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
3006
+ // Do nothing here - this will be maybe fixed in the non-matching mapping
3007
+ }
3008
+ }
3009
+ }
3010
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3011
+ finally {
3012
+ try {
3013
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3014
+ }
3015
+ finally { if (e_1) throw e_1.error; }
3016
+ }
3017
+ if (expectedParameterNames.size === 0) {
3018
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3019
+ Object.freeze(mappedParameters);
3020
+ return mappedParameters;
3021
+ }
3022
+ // Phase 2️⃣: Non-matching mapping
3023
+ if (expectedParameterNames.size !== availableParametersNames.size) {
3024
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n Can not map available parameters to expected parameters\n\n Mapped parameters:\n ".concat(block(Object.keys(mappedParameters)
3025
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3026
+ .join('\n')), "\n\n Expected parameters which can not be mapped:\n ").concat(block(Array.from(expectedParameterNames)
3027
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3028
+ .join('\n')), "\n\n Remaining available parameters:\n ").concat(block(Array.from(availableParametersNames)
3029
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
3030
+ .join('\n')), "\n\n "); }));
3031
+ }
3032
+ var expectedParameterNamesArray = Array.from(expectedParameterNames);
3033
+ var availableParametersNamesArray = Array.from(availableParametersNames);
3034
+ for (var i = 0; i < expectedParameterNames.size; i++) {
3035
+ mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
3036
+ }
3037
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3038
+ Object.freeze(mappedParameters);
3039
+ return mappedParameters;
3040
+ }
3041
+
3042
+ /**
3043
+ * Extracts all code blocks from markdown.
3044
+ *
3045
+ * Note: There are multiple simmilar function:
3046
+ * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
3047
+ * - `extractJsonBlock` extracts exactly one valid JSON code block
3048
+ * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
3049
+ * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
3050
+ *
3051
+ * @param markdown any valid markdown
3052
+ * @returns code blocks with language and content
3053
+ * @throws {ParseError} if block is not closed properly
3054
+ * @public exported from `@promptbook/markdown-utils`
3055
+ */
3056
+ function extractAllBlocksFromMarkdown(markdown) {
3057
+ var e_1, _a;
3058
+ var codeBlocks = [];
3059
+ var lines = markdown.split('\n');
3060
+ // Note: [0] Ensure that the last block notated by gt > will be closed
3061
+ lines.push('');
3062
+ var currentCodeBlock = null;
3063
+ try {
3064
+ for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) {
3065
+ var line = lines_1_1.value;
3066
+ if (line.startsWith('> ') || line === '>') {
3067
+ if (currentCodeBlock === null) {
3068
+ currentCodeBlock = { blockNotation: '>', language: null, content: '' };
3069
+ } /* not else */
3070
+ if (currentCodeBlock.blockNotation === '>') {
3071
+ if (currentCodeBlock.content !== '') {
3072
+ currentCodeBlock.content += '\n';
3073
+ }
3074
+ currentCodeBlock.content += line.slice(2);
3075
+ }
3076
+ }
3077
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) {
3078
+ codeBlocks.push(currentCodeBlock);
3079
+ currentCodeBlock = null;
3080
+ }
3081
+ /* not else */
3082
+ if (line.startsWith('```')) {
3083
+ var language = line.slice(3).trim() || null;
3084
+ if (currentCodeBlock === null) {
3085
+ currentCodeBlock = { blockNotation: '```', language: language, content: '' };
3086
+ }
3087
+ else {
3088
+ if (language !== null) {
3089
+ throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
3090
+ }
3091
+ codeBlocks.push(currentCodeBlock);
3092
+ currentCodeBlock = null;
3093
+ }
3094
+ }
3095
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') {
3096
+ if (currentCodeBlock.content !== '') {
3097
+ currentCodeBlock.content += '\n';
3098
+ }
3099
+ currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
3100
+ }
3101
+ }
3102
+ }
3103
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3104
+ finally {
3105
+ try {
3106
+ if (lines_1_1 && !lines_1_1.done && (_a = lines_1.return)) _a.call(lines_1);
3107
+ }
3108
+ finally { if (e_1) throw e_1.error; }
3109
+ }
3110
+ if (currentCodeBlock !== null) {
3111
+ throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
3112
+ }
3113
+ return codeBlocks;
3114
+ }
3115
+ /**
3116
+ * TODO: Maybe name for `blockNotation` instead of '```' and '>'
3117
+ */
3118
+
3119
+ /**
3120
+ * Extracts extracts exactly one valid JSON code block
3121
+ *
3122
+ * - When given string is a valid JSON as it is, it just returns it
3123
+ * - When there is no JSON code block the function throws a `ParseError`
3124
+ * - When there are multiple JSON code blocks the function throws a `ParseError`
3125
+ *
3126
+ * Note: It is not important if marked as ```json BUT if it is VALID JSON
3127
+ * Note: There are multiple simmilar function:
3128
+ * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
3129
+ * - `extractJsonBlock` extracts exactly one valid JSON code block
3130
+ * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
3131
+ * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
3132
+ *
3133
+ * @public exported from `@promptbook/markdown-utils`
3134
+ * @throws {ParseError} if there is no valid JSON block in the markdown
3135
+ */
3136
+ function extractJsonBlock(markdown) {
3137
+ if (isValidJsonString(markdown)) {
3138
+ return markdown;
3139
+ }
3140
+ var codeBlocks = extractAllBlocksFromMarkdown(markdown);
3141
+ var jsonBlocks = codeBlocks.filter(function (_a) {
3142
+ var content = _a.content;
3143
+ return isValidJsonString(content);
3144
+ });
3145
+ if (jsonBlocks.length === 0) {
3146
+ throw new Error('There is no valid JSON block in the markdown');
3147
+ }
3148
+ if (jsonBlocks.length > 1) {
3149
+ throw new Error('There are multiple JSON code blocks in the markdown');
3150
+ }
3151
+ return jsonBlocks[0].content;
3152
+ }
3153
+ /**
3154
+ * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
3155
+ * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
3156
+ */
3157
+
3158
+ /**
3159
+ * Just says that the variable is not used but should be kept
3160
+ * No side effects.
3161
+ *
3162
+ * Note: It can be usefull for:
3163
+ *
3164
+ * 1) Suppressing eager optimization of unused imports
3165
+ * 2) Suppressing eslint errors of unused variables in the tests
3166
+ * 3) Keeping the type of the variable for type testing
3167
+ *
3168
+ * @param value any values
3169
+ * @returns void
3170
+ * @private within the repository
3171
+ */
3172
+ function keepUnused() {
3173
+ var valuesToKeep = [];
3174
+ for (var _i = 0; _i < arguments.length; _i++) {
3175
+ valuesToKeep[_i] = arguments[_i];
3176
+ }
3177
+ }
3178
+
3179
+ /**
3180
+ * Replaces parameters in template with values from parameters object
3181
+ *
3182
+ * @param template the template with parameters in {curly} braces
3183
+ * @param parameters the object with parameters
3184
+ * @returns the template with replaced parameters
3185
+ * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened
3186
+ * @public exported from `@promptbook/utils`
3187
+ */
3188
+ function replaceParameters(template, parameters) {
3189
+ var e_1, _a;
3190
+ try {
3191
+ for (var _b = __values(Object.entries(parameters)), _c = _b.next(); !_c.done; _c = _b.next()) {
3192
+ var _d = __read(_c.value, 2), parameterName = _d[0], parameterValue = _d[1];
3193
+ if (parameterValue === RESERVED_PARAMETER_MISSING_VALUE) {
3194
+ throw new UnexpectedError("Parameter {".concat(parameterName, "} has missing value"));
3195
+ }
3196
+ else if (parameterValue === RESERVED_PARAMETER_RESTRICTED) {
3197
+ // TODO: [🍵]
3198
+ throw new UnexpectedError("Parameter {".concat(parameterName, "} is restricted to use"));
3199
+ }
3200
+ }
3201
+ }
3202
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3203
+ finally {
2712
3204
  try {
2713
3205
  if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2714
3206
  }
@@ -2763,81 +3255,6 @@ function replaceParameters(template, parameters) {
2763
3255
  return replacedTemplate;
2764
3256
  }
2765
3257
 
2766
- /**
2767
- * Create difference set of two sets.
2768
- *
2769
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2770
- * @public exported from `@promptbook/utils`
2771
- */
2772
- function difference(a, b, isEqual) {
2773
- var e_1, _a;
2774
- if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
2775
- var diff = new Set();
2776
- var _loop_1 = function (itemA) {
2777
- if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
2778
- diff.add(itemA);
2779
- }
2780
- };
2781
- try {
2782
- for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
2783
- var itemA = _c.value;
2784
- _loop_1(itemA);
2785
- }
2786
- }
2787
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2788
- finally {
2789
- try {
2790
- if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2791
- }
2792
- finally { if (e_1) throw e_1.error; }
2793
- }
2794
- return diff;
2795
- }
2796
- /**
2797
- * TODO: [🧠][💯] Maybe also implement symmetricDifference
2798
- */
2799
-
2800
- /**
2801
- * Creates a new set with all elements that are present in either set
2802
- *
2803
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2804
- * @public exported from `@promptbook/utils`
2805
- */
2806
- function union() {
2807
- var e_1, _a, e_2, _b;
2808
- var sets = [];
2809
- for (var _i = 0; _i < arguments.length; _i++) {
2810
- sets[_i] = arguments[_i];
2811
- }
2812
- var union = new Set();
2813
- try {
2814
- for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
2815
- var set = sets_1_1.value;
2816
- try {
2817
- for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
2818
- var item = _d.value;
2819
- union.add(item);
2820
- }
2821
- }
2822
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
2823
- finally {
2824
- try {
2825
- if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
2826
- }
2827
- finally { if (e_2) throw e_2.error; }
2828
- }
2829
- }
2830
- }
2831
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2832
- finally {
2833
- try {
2834
- if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
2835
- }
2836
- finally { if (e_1) throw e_1.error; }
2837
- }
2838
- return union;
2839
- }
2840
-
2841
3258
  /**
2842
3259
  * Counts number of characters in the text
2843
3260
  *
@@ -2927,6 +3344,9 @@ var CountUtils = {
2927
3344
  LINES: countLines,
2928
3345
  PAGES: countPages,
2929
3346
  };
3347
+ /**
3348
+ * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
3349
+ */
2930
3350
 
2931
3351
  /**
2932
3352
  * Function checkExpectations will check if the expectations on given value are met
@@ -2948,447 +3368,250 @@ function checkExpectations(expectations, value) {
2948
3368
  if (min && amount < min) {
2949
3369
  throw new ExpectError("Expected at least ".concat(min, " ").concat(unit, " but got ").concat(amount));
2950
3370
  } /* not else */
2951
- if (max && amount > max) {
2952
- throw new ExpectError("Expected at most ".concat(max, " ").concat(unit, " but got ").concat(amount));
2953
- }
2954
- }
2955
- }
2956
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2957
- finally {
2958
- try {
2959
- if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2960
- }
2961
- finally { if (e_1) throw e_1.error; }
2962
- }
2963
- }
2964
- /**
2965
- * TODO: [💝] Unite object for expecting amount and format
2966
- */
2967
-
2968
- /**
2969
- * Just marks a place of place where should be something implemented
2970
- * No side effects.
2971
- *
2972
- * Note: It can be usefull suppressing eslint errors of unused variables
2973
- *
2974
- * @param value any values
2975
- * @returns void
2976
- * @private within the repository
2977
- */
2978
- function TODO_USE() {
2979
- var value = [];
2980
- for (var _i = 0; _i < arguments.length; _i++) {
2981
- value[_i] = arguments[_i];
2982
- }
2983
- }
2984
-
2985
- /**
2986
- * @@@
2987
- *
2988
- * @private internal utility of `createPipelineExecutor`
2989
- */
2990
- function getContextForTemplate(template) {
2991
- return __awaiter(this, void 0, void 0, function () {
2992
- return __generator(this, function (_a) {
2993
- TODO_USE(template);
2994
- return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */];
2995
- });
2996
- });
2997
- }
2998
-
2999
- /**
3000
- * @@@
3001
- *
3002
- * @private internal utility of `createPipelineExecutor`
3003
- */
3004
- function getKnowledgeForTemplate(options) {
3005
- return __awaiter(this, void 0, void 0, function () {
3006
- var preparedPipeline, template;
3007
- return __generator(this, function (_a) {
3008
- preparedPipeline = options.preparedPipeline, template = options.template;
3009
- // TODO: [♨] Implement Better - use real index and keyword search from `template` and {samples}
3010
- TODO_USE(template);
3011
- return [2 /*return*/, preparedPipeline.knowledgePieces.map(function (_a) {
3012
- var content = _a.content;
3013
- return "- ".concat(content);
3014
- }).join('\n')];
3015
- });
3016
- });
3017
- }
3018
-
3019
- /**
3020
- * @@@
3021
- *
3022
- * @private internal utility of `createPipelineExecutor`
3023
- */
3024
- function getSamplesForTemplate(template) {
3025
- return __awaiter(this, void 0, void 0, function () {
3026
- return __generator(this, function (_a) {
3027
- // TODO: [♨] Implement Better - use real index and keyword search
3028
- TODO_USE(template);
3029
- return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */];
3030
- });
3031
- });
3032
- }
3033
-
3034
- /**
3035
- * @@@
3036
- *
3037
- * @private internal utility of `createPipelineExecutor`
3038
- */
3039
- function getReservedParametersForTemplate(options) {
3040
- return __awaiter(this, void 0, void 0, function () {
3041
- var preparedPipeline, template, pipelineIdentification, context, knowledge, samples, currentDate, modelName, reservedParameters, _loop_1, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName;
3042
- var e_1, _a;
3043
- return __generator(this, function (_b) {
3044
- switch (_b.label) {
3045
- case 0:
3046
- preparedPipeline = options.preparedPipeline, template = options.template, pipelineIdentification = options.pipelineIdentification;
3047
- return [4 /*yield*/, getContextForTemplate(template)];
3048
- case 1:
3049
- context = _b.sent();
3050
- return [4 /*yield*/, getKnowledgeForTemplate({ preparedPipeline: preparedPipeline, template: template })];
3051
- case 2:
3052
- knowledge = _b.sent();
3053
- return [4 /*yield*/, getSamplesForTemplate(template)];
3054
- case 3:
3055
- samples = _b.sent();
3056
- currentDate = new Date().toISOString();
3057
- modelName = RESERVED_PARAMETER_MISSING_VALUE;
3058
- reservedParameters = {
3059
- content: RESERVED_PARAMETER_RESTRICTED,
3060
- context: context,
3061
- knowledge: knowledge,
3062
- samples: samples,
3063
- currentDate: currentDate,
3064
- modelName: modelName,
3065
- };
3066
- _loop_1 = function (parameterName) {
3067
- if (reservedParameters[parameterName] === undefined) {
3068
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Reserved parameter {".concat(parameterName, "} is not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3069
- }
3070
- };
3071
- try {
3072
- // Note: Doublecheck that ALL reserved parameters are defined:
3073
- for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
3074
- parameterName = RESERVED_PARAMETER_NAMES_1_1.value;
3075
- _loop_1(parameterName);
3076
- }
3077
- }
3078
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
3079
- finally {
3080
- try {
3081
- if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1);
3082
- }
3083
- finally { if (e_1) throw e_1.error; }
3084
- }
3085
- return [2 /*return*/, reservedParameters];
3371
+ if (max && amount > max) {
3372
+ throw new ExpectError("Expected at most ".concat(max, " ").concat(unit, " but got ").concat(amount));
3086
3373
  }
3087
- });
3088
- });
3374
+ }
3375
+ }
3376
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3377
+ finally {
3378
+ try {
3379
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3380
+ }
3381
+ finally { if (e_1) throw e_1.error; }
3382
+ }
3089
3383
  }
3384
+ /**
3385
+ * TODO: [💝] Unite object for expecting amount and format
3386
+ * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
3387
+ * Note: [💝] and [🤠] are interconnected together
3388
+ */
3090
3389
 
3091
3390
  /**
3092
3391
  * @@@
3093
3392
  *
3094
3393
  * @private internal utility of `createPipelineExecutor`
3095
3394
  */
3096
- function executeTemplate(options) {
3395
+ function executeAttempts(options) {
3097
3396
  return __awaiter(this, void 0, void 0, function () {
3098
- var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, $ongoingResult, maxAttempts, jokerParameterNames, preparedContent, _loop_2, attempt, state_1;
3099
- var e_1, _f, _g;
3100
- return __generator(this, function (_h) {
3101
- switch (_h.label) {
3397
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
3398
+ return __generator(this, function (_a) {
3399
+ switch (_a.label) {
3102
3400
  case 0:
3103
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, llmTools = options.llmTools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3401
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3104
3402
  maxExecutionAttempts = settings.maxExecutionAttempts;
3105
- name = "pipeline-executor-frame-".concat(currentTemplate.name);
3106
- title = currentTemplate.title;
3107
- priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
3108
- return [4 /*yield*/, onProgress({
3109
- name: name,
3110
- title: title,
3111
- isStarted: false,
3112
- isDone: false,
3113
- templateType: currentTemplate.templateType,
3114
- parameterName: currentTemplate.resultingParameterName,
3115
- parameterValue: null,
3116
- // <- [🍸]
3117
- })];
3118
- case 1:
3119
- _h.sent();
3120
- usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
3121
- dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
3122
- if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
3123
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Dependent parameters:\n ").concat(Array.from(dependentParameterNames)
3124
- .map(function (name) { return "{".concat(name, "}"); })
3125
- .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames)
3126
- .map(function (name) { return "{".concat(name, "}"); })
3127
- .join(', '), "\n\n "); }));
3128
- }
3129
- _b = (_a = Object).freeze;
3130
- _c = [{}];
3131
- return [4 /*yield*/, getReservedParametersForTemplate({
3132
- preparedPipeline: preparedPipeline,
3133
- template: currentTemplate,
3134
- pipelineIdentification: pipelineIdentification,
3135
- })];
3136
- case 2:
3137
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
3138
- definedParameterNames = new Set(Object.keys(definedParameters));
3139
- parameters = {};
3140
- _loop_1 = function (parameterName) {
3141
- // Situation: Parameter is defined and used
3142
- if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3143
- parameters[parameterName] = definedParameters[parameterName];
3144
- }
3145
- // Situation: Parameter is defined but NOT used
3146
- else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) ;
3147
- // Situation: Parameter is NOT defined BUT used
3148
- else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3149
- // Houston, we have a problem
3150
- // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck
3151
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is NOT defined\n BUT used in template \"").concat(currentTemplate.title || currentTemplate.name, "\"\n\n This should be catched in `validatePipeline`\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3152
- }
3153
- };
3154
- try {
3155
- // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
3156
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
3157
- parameterName = _e.value;
3158
- _loop_1(parameterName);
3159
- }
3160
- }
3161
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
3162
- finally {
3163
- try {
3164
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
3165
- }
3166
- finally { if (e_1) throw e_1.error; }
3167
- }
3168
- // Note: Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
3169
- Object.freeze(parameters);
3170
- $ongoingResult = {
3403
+ $ongoingTemplateResult = {
3171
3404
  $result: null,
3172
3405
  $resultString: null,
3173
3406
  $expectError: null,
3174
3407
  $scriptPipelineExecutionErrors: [],
3175
3408
  };
3176
- maxAttempts = currentTemplate.templateType === 'DIALOG_TEMPLATE' ? Infinity : maxExecutionAttempts;
3177
- jokerParameterNames = currentTemplate.jokerParameterNames || [];
3178
- preparedContent = (currentTemplate.preparedContent || '{content}')
3179
- .split('{content}')
3180
- .join(currentTemplate.content);
3181
- _loop_2 = function (attempt) {
3182
- var isJokerAttempt, jokerParameterName, _j, modelRequirements, _k, _l, _m, _o, _p, _q, scriptTools, _r, error_1, e_2_1, _s, _t, _u, functionName, postprocessingError, _v, _w, scriptTools, _x, error_2, e_3_1, e_4_1, error_3;
3183
- var e_2, _y, e_4, _z, e_3, _0;
3184
- return __generator(this, function (_1) {
3185
- switch (_1.label) {
3409
+ _loop_1 = function (attempt) {
3410
+ var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
3411
+ var e_1, _q, e_3, _r, e_2, _s;
3412
+ return __generator(this, function (_t) {
3413
+ switch (_t.label) {
3186
3414
  case 0:
3187
3415
  isJokerAttempt = attempt < 0;
3188
3416
  jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
3189
- // TODO: [🧠] !!!!!! JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3417
+ // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3190
3418
  if (isJokerAttempt && !jokerParameterName) {
3191
3419
  throw new UnexpectedError(spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
3192
3420
  }
3193
- $ongoingResult.$result = null;
3194
- $ongoingResult.$resultString = null;
3195
- $ongoingResult.$expectError = null;
3421
+ $ongoingTemplateResult.$result = null;
3422
+ $ongoingTemplateResult.$resultString = null;
3423
+ $ongoingTemplateResult.$expectError = null;
3196
3424
  if (isJokerAttempt) {
3197
3425
  if (parameters[jokerParameterName] === undefined) {
3198
3426
  throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3199
3427
  // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
3200
3428
  }
3201
3429
  else {
3202
- $ongoingResult.$resultString = parameters[jokerParameterName];
3430
+ $ongoingTemplateResult.$resultString = parameters[jokerParameterName];
3203
3431
  }
3204
3432
  }
3205
- _1.label = 1;
3433
+ _t.label = 1;
3206
3434
  case 1:
3207
- _1.trys.push([1, 44, 45, 46]);
3208
- if (!!isJokerAttempt) return [3 /*break*/, 26];
3209
- _j = currentTemplate.templateType;
3210
- switch (_j) {
3435
+ _t.trys.push([1, 43, 44, 45]);
3436
+ if (!!isJokerAttempt) return [3 /*break*/, 25];
3437
+ _b = template.templateType;
3438
+ switch (_b) {
3211
3439
  case 'SIMPLE_TEMPLATE': return [3 /*break*/, 2];
3212
3440
  case 'PROMPT_TEMPLATE': return [3 /*break*/, 3];
3213
- case 'SCRIPT_TEMPLATE': return [3 /*break*/, 12];
3214
- case 'DIALOG_TEMPLATE': return [3 /*break*/, 23];
3441
+ case 'SCRIPT_TEMPLATE': return [3 /*break*/, 11];
3442
+ case 'DIALOG_TEMPLATE': return [3 /*break*/, 22];
3215
3443
  }
3216
- return [3 /*break*/, 25];
3444
+ return [3 /*break*/, 24];
3217
3445
  case 2:
3218
- $ongoingResult.$resultString = replaceParameters(preparedContent, parameters);
3219
- return [3 /*break*/, 26];
3446
+ $ongoingTemplateResult.$resultString = replaceParameters(preparedContent, parameters);
3447
+ return [3 /*break*/, 25];
3220
3448
  case 3:
3221
- modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (preparedPipeline.defaultModelRequirements || {})), (currentTemplate.modelRequirements || {}));
3222
- $ongoingResult.$prompt = {
3223
- title: currentTemplate.title,
3449
+ modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (preparedPipeline.defaultModelRequirements || {})), (template.modelRequirements || {}));
3450
+ $ongoingTemplateResult.$prompt = {
3451
+ title: template.title,
3224
3452
  pipelineUrl: "".concat(preparedPipeline.pipelineUrl
3225
3453
  ? preparedPipeline.pipelineUrl
3226
- : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(currentTemplate.name),
3454
+ : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name
3455
+ // <- TODO: Here should be maybe also subformat index to distinguish between same template with different subformat values
3456
+ ),
3227
3457
  parameters: parameters,
3228
3458
  content: preparedContent,
3229
3459
  modelRequirements: modelRequirements,
3230
3460
  expectations: __assign(__assign({}, (preparedPipeline.personas.find(function (_a) {
3231
3461
  var name = _a.name;
3232
- return name === currentTemplate.personaName;
3233
- }) || {})), currentTemplate.expectations),
3234
- format: currentTemplate.format,
3235
- postprocessingFunctionNames: currentTemplate.postprocessingFunctionNames,
3462
+ return name === template.personaName;
3463
+ }) ||
3464
+ {})), template.expectations),
3465
+ format: template.format,
3466
+ postprocessingFunctionNames: template.postprocessingFunctionNames,
3236
3467
  }; // <- TODO: Not very good type guard
3237
- _k = modelRequirements.modelVariant;
3238
- switch (_k) {
3468
+ _c = modelRequirements.modelVariant;
3469
+ switch (_c) {
3239
3470
  case 'CHAT': return [3 /*break*/, 4];
3240
3471
  case 'COMPLETION': return [3 /*break*/, 6];
3241
3472
  case 'EMBEDDING': return [3 /*break*/, 8];
3242
3473
  }
3243
- return [3 /*break*/, 10];
3474
+ return [3 /*break*/, 9];
3244
3475
  case 4:
3245
- _l = $ongoingResult;
3246
- return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingResult.$prompt))];
3476
+ _d = $ongoingTemplateResult;
3477
+ return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3247
3478
  case 5:
3248
- _l.$chatResult = _1.sent();
3479
+ _d.$chatResult = _t.sent();
3249
3480
  // TODO: [🍬] Destroy chatThread
3250
- $ongoingResult.$result = $ongoingResult.$chatResult;
3251
- $ongoingResult.$resultString = $ongoingResult.$chatResult.content;
3252
- return [3 /*break*/, 11];
3481
+ $ongoingTemplateResult.$result = $ongoingTemplateResult.$chatResult;
3482
+ $ongoingTemplateResult.$resultString = $ongoingTemplateResult.$chatResult.content;
3483
+ return [3 /*break*/, 10];
3253
3484
  case 6:
3254
- _m = $ongoingResult;
3255
- return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingResult.$prompt))];
3485
+ _e = $ongoingTemplateResult;
3486
+ return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3256
3487
  case 7:
3257
- _m.$completionResult = _1.sent();
3258
- $ongoingResult.$result = $ongoingResult.$completionResult;
3259
- $ongoingResult.$resultString = $ongoingResult.$completionResult.content;
3260
- return [3 /*break*/, 11];
3261
- case 8:
3262
- // TODO: [🧠] This is weird, embedding model can not be used such a way in the pipeline
3263
- _o = $ongoingResult;
3264
- return [4 /*yield*/, llmTools.callEmbeddingModel($deepFreeze($ongoingResult.$prompt))];
3265
- case 9:
3266
- // TODO: [🧠] This is weird, embedding model can not be used such a way in the pipeline
3267
- _o.$embeddingResult = _1.sent();
3268
- $ongoingResult.$result = $ongoingResult.$embeddingResult;
3269
- $ongoingResult.$resultString = $ongoingResult.$embeddingResult.content.join(',');
3270
- return [3 /*break*/, 11];
3271
- case 10: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown model variant \"".concat(currentTemplate.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3272
- case 11: return [3 /*break*/, 26];
3273
- case 12:
3488
+ _e.$completionResult = _t.sent();
3489
+ $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
3490
+ $ongoingTemplateResult.$resultString =
3491
+ $ongoingTemplateResult.$completionResult.content;
3492
+ return [3 /*break*/, 10];
3493
+ case 8: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Embedding model can not be used in pipeline\n\n This should be catched during parsing\n\n ".concat(block(pipelineIdentification), "\n\n "); }));
3494
+ case 9: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown model variant \"".concat(template.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3495
+ case 10: return [3 /*break*/, 25];
3496
+ case 11:
3274
3497
  if (arrayableToArray(tools.script).length === 0) {
3275
3498
  throw new PipelineExecutionError(spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3276
3499
  }
3277
- if (!currentTemplate.contentLanguage) {
3278
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(currentTemplate.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3500
+ if (!template.contentLanguage) {
3501
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(template.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3279
3502
  }
3280
- _1.label = 13;
3503
+ _t.label = 12;
3504
+ case 12:
3505
+ _t.trys.push([12, 19, 20, 21]);
3506
+ _f = (e_1 = void 0, __values(arrayableToArray(tools.script))), _g = _f.next();
3507
+ _t.label = 13;
3281
3508
  case 13:
3282
- _1.trys.push([13, 20, 21, 22]);
3283
- _p = (e_2 = void 0, __values(arrayableToArray(tools.script))), _q = _p.next();
3284
- _1.label = 14;
3509
+ if (!!_g.done) return [3 /*break*/, 18];
3510
+ scriptTools = _g.value;
3511
+ _t.label = 14;
3285
3512
  case 14:
3286
- if (!!_q.done) return [3 /*break*/, 19];
3287
- scriptTools = _q.value;
3288
- _1.label = 15;
3289
- case 15:
3290
- _1.trys.push([15, 17, , 18]);
3291
- _r = $ongoingResult;
3513
+ _t.trys.push([14, 16, , 17]);
3514
+ _h = $ongoingTemplateResult;
3292
3515
  return [4 /*yield*/, scriptTools.execute($deepFreeze({
3293
- scriptLanguage: currentTemplate.contentLanguage,
3516
+ scriptLanguage: template.contentLanguage,
3294
3517
  script: preparedContent,
3295
3518
  parameters: parameters,
3296
3519
  }))];
3520
+ case 15:
3521
+ _h.$resultString = _t.sent();
3522
+ return [3 /*break*/, 18];
3297
3523
  case 16:
3298
- _r.$resultString = _1.sent();
3299
- return [3 /*break*/, 19];
3300
- case 17:
3301
- error_1 = _1.sent();
3524
+ error_1 = _t.sent();
3302
3525
  if (!(error_1 instanceof Error)) {
3303
3526
  throw error_1;
3304
3527
  }
3305
3528
  if (error_1 instanceof UnexpectedError) {
3306
3529
  throw error_1;
3307
3530
  }
3308
- $ongoingResult.$scriptPipelineExecutionErrors.push(error_1);
3309
- return [3 /*break*/, 18];
3310
- case 18:
3311
- _q = _p.next();
3312
- return [3 /*break*/, 14];
3313
- case 19: return [3 /*break*/, 22];
3531
+ $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_1);
3532
+ return [3 /*break*/, 17];
3533
+ case 17:
3534
+ _g = _f.next();
3535
+ return [3 /*break*/, 13];
3536
+ case 18: return [3 /*break*/, 21];
3537
+ case 19:
3538
+ e_1_1 = _t.sent();
3539
+ e_1 = { error: e_1_1 };
3540
+ return [3 /*break*/, 21];
3314
3541
  case 20:
3315
- e_2_1 = _1.sent();
3316
- e_2 = { error: e_2_1 };
3317
- return [3 /*break*/, 22];
3318
- case 21:
3319
3542
  try {
3320
- if (_q && !_q.done && (_y = _p.return)) _y.call(_p);
3543
+ if (_g && !_g.done && (_q = _f.return)) _q.call(_f);
3321
3544
  }
3322
- finally { if (e_2) throw e_2.error; }
3545
+ finally { if (e_1) throw e_1.error; }
3323
3546
  return [7 /*endfinally*/];
3324
- case 22:
3325
- if ($ongoingResult.$resultString !== null) {
3326
- return [3 /*break*/, 26];
3547
+ case 21:
3548
+ if ($ongoingTemplateResult.$resultString !== null) {
3549
+ return [3 /*break*/, 25];
3327
3550
  }
3328
- if ($ongoingResult.$scriptPipelineExecutionErrors.length === 1) {
3329
- throw $ongoingResult.$scriptPipelineExecutionErrors[0];
3551
+ if ($ongoingTemplateResult.$scriptPipelineExecutionErrors.length === 1) {
3552
+ throw $ongoingTemplateResult.$scriptPipelineExecutionErrors[0];
3330
3553
  }
3331
3554
  else {
3332
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingResult.$scriptPipelineExecutionErrors
3555
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTemplateResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTemplateResult.$scriptPipelineExecutionErrors
3333
3556
  .map(function (error) { return '- ' + error.message; })
3334
3557
  .join('\n\n')), "\n "); }));
3335
3558
  }
3336
- case 23:
3559
+ case 22:
3337
3560
  if (tools.userInterface === undefined) {
3338
3561
  throw new PipelineExecutionError(spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3339
3562
  }
3340
3563
  // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3341
- _s = $ongoingResult;
3564
+ _j = $ongoingTemplateResult;
3342
3565
  return [4 /*yield*/, tools.userInterface.promptDialog($deepFreeze({
3343
- promptTitle: currentTemplate.title,
3344
- promptMessage: replaceParameters(currentTemplate.description || '', parameters),
3566
+ promptTitle: template.title,
3567
+ promptMessage: replaceParameters(template.description || '', parameters),
3345
3568
  defaultValue: replaceParameters(preparedContent, parameters),
3346
3569
  // TODO: [🧠] !! Figure out how to define placeholder in .ptbk.md file
3347
3570
  placeholder: undefined,
3348
3571
  priority: priority,
3349
3572
  }))];
3350
- case 24:
3573
+ case 23:
3351
3574
  // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3352
- _s.$resultString = _1.sent();
3353
- return [3 /*break*/, 26];
3354
- case 25: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown execution type \"".concat(currentTemplate.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3575
+ _j.$resultString = _t.sent();
3576
+ return [3 /*break*/, 25];
3577
+ case 24: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown execution type \"".concat(template.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3578
+ case 25:
3579
+ if (!(!isJokerAttempt && template.postprocessingFunctionNames)) return [3 /*break*/, 42];
3580
+ _t.label = 26;
3355
3581
  case 26:
3356
- if (!(!isJokerAttempt && currentTemplate.postprocessingFunctionNames)) return [3 /*break*/, 43];
3357
- _1.label = 27;
3582
+ _t.trys.push([26, 40, 41, 42]);
3583
+ _k = (e_3 = void 0, __values(template.postprocessingFunctionNames)), _l = _k.next();
3584
+ _t.label = 27;
3358
3585
  case 27:
3359
- _1.trys.push([27, 41, 42, 43]);
3360
- _t = (e_4 = void 0, __values(currentTemplate.postprocessingFunctionNames)), _u = _t.next();
3361
- _1.label = 28;
3362
- case 28:
3363
- if (!!_u.done) return [3 /*break*/, 40];
3364
- functionName = _u.value;
3586
+ if (!!_l.done) return [3 /*break*/, 39];
3587
+ functionName = _l.value;
3365
3588
  postprocessingError = null;
3366
- _1.label = 29;
3589
+ _t.label = 28;
3590
+ case 28:
3591
+ _t.trys.push([28, 35, 36, 37]);
3592
+ _m = (e_2 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next();
3593
+ _t.label = 29;
3367
3594
  case 29:
3368
- _1.trys.push([29, 36, 37, 38]);
3369
- _v = (e_3 = void 0, __values(arrayableToArray(tools.script))), _w = _v.next();
3370
- _1.label = 30;
3595
+ if (!!_o.done) return [3 /*break*/, 34];
3596
+ scriptTools = _o.value;
3597
+ _t.label = 30;
3371
3598
  case 30:
3372
- if (!!_w.done) return [3 /*break*/, 35];
3373
- scriptTools = _w.value;
3374
- _1.label = 31;
3375
- case 31:
3376
- _1.trys.push([31, 33, , 34]);
3377
- _x = $ongoingResult;
3599
+ _t.trys.push([30, 32, , 33]);
3600
+ _p = $ongoingTemplateResult;
3378
3601
  return [4 /*yield*/, scriptTools.execute({
3379
3602
  scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
3380
3603
  script: "".concat(functionName, "(resultString)"),
3381
3604
  parameters: {
3382
- resultString: $ongoingResult.$resultString || '',
3605
+ resultString: $ongoingTemplateResult.$resultString || '',
3383
3606
  // Note: No ...parametersForTemplate, because working with result only
3384
3607
  },
3385
3608
  })];
3386
- case 32:
3387
- _x.$resultString = _1.sent();
3609
+ case 31:
3610
+ _p.$resultString = _t.sent();
3388
3611
  postprocessingError = null;
3389
- return [3 /*break*/, 35];
3390
- case 33:
3391
- error_2 = _1.sent();
3612
+ return [3 /*break*/, 34];
3613
+ case 32:
3614
+ error_2 = _t.sent();
3392
3615
  if (!(error_2 instanceof Error)) {
3393
3616
  throw error_2;
3394
3617
  }
@@ -3396,49 +3619,49 @@ function executeTemplate(options) {
3396
3619
  throw error_2;
3397
3620
  }
3398
3621
  postprocessingError = error_2;
3399
- $ongoingResult.$scriptPipelineExecutionErrors.push(error_2);
3400
- return [3 /*break*/, 34];
3401
- case 34:
3402
- _w = _v.next();
3403
- return [3 /*break*/, 30];
3404
- case 35: return [3 /*break*/, 38];
3622
+ $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_2);
3623
+ return [3 /*break*/, 33];
3624
+ case 33:
3625
+ _o = _m.next();
3626
+ return [3 /*break*/, 29];
3627
+ case 34: return [3 /*break*/, 37];
3628
+ case 35:
3629
+ e_2_1 = _t.sent();
3630
+ e_2 = { error: e_2_1 };
3631
+ return [3 /*break*/, 37];
3405
3632
  case 36:
3406
- e_3_1 = _1.sent();
3407
- e_3 = { error: e_3_1 };
3408
- return [3 /*break*/, 38];
3409
- case 37:
3410
3633
  try {
3411
- if (_w && !_w.done && (_0 = _v.return)) _0.call(_v);
3634
+ if (_o && !_o.done && (_s = _m.return)) _s.call(_m);
3412
3635
  }
3413
- finally { if (e_3) throw e_3.error; }
3636
+ finally { if (e_2) throw e_2.error; }
3414
3637
  return [7 /*endfinally*/];
3415
- case 38:
3638
+ case 37:
3416
3639
  if (postprocessingError) {
3417
3640
  throw postprocessingError;
3418
3641
  }
3419
- _1.label = 39;
3420
- case 39:
3421
- _u = _t.next();
3422
- return [3 /*break*/, 28];
3423
- case 40: return [3 /*break*/, 43];
3642
+ _t.label = 38;
3643
+ case 38:
3644
+ _l = _k.next();
3645
+ return [3 /*break*/, 27];
3646
+ case 39: return [3 /*break*/, 42];
3647
+ case 40:
3648
+ e_3_1 = _t.sent();
3649
+ e_3 = { error: e_3_1 };
3650
+ return [3 /*break*/, 42];
3424
3651
  case 41:
3425
- e_4_1 = _1.sent();
3426
- e_4 = { error: e_4_1 };
3427
- return [3 /*break*/, 43];
3428
- case 42:
3429
3652
  try {
3430
- if (_u && !_u.done && (_z = _t.return)) _z.call(_t);
3653
+ if (_l && !_l.done && (_r = _k.return)) _r.call(_k);
3431
3654
  }
3432
- finally { if (e_4) throw e_4.error; }
3655
+ finally { if (e_3) throw e_3.error; }
3433
3656
  return [7 /*endfinally*/];
3434
- case 43:
3657
+ case 42:
3435
3658
  // TODO: [💝] Unite object for expecting amount and format
3436
- if (currentTemplate.format) {
3437
- if (currentTemplate.format === 'JSON') {
3438
- if (!isValidJsonString($ongoingResult.$resultString || '')) {
3659
+ if (template.format) {
3660
+ if (template.format === 'JSON') {
3661
+ if (!isValidJsonString($ongoingTemplateResult.$resultString || '')) {
3439
3662
  // TODO: [🏢] Do more universally via `FormatDefinition`
3440
3663
  try {
3441
- $ongoingResult.$resultString = extractJsonBlock($ongoingResult.$resultString || '');
3664
+ $ongoingTemplateResult.$resultString = extractJsonBlock($ongoingTemplateResult.$resultString || '');
3442
3665
  }
3443
3666
  catch (error) {
3444
3667
  keepUnused(error);
@@ -3448,49 +3671,51 @@ function executeTemplate(options) {
3448
3671
  }
3449
3672
  }
3450
3673
  else {
3451
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Unknown format \"".concat(currentTemplate.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3674
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Unknown format \"".concat(template.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3452
3675
  }
3453
3676
  }
3454
3677
  // TODO: [💝] Unite object for expecting amount and format
3455
- if (currentTemplate.expectations) {
3456
- checkExpectations(currentTemplate.expectations, $ongoingResult.$resultString || '');
3678
+ if (template.expectations) {
3679
+ checkExpectations(template.expectations, $ongoingTemplateResult.$resultString || '');
3457
3680
  }
3458
3681
  return [2 /*return*/, "break-attempts"];
3459
- case 44:
3460
- error_3 = _1.sent();
3682
+ case 43:
3683
+ error_3 = _t.sent();
3461
3684
  if (!(error_3 instanceof ExpectError)) {
3462
3685
  throw error_3;
3463
3686
  }
3464
- $ongoingResult.$expectError = error_3;
3465
- return [3 /*break*/, 46];
3466
- case 45:
3687
+ $ongoingTemplateResult.$expectError = error_3;
3688
+ return [3 /*break*/, 45];
3689
+ case 44:
3467
3690
  if (!isJokerAttempt &&
3468
- currentTemplate.templateType === 'PROMPT_TEMPLATE' &&
3469
- $ongoingResult.$prompt
3691
+ template.templateType === 'PROMPT_TEMPLATE' &&
3692
+ $ongoingTemplateResult.$prompt
3470
3693
  // <- Note: [2] When some expected parameter is not defined, error will occur in replaceParameters
3471
3694
  // In that case we don’t want to make a report about it because it’s not a llm execution error
3472
3695
  ) {
3473
3696
  // TODO: [🧠] Maybe put other templateTypes into report
3474
3697
  $executionReport.promptExecutions.push({
3475
- prompt: __assign({}, $ongoingResult.$prompt),
3476
- result: $ongoingResult.$result || undefined,
3477
- error: $ongoingResult.$expectError === null ? undefined : serializeError($ongoingResult.$expectError),
3698
+ prompt: __assign({}, $ongoingTemplateResult.$prompt),
3699
+ result: $ongoingTemplateResult.$result || undefined,
3700
+ error: $ongoingTemplateResult.$expectError === null
3701
+ ? undefined
3702
+ : serializeError($ongoingTemplateResult.$expectError),
3478
3703
  });
3479
3704
  }
3480
3705
  return [7 /*endfinally*/];
3481
- case 46:
3482
- if ($ongoingResult.$expectError !== null && attempt === maxAttempts - 1) {
3706
+ case 45:
3707
+ if ($ongoingTemplateResult.$expectError !== null && attempt === maxAttempts - 1) {
3483
3708
  throw new PipelineExecutionError(spaceTrim(function (block) {
3484
3709
  var _a, _b, _c;
3485
- return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3710
+ return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTemplateResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3486
3711
  .split('\n')
3487
3712
  .map(function (line) { return "> ".concat(line); })
3488
- .join('\n')), "\n\n Last error ").concat(((_b = $ongoingResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3713
+ .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTemplateResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTemplateResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3489
3714
  .split('\n')
3490
3715
  .map(function (line) { return "> ".concat(line); })
3491
- .join('\n')), "\n\n Last result:\n ").concat(block($ongoingResult.$resultString === null
3716
+ .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTemplateResult.$resultString === null
3492
3717
  ? 'null'
3493
- : $ongoingResult.$resultString
3718
+ : $ongoingTemplateResult.$resultString
3494
3719
  .split('\n')
3495
3720
  .map(function (line) { return "> ".concat(line); })
3496
3721
  .join('\n')), "\n ---\n ");
@@ -3501,36 +3726,322 @@ function executeTemplate(options) {
3501
3726
  });
3502
3727
  };
3503
3728
  attempt = -jokerParameterNames.length;
3504
- _h.label = 3;
3729
+ _a.label = 1;
3730
+ case 1:
3731
+ if (!(attempt < maxAttempts)) return [3 /*break*/, 4];
3732
+ return [5 /*yield**/, _loop_1(attempt)];
3733
+ case 2:
3734
+ state_1 = _a.sent();
3735
+ switch (state_1) {
3736
+ case "break-attempts": return [3 /*break*/, 4];
3737
+ }
3738
+ _a.label = 3;
3739
+ case 3:
3740
+ attempt++;
3741
+ return [3 /*break*/, 1];
3742
+ case 4:
3743
+ if ($ongoingTemplateResult.$resultString === null) {
3744
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3745
+ }
3746
+ return [2 /*return*/, $ongoingTemplateResult.$resultString];
3747
+ }
3748
+ });
3749
+ });
3750
+ }
3751
+ /**
3752
+ * TODO: Break into smaller functions
3753
+ */
3754
+
3755
+ /**
3756
+ * @@@
3757
+ *
3758
+ * @private internal utility of `createPipelineExecutor`
3759
+ */
3760
+ function executeFormatSubvalues(options) {
3761
+ return __awaiter(this, void 0, void 0, function () {
3762
+ var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3763
+ var _this = this;
3764
+ return __generator(this, function (_a) {
3765
+ switch (_a.label) {
3766
+ case 0:
3767
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3768
+ if (template.foreach === undefined) {
3769
+ return [2 /*return*/, /* not await */ executeAttempts(options)];
3770
+ }
3771
+ if (jokerParameterNames.length !== 0) {
3772
+ throw new UnexpectedError(spaceTrim$1(function (block) { return "\n JOKER parameters are not supported together with FOREACH command\n\n [\uD83E\uDDDE\u200D\u2640\uFE0F] This should be prevented in `validatePipeline`\n\n ".concat(block(pipelineIdentification), "\n "); }));
3773
+ }
3774
+ parameterValue = parameters[template.foreach.parameterName] || '';
3775
+ formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
3776
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(template.foreach.formatName);
3777
+ });
3778
+ if (formatDefinition === undefined) {
3779
+ throw new UnexpectedError(
3780
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3781
+ spaceTrim$1(function (block) { return "\n Unsupported format \"".concat(template.foreach.formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
3782
+ .map(function (formatName) { return "- ".concat(formatName); })
3783
+ .join('\n')), "\n\n [\u26F7] This should never happen because format name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3784
+ }
3785
+ subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
3786
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(template.foreach.subformatName);
3787
+ });
3788
+ if (subvalueDefinition === undefined) {
3789
+ throw new UnexpectedError(
3790
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3791
+ spaceTrim$1(function (block) { return "\n Unsupported subformat name \"".concat(template.foreach.subformatName, "\" for format \"").concat(template.foreach.formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
3792
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
3793
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
3794
+ .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3795
+ }
3796
+ if (formatDefinition.formatName === 'CSV') {
3797
+ formatSettings = settings.csvSettings;
3798
+ // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
3799
+ }
3800
+ return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
3801
+ var mappedParameters, allSubparameters, subresultString;
3802
+ return __generator(this, function (_a) {
3803
+ switch (_a.label) {
3804
+ case 0:
3805
+ // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
3806
+ // TODO: When done [🐚] Report progress also for each subvalue here
3807
+ try {
3808
+ mappedParameters = mapAvailableToExpectedParameters({
3809
+ expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
3810
+ availableParameters: subparameters,
3811
+ });
3812
+ }
3813
+ catch (error) {
3814
+ if (!(error instanceof PipelineExecutionError)) {
3815
+ throw error;
3816
+ }
3817
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n ".concat(error.message, "\n\n This is error in FOREACH command\n You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n ").concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }));
3818
+ }
3819
+ allSubparameters = __assign(__assign({}, parameters), mappedParameters);
3820
+ // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
3821
+ Object.freeze(allSubparameters);
3822
+ return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }) }))];
3823
+ case 1:
3824
+ subresultString = _a.sent();
3825
+ return [2 /*return*/, subresultString];
3826
+ }
3827
+ });
3828
+ }); })];
3829
+ case 1:
3830
+ resultString = _a.sent();
3831
+ return [2 /*return*/, resultString];
3832
+ }
3833
+ });
3834
+ });
3835
+ }
3836
+
3837
+ /**
3838
+ * @@@
3839
+ *
3840
+ * @private internal utility of `createPipelineExecutor`
3841
+ */
3842
+ function getContextForTemplate(template) {
3843
+ return __awaiter(this, void 0, void 0, function () {
3844
+ return __generator(this, function (_a) {
3845
+ TODO_USE(template);
3846
+ return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */];
3847
+ });
3848
+ });
3849
+ }
3850
+
3851
+ /**
3852
+ * @@@
3853
+ *
3854
+ * @private internal utility of `createPipelineExecutor`
3855
+ */
3856
+ function getKnowledgeForTemplate(options) {
3857
+ return __awaiter(this, void 0, void 0, function () {
3858
+ var preparedPipeline, template;
3859
+ return __generator(this, function (_a) {
3860
+ preparedPipeline = options.preparedPipeline, template = options.template;
3861
+ // TODO: [♨] Implement Better - use real index and keyword search from `template` and {samples}
3862
+ TODO_USE(template);
3863
+ return [2 /*return*/, preparedPipeline.knowledgePieces.map(function (_a) {
3864
+ var content = _a.content;
3865
+ return "- ".concat(content);
3866
+ }).join('\n')];
3867
+ });
3868
+ });
3869
+ }
3870
+
3871
+ /**
3872
+ * @@@
3873
+ *
3874
+ * @private internal utility of `createPipelineExecutor`
3875
+ */
3876
+ function getSamplesForTemplate(template) {
3877
+ return __awaiter(this, void 0, void 0, function () {
3878
+ return __generator(this, function (_a) {
3879
+ // TODO: [♨] Implement Better - use real index and keyword search
3880
+ TODO_USE(template);
3881
+ return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */];
3882
+ });
3883
+ });
3884
+ }
3885
+
3886
+ /**
3887
+ * @@@
3888
+ *
3889
+ * @private internal utility of `createPipelineExecutor`
3890
+ */
3891
+ function getReservedParametersForTemplate(options) {
3892
+ return __awaiter(this, void 0, void 0, function () {
3893
+ var preparedPipeline, template, pipelineIdentification, context, knowledge, samples, currentDate, modelName, reservedParameters, _loop_1, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName;
3894
+ var e_1, _a;
3895
+ return __generator(this, function (_b) {
3896
+ switch (_b.label) {
3897
+ case 0:
3898
+ preparedPipeline = options.preparedPipeline, template = options.template, pipelineIdentification = options.pipelineIdentification;
3899
+ return [4 /*yield*/, getContextForTemplate(template)];
3900
+ case 1:
3901
+ context = _b.sent();
3902
+ return [4 /*yield*/, getKnowledgeForTemplate({ preparedPipeline: preparedPipeline, template: template })];
3903
+ case 2:
3904
+ knowledge = _b.sent();
3905
+ return [4 /*yield*/, getSamplesForTemplate(template)];
3505
3906
  case 3:
3506
- if (!(attempt < maxAttempts)) return [3 /*break*/, 6];
3507
- return [5 /*yield**/, _loop_2(attempt)];
3508
- case 4:
3509
- state_1 = _h.sent();
3510
- switch (state_1) {
3511
- case "break-attempts": return [3 /*break*/, 6];
3907
+ samples = _b.sent();
3908
+ currentDate = new Date().toISOString();
3909
+ modelName = RESERVED_PARAMETER_MISSING_VALUE;
3910
+ reservedParameters = {
3911
+ content: RESERVED_PARAMETER_RESTRICTED,
3912
+ context: context,
3913
+ knowledge: knowledge,
3914
+ samples: samples,
3915
+ currentDate: currentDate,
3916
+ modelName: modelName,
3917
+ };
3918
+ _loop_1 = function (parameterName) {
3919
+ if (reservedParameters[parameterName] === undefined) {
3920
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Reserved parameter {".concat(parameterName, "} is not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3921
+ }
3922
+ };
3923
+ try {
3924
+ // Note: Doublecheck that ALL reserved parameters are defined:
3925
+ for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
3926
+ parameterName = RESERVED_PARAMETER_NAMES_1_1.value;
3927
+ _loop_1(parameterName);
3928
+ }
3512
3929
  }
3513
- _h.label = 5;
3514
- case 5:
3515
- attempt++;
3516
- return [3 /*break*/, 3];
3517
- case 6:
3518
- //------------------------------------
3519
- /*
3520
-
3521
-
3522
-
3523
-
3524
-
3525
-
3526
-
3527
-
3528
-
3529
- */
3530
- //------------------------------------
3531
- if ($ongoingResult.$resultString === null) {
3532
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3930
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3931
+ finally {
3932
+ try {
3933
+ if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1);
3934
+ }
3935
+ finally { if (e_1) throw e_1.error; }
3936
+ }
3937
+ return [2 /*return*/, reservedParameters];
3938
+ }
3939
+ });
3940
+ });
3941
+ }
3942
+
3943
+ /**
3944
+ * @@@
3945
+ *
3946
+ * @private internal utility of `createPipelineExecutor`
3947
+ */
3948
+ function executeTemplate(options) {
3949
+ return __awaiter(this, void 0, void 0, function () {
3950
+ var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
3951
+ var e_1, _f, _g;
3952
+ return __generator(this, function (_h) {
3953
+ switch (_h.label) {
3954
+ case 0:
3955
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, llmTools = options.llmTools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3956
+ maxExecutionAttempts = settings.maxExecutionAttempts;
3957
+ name = "pipeline-executor-frame-".concat(currentTemplate.name);
3958
+ title = currentTemplate.title;
3959
+ priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
3960
+ return [4 /*yield*/, onProgress({
3961
+ name: name,
3962
+ title: title,
3963
+ isStarted: false,
3964
+ isDone: false,
3965
+ templateType: currentTemplate.templateType,
3966
+ parameterName: currentTemplate.resultingParameterName,
3967
+ parameterValue: null,
3968
+ // <- [🍸]
3969
+ })];
3970
+ case 1:
3971
+ _h.sent();
3972
+ usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
3973
+ dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
3974
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3975
+ if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
3976
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames)
3977
+ .map(function (name) { return "{".concat(name, "}"); })
3978
+ .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames)
3979
+ .map(function (name) { return "{".concat(name, "}"); })
3980
+ .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3981
+ }
3982
+ _b = (_a = Object).freeze;
3983
+ _c = [{}];
3984
+ return [4 /*yield*/, getReservedParametersForTemplate({
3985
+ preparedPipeline: preparedPipeline,
3986
+ template: currentTemplate,
3987
+ pipelineIdentification: pipelineIdentification,
3988
+ })];
3989
+ case 2:
3990
+ definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
3991
+ definedParameterNames = new Set(Object.keys(definedParameters));
3992
+ parameters = {};
3993
+ _loop_1 = function (parameterName) {
3994
+ // Situation: Parameter is defined and used
3995
+ if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3996
+ parameters[parameterName] = definedParameters[parameterName];
3997
+ }
3998
+ // Situation: Parameter is defined but NOT used
3999
+ else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) ;
4000
+ // Situation: Parameter is NOT defined BUT used
4001
+ else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
4002
+ // Houston, we have a problem
4003
+ // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck
4004
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is NOT defined\n BUT used in template \"").concat(currentTemplate.title || currentTemplate.name, "\"\n\n This should be catched in `validatePipeline`\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4005
+ }
4006
+ };
4007
+ try {
4008
+ // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4009
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4010
+ for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4011
+ parameterName = _e.value;
4012
+ _loop_1(parameterName);
4013
+ }
4014
+ }
4015
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4016
+ finally {
4017
+ try {
4018
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4019
+ }
4020
+ finally { if (e_1) throw e_1.error; }
3533
4021
  }
4022
+ // Note: [👨‍👨‍👧] Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
4023
+ Object.freeze(parameters);
4024
+ maxAttempts = currentTemplate.templateType === 'DIALOG_TEMPLATE' ? Infinity : maxExecutionAttempts;
4025
+ jokerParameterNames = currentTemplate.jokerParameterNames || [];
4026
+ preparedContent = (currentTemplate.preparedContent || '{content}')
4027
+ .split('{content}')
4028
+ .join(currentTemplate.content);
4029
+ return [4 /*yield*/, executeFormatSubvalues({
4030
+ jokerParameterNames: jokerParameterNames,
4031
+ priority: priority,
4032
+ maxAttempts: maxAttempts,
4033
+ preparedContent: preparedContent,
4034
+ parameters: parameters,
4035
+ template: currentTemplate,
4036
+ preparedPipeline: preparedPipeline,
4037
+ tools: tools,
4038
+ llmTools: llmTools,
4039
+ settings: settings,
4040
+ $executionReport: $executionReport,
4041
+ pipelineIdentification: pipelineIdentification,
4042
+ })];
4043
+ case 3:
4044
+ resultString = _h.sent();
3534
4045
  return [4 /*yield*/, onProgress({
3535
4046
  name: name,
3536
4047
  title: title,
@@ -3538,13 +4049,15 @@ function executeTemplate(options) {
3538
4049
  isDone: true,
3539
4050
  templateType: currentTemplate.templateType,
3540
4051
  parameterName: currentTemplate.resultingParameterName,
3541
- parameterValue: $ongoingResult.$resultString,
4052
+ parameterValue: resultString,
3542
4053
  // <- [🍸]
3543
4054
  })];
3544
- case 7:
4055
+ case 4:
3545
4056
  _h.sent();
3546
4057
  return [2 /*return*/, Object.freeze((_g = {},
3547
- _g[currentTemplate.resultingParameterName] = $ongoingResult.$resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */,
4058
+ _g[currentTemplate.resultingParameterName] =
4059
+ // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4060
+ resultString,
3548
4061
  _g))];
3549
4062
  }
3550
4063
  });
@@ -3553,6 +4066,9 @@ function executeTemplate(options) {
3553
4066
  /**
3554
4067
  * TODO: [🤹‍♂️]
3555
4068
  */
4069
+ /**
4070
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4071
+ */
3556
4072
 
3557
4073
  /**
3558
4074
  * @@@
@@ -3573,6 +4089,7 @@ function filterJustOutputParameters(options) {
3573
4089
  };
3574
4090
  try {
3575
4091
  // Note: Filter ONLY output parameters
4092
+ // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters`
3576
4093
  for (var _b = __values(preparedPipeline.parameters.filter(function (_a) {
3577
4094
  var isOutput = _a.isOutput;
3578
4095
  return isOutput;
@@ -3687,7 +4204,7 @@ function executePipeline(options) {
3687
4204
  return name === parameterName;
3688
4205
  });
3689
4206
  if (!(parameter === undefined)) return [3 /*break*/, 1];
3690
- warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
4207
+ warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
3691
4208
  return [3 /*break*/, 4];
3692
4209
  case 1:
3693
4210
  if (!(parameter.isInput === false)) return [3 /*break*/, 4];
@@ -3699,10 +4216,10 @@ function executePipeline(options) {
3699
4216
  // Note: Wait a short time to prevent race conditions
3700
4217
  _h.sent();
3701
4218
  _h.label = 3;
3702
- case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
4219
+ case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
3703
4220
  isSuccessful: false,
3704
4221
  errors: __spreadArray([
3705
- new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
4222
+ new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
3706
4223
  ], __read(errors), false).map(serializeError),
3707
4224
  warnings: warnings.map(serializeError),
3708
4225
  executionReport: executionReport,
@@ -3766,7 +4283,7 @@ function executePipeline(options) {
3766
4283
  case 0:
3767
4284
  if (loopLimit-- < 0) {
3768
4285
  // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
3769
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
4286
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
3770
4287
  }
3771
4288
  currentTemplate = unresovedTemplates_1.find(function (template) {
3772
4289
  return template.dependentParameterNames.every(function (name) {
@@ -3776,14 +4293,14 @@ function executePipeline(options) {
3776
4293
  if (!(!currentTemplate && resolving_1.length === 0)) return [3 /*break*/, 1];
3777
4294
  throw new UnexpectedError(
3778
4295
  // TODO: [🐎] DRY
3779
- spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
4296
+ spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
3780
4297
  .map(function (_a) {
3781
4298
  var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
3782
4299
  return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
3783
4300
  .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
3784
4301
  .join(' and '));
3785
4302
  })
3786
- .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
4303
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
3787
4304
  case 1:
3788
4305
  if (!!currentTemplate) return [3 /*break*/, 3];
3789
4306
  /* [🤹‍♂️] */ return [4 /*yield*/, Promise.race(resolving_1)];
@@ -3800,10 +4317,10 @@ function executePipeline(options) {
3800
4317
  llmTools: llmTools,
3801
4318
  onProgress: function (progress) {
3802
4319
  if (isReturned) {
3803
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4320
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
3804
4321
  .split('\n')
3805
4322
  .map(function (line) { return "> ".concat(line); })
3806
- .join('\n')), "\n "); }));
4323
+ .join('\n')), "\n "); }));
3807
4324
  }
3808
4325
  if (onProgress) {
3809
4326
  onProgress(progress);
@@ -3811,7 +4328,7 @@ function executePipeline(options) {
3811
4328
  },
3812
4329
  settings: settings,
3813
4330
  $executionReport: executionReport,
3814
- pipelineIdentification: pipelineIdentification,
4331
+ pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
3815
4332
  })
3816
4333
  .then(function (newParametersToPass) {
3817
4334
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
@@ -3820,6 +4337,8 @@ function executePipeline(options) {
3820
4337
  .then(function () {
3821
4338
  resolving_1 = resolving_1.filter(function (w) { return w !== work_1; });
3822
4339
  });
4340
+ // <- Note: Errors are catched here [3]
4341
+ // TODO: BUT if in multiple templates are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here
3823
4342
  resolving_1.push(work_1);
3824
4343
  _j.label = 4;
3825
4344
  case 4: return [2 /*return*/];
@@ -3901,6 +4420,9 @@ function executePipeline(options) {
3901
4420
  });
3902
4421
  });
3903
4422
  }
4423
+ /**
4424
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4425
+ */
3904
4426
 
3905
4427
  /**
3906
4428
  * Creates executor function from pipeline and execution tools.
@@ -3912,7 +4434,7 @@ function executePipeline(options) {
3912
4434
  function createPipelineExecutor(options) {
3913
4435
  var _this = this;
3914
4436
  var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
3915
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d, _e = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e;
4437
+ var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f;
3916
4438
  validatePipeline(pipeline);
3917
4439
  var pipelineIdentification = (function () {
3918
4440
  // Note: This is a 😐 implementation of [🚞]
@@ -3932,9 +4454,11 @@ function createPipelineExecutor(options) {
3932
4454
  else if (isNotPreparedWarningSupressed !== true) {
3933
4455
  console.warn(spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
3934
4456
  }
4457
+ var runCount = 0;
3935
4458
  var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
3936
4459
  return __generator(this, function (_a) {
3937
- return [2 /*return*/, executePipeline({
4460
+ runCount++;
4461
+ return [2 /*return*/, /* not await */ executePipeline({
3938
4462
  pipeline: pipeline,
3939
4463
  preparedPipeline: preparedPipeline,
3940
4464
  setPreparedPipeline: function (newPreparedPipeline) {
@@ -3943,10 +4467,11 @@ function createPipelineExecutor(options) {
3943
4467
  inputParameters: inputParameters,
3944
4468
  tools: tools,
3945
4469
  onProgress: onProgress,
3946
- pipelineIdentification: pipelineIdentification,
4470
+ pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
3947
4471
  settings: {
3948
4472
  maxExecutionAttempts: maxExecutionAttempts,
3949
4473
  maxParallelCount: maxParallelCount,
4474
+ csvSettings: csvSettings,
3950
4475
  isVerbose: isVerbose,
3951
4476
  isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
3952
4477
  },
@@ -3955,6 +4480,9 @@ function createPipelineExecutor(options) {
3955
4480
  }); };
3956
4481
  return pipelineExecutor;
3957
4482
  }
4483
+ /**
4484
+ * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4485
+ */
3958
4486
 
3959
4487
  /**
3960
4488
  * @@@
@@ -4006,7 +4534,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
4006
4534
  outputParameters = result.outputParameters;
4007
4535
  knowledgePiecesRaw = outputParameters.knowledgePieces;
4008
4536
  knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
4009
- // <- TODO: !!!!! Smarter split and filter out empty pieces
4537
+ // <- TODO: [main] !!!!! Smarter split and filter out empty pieces
4010
4538
  if (isVerbose) {
4011
4539
  console.info('knowledgeTextPieces:', knowledgeTextPieces);
4012
4540
  }
@@ -4064,8 +4592,13 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
4064
4592
  case 6: return [3 /*break*/, 8];
4065
4593
  case 7:
4066
4594
  error_1 = _c.sent();
4595
+ // Note: Here is expected error:
4596
+ // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
4597
+ if (!(error_1 instanceof PipelineExecutionError)) {
4598
+ throw error_1;
4599
+ }
4067
4600
  // TODO: [🟥] Detect browser / node and make it colorfull
4068
- console.error(error_1);
4601
+ console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
4069
4602
  return [3 /*break*/, 8];
4070
4603
  case 8: return [2 /*return*/, {
4071
4604
  name: name,
@@ -4086,7 +4619,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
4086
4619
  });
4087
4620
  }
4088
4621
  /**
4089
- * TODO: [🐝][🔼] !!! Export via `@promptbook/markdown`
4622
+ * TODO: [🐝][🔼][main] !!! Export via `@promptbook/markdown`
4090
4623
  * TODO: [🪂] Do it in parallel 11:11
4091
4624
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4092
4625
  */
@@ -4110,7 +4643,7 @@ function prepareKnowledgePieces(knowledgeSources, options) {
4110
4643
  var partialPieces, pieces;
4111
4644
  return __generator(this, function (_a) {
4112
4645
  switch (_a.label) {
4113
- case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4646
+ case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝][main] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4114
4647
  options)];
4115
4648
  case 1:
4116
4649
  partialPieces = _a.sent();
@@ -4302,7 +4835,7 @@ function preparePersona(personaDescription, options) {
4302
4835
  });
4303
4836
  }
4304
4837
  /**
4305
- * TODO: [🔃] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4838
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4306
4839
  * TODO: [🏢] !! Check validity of `modelName` in pipeline
4307
4840
  * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4308
4841
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
@@ -4351,7 +4884,7 @@ function prepareTemplates(pipeline, options) {
4351
4884
  case 0:
4352
4885
  _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
4353
4886
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
4354
- // TODO: !!!!! Apply samples to each template (if missing and is for the template defined)
4887
+ // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
4355
4888
  TODO_USE(parameters);
4356
4889
  templatesPrepared = new Array(
4357
4890
  // <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -4383,7 +4916,7 @@ function prepareTemplates(pipeline, options) {
4383
4916
  /**
4384
4917
  * TODO: [🧠] Add context to each template (if missing)
4385
4918
  * TODO: [🧠] What is better name `prepareTemplate` or `prepareTemplateAndParameters`
4386
- * TODO: [♨] !!! Prepare index the samples and maybe templates
4919
+ * TODO: [♨][main] !!! Prepare index the samples and maybe templates
4387
4920
  * TODO: Write tests for `preparePipeline`
4388
4921
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
4389
4922
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
@@ -4555,7 +5088,7 @@ var knowledgeCommandParser = {
4555
5088
  if (sourceContent === '') {
4556
5089
  throw new ParseError("Source is not defined");
4557
5090
  }
4558
- // TODO: !!!! Following checks should be applied every link in the `sourceContent`
5091
+ // TODO: [main] !!!! Following checks should be applied every link in the `sourceContent`
4559
5092
  if (sourceContent.startsWith('http://')) {
4560
5093
  throw new ParseError("Source is not secure");
4561
5094
  }
@@ -4758,7 +5291,7 @@ var templateCommandParser = {
4758
5291
  if (command.templateType === 'KNOWLEDGE') {
4759
5292
  knowledgeCommandParser.$applyToPipelineJson({
4760
5293
  type: 'KNOWLEDGE',
4761
- sourceContent: $templateJson.content, // <- TODO: [🐝] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
5294
+ sourceContent: $templateJson.content, // <- TODO: [🐝][main] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
4762
5295
  }, $pipelineJson);
4763
5296
  $templateJson.isTemplate = false;
4764
5297
  return;
@@ -5108,6 +5641,171 @@ function normalizeTo_SCREAMING_CASE(text) {
5108
5641
  * TODO: [🌺] Use some intermediate util splitWords
5109
5642
  */
5110
5643
 
5644
+ /**
5645
+ * @@@
5646
+ *
5647
+ * @param text @@@
5648
+ * @param _isFirstLetterCapital @@@
5649
+ * @returns @@@
5650
+ * @example 'helloWorld'
5651
+ * @example 'iLovePromptbook'
5652
+ * @public exported from `@promptbook/utils`
5653
+ */
5654
+ function normalizeTo_camelCase(text, _isFirstLetterCapital) {
5655
+ var e_1, _a;
5656
+ if (_isFirstLetterCapital === void 0) { _isFirstLetterCapital = false; }
5657
+ var charType;
5658
+ var lastCharType = null;
5659
+ var normalizedName = '';
5660
+ try {
5661
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5662
+ var char = text_1_1.value;
5663
+ var normalizedChar = void 0;
5664
+ if (/^[a-z]$/.test(char)) {
5665
+ charType = 'LOWERCASE';
5666
+ normalizedChar = char;
5667
+ }
5668
+ else if (/^[A-Z]$/.test(char)) {
5669
+ charType = 'UPPERCASE';
5670
+ normalizedChar = char.toLowerCase();
5671
+ }
5672
+ else if (/^[0-9]$/.test(char)) {
5673
+ charType = 'NUMBER';
5674
+ normalizedChar = char;
5675
+ }
5676
+ else {
5677
+ charType = 'OTHER';
5678
+ normalizedChar = '';
5679
+ }
5680
+ if (!lastCharType) {
5681
+ if (_isFirstLetterCapital) {
5682
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: DRY
5683
+ }
5684
+ }
5685
+ else if (charType !== lastCharType &&
5686
+ !(charType === 'LOWERCASE' && lastCharType === 'UPPERCASE') &&
5687
+ !(lastCharType === 'NUMBER') &&
5688
+ !(charType === 'NUMBER')) {
5689
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: [🌺] DRY
5690
+ }
5691
+ normalizedName += normalizedChar;
5692
+ lastCharType = charType;
5693
+ }
5694
+ }
5695
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5696
+ finally {
5697
+ try {
5698
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5699
+ }
5700
+ finally { if (e_1) throw e_1.error; }
5701
+ }
5702
+ return normalizedName;
5703
+ }
5704
+ /**
5705
+ * TODO: [🌺] Use some intermediate util splitWords
5706
+ */
5707
+
5708
+ /**
5709
+ * Removes quotes from a string
5710
+ *
5711
+ * Tip: This is very usefull for post-processing of the result of the LLM model
5712
+ * Note: This function removes only the same quotes from the beginning and the end of the string
5713
+ * Note: There are two simmilar functions:
5714
+ * - `removeQuotes` which removes only bounding quotes
5715
+ * - `unwrapResult` which removes whole introduce sentence
5716
+ *
5717
+ * @param text optionally quoted text
5718
+ * @returns text without quotes
5719
+ * @public exported from `@promptbook/utils`
5720
+ */
5721
+ function removeQuotes(text) {
5722
+ if (text.startsWith('"') && text.endsWith('"')) {
5723
+ return text.slice(1, -1);
5724
+ }
5725
+ if (text.startsWith('\'') && text.endsWith('\'')) {
5726
+ return text.slice(1, -1);
5727
+ }
5728
+ return text;
5729
+ }
5730
+
5731
+ /**
5732
+ * Function `validateParameterName` will @@@
5733
+ *
5734
+ * @param parameterName @@@
5735
+ * @returns @@@
5736
+ * @throws {ParseError} @@@
5737
+ * @private within the repository
5738
+ */
5739
+ function validateParameterName(parameterName) {
5740
+ var e_1, _a;
5741
+ var rawParameterName = parameterName;
5742
+ try {
5743
+ for (var _b = __values([
5744
+ ['`', '`'],
5745
+ ['{', '}'],
5746
+ ['[', ']'],
5747
+ ['(', ')'],
5748
+ ['<', '>'],
5749
+ ]), _c = _b.next(); !_c.done; _c = _b.next()) {
5750
+ var _d = __read(_c.value, 2), start = _d[0], end = _d[1];
5751
+ if (parameterName.substring(0, 1) === start &&
5752
+ parameterName.substring(parameterName.length - 1, parameterName.length) === end
5753
+ // <- TODO: More universal that 1 character
5754
+ ) {
5755
+ parameterName = parameterName.substring(1, parameterName.length - 1);
5756
+ // <- TODO: More universal that 1 character
5757
+ }
5758
+ }
5759
+ }
5760
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5761
+ finally {
5762
+ try {
5763
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
5764
+ }
5765
+ finally { if (e_1) throw e_1.error; }
5766
+ }
5767
+ // TODO: [🐠] Following try-catch block should be part of common validators logic
5768
+ try {
5769
+ /*
5770
+ Note: We don't need to check for spaces because we are going to normalize the parameter name to camelCase
5771
+ if (parameterName.includes(' ')) {
5772
+ throw new ParseError(`Parameter name cannot contain spaces`);
5773
+ }
5774
+ */
5775
+ if (parameterName.includes('.')) {
5776
+ throw new ParseError("Parameter name cannot contain dots");
5777
+ }
5778
+ if (parameterName.includes('/') || parameterName.includes('\\')) {
5779
+ throw new ParseError("Parameter name cannot contain slashes");
5780
+ }
5781
+ if (parameterName.includes('(') ||
5782
+ parameterName.includes(')') ||
5783
+ parameterName.includes('{') ||
5784
+ parameterName.includes('}') ||
5785
+ parameterName.includes('[') ||
5786
+ parameterName.includes(']')) {
5787
+ throw new ParseError("Parameter name cannot contain braces");
5788
+ }
5789
+ parameterName = removeDiacritics(parameterName);
5790
+ parameterName = removeEmojis(parameterName);
5791
+ parameterName = removeQuotes(parameterName);
5792
+ parameterName = normalizeTo_camelCase(parameterName);
5793
+ if (parameterName === '') {
5794
+ throw new ParseError("Parameter name cannot be empty");
5795
+ }
5796
+ if (RESERVED_PARAMETER_NAMES.includes(parameterName)) {
5797
+ throw new ParseError("{".concat(parameterName, "} is a reserved parameter name"));
5798
+ }
5799
+ }
5800
+ catch (error) {
5801
+ if (!(error instanceof ParseError)) {
5802
+ throw error;
5803
+ }
5804
+ throw new ParseError(spaceTrim$1(function (block) { return "\n ".concat(block(error.message), "\n\n Tried to validate parameter name:\n ").concat(block(rawParameterName), "\n "); }));
5805
+ }
5806
+ return parameterName;
5807
+ }
5808
+
5111
5809
  /**
5112
5810
  * Parses the foreach command
5113
5811
  *
@@ -5137,15 +5835,16 @@ var foreachCommandParser = {
5137
5835
  /**
5138
5836
  * Link to discussion
5139
5837
  */
5140
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5838
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/148',
5141
5839
  /**
5142
5840
  * Example usages of the FOREACH command
5143
5841
  */
5144
5842
  examples: [
5145
- 'FOREACH List Line `{customers}` -> `{customer}`',
5146
- 'FOR List Line `{customers}` -> `{customer}`',
5147
- 'EACH List Line `{customers}` -> `{customer}`',
5148
- // <- TODO: [🍭] !!!!!! More
5843
+ 'FOREACH Text Line `{customers}` -> `{customer}`',
5844
+ 'FOREACH Csv Cell `{customers}` -> `{cell}`',
5845
+ 'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
5846
+ 'FOR Text Line `{customers}` -> `{customer}`',
5847
+ 'EACH Text Line `{customers}` -> `{customer}`',
5149
5848
  ],
5150
5849
  /**
5151
5850
  * Parses the FOREACH command
@@ -5153,55 +5852,75 @@ var foreachCommandParser = {
5153
5852
  parse: function (input) {
5154
5853
  var args = input.args;
5155
5854
  var formatName = normalizeTo_SCREAMING_CASE(args[0] || '');
5156
- var cellName = normalizeTo_SCREAMING_CASE(args[1] || '');
5157
- var parameterNameWrapped = args[2];
5855
+ var subformatName = normalizeTo_SCREAMING_CASE(args[1] || '');
5856
+ var parameterNameArg = args[2] || '';
5158
5857
  var assignSign = args[3];
5159
- var subparameterNameWrapped = args[4];
5160
- if (![
5161
- 'LIST',
5162
- 'CSV',
5163
- // <- TODO: [🏢] Unhardcode formats
5164
- ].includes(formatName)) {
5165
- console.info({ args: args, formatName: formatName });
5166
- throw new Error("Unsupported format \"".concat(formatName, "\""));
5858
+ var formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
5859
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(formatName);
5860
+ });
5861
+ if (formatDefinition === undefined) {
5862
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unsupported format \"".concat(formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
5863
+ .map(function (formatName) { return "- ".concat(formatName); })
5864
+ .join('\n')), "\n "); }));
5167
5865
  // <- TODO: [🏢] List all supported format names
5168
5866
  }
5169
- if (![
5170
- 'LINE',
5171
- 'ROW',
5172
- 'COLUMN',
5173
- 'CELL',
5174
- // <- TODO: [🏢] Unhardcode format cells
5175
- ].includes(cellName)) {
5176
- console.info({ args: args, cellName: cellName });
5177
- throw new Error("Format ".concat(formatName, " does not support cell \"").concat(cellName, "\""));
5178
- // <- TODO: [🏢] List all supported cell names for the format
5867
+ var subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
5868
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(subformatName);
5869
+ });
5870
+ if (subvalueDefinition === undefined) {
5871
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unsupported subformat name \"".concat(subformatName, "\" for format \"").concat(formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
5872
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
5873
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
5874
+ .join('\n')), "\n "); }));
5875
+ // <- TODO: [🏢] List all supported subformat names for the format
5179
5876
  }
5180
5877
  if (assignSign !== '->') {
5181
- console.info({ args: args, assignSign: assignSign });
5182
- throw new Error("FOREACH command must have '->' to assign the value to the parameter");
5183
- }
5184
- // TODO: !!!!!! Replace with propper parameter name validation
5185
- if ((parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1)) !== '{' ||
5186
- (parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length)) !== '}') {
5187
- console.info({ args: args, parameterNameWrapped: parameterNameWrapped }, parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1), parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length));
5188
- throw new Error("!!!!!! 1 Here will be error (with rules and precise error) from validateParameterName");
5189
- }
5190
- var parameterName = parameterNameWrapped.substring(1, parameterNameWrapped.length - 1);
5191
- // TODO: !!!!!! Replace with propper parameter name validation
5192
- if ((subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(0, 1)) !== '{' ||
5193
- (subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(subparameterNameWrapped.length - 1, subparameterNameWrapped.length)) !==
5194
- '}') {
5195
- console.info({ args: args, subparameterNameWrapped: subparameterNameWrapped });
5196
- throw new Error("!!!!!! 2 Here will be error (with rules and precise error) from validateParameterName");
5197
- }
5198
- var subparameterName = subparameterNameWrapped.substring(1, subparameterNameWrapped.length - 1);
5878
+ throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
5879
+ }
5880
+ var parameterName = validateParameterName(parameterNameArg);
5881
+ var outputSubparameterName = null;
5882
+ // TODO: [4] DRY
5883
+ var inputSubparameterNames = args
5884
+ .slice(4)
5885
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5886
+ .filter(function (parameterName) { return !parameterName.includes('+'); })
5887
+ .filter(function (parameterName) { return parameterName !== ''; })
5888
+ .map(validateParameterName);
5889
+ // TODO: [4] DRY
5890
+ var outputSubparameterNames = args
5891
+ .slice(4)
5892
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5893
+ .filter(function (parameterName) { return parameterName.includes('+'); })
5894
+ .map(function (parameterName) { return parameterName.split('+').join(''); })
5895
+ .map(validateParameterName);
5896
+ if (outputSubparameterNames.length === 1) {
5897
+ outputSubparameterName = outputSubparameterNames[0];
5898
+ }
5899
+ else if (outputSubparameterNames.length > 1) {
5900
+ throw new ParseError("FOREACH command can not have more than one output subparameter");
5901
+ }
5902
+ if (inputSubparameterNames.length === 0) {
5903
+ throw new ParseError("FOREACH command must have at least one input subparameter");
5904
+ }
5905
+ if (outputSubparameterName === null) {
5906
+ // TODO: Following code should be unhardcoded from here and moved to the format definition
5907
+ if (formatName === 'CSV' && subformatName === 'CELL') {
5908
+ outputSubparameterName = 'newCell';
5909
+ }
5910
+ else if (formatName === 'TEXT' && subformatName === 'LINE') {
5911
+ outputSubparameterName = 'newLine';
5912
+ }
5913
+ else {
5914
+ throw new ParseError(spaceTrim$1("\n FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n Correct example:\n - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n ")));
5915
+ }
5916
+ }
5199
5917
  return {
5200
5918
  type: 'FOREACH',
5201
5919
  formatName: formatName,
5202
- cellName: cellName,
5920
+ subformatName: subformatName,
5203
5921
  parameterName: parameterName,
5204
- subparameterName: subparameterName,
5922
+ inputSubparameterNames: inputSubparameterNames,
5923
+ outputSubparameterName: outputSubparameterName,
5205
5924
  };
5206
5925
  },
5207
5926
  /**
@@ -5210,11 +5929,17 @@ var foreachCommandParser = {
5210
5929
  * Note: `$` is used to indicate that this function mutates given `templateJson`
5211
5930
  */
5212
5931
  $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
5213
- var formatName = command.formatName, cellName = command.cellName, parameterName = command.parameterName, subparameterName = command.subparameterName;
5214
- // TODO: !!!!!! Detect double use
5215
- // TODO: !!!!!! Detect usage with JOKER and don't allow it
5216
- $templateJson.foreach = { formatName: formatName, cellName: cellName, parameterName: parameterName, subparameterName: subparameterName };
5217
- keepUnused($pipelineJson); // <- TODO: !!!!!! BUT Maybe register subparameter from foreach into parameters of the pipeline
5932
+ var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
5933
+ // TODO: [🍭] Detect double use
5934
+ // TODO: [🍭] Detect usage with JOKER and don't allow it
5935
+ $templateJson.foreach = {
5936
+ formatName: formatName,
5937
+ subformatName: subformatName,
5938
+ parameterName: parameterName,
5939
+ inputSubparameterNames: inputSubparameterNames,
5940
+ outputSubparameterName: outputSubparameterName,
5941
+ };
5942
+ keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
5218
5943
  // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
5219
5944
  },
5220
5945
  /**
@@ -5237,8 +5962,7 @@ var foreachCommandParser = {
5237
5962
  },
5238
5963
  };
5239
5964
  /**
5240
- * TODO: !!!!!! Comment console logs
5241
- * TODO: [🍭] !!!!!! Make .ptbk.md file with examples of the FOREACH command and also with wrong parsing and logic
5965
+ * TODO: [🍭] Make .ptbk.md file with examples of the FOREACH with wrong parsing and logic
5242
5966
  */
5243
5967
 
5244
5968
  /**
@@ -5348,12 +6072,11 @@ var jokerCommandParser = {
5348
6072
  */
5349
6073
  parse: function (input) {
5350
6074
  var args = input.args;
5351
- // TODO: !!!!!! Replace with propper parameter name validation
5352
- var parametersMatch = (args.pop() || '').match(/^\{(?<parameterName>[a-z0-9_]+)\}$/im);
5353
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5354
- throw new ParseError("Invalid joker");
6075
+ if (args.length !== 1) {
6076
+ throw new ParseError("JOKE command expects exactly one parameter name");
5355
6077
  }
5356
- var parameterName = parametersMatch.groups.parameterName;
6078
+ var parameterNameArg = args[0] || '';
6079
+ var parameterName = validateParameterName(parameterNameArg);
5357
6080
  return {
5358
6081
  type: 'JOKER',
5359
6082
  parameterName: parameterName,
@@ -5428,6 +6151,9 @@ var modelCommandParser = {
5428
6151
  */
5429
6152
  parse: function (input) {
5430
6153
  var args = input.args, normalized = input.normalized;
6154
+ var availableVariantsMessage = spaceTrim$1(function (block) { return "\n Available variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) {
6155
+ return "- ".concat(variantName).concat(variantName !== 'EMBEDDING' ? '' : ' (Not available in pipeline)');
6156
+ }).join('\n')), "\n "); });
5431
6157
  // TODO: Make this more elegant and dynamically
5432
6158
  if (normalized.startsWith('MODEL_VARIANT')) {
5433
6159
  if (normalized === 'MODEL_VARIANT_CHAT') {
@@ -5443,17 +6169,13 @@ var modelCommandParser = {
5443
6169
  key: 'modelVariant',
5444
6170
  value: 'COMPLETION',
5445
6171
  };
6172
+ // <- Note: [🤖]
5446
6173
  }
5447
6174
  else if (normalized.startsWith('MODEL_VARIANT_EMBED')) {
5448
- return {
5449
- type: 'MODEL',
5450
- key: 'modelVariant',
5451
- value: 'EMBEDDING',
5452
- };
5453
- // <- Note: [🤖]
6175
+ spaceTrim$1(function (block) { return "\n Embedding model can not be used in pipeline\n\n ".concat(block(availableVariantsMessage), "\n "); });
5454
6176
  }
5455
6177
  else {
5456
- throw new ParseError(spaceTrim$1(function (block) { return "\n Unknown model variant in command:\n\n Supported variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) { return "- ".concat(variantName); }).join('\n')), "\n "); }));
6178
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unknown model variant in command:\n\n ".concat(block(availableVariantsMessage), "\n "); }));
5457
6179
  }
5458
6180
  }
5459
6181
  if (normalized.startsWith('MODEL_NAME')) {
@@ -5578,14 +6300,13 @@ var parameterCommandParser = {
5578
6300
  * Parses the PARAMETER command
5579
6301
  */
5580
6302
  parse: function (input) {
5581
- var normalized = input.normalized, raw = input.raw;
5582
- var parametersMatch = raw.match(/\{(?<parameterName>[a-z0-9_]+)\}[^\S\r\n]*(?<parameterDescription>.*)$/im);
5583
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5584
- throw new ParseError("Invalid parameter");
5585
- }
5586
- var _a = parametersMatch.groups, parameterName = _a.parameterName, parameterDescription = _a.parameterDescription;
5587
- if (parameterDescription && parameterDescription.match(/\{(?<parameterName>[a-z0-9_]+)\}/im)) {
5588
- throw new ParseError("Parameter {".concat(parameterName, "} can not contain another parameter in description"));
6303
+ var normalized = input.normalized, args = input.args, raw = input.raw;
6304
+ var parameterNameRaw = args.shift() || '';
6305
+ var parameterDescriptionRaw = args.join(' ');
6306
+ // <- TODO: When [🥶] fixed, change to:
6307
+ // > const parameterDescriptionRaw = rawArgs.split(parameterNameRaw).join('').trim();
6308
+ if (parameterDescriptionRaw && parameterDescriptionRaw.match(/\{(?<embeddedParameterName>[a-z0-9_]+)\}/im)) {
6309
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Parameter {".concat(parameterNameRaw, "} can not contain another parameter in description\n\n The description:\n ").concat(block(parameterDescriptionRaw), "\n "); }));
5589
6310
  }
5590
6311
  var isInput = normalized.startsWith('INPUT');
5591
6312
  var isOutput = normalized.startsWith('OUTPUT');
@@ -5593,11 +6314,12 @@ var parameterCommandParser = {
5593
6314
  isInput = false;
5594
6315
  isOutput = false;
5595
6316
  }
5596
- // TODO: !!!!!! Add parameter name validation
6317
+ var parameterName = validateParameterName(parameterNameRaw);
6318
+ var parameterDescription = parameterDescriptionRaw.trim() || null;
5597
6319
  return {
5598
6320
  type: 'PARAMETER',
5599
6321
  parameterName: parameterName,
5600
- parameterDescription: parameterDescription.trim() || null,
6322
+ parameterDescription: parameterDescription,
5601
6323
  isInput: isInput,
5602
6324
  isOutput: isOutput,
5603
6325
  };
@@ -5922,6 +6644,7 @@ var promptbookVersionCommandParser = {
5922
6644
  * Note: `$` is used to indicate that this function mutates given `pipelineJson`
5923
6645
  */
5924
6646
  $applyToPipelineJson: function (command, $pipelineJson) {
6647
+ // TODO: Warn if the version is overridden
5925
6648
  $pipelineJson.promptbookVersion = command.promptbookVersion;
5926
6649
  },
5927
6650
  /**
@@ -6404,7 +7127,9 @@ function parseCommand(raw, usagePlace) {
6404
7127
  for (var commandNameSegmentsCount = 0; commandNameSegmentsCount < Math.min(items.length, 3); commandNameSegmentsCount++) {
6405
7128
  var commandNameRaw = items.slice(0, commandNameSegmentsCount + 1).join('_');
6406
7129
  var args = items.slice(commandNameSegmentsCount + 1);
6407
- var rawArgs = raw.substring(commandNameRaw.length).trim();
7130
+ var rawArgs = raw
7131
+ .substring(commandNameRaw.length)
7132
+ .trim();
6408
7133
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6409
7134
  if (command !== null) {
6410
7135
  return command;
@@ -6415,7 +7140,9 @@ function parseCommand(raw, usagePlace) {
6415
7140
  {
6416
7141
  var commandNameRaw = items.slice(-1).join('_');
6417
7142
  var args = items.slice(0, -1); // <- Note: This is probbably not correct
6418
- var rawArgs = raw.substring(0, raw.length - commandNameRaw.length).trim();
7143
+ var rawArgs = raw
7144
+ .substring(0, raw.length - commandNameRaw.length)
7145
+ .trim();
6419
7146
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6420
7147
  if (command !== null) {
6421
7148
  return command;
@@ -6555,7 +7282,7 @@ function extractAllListItemsFromMarkdown(markdown) {
6555
7282
  function extractOneBlockFromMarkdown(markdown) {
6556
7283
  var codeBlocks = extractAllBlocksFromMarkdown(markdown);
6557
7284
  if (codeBlocks.length !== 1) {
6558
- throw new ParseError(spaceTrim$1(function (block) { return "\n There should be exactly 1 code block, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
7285
+ throw new ParseError(spaceTrim$1(function (block) { return "\n There should be exactly 1 code block in template, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
6559
7286
  }
6560
7287
  return codeBlocks[0];
6561
7288
  }
@@ -6742,7 +7469,7 @@ function pipelineStringToJsonSync(pipelineString) {
6742
7469
  var $pipelineJson = {
6743
7470
  title: undefined /* <- Note: [🍙] Putting here placeholder to keep `title` on top at final JSON */,
6744
7471
  pipelineUrl: undefined /* <- Note: Putting here placeholder to keep `pipelineUrl` on top at final JSON */,
6745
- promptbookVersion: PROMPTBOOK_VERSION,
7472
+ promptbookVersion: undefined /* <- Note: By default no explicit version */,
6746
7473
  description: undefined /* <- Note: [🍙] Putting here placeholder to keep `description` on top at final JSON */,
6747
7474
  parameters: [],
6748
7475
  templates: [],
@@ -7033,7 +7760,7 @@ function pipelineStringToJsonSync(pipelineString) {
7033
7760
  return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
7034
7761
  }
7035
7762
  /**
7036
- * TODO: !!!! Warn if used only sync version
7763
+ * TODO: [main] !!!! Warn if used only sync version
7037
7764
  * TODO: [🚞] Report here line/column of error
7038
7765
  * TODO: Use spaceTrim more effectively
7039
7766
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
@@ -7816,7 +8543,7 @@ function isSerializableAsJson(value) {
7816
8543
  }
7817
8544
  }
7818
8545
  /**
7819
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
8546
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
7820
8547
  * TODO: [🧠][💺] Can be done this on type-level?
7821
8548
  */
7822
8549