@promptbook/node 0.69.0-2 → 0.69.0-21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +21 -58
  2. package/esm/index.es.js +1855 -1069
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/promptbook-collection/index.d.ts +0 -3
  5. package/esm/typings/src/_packages/core.index.d.ts +9 -1
  6. package/esm/typings/src/_packages/types.index.d.ts +10 -0
  7. package/esm/typings/src/_packages/utils.index.d.ts +4 -4
  8. package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
  9. package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +1 -1
  10. package/esm/typings/src/commands/FOREACH/ForeachCommand.d.ts +1 -1
  11. package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -5
  12. package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -2
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +1 -1
  14. package/esm/typings/src/config.d.ts +11 -4
  15. package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +1 -1
  16. package/esm/typings/src/conversion/prettify/renderPipelineMermaidOptions.d.ts +3 -3
  17. package/esm/typings/src/conversion/utils/extractParameterNamesFromTemplate.d.ts +2 -1
  18. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +6 -5
  19. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +9 -8
  20. package/esm/typings/src/execution/createPipelineExecutor/$OngoingTemplateResult.d.ts +45 -0
  21. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +20 -0
  22. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +40 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +10 -0
  24. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +55 -0
  25. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +62 -0
  26. package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatCells.d.ts +19 -0
  27. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +74 -0
  28. package/esm/typings/src/execution/createPipelineExecutor/filterJustOutputParameters.d.ts +34 -0
  29. package/esm/typings/src/execution/createPipelineExecutor/getContextForTemplate.d.ts +10 -0
  30. package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTemplate.d.ts +27 -0
  31. package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTemplate.d.ts +30 -0
  32. package/esm/typings/src/execution/createPipelineExecutor/getSamplesForTemplate.d.ts +10 -0
  33. package/esm/typings/src/execution/embeddingVectorToString.d.ts +1 -1
  34. package/esm/typings/src/execution/utils/checkExpectations.d.ts +2 -0
  35. package/esm/typings/src/execution/utils/usageToHuman.d.ts +3 -4
  36. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +14 -15
  37. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +31 -0
  38. package/esm/typings/src/formats/csv/{ListFormatDefinition.d.ts → CsvFormatDefinition.d.ts} +6 -3
  39. package/esm/typings/src/formats/csv/CsvSettings.d.ts +13 -0
  40. package/esm/typings/src/formats/index.d.ts +1 -1
  41. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +4 -3
  42. package/esm/typings/src/formats/text/TextFormatDefinition.d.ts +19 -0
  43. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +4 -3
  44. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +1 -1
  45. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +1 -1
  46. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +1 -1
  47. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  48. package/esm/typings/src/llm-providers/anthropic-claude/createAnthropicClaudeExecutionTools.d.ts +2 -2
  49. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +2 -2
  50. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +1 -1
  51. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  52. package/esm/typings/src/prepare/isPipelinePrepared.d.ts +1 -1
  53. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  54. package/esm/typings/src/types/PipelineJson/ParameterJson.d.ts +1 -1
  55. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +1 -1
  56. package/esm/typings/src/types/execution-report/executionReportJsonToString.d.ts +2 -1
  57. package/esm/typings/src/types/typeAliases.d.ts +1 -1
  58. package/esm/typings/src/utils/expectation-counters/index.d.ts +3 -0
  59. package/esm/typings/src/utils/organization/{f.d.ts → empty_object.d.ts} +5 -1
  60. package/esm/typings/src/utils/organization/just_empty_object.d.ts +12 -0
  61. package/esm/typings/src/utils/{extractParameterNames.d.ts → parameters/extractParameterNames.d.ts} +2 -2
  62. package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +27 -0
  63. package/esm/typings/src/utils/{replaceParameters.d.ts → parameters/replaceParameters.d.ts} +2 -2
  64. package/esm/typings/src/utils/serialization/checkSerializableAsJson.d.ts +1 -1
  65. package/esm/typings/src/utils/serialization/isSerializableAsJson.d.ts +1 -1
  66. package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +10 -0
  67. package/package.json +18 -13
  68. package/umd/index.umd.js +1858 -1073
  69. package/umd/index.umd.js.map +1 -1
  70. package/esm/typings/src/execution/createPipelineExecutor.d.ts +0 -72
  71. package/esm/typings/src/formats/list/ListFormatDefinition.d.ts +0 -16
  72. /package/esm/typings/src/utils/{extractParameterNames.test.d.ts → parameters/extractParameterNames.test.d.ts} +0 -0
  73. /package/esm/typings/src/{execution/utils/usageToHuman.test.d.ts → utils/parameters/mapAvailableToExpectedParameters.test.d.ts} +0 -0
  74. /package/esm/typings/src/utils/{replaceParameters.test.d.ts → parameters/replaceParameters.test.d.ts} +0 -0
  75. /package/esm/typings/src/{personas/preparePersona.test.d.ts → utils/validators/parameterName/validateParameterName.test.d.ts} +0 -0
package/esm/index.es.js CHANGED
@@ -5,6 +5,7 @@ import spaceTrim$1, { spaceTrim } from 'spacetrim';
5
5
  import { format } from 'prettier';
6
6
  import parserHtml from 'prettier/parser-html';
7
7
  import { forTime } from 'waitasecond';
8
+ import { unparse, parse } from 'papaparse';
8
9
  import hexEncoder from 'crypto-js/enc-hex';
9
10
  import sha256 from 'crypto-js/sha256';
10
11
  import { join } from 'path/posix';
@@ -14,8 +15,8 @@ import * as dotenv from 'dotenv';
14
15
  /**
15
16
  * The version of the Promptbook library
16
17
  */
17
- var PROMPTBOOK_VERSION = '0.69.0-1';
18
- // TODO: !!!! List here all the versions and annotate + put into script
18
+ var PROMPTBOOK_VERSION = '0.69.0-20';
19
+ // TODO:[main] !!!! List here all the versions and annotate + put into script
19
20
 
20
21
  /*! *****************************************************************************
21
22
  Copyright (c) Microsoft Corporation.
@@ -325,7 +326,7 @@ function checkSerializableAsJson(name, value) {
325
326
  }
326
327
  /**
327
328
  * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
328
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
329
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
329
330
  * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
330
331
  */
331
332
 
@@ -367,13 +368,13 @@ var IMMEDIATE_TIME = 10;
367
368
  *
368
369
  * @public exported from `@promptbook/core`
369
370
  */
370
- var MAX_PARALLEL_COUNT = 5;
371
+ var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
371
372
  /**
372
373
  * The maximum number of attempts to execute LLM task before giving up
373
374
  *
374
375
  * @public exported from `@promptbook/core`
375
376
  */
376
- var MAX_EXECUTION_ATTEMPTS = 3;
377
+ var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
377
378
  /**
378
379
  * The maximum length of the (generated) filename
379
380
  *
@@ -404,6 +405,7 @@ var RESERVED_PARAMETER_NAMES = $asDeeplyFrozenSerializableJson('RESERVED_PARAMET
404
405
  'samples',
405
406
  'modelName',
406
407
  'currentDate',
408
+ // <- TODO: !!!!! list here all command names
407
409
  // <- TODO: Add more like 'date', 'modelName',...
408
410
  // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
409
411
  ]);
@@ -420,6 +422,17 @@ var RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
420
422
  */
421
423
  var RESERVED_PARAMETER_RESTRICTED = 'RESTRICTED-' + REPLACING_NONCE;
422
424
  // <- TODO: [🧜‍♂️]
425
+ /**
426
+ * @@@
427
+ *
428
+ * @public exported from `@promptbook/core`
429
+ */
430
+ var DEFAULT_CSV_SETTINGS = Object.freeze({
431
+ delimiter: ',',
432
+ quoteChar: '"',
433
+ newline: '\n',
434
+ skipEmptyLines: true,
435
+ });
423
436
  /**
424
437
  * @@@
425
438
  *
@@ -503,7 +516,7 @@ function pipelineJsonToString(pipelineJson) {
503
516
  commands.push("PIPELINE URL ".concat(pipelineUrl));
504
517
  }
505
518
  commands.push("PROMPTBOOK VERSION ".concat(promptbookVersion));
506
- // TODO: !!! This increase size of the bundle and is probbably not necessary
519
+ // TODO:[main] !!! This increase size of the bundle and is probbably not necessary
507
520
  pipelineString = prettifyMarkdown(pipelineString);
508
521
  try {
509
522
  for (var _g = __values(parameters.filter(function (_a) {
@@ -651,12 +664,12 @@ function pipelineJsonToString(pipelineJson) {
651
664
  pipelineString += '```' + contentLanguage;
652
665
  pipelineString += '\n';
653
666
  pipelineString += spaceTrim$1(content);
654
- // <- TODO: !!! Escape
667
+ // <- TODO:[main] !!! Escape
655
668
  // <- TODO: [🧠] Some clear strategy how to spaceTrim the blocks
656
669
  pipelineString += '\n';
657
670
  pipelineString += '```';
658
671
  pipelineString += '\n\n';
659
- pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: !!! If the parameter here has description, add it and use templateParameterJsonToString
672
+ pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO:[main] !!! If the parameter here has description, add it and use templateParameterJsonToString
660
673
  }
661
674
  }
662
675
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
@@ -883,7 +896,7 @@ function forEachAsync(array, options, callbackfunction) {
883
896
  });
884
897
  }
885
898
 
886
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.69.0-1",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.69.0-1",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.69.0-1",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.69.0-1",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
899
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
887
900
 
888
901
  /**
889
902
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -958,7 +971,7 @@ function isValidPromptbookVersion(version) {
958
971
  if ( /* version === '1.0.0' || */version === '2.0.0' || version === '3.0.0') {
959
972
  return false;
960
973
  }
961
- // <- TODO: !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
974
+ // <- TODO:[main] !!! Check isValidPromptbookVersion against PROMPTBOOK_VERSIONS
962
975
  return true;
963
976
  }
964
977
 
@@ -1127,19 +1140,19 @@ function validatePipelineCore(pipeline) {
1127
1140
  // <- Note: [🚲]
1128
1141
  throw new PipelineLogicError(spaceTrim(function (block) { return "\n Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1129
1142
  }
1130
- if (!isValidPromptbookVersion(pipeline.promptbookVersion)) {
1143
+ if (pipeline.promptbookVersion !== undefined && !isValidPromptbookVersion(pipeline.promptbookVersion)) {
1131
1144
  // <- Note: [🚲]
1132
1145
  throw new PipelineLogicError(spaceTrim(function (block) { return "\n Invalid Promptbook Version \"".concat(pipeline.promptbookVersion, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
1133
1146
  }
1134
1147
  // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
1135
1148
  if (!Array.isArray(pipeline.parameters)) {
1136
1149
  // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
1137
- throw new ParseError(spaceTrim(function (block) { return "\n Promptbook is valid JSON but with wrong structure\n\n `promptbook.parameters` expected to be an array, but got ".concat(typeof pipeline.parameters, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
1150
+ throw new ParseError(spaceTrim(function (block) { return "\n Pipeline is valid JSON but with wrong structure\n\n `PipelineJson.parameters` expected to be an array, but got ".concat(typeof pipeline.parameters, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
1138
1151
  }
1139
1152
  // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
1140
1153
  if (!Array.isArray(pipeline.templates)) {
1141
1154
  // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
1142
- throw new ParseError(spaceTrim(function (block) { return "\n Promptbook is valid JSON but with wrong structure\n\n `promptbook.templates` expected to be an array, but got ".concat(typeof pipeline.templates, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
1155
+ throw new ParseError(spaceTrim(function (block) { return "\n Pipeline is valid JSON but with wrong structure\n\n `PipelineJson.templates` expected to be an array, but got ".concat(typeof pipeline.templates, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
1143
1156
  }
1144
1157
  var _loop_1 = function (parameter) {
1145
1158
  if (parameter.isInput && parameter.isOutput) {
@@ -1310,6 +1323,7 @@ function validatePipelineCore(pipeline) {
1310
1323
  }
1311
1324
  }
1312
1325
  /**
1326
+ * TODO: !!!!! [🧞‍♀️] Do not allow joker + foreach
1313
1327
  * TODO: [🧠] Work with promptbookVersion
1314
1328
  * TODO: Use here some json-schema, Zod or something similar and change it to:
1315
1329
  * > /**
@@ -1321,11 +1335,11 @@ function validatePipelineCore(pipeline) {
1321
1335
  * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
1322
1336
  */
1323
1337
  /**
1324
- * TODO: [🐣] !!!! Validate that all samples match expectations
1325
- * TODO: [🐣][🐝] !!!! Validate that knowledge is valid (non-void)
1326
- * TODO: [🐣] !!!! Validate that persona can be used only with CHAT variant
1327
- * TODO: [🐣] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1328
- * TODO: [🐣] !!!! Validate that reserved parameter is not used as joker
1338
+ * TODO: [🐣][main] !!!! Validate that all samples match expectations
1339
+ * TODO: [🐣][🐝][main] !!!! Validate that knowledge is valid (non-void)
1340
+ * TODO: [🐣][main] !!!! Validate that persona can be used only with CHAT variant
1341
+ * TODO: [🐣][main] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
1342
+ * TODO: [🐣][main] !!!! Validate that reserved parameter is not used as joker
1329
1343
  * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists
1330
1344
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
1331
1345
  */
@@ -2047,121 +2061,35 @@ function assertsExecutionSuccessful(executionResult) {
2047
2061
  */
2048
2062
 
2049
2063
  /**
2050
- * Parses the given script and returns the list of all used variables that are not defined in the script
2051
- *
2052
- * @param script from which to extract the variables
2053
- * @returns the list of variable names
2054
- * @throws {ParseError} if the script is invalid
2055
- * @public exported from `@promptbook/utils`
2056
- */
2057
- function extractVariables(script) {
2058
- var variables = new Set();
2059
- script = "(()=>{".concat(script, "})()");
2060
- try {
2061
- for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
2062
- try {
2063
- eval(script);
2064
- }
2065
- catch (error) {
2066
- if (!(error instanceof ReferenceError)) {
2067
- throw error;
2068
- }
2069
- var undefinedName = error.message.split(' ')[0];
2070
- /*
2071
- Note: Parsing the error
2072
- [PipelineUrlError: thing is not defined]
2073
- */
2074
- if (!undefinedName) {
2075
- throw error;
2076
- }
2077
- if (script.includes(undefinedName + '(')) {
2078
- script = "const ".concat(undefinedName, " = ()=>'';") + script;
2079
- }
2080
- else {
2081
- variables.add(undefinedName);
2082
- script = "const ".concat(undefinedName, " = '';") + script;
2083
- }
2084
- }
2085
- }
2086
- catch (error) {
2087
- if (!(error instanceof Error)) {
2088
- throw error;
2089
- }
2090
- throw new ParseError(spaceTrim(function (block) { return "\n Can not extract variables from the script\n\n ".concat(block(error.toString()), "}\n "); }));
2091
- }
2092
- return variables;
2093
- }
2094
- /**
2095
- * TODO: [🔣] Support for multiple languages - python, java,...
2096
- */
2097
-
2098
- /**
2099
- * Parses the template and returns the set of all used parameters
2064
+ * Determine if the pipeline is fully prepared
2100
2065
  *
2101
- * @param template the template with used parameters
2102
- * @returns the set of parameter names
2103
- * @throws {ParseError} if the script is invalid
2104
- * @public exported from `@promptbook/utils`
2066
+ * @public exported from `@promptbook/core`
2105
2067
  */
2106
- function extractParameterNamesFromTemplate(template) {
2107
- var e_1, _a, e_2, _b, e_3, _c;
2108
- var title = template.title, description = template.description, templateType = template.templateType, content = template.content, preparedContent = template.preparedContent, jokerParameterNames = template.jokerParameterNames, foreach = template.foreach;
2109
- var parameterNames = new Set();
2110
- try {
2111
- for (var _d = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _e = _d.next(); !_e.done; _e = _d.next()) {
2112
- var parameterName = _e.value;
2113
- parameterNames.add(parameterName);
2114
- }
2115
- }
2116
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2117
- finally {
2118
- try {
2119
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2120
- }
2121
- finally { if (e_1) throw e_1.error; }
2122
- }
2123
- if (templateType === 'SCRIPT_TEMPLATE') {
2124
- try {
2125
- for (var _f = __values(extractVariables(content)), _g = _f.next(); !_g.done; _g = _f.next()) {
2126
- var parameterName = _g.value;
2127
- parameterNames.add(parameterName);
2128
- }
2129
- }
2130
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
2131
- finally {
2132
- try {
2133
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
2134
- }
2135
- finally { if (e_2) throw e_2.error; }
2136
- }
2137
- }
2138
- try {
2139
- for (var _h = __values(jokerParameterNames || []), _j = _h.next(); !_j.done; _j = _h.next()) {
2140
- var jokerName = _j.value;
2141
- parameterNames.add(jokerName);
2142
- }
2143
- }
2144
- catch (e_3_1) { e_3 = { error: e_3_1 }; }
2145
- finally {
2146
- try {
2147
- if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
2148
- }
2149
- finally { if (e_3) throw e_3.error; }
2068
+ function isPipelinePrepared(pipeline) {
2069
+ // Note: Ignoring `pipeline.preparations` @@@
2070
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2071
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2072
+ return false;
2150
2073
  }
2151
- parameterNames.delete('content');
2152
- // <- Note {websiteContent} is used in `preparedContent`
2153
- // Note: [🍭] Fixing dependent subparameterName from FOREACH command
2154
- if (foreach !== undefined) {
2155
- if (parameterNames.has(foreach.subparameterName)) {
2156
- parameterNames.delete(foreach.subparameterName);
2157
- parameterNames.add(foreach.parameterName);
2158
- // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2159
- }
2074
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2075
+ return false;
2160
2076
  }
2161
- return parameterNames;
2077
+ /*
2078
+ TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2079
+ > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2080
+ > return false;
2081
+ > }
2082
+ */
2083
+ return true;
2162
2084
  }
2163
2085
  /**
2164
- * TODO: [🔣] If script require contentLanguage
2086
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2087
+ * TODO: [🐠] Maybe base this on `makeValidator`
2088
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2089
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2090
+ * - [🏍] ? Is context in each template
2091
+ * - [♨] Are samples prepared
2092
+ * - [♨] Are templates prepared
2165
2093
  */
2166
2094
 
2167
2095
  /**
@@ -2181,27 +2109,6 @@ function serializeError(error) {
2181
2109
  };
2182
2110
  }
2183
2111
 
2184
- /**
2185
- * Function isValidJsonString will tell you if the string is valid JSON or not
2186
- *
2187
- * @public exported from `@promptbook/utils`
2188
- */
2189
- function isValidJsonString(value /* <- [👨‍⚖️] */) {
2190
- try {
2191
- JSON.parse(value);
2192
- return true;
2193
- }
2194
- catch (error) {
2195
- if (!(error instanceof Error)) {
2196
- throw error;
2197
- }
2198
- if (error.message.includes('Unexpected token')) {
2199
- return false;
2200
- }
2201
- return false;
2202
- }
2203
- }
2204
-
2205
2112
  /**
2206
2113
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2207
2114
  *
@@ -2228,9 +2135,10 @@ var MultipleLlmExecutionTools = /** @class */ (function () {
2228
2135
  });
2229
2136
  Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2230
2137
  get: function () {
2231
- return this.llmExecutionTools
2232
- .map(function (tools, index) { return "".concat(index + 1, ") ").concat(tools.title, " ").concat(tools.description || ''); })
2233
- .join('\n');
2138
+ return this.llmExecutionTools.map(function (_a, index) {
2139
+ var title = _a.title;
2140
+ return "".concat(index + 1, ") `").concat(title, "`");
2141
+ }).join('\n');
2234
2142
  },
2235
2143
  enumerable: false,
2236
2144
  configurable: true
@@ -2428,9 +2336,7 @@ var MultipleLlmExecutionTools = /** @class */ (function () {
2428
2336
  throw new PipelineExecutionError("You have not provided any `LlmExecutionTools`");
2429
2337
  }
2430
2338
  else {
2431
- throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.llmExecutionTools
2432
- .map(function (tools) { return "- ".concat(tools.title, " ").concat(tools.description || ''); })
2433
- .join('\n')), "\n\n "); }));
2339
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\"\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.description), "\n\n "); }));
2434
2340
  }
2435
2341
  }
2436
2342
  });
@@ -2496,75 +2402,680 @@ function joinLlmExecutionTools() {
2496
2402
  */
2497
2403
 
2498
2404
  /**
2499
- * Extracts all code blocks from markdown.
2405
+ * Takes an item or an array of items and returns an array of items
2500
2406
  *
2501
- * Note: There are multiple simmilar function:
2502
- * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
2503
- * - `extractJsonBlock` extracts exactly one valid JSON code block
2504
- * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
2505
- * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
2407
+ * 1) Any item except array and undefined returns array with that one item (also null)
2408
+ * 2) Undefined returns empty array
2409
+ * 3) Array returns itself
2506
2410
  *
2507
- * @param markdown any valid markdown
2508
- * @returns code blocks with language and content
2509
- * @throws {ParseError} if block is not closed properly
2510
- * @public exported from `@promptbook/markdown-utils`
2411
+ * @private internal utility
2511
2412
  */
2512
- function extractAllBlocksFromMarkdown(markdown) {
2513
- var e_1, _a;
2514
- var codeBlocks = [];
2515
- var lines = markdown.split('\n');
2516
- // Note: [0] Ensure that the last block notated by gt > will be closed
2517
- lines.push('');
2518
- var currentCodeBlock = null;
2413
+ function arrayableToArray(input) {
2414
+ if (input === undefined) {
2415
+ return [];
2416
+ }
2417
+ if (input instanceof Array) {
2418
+ return input;
2419
+ }
2420
+ return [input];
2421
+ }
2422
+
2423
+ /**
2424
+ * Parses the given script and returns the list of all used variables that are not defined in the script
2425
+ *
2426
+ * @param script from which to extract the variables
2427
+ * @returns the list of variable names
2428
+ * @throws {ParseError} if the script is invalid
2429
+ * @public exported from `@promptbook/utils`
2430
+ */
2431
+ function extractVariables(script) {
2432
+ var variables = new Set();
2433
+ script = "(()=>{".concat(script, "})()");
2519
2434
  try {
2520
- for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) {
2521
- var line = lines_1_1.value;
2522
- if (line.startsWith('> ') || line === '>') {
2523
- if (currentCodeBlock === null) {
2524
- currentCodeBlock = { blockNotation: '>', language: null, content: '' };
2525
- } /* not else */
2526
- if (currentCodeBlock.blockNotation === '>') {
2527
- if (currentCodeBlock.content !== '') {
2528
- currentCodeBlock.content += '\n';
2529
- }
2530
- currentCodeBlock.content += line.slice(2);
2531
- }
2532
- }
2533
- else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) {
2534
- codeBlocks.push(currentCodeBlock);
2535
- currentCodeBlock = null;
2435
+ for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
2436
+ try {
2437
+ eval(script);
2536
2438
  }
2537
- /* not else */
2538
- if (line.startsWith('```')) {
2539
- var language = line.slice(3).trim() || null;
2540
- if (currentCodeBlock === null) {
2541
- currentCodeBlock = { blockNotation: '```', language: language, content: '' };
2439
+ catch (error) {
2440
+ if (!(error instanceof ReferenceError)) {
2441
+ throw error;
2542
2442
  }
2543
- else {
2544
- if (language !== null) {
2545
- throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
2546
- }
2547
- codeBlocks.push(currentCodeBlock);
2548
- currentCodeBlock = null;
2443
+ var undefinedName = error.message.split(' ')[0];
2444
+ /*
2445
+ Note: Parsing the error
2446
+ [PipelineUrlError: thing is not defined]
2447
+ */
2448
+ if (!undefinedName) {
2449
+ throw error;
2549
2450
  }
2550
- }
2551
- else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') {
2552
- if (currentCodeBlock.content !== '') {
2553
- currentCodeBlock.content += '\n';
2451
+ if (script.includes(undefinedName + '(')) {
2452
+ script = "const ".concat(undefinedName, " = ()=>'';") + script;
2453
+ }
2454
+ else {
2455
+ variables.add(undefinedName);
2456
+ script = "const ".concat(undefinedName, " = '';") + script;
2554
2457
  }
2555
- currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
2556
2458
  }
2557
- }
2558
2459
  }
2559
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2560
- finally {
2561
- try {
2562
- if (lines_1_1 && !lines_1_1.done && (_a = lines_1.return)) _a.call(lines_1);
2460
+ catch (error) {
2461
+ if (!(error instanceof Error)) {
2462
+ throw error;
2563
2463
  }
2564
- finally { if (e_1) throw e_1.error; }
2565
- }
2566
- if (currentCodeBlock !== null) {
2567
- throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
2464
+ throw new ParseError(spaceTrim(function (block) { return "\n Can not extract variables from the script\n\n ".concat(block(error.toString()), "}\n "); }));
2465
+ }
2466
+ return variables;
2467
+ }
2468
+ /**
2469
+ * TODO: [🔣] Support for multiple languages - python, java,...
2470
+ */
2471
+
2472
+ /**
2473
+ * Parses the template and returns the set of all used parameters
2474
+ *
2475
+ * @param template the template with used parameters
2476
+ * @returns the set of parameter names
2477
+ * @throws {ParseError} if the script is invalid
2478
+ * @public exported from `@promptbook/utils`
2479
+ */
2480
+ function extractParameterNamesFromTemplate(template) {
2481
+ var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
2482
+ var title = template.title, description = template.description, templateType = template.templateType, content = template.content, preparedContent = template.preparedContent, jokerParameterNames = template.jokerParameterNames, foreach = template.foreach;
2483
+ var parameterNames = new Set();
2484
+ try {
2485
+ for (var _e = __values(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameterNames(title)), false), __read(extractParameterNames(description || '')), false), __read(extractParameterNames(content)), false), __read(extractParameterNames(preparedContent || '')), false)), _f = _e.next(); !_f.done; _f = _e.next()) {
2486
+ var parameterName = _f.value;
2487
+ parameterNames.add(parameterName);
2488
+ }
2489
+ }
2490
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2491
+ finally {
2492
+ try {
2493
+ if (_f && !_f.done && (_a = _e.return)) _a.call(_e);
2494
+ }
2495
+ finally { if (e_1) throw e_1.error; }
2496
+ }
2497
+ if (templateType === 'SCRIPT_TEMPLATE') {
2498
+ try {
2499
+ for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
2500
+ var parameterName = _h.value;
2501
+ parameterNames.add(parameterName);
2502
+ }
2503
+ }
2504
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2505
+ finally {
2506
+ try {
2507
+ if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
2508
+ }
2509
+ finally { if (e_2) throw e_2.error; }
2510
+ }
2511
+ }
2512
+ try {
2513
+ for (var _j = __values(jokerParameterNames || []), _k = _j.next(); !_k.done; _k = _j.next()) {
2514
+ var jokerName = _k.value;
2515
+ parameterNames.add(jokerName);
2516
+ }
2517
+ }
2518
+ catch (e_3_1) { e_3 = { error: e_3_1 }; }
2519
+ finally {
2520
+ try {
2521
+ if (_k && !_k.done && (_c = _j.return)) _c.call(_j);
2522
+ }
2523
+ finally { if (e_3) throw e_3.error; }
2524
+ }
2525
+ parameterNames.delete('content');
2526
+ // <- Note {websiteContent} is used in `preparedContent`
2527
+ // Note: [🍭] Fixing dependent subparameterName from FOREACH command
2528
+ if (foreach !== undefined) {
2529
+ try {
2530
+ for (var _l = __values(foreach.inputSubparameterNames), _m = _l.next(); !_m.done; _m = _l.next()) {
2531
+ var subparameterName = _m.value;
2532
+ if (parameterNames.has(subparameterName)) {
2533
+ parameterNames.delete(subparameterName);
2534
+ parameterNames.add(foreach.parameterName);
2535
+ // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
2536
+ }
2537
+ }
2538
+ }
2539
+ catch (e_4_1) { e_4 = { error: e_4_1 }; }
2540
+ finally {
2541
+ try {
2542
+ if (_m && !_m.done && (_d = _l.return)) _d.call(_l);
2543
+ }
2544
+ finally { if (e_4) throw e_4.error; }
2545
+ }
2546
+ }
2547
+ return parameterNames;
2548
+ }
2549
+ /**
2550
+ * TODO: [🔣] If script require contentLanguage
2551
+ */
2552
+
2553
+ /**
2554
+ * Create difference set of two sets.
2555
+ *
2556
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2557
+ * @public exported from `@promptbook/utils`
2558
+ */
2559
+ function difference(a, b, isEqual) {
2560
+ var e_1, _a;
2561
+ if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
2562
+ var diff = new Set();
2563
+ var _loop_1 = function (itemA) {
2564
+ if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
2565
+ diff.add(itemA);
2566
+ }
2567
+ };
2568
+ try {
2569
+ for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
2570
+ var itemA = _c.value;
2571
+ _loop_1(itemA);
2572
+ }
2573
+ }
2574
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2575
+ finally {
2576
+ try {
2577
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2578
+ }
2579
+ finally { if (e_1) throw e_1.error; }
2580
+ }
2581
+ return diff;
2582
+ }
2583
+ /**
2584
+ * TODO: [🧠][💯] Maybe also implement symmetricDifference
2585
+ */
2586
+
2587
+ /**
2588
+ * Creates a new set with all elements that are present in either set
2589
+ *
2590
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2591
+ * @public exported from `@promptbook/utils`
2592
+ */
2593
+ function union() {
2594
+ var e_1, _a, e_2, _b;
2595
+ var sets = [];
2596
+ for (var _i = 0; _i < arguments.length; _i++) {
2597
+ sets[_i] = arguments[_i];
2598
+ }
2599
+ var union = new Set();
2600
+ try {
2601
+ for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
2602
+ var set = sets_1_1.value;
2603
+ try {
2604
+ for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
2605
+ var item = _d.value;
2606
+ union.add(item);
2607
+ }
2608
+ }
2609
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2610
+ finally {
2611
+ try {
2612
+ if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
2613
+ }
2614
+ finally { if (e_2) throw e_2.error; }
2615
+ }
2616
+ }
2617
+ }
2618
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2619
+ finally {
2620
+ try {
2621
+ if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
2622
+ }
2623
+ finally { if (e_1) throw e_1.error; }
2624
+ }
2625
+ return union;
2626
+ }
2627
+
2628
+ /**
2629
+ * Just marks a place of place where should be something implemented
2630
+ * No side effects.
2631
+ *
2632
+ * Note: It can be usefull suppressing eslint errors of unused variables
2633
+ *
2634
+ * @param value any values
2635
+ * @returns void
2636
+ * @private within the repository
2637
+ */
2638
+ function TODO_USE() {
2639
+ var value = [];
2640
+ for (var _i = 0; _i < arguments.length; _i++) {
2641
+ value[_i] = arguments[_i];
2642
+ }
2643
+ }
2644
+
2645
+ /**
2646
+ * @@@
2647
+ *
2648
+ * @public exported from `@promptbook/core`
2649
+ */
2650
+ var MANDATORY_CSV_SETTINGS = Object.freeze({
2651
+ header: true,
2652
+ // encoding: 'utf8',
2653
+ });
2654
+
2655
+ /**
2656
+ * Definition for CSV spreadsheet
2657
+ *
2658
+ * @public exported from `@promptbook/core`
2659
+ * <- TODO: [🏢] Export from package `@promptbook/csv`
2660
+ */
2661
+ var CsvFormatDefinition = {
2662
+ formatName: 'CSV',
2663
+ aliases: ['SPREADSHEET', 'TABLE'],
2664
+ isValid: function (value, settings, schema) {
2665
+ // TODO: !!!!!! Implement CSV validation
2666
+ TODO_USE(value /* <- TODO: Use value here */);
2667
+ TODO_USE(settings /* <- TODO: Use settings here */);
2668
+ TODO_USE(schema /* <- TODO: Use schema here */);
2669
+ return true;
2670
+ },
2671
+ canBeValid: function (partialValue, settings, schema) {
2672
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2673
+ TODO_USE(settings /* <- TODO: Use settings here */);
2674
+ TODO_USE(schema /* <- TODO: Use schema here */);
2675
+ return true;
2676
+ },
2677
+ heal: function (value, settings, schema) {
2678
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2679
+ TODO_USE(settings /* <- TODO: Use settings here */);
2680
+ TODO_USE(schema /* <- TODO: Use schema here */);
2681
+ throw new Error('Not implemented');
2682
+ },
2683
+ subvalueDefinitions: [
2684
+ {
2685
+ subvalueName: 'ROW',
2686
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2687
+ return __awaiter(this, void 0, void 0, function () {
2688
+ var csv, mappedData;
2689
+ var _this = this;
2690
+ return __generator(this, function (_a) {
2691
+ switch (_a.label) {
2692
+ case 0:
2693
+ csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2694
+ if (csv.errors.length !== 0) {
2695
+ throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
2696
+ spaceTrim$1(function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2697
+ }
2698
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
2699
+ var _a, _b;
2700
+ var _c;
2701
+ return __generator(this, function (_d) {
2702
+ switch (_d.label) {
2703
+ case 0:
2704
+ if (row[outputParameterName]) {
2705
+ throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
2706
+ "Can not overwrite existing column \"".concat(outputParameterName, "\" in CSV row"));
2707
+ }
2708
+ _a = [__assign({}, row)];
2709
+ _c = {};
2710
+ _b = outputParameterName;
2711
+ return [4 /*yield*/, mapCallback(row, index)];
2712
+ case 1: return [2 /*return*/, __assign.apply(void 0, _a.concat([(_c[_b] = _d.sent(), _c)]))];
2713
+ }
2714
+ });
2715
+ }); }))];
2716
+ case 1:
2717
+ mappedData = _a.sent();
2718
+ return [2 /*return*/, unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2719
+ }
2720
+ });
2721
+ });
2722
+ },
2723
+ },
2724
+ {
2725
+ subvalueName: 'CELL',
2726
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2727
+ return __awaiter(this, void 0, void 0, function () {
2728
+ var csv, mappedData;
2729
+ var _this = this;
2730
+ return __generator(this, function (_a) {
2731
+ switch (_a.label) {
2732
+ case 0:
2733
+ csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
2734
+ if (csv.errors.length !== 0) {
2735
+ throw new ParseError(// <- TODO: !!!!!! Split PipelineParseError and FormatParseError -> CsvParseError
2736
+ spaceTrim$1(function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
2737
+ }
2738
+ return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
2739
+ var _this = this;
2740
+ return __generator(this, function (_a) {
2741
+ return [2 /*return*/, /* not await */ Promise.all(Object.entries(row).map(function (_a, columnIndex) {
2742
+ var _b = __read(_a, 2), key = _b[0], value = _b[1];
2743
+ return __awaiter(_this, void 0, void 0, function () {
2744
+ var index;
2745
+ var _c;
2746
+ return __generator(this, function (_d) {
2747
+ index = rowIndex * Object.keys(row).length + columnIndex;
2748
+ return [2 /*return*/, /* not await */ mapCallback((_c = {}, _c[key] = value, _c), index)];
2749
+ });
2750
+ });
2751
+ }))];
2752
+ });
2753
+ }); }))];
2754
+ case 1:
2755
+ mappedData = _a.sent();
2756
+ return [2 /*return*/, unparse(mappedData, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS))];
2757
+ }
2758
+ });
2759
+ });
2760
+ },
2761
+ },
2762
+ ],
2763
+ };
2764
+ /**
2765
+ * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
2766
+ * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
2767
+ * TODO: [🍓] In `CsvFormatDefinition` implement `heal
2768
+ * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
2769
+ * TODO: [🏢] Allow to expect something inside CSV objects and other formats
2770
+ */
2771
+
2772
+ /**
2773
+ * Function isValidJsonString will tell you if the string is valid JSON or not
2774
+ *
2775
+ * @public exported from `@promptbook/utils`
2776
+ */
2777
+ function isValidJsonString(value /* <- [👨‍⚖️] */) {
2778
+ try {
2779
+ JSON.parse(value);
2780
+ return true;
2781
+ }
2782
+ catch (error) {
2783
+ if (!(error instanceof Error)) {
2784
+ throw error;
2785
+ }
2786
+ if (error.message.includes('Unexpected token')) {
2787
+ return false;
2788
+ }
2789
+ return false;
2790
+ }
2791
+ }
2792
+
2793
+ /**
2794
+ * Definition for JSON format
2795
+ *
2796
+ * @private still in development [🏢]
2797
+ */
2798
+ var JsonFormatDefinition = {
2799
+ formatName: 'JSON',
2800
+ mimeType: 'application/json',
2801
+ isValid: function (value, settings, schema) {
2802
+ TODO_USE(schema /* <- TODO: Use schema here */);
2803
+ TODO_USE(settings /* <- TODO: Use settings here */);
2804
+ return isValidJsonString(value);
2805
+ },
2806
+ canBeValid: function (partialValue, settings, schema) {
2807
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2808
+ TODO_USE(settings /* <- TODO: Use settings here */);
2809
+ TODO_USE(schema /* <- TODO: Use schema here */);
2810
+ return true;
2811
+ },
2812
+ heal: function (value, settings, schema) {
2813
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2814
+ TODO_USE(settings /* <- TODO: Use settings here */);
2815
+ TODO_USE(schema /* <- TODO: Use schema here */);
2816
+ throw new Error('Not implemented');
2817
+ },
2818
+ subvalueDefinitions: [],
2819
+ };
2820
+ /**
2821
+ * TODO: [🧠] Maybe propper instance of object
2822
+ * TODO: [0] Make string_serialized_json
2823
+ * TODO: [1] Make type for JSON Settings and Schema
2824
+ * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
2825
+ * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
2826
+ * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
2827
+ * TODO: [🍓] In `JsonFormatDefinition` implement `heal
2828
+ * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
2829
+ * TODO: [🏢] Allow to expect something inside JSON objects and other formats
2830
+ */
2831
+
2832
+ /**
2833
+ * Definition for any text - this will be always valid
2834
+ *
2835
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
2836
+ *
2837
+ * @public exported from `@promptbook/core`
2838
+ */
2839
+ var TextFormatDefinition = {
2840
+ formatName: 'TEXT',
2841
+ isValid: function (value) {
2842
+ return typeof value === 'string';
2843
+ },
2844
+ canBeValid: function (partialValue) {
2845
+ return typeof partialValue === 'string';
2846
+ },
2847
+ heal: function () {
2848
+ throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
2849
+ },
2850
+ subvalueDefinitions: [
2851
+ {
2852
+ subvalueName: 'LINE',
2853
+ mapValues: function (value, outputParameterName, settings, mapCallback) {
2854
+ return __awaiter(this, void 0, void 0, function () {
2855
+ var lines, mappedLines;
2856
+ return __generator(this, function (_a) {
2857
+ switch (_a.label) {
2858
+ case 0:
2859
+ lines = value.split('\n');
2860
+ return [4 /*yield*/, Promise.all(lines.map(function (lineContent, lineNumber) {
2861
+ // TODO: [🧠] Maybe option to skip empty line
2862
+ /* not await */ return mapCallback({
2863
+ lineContent: lineContent,
2864
+ // TODO: [🧠] Maybe also put here `lineNumber`
2865
+ }, lineNumber);
2866
+ }))];
2867
+ case 1:
2868
+ mappedLines = _a.sent();
2869
+ return [2 /*return*/, mappedLines.join('\n')];
2870
+ }
2871
+ });
2872
+ });
2873
+ },
2874
+ },
2875
+ // <- TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2876
+ ],
2877
+ };
2878
+ /**
2879
+ * TODO: [1] Make type for XML Text and Schema
2880
+ * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages aviable as subvalues
2881
+ * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
2882
+ * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
2883
+ * TODO: [🍓] In `TextFormatDefinition` implement `heal
2884
+ * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
2885
+ * TODO: [🏢] Allow to expect something inside each item of list and other formats
2886
+ */
2887
+
2888
+ /**
2889
+ * Definition for XML format
2890
+ *
2891
+ * @private still in development [🏢]
2892
+ */
2893
+ var XmlFormatDefinition = {
2894
+ formatName: 'XML',
2895
+ mimeType: 'application/xml',
2896
+ isValid: function (value, settings, schema) {
2897
+ TODO_USE(value /* <- TODO: Use value here */);
2898
+ TODO_USE(settings /* <- TODO: Use settings here */);
2899
+ TODO_USE(schema /* <- TODO: Use schema here */);
2900
+ return true;
2901
+ },
2902
+ canBeValid: function (partialValue, settings, schema) {
2903
+ TODO_USE(partialValue /* <- TODO: Use partialValue here */);
2904
+ TODO_USE(settings /* <- TODO: Use settings here */);
2905
+ TODO_USE(schema /* <- TODO: Use schema here */);
2906
+ return true;
2907
+ },
2908
+ heal: function (value, settings, schema) {
2909
+ TODO_USE(value /* <- TODO: Use partialValue here */);
2910
+ TODO_USE(settings /* <- TODO: Use settings here */);
2911
+ TODO_USE(schema /* <- TODO: Use schema here */);
2912
+ throw new Error('Not implemented');
2913
+ },
2914
+ subvalueDefinitions: [],
2915
+ };
2916
+ /**
2917
+ * TODO: [🧠] Maybe propper instance of object
2918
+ * TODO: [0] Make string_serialized_xml
2919
+ * TODO: [1] Make type for XML Settings and Schema
2920
+ * TODO: [🧠] What to use for validating XMLs - XSD,...
2921
+ * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
2922
+ * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
2923
+ * TODO: [🍓] In `XmlFormatDefinition` implement `heal
2924
+ * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
2925
+ * TODO: [🏢] Allow to expect something inside XML and other formats
2926
+ */
2927
+
2928
+ /**
2929
+ * Definitions for all formats supported by Promptbook
2930
+ *
2931
+ * @private internal index of `...` <- TODO [🏢]
2932
+ */
2933
+ var FORMAT_DEFINITIONS = [
2934
+ JsonFormatDefinition,
2935
+ XmlFormatDefinition,
2936
+ TextFormatDefinition,
2937
+ CsvFormatDefinition,
2938
+ ];
2939
+
2940
+ /**
2941
+ * Maps available parameters to expected parameters
2942
+ *
2943
+ * The strategy is:
2944
+ * 1) @@@
2945
+ * 2) @@@
2946
+ *
2947
+ * @throws {PipelineExecutionError} @@@
2948
+ * @private within the repository used in `createPipelineExecutor`
2949
+ */
2950
+ function mapAvailableToExpectedParameters(options) {
2951
+ var e_1, _a;
2952
+ var expectedParameters = options.expectedParameters, availableParameters = options.availableParameters;
2953
+ var availableParametersNames = new Set(Object.keys(availableParameters));
2954
+ var expectedParameterNames = new Set(Object.keys(expectedParameters));
2955
+ var mappedParameters = {};
2956
+ try {
2957
+ // Phase 1️⃣: Matching mapping
2958
+ for (var _b = __values(Array.from(union(availableParametersNames, expectedParameterNames))), _c = _b.next(); !_c.done; _c = _b.next()) {
2959
+ var parameterName = _c.value;
2960
+ // Situation: Parameter is available and expected
2961
+ if (availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
2962
+ mappedParameters[parameterName] = availableParameters[parameterName];
2963
+ // <- Note: [👩‍👩‍👧] Maybe detect parameter collision here?
2964
+ availableParametersNames.delete(parameterName);
2965
+ expectedParameterNames.delete(parameterName);
2966
+ }
2967
+ // Situation: Parameter is available but NOT expected
2968
+ else if (availableParametersNames.has(parameterName) && !expectedParameterNames.has(parameterName)) {
2969
+ // [🐱‍👤] Do not pass this parameter to prompt - Maybe use it non-matching mapping
2970
+ }
2971
+ // Situation: Parameter is NOT available BUT expected
2972
+ else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) {
2973
+ // Do nothing here - this will be maybe fixed in the non-matching mapping
2974
+ }
2975
+ }
2976
+ }
2977
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2978
+ finally {
2979
+ try {
2980
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2981
+ }
2982
+ finally { if (e_1) throw e_1.error; }
2983
+ }
2984
+ if (expectedParameterNames.size === 0) {
2985
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
2986
+ Object.freeze(mappedParameters);
2987
+ return mappedParameters;
2988
+ }
2989
+ // Phase 2️⃣: Non-matching mapping
2990
+ if (expectedParameterNames.size !== availableParametersNames.size) {
2991
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n Can not map available parameters to expected parameters\n\n Mapped parameters:\n ".concat(block(Object.keys(mappedParameters)
2992
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
2993
+ .join('\n')), "\n\n Expected parameters which can not be mapped:\n ").concat(block(Array.from(expectedParameterNames)
2994
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
2995
+ .join('\n')), "\n\n Remaining available parameters:\n ").concat(block(Array.from(availableParametersNames)
2996
+ .map(function (parameterName) { return "- {".concat(parameterName, "}"); })
2997
+ .join('\n')), "\n\n "); }));
2998
+ }
2999
+ var expectedParameterNamesArray = Array.from(expectedParameterNames);
3000
+ var availableParametersNamesArray = Array.from(availableParametersNames);
3001
+ for (var i = 0; i < expectedParameterNames.size; i++) {
3002
+ mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
3003
+ }
3004
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
3005
+ Object.freeze(mappedParameters);
3006
+ return mappedParameters;
3007
+ }
3008
+
3009
+ /**
3010
+ * Extracts all code blocks from markdown.
3011
+ *
3012
+ * Note: There are multiple simmilar function:
3013
+ * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
3014
+ * - `extractJsonBlock` extracts exactly one valid JSON code block
3015
+ * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
3016
+ * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
3017
+ *
3018
+ * @param markdown any valid markdown
3019
+ * @returns code blocks with language and content
3020
+ * @throws {ParseError} if block is not closed properly
3021
+ * @public exported from `@promptbook/markdown-utils`
3022
+ */
3023
+ function extractAllBlocksFromMarkdown(markdown) {
3024
+ var e_1, _a;
3025
+ var codeBlocks = [];
3026
+ var lines = markdown.split('\n');
3027
+ // Note: [0] Ensure that the last block notated by gt > will be closed
3028
+ lines.push('');
3029
+ var currentCodeBlock = null;
3030
+ try {
3031
+ for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) {
3032
+ var line = lines_1_1.value;
3033
+ if (line.startsWith('> ') || line === '>') {
3034
+ if (currentCodeBlock === null) {
3035
+ currentCodeBlock = { blockNotation: '>', language: null, content: '' };
3036
+ } /* not else */
3037
+ if (currentCodeBlock.blockNotation === '>') {
3038
+ if (currentCodeBlock.content !== '') {
3039
+ currentCodeBlock.content += '\n';
3040
+ }
3041
+ currentCodeBlock.content += line.slice(2);
3042
+ }
3043
+ }
3044
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) {
3045
+ codeBlocks.push(currentCodeBlock);
3046
+ currentCodeBlock = null;
3047
+ }
3048
+ /* not else */
3049
+ if (line.startsWith('```')) {
3050
+ var language = line.slice(3).trim() || null;
3051
+ if (currentCodeBlock === null) {
3052
+ currentCodeBlock = { blockNotation: '```', language: language, content: '' };
3053
+ }
3054
+ else {
3055
+ if (language !== null) {
3056
+ throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
3057
+ }
3058
+ codeBlocks.push(currentCodeBlock);
3059
+ currentCodeBlock = null;
3060
+ }
3061
+ }
3062
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') {
3063
+ if (currentCodeBlock.content !== '') {
3064
+ currentCodeBlock.content += '\n';
3065
+ }
3066
+ currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
3067
+ }
3068
+ }
3069
+ }
3070
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3071
+ finally {
3072
+ try {
3073
+ if (lines_1_1 && !lines_1_1.done && (_a = lines_1.return)) _a.call(lines_1);
3074
+ }
3075
+ finally { if (e_1) throw e_1.error; }
3076
+ }
3077
+ if (currentCodeBlock !== null) {
3078
+ throw new ParseError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
2568
3079
  }
2569
3080
  return codeBlocks;
2570
3081
  }
@@ -2611,57 +3122,6 @@ function extractJsonBlock(markdown) {
2611
3122
  * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
2612
3123
  */
2613
3124
 
2614
- /**
2615
- * Determine if the pipeline is fully prepared
2616
- *
2617
- * @public exported from `@promptbook/core`
2618
- */
2619
- function isPipelinePrepared(pipeline) {
2620
- // Note: Ignoring `pipeline.preparations` @@@
2621
- // Note: Ignoring `pipeline.knowledgePieces` @@@
2622
- if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2623
- return false;
2624
- }
2625
- if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2626
- return false;
2627
- }
2628
- /*
2629
- TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2630
- > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2631
- > return false;
2632
- > }
2633
- */
2634
- return true;
2635
- }
2636
- /**
2637
- * TODO: [🔃] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2638
- * TODO: [🐠] Maybe base this on `makeValidator`
2639
- * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2640
- * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2641
- * - [🏍] ? Is context in each template
2642
- * - [♨] Are samples prepared
2643
- * - [♨] Are templates prepared
2644
- */
2645
-
2646
- /**
2647
- * Takes an item or an array of items and returns an array of items
2648
- *
2649
- * 1) Any item except array and undefined returns array with that one item (also null)
2650
- * 2) Undefined returns empty array
2651
- * 3) Array returns itself
2652
- *
2653
- * @private internal utility
2654
- */
2655
- function arrayableToArray(input) {
2656
- if (input === undefined) {
2657
- return [];
2658
- }
2659
- if (input instanceof Array) {
2660
- return input;
2661
- }
2662
- return [input];
2663
- }
2664
-
2665
3125
  /**
2666
3126
  * Just says that the variable is not used but should be kept
2667
3127
  * No side effects.
@@ -2683,23 +3143,6 @@ function keepUnused() {
2683
3143
  }
2684
3144
  }
2685
3145
 
2686
- /**
2687
- * Just marks a place of place where should be something implemented
2688
- * No side effects.
2689
- *
2690
- * Note: It can be usefull suppressing eslint errors of unused variables
2691
- *
2692
- * @param value any values
2693
- * @returns void
2694
- * @private within the repository
2695
- */
2696
- function TODO_USE() {
2697
- var value = [];
2698
- for (var _i = 0; _i < arguments.length; _i++) {
2699
- value[_i] = arguments[_i];
2700
- }
2701
- }
2702
-
2703
3146
  /**
2704
3147
  * Replaces parameters in template with values from parameters object
2705
3148
  *
@@ -2759,99 +3202,24 @@ function replaceParameters(template, parameters) {
2759
3202
  .map(function (line, index) { return (index === 0 ? line : "".concat(precol).concat(line)); })
2760
3203
  .join('\n');
2761
3204
  }
2762
- replacedTemplate =
2763
- replacedTemplate.substring(0, match.index + precol.length) +
2764
- parameterValue +
2765
- replacedTemplate.substring(match.index + precol.length + parameterName.length + 2);
2766
- };
2767
- while ((match = /^(?<precol>.*){(?<parameterName>\w+)}(.*)/m /* <- Not global */
2768
- .exec(replacedTemplate))) {
2769
- _loop_1();
2770
- }
2771
- // [💫] Check if there are parameters that are not closed properly
2772
- if (/{\w+$/.test(replacedTemplate)) {
2773
- throw new PipelineExecutionError('Parameter is not closed');
2774
- }
2775
- // [💫] Check if there are parameters that are not opened properly
2776
- if (/^\w+}/.test(replacedTemplate)) {
2777
- throw new PipelineExecutionError('Parameter is not opened');
2778
- }
2779
- return replacedTemplate;
2780
- }
2781
-
2782
- /**
2783
- * Create difference set of two sets.
2784
- *
2785
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2786
- * @public exported from `@promptbook/utils`
2787
- */
2788
- function difference(a, b, isEqual) {
2789
- var e_1, _a;
2790
- if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
2791
- var diff = new Set();
2792
- var _loop_1 = function (itemA) {
2793
- if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
2794
- diff.add(itemA);
2795
- }
2796
- };
2797
- try {
2798
- for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
2799
- var itemA = _c.value;
2800
- _loop_1(itemA);
2801
- }
2802
- }
2803
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2804
- finally {
2805
- try {
2806
- if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
2807
- }
2808
- finally { if (e_1) throw e_1.error; }
2809
- }
2810
- return diff;
2811
- }
2812
- /**
2813
- * TODO: [🧠][💯] Maybe also implement symmetricDifference
2814
- */
2815
-
2816
- /**
2817
- * Creates a new set with all elements that are present in either set
2818
- *
2819
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2820
- * @public exported from `@promptbook/utils`
2821
- */
2822
- function union() {
2823
- var e_1, _a, e_2, _b;
2824
- var sets = [];
2825
- for (var _i = 0; _i < arguments.length; _i++) {
2826
- sets[_i] = arguments[_i];
2827
- }
2828
- var union = new Set();
2829
- try {
2830
- for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
2831
- var set = sets_1_1.value;
2832
- try {
2833
- for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
2834
- var item = _d.value;
2835
- union.add(item);
2836
- }
2837
- }
2838
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
2839
- finally {
2840
- try {
2841
- if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
2842
- }
2843
- finally { if (e_2) throw e_2.error; }
2844
- }
2845
- }
2846
- }
2847
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2848
- finally {
2849
- try {
2850
- if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
2851
- }
2852
- finally { if (e_1) throw e_1.error; }
3205
+ replacedTemplate =
3206
+ replacedTemplate.substring(0, match.index + precol.length) +
3207
+ parameterValue +
3208
+ replacedTemplate.substring(match.index + precol.length + parameterName.length + 2);
3209
+ };
3210
+ while ((match = /^(?<precol>.*){(?<parameterName>\w+)}(.*)/m /* <- Not global */
3211
+ .exec(replacedTemplate))) {
3212
+ _loop_1();
2853
3213
  }
2854
- return union;
3214
+ // [💫] Check if there are parameters that are not closed properly
3215
+ if (/{\w+$/.test(replacedTemplate)) {
3216
+ throw new PipelineExecutionError('Parameter is not closed');
3217
+ }
3218
+ // [💫] Check if there are parameters that are not opened properly
3219
+ if (/^\w+}/.test(replacedTemplate)) {
3220
+ throw new PipelineExecutionError('Parameter is not opened');
3221
+ }
3222
+ return replacedTemplate;
2855
3223
  }
2856
3224
 
2857
3225
  /**
@@ -2943,6 +3311,9 @@ var CountUtils = {
2943
3311
  LINES: countLines,
2944
3312
  PAGES: countPages,
2945
3313
  };
3314
+ /**
3315
+ * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
3316
+ */
2946
3317
 
2947
3318
  /**
2948
3319
  * Function checkExpectations will check if the expectations on given value are met
@@ -2979,611 +3350,748 @@ function checkExpectations(expectations, value) {
2979
3350
  }
2980
3351
  /**
2981
3352
  * TODO: [💝] Unite object for expecting amount and format
3353
+ * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
3354
+ * Note: [💝] and [🤠] are interconnected together
2982
3355
  */
2983
3356
 
2984
3357
  /**
2985
- * Creates executor function from pipeline and execution tools.
3358
+ * @@@
2986
3359
  *
2987
- * @returns The executor function
2988
- * @throws {PipelineLogicError} on logical error in the pipeline
2989
- * @public exported from `@promptbook/core`
3360
+ * @private internal utility of `createPipelineExecutor`
2990
3361
  */
2991
- function createPipelineExecutor(options) {
2992
- var _this = this;
2993
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
2994
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d, _e = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e;
2995
- validatePipeline(pipeline);
2996
- var pipelineIdentification = (function () {
2997
- // Note: This is a 😐 implementation of [🚞]
2998
- var _ = [];
2999
- if (pipeline.sourceFile !== undefined) {
3000
- _.push("File: ".concat(pipeline.sourceFile));
3001
- }
3002
- if (pipeline.pipelineUrl !== undefined) {
3003
- _.push("Url: ".concat(pipeline.pipelineUrl));
3004
- }
3005
- return _.join('\n');
3006
- })();
3007
- var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
3008
- var preparedPipeline;
3009
- if (isPipelinePrepared(pipeline)) {
3010
- preparedPipeline = pipeline;
3011
- }
3012
- else if (isNotPreparedWarningSupressed !== true) {
3013
- console.warn(spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
3014
- }
3015
- var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
3016
- // TODO: !!! Extract to separate functions and files - ALL FUNCTIONS BELOW
3017
- function getContextForTemplate(template) {
3018
- return __awaiter(this, void 0, void 0, function () {
3019
- return __generator(this, function (_a) {
3020
- TODO_USE(template);
3021
- return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */];
3022
- });
3023
- });
3024
- }
3025
- function getKnowledgeForTemplate(template) {
3026
- return __awaiter(this, void 0, void 0, function () {
3027
- return __generator(this, function (_a) {
3028
- // TODO: [♨] Implement Better - use real index and keyword search from `template` and {samples}
3029
- TODO_USE(template);
3030
- return [2 /*return*/, preparedPipeline.knowledgePieces.map(function (_a) {
3031
- var content = _a.content;
3032
- return "- ".concat(content);
3033
- }).join('\n')];
3034
- });
3035
- });
3036
- }
3037
- function getSamplesForTemplate(template) {
3038
- return __awaiter(this, void 0, void 0, function () {
3039
- return __generator(this, function (_a) {
3040
- // TODO: [♨] Implement Better - use real index and keyword search
3041
- TODO_USE(template);
3042
- return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */];
3043
- });
3044
- });
3045
- }
3046
- function getReservedParametersForTemplate(template) {
3047
- return __awaiter(this, void 0, void 0, function () {
3048
- var context, knowledge, samples, currentDate, modelName, reservedParameters, _loop_3, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName;
3049
- var e_3, _a;
3050
- return __generator(this, function (_b) {
3051
- switch (_b.label) {
3052
- case 0: return [4 /*yield*/, getContextForTemplate(template)];
3053
- case 1:
3054
- context = _b.sent();
3055
- return [4 /*yield*/, getKnowledgeForTemplate(template)];
3056
- case 2:
3057
- knowledge = _b.sent();
3058
- return [4 /*yield*/, getSamplesForTemplate(template)];
3059
- case 3:
3060
- samples = _b.sent();
3061
- currentDate = new Date().toISOString();
3062
- modelName = RESERVED_PARAMETER_MISSING_VALUE;
3063
- reservedParameters = {
3064
- content: RESERVED_PARAMETER_RESTRICTED,
3065
- context: context,
3066
- knowledge: knowledge,
3067
- samples: samples,
3068
- currentDate: currentDate,
3069
- modelName: modelName,
3070
- };
3071
- _loop_3 = function (parameterName) {
3072
- if (reservedParameters[parameterName] === undefined) {
3073
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Reserved parameter {".concat(parameterName, "} is not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3074
- }
3075
- };
3076
- try {
3077
- // Note: Doublecheck that ALL reserved parameters are defined:
3078
- for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
3079
- parameterName = RESERVED_PARAMETER_NAMES_1_1.value;
3080
- _loop_3(parameterName);
3081
- }
3082
- }
3083
- catch (e_3_1) { e_3 = { error: e_3_1 }; }
3084
- finally {
3085
- try {
3086
- if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1);
3087
- }
3088
- finally { if (e_3) throw e_3.error; }
3089
- }
3090
- return [2 /*return*/, reservedParameters];
3091
- }
3092
- });
3093
- });
3094
- }
3095
- function executeSingleTemplate(currentTemplate) {
3096
- return __awaiter(this, void 0, void 0, function () {
3097
- var name, title, priority, progress_1, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_4, _d, _e, parameterName, prompt, chatResult, completionResult, embeddingResult, result, resultString, expectError, scriptPipelineExecutionErrors, maxAttempts, jokerParameterNames, preparedContent, _loop_5, attempt, state_2, progress_2;
3098
- var e_4, _f, _g;
3099
- return __generator(this, function (_h) {
3100
- switch (_h.label) {
3101
- case 0:
3102
- name = "pipeline-executor-frame-".concat(currentTemplate.name);
3103
- title = currentTemplate.title;
3104
- priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
3105
- if (!(onProgress !== undefined) /* <- [3] */) return [3 /*break*/, 2]; /* <- [3] */
3106
- progress_1 = {
3107
- name: name,
3108
- title: title,
3109
- isStarted: false,
3110
- isDone: false,
3111
- templateType: currentTemplate.templateType,
3112
- parameterName: currentTemplate.resultingParameterName,
3113
- parameterValue: null,
3114
- // <- [3]
3115
- };
3116
- if (isReturned) {
3117
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished \uD83C\uDF4F\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress_1, null, 4)
3118
- .split('\n')
3119
- .map(function (line) { return "> ".concat(line); })
3120
- .join('\n')), "\n "); }));
3121
- }
3122
- return [4 /*yield*/, onProgress(progress_1)];
3123
- case 1:
3124
- _h.sent();
3125
- _h.label = 2;
3126
- case 2:
3127
- usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
3128
- dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
3129
- if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
3130
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Dependent parameters:\n ").concat(Array.from(dependentParameterNames)
3131
- .map(function (name) { return "{".concat(name, "}"); })
3132
- .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames)
3133
- .map(function (name) { return "{".concat(name, "}"); })
3134
- .join(', '), "\n\n "); }));
3135
- }
3136
- _b = (_a = Object).freeze;
3137
- _c = [{}];
3138
- return [4 /*yield*/, getReservedParametersForTemplate(currentTemplate)];
3139
- case 3:
3140
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
3141
- definedParameterNames = new Set(Object.keys(definedParameters));
3142
- parameters = {};
3143
- _loop_4 = function (parameterName) {
3144
- // Situation: Parameter is defined and used
3145
- if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3146
- parameters[parameterName] = definedParameters[parameterName];
3147
- }
3148
- // Situation: Parameter is defined but NOT used
3149
- else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) ;
3150
- // Situation: Parameter is NOT defined BUT used
3151
- else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3152
- // Houston, we have a problem
3153
- // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck
3154
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is NOT defined\n BUT used in template \"").concat(currentTemplate.title || currentTemplate.name, "\"\n\n This should be catched in `validatePipeline`\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3155
- }
3156
- };
3157
- try {
3158
- // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
3159
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
3160
- parameterName = _e.value;
3161
- _loop_4(parameterName);
3162
- }
3163
- }
3164
- catch (e_4_1) { e_4 = { error: e_4_1 }; }
3165
- finally {
3166
- try {
3167
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
3168
- }
3169
- finally { if (e_4) throw e_4.error; }
3170
- }
3171
- // Note: Now we can freeze `parameters` because we are sure that all and only used parameters are defined
3172
- Object.freeze(parameters);
3173
- result = null;
3174
- resultString = null;
3175
- expectError = null;
3176
- maxAttempts = currentTemplate.templateType === 'DIALOG_TEMPLATE' ? Infinity : maxExecutionAttempts;
3177
- jokerParameterNames = currentTemplate.jokerParameterNames || [];
3178
- preparedContent = (currentTemplate.preparedContent || '{content}')
3179
- .split('{content}')
3180
- .join(currentTemplate.content);
3181
- _loop_5 = function (attempt) {
3182
- var isJokerAttempt, jokerParameterName, _j, modelRequirements, _k, _l, _m, scriptTools, error_2, e_5_1, _o, _p, functionName, postprocessingError, _q, _r, scriptTools, error_3, e_6_1, e_7_1, error_4;
3183
- var e_5, _s, e_7, _t, e_6, _u;
3184
- return __generator(this, function (_v) {
3185
- switch (_v.label) {
3186
- case 0:
3187
- isJokerAttempt = attempt < 0;
3188
- jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
3189
- // TODO: [🧠] !!!!!! JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3190
- if (isJokerAttempt && !jokerParameterName) {
3191
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
3192
- }
3193
- result = null;
3194
- resultString = null;
3195
- expectError = null;
3196
- if (isJokerAttempt) {
3197
- if (parameters[jokerParameterName] === undefined) {
3198
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3199
- // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
3200
- }
3201
- else {
3202
- resultString = parameters[jokerParameterName];
3203
- }
3204
- }
3205
- _v.label = 1;
3206
- case 1:
3207
- _v.trys.push([1, 44, 45, 46]);
3208
- if (!!isJokerAttempt) return [3 /*break*/, 26];
3209
- _j = currentTemplate.templateType;
3210
- switch (_j) {
3211
- case 'SIMPLE_TEMPLATE': return [3 /*break*/, 2];
3212
- case 'PROMPT_TEMPLATE': return [3 /*break*/, 3];
3213
- case 'SCRIPT_TEMPLATE': return [3 /*break*/, 12];
3214
- case 'DIALOG_TEMPLATE': return [3 /*break*/, 23];
3215
- }
3216
- return [3 /*break*/, 25];
3217
- case 2:
3218
- resultString = replaceParameters(preparedContent, parameters);
3219
- return [3 /*break*/, 26];
3220
- case 3:
3221
- modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (pipeline.defaultModelRequirements || {})), (currentTemplate.modelRequirements || {}));
3222
- prompt = {
3223
- title: currentTemplate.title,
3224
- pipelineUrl: "".concat(preparedPipeline.pipelineUrl
3225
- ? preparedPipeline.pipelineUrl
3226
- : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(currentTemplate.name),
3227
- parameters: parameters,
3228
- content: preparedContent,
3229
- modelRequirements: modelRequirements,
3230
- expectations: __assign(__assign({}, (preparedPipeline.personas.find(function (_a) {
3231
- var name = _a.name;
3232
- return name === currentTemplate.personaName;
3233
- }) || {})), currentTemplate.expectations),
3234
- format: currentTemplate.format,
3235
- postprocessingFunctionNames: currentTemplate.postprocessingFunctionNames,
3236
- }; // <- TODO: Not very good type guard
3237
- _k = modelRequirements.modelVariant;
3238
- switch (_k) {
3239
- case 'CHAT': return [3 /*break*/, 4];
3240
- case 'COMPLETION': return [3 /*break*/, 6];
3241
- case 'EMBEDDING': return [3 /*break*/, 8];
3242
- }
3243
- return [3 /*break*/, 10];
3244
- case 4: return [4 /*yield*/, llmTools.callChatModel($deepFreeze(prompt))];
3245
- case 5:
3246
- chatResult = _v.sent();
3247
- // TODO: [🍬] Destroy chatThread
3248
- result = chatResult;
3249
- resultString = chatResult.content;
3250
- return [3 /*break*/, 11];
3251
- case 6: return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze(prompt))];
3252
- case 7:
3253
- completionResult = _v.sent();
3254
- result = completionResult;
3255
- resultString = completionResult.content;
3256
- return [3 /*break*/, 11];
3257
- case 8: return [4 /*yield*/, llmTools.callEmbeddingModel($deepFreeze(prompt))];
3258
- case 9:
3259
- embeddingResult = _v.sent();
3260
- result = embeddingResult;
3261
- resultString = embeddingResult.content.join(',');
3262
- return [3 /*break*/, 11];
3263
- case 10: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown model variant \"".concat(currentTemplate.modelRequirements
3264
- .modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3265
- case 11: return [3 /*break*/, 26];
3266
- case 12:
3267
- if (arrayableToArray(tools.script).length === 0) {
3268
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3269
- }
3270
- if (!currentTemplate.contentLanguage) {
3271
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(currentTemplate.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3272
- }
3273
- // TODO: DRY [1]
3274
- scriptPipelineExecutionErrors = [];
3275
- _v.label = 13;
3276
- case 13:
3277
- _v.trys.push([13, 20, 21, 22]);
3278
- _l = (e_5 = void 0, __values(arrayableToArray(tools.script))), _m = _l.next();
3279
- _v.label = 14;
3280
- case 14:
3281
- if (!!_m.done) return [3 /*break*/, 19];
3282
- scriptTools = _m.value;
3283
- _v.label = 15;
3284
- case 15:
3285
- _v.trys.push([15, 17, , 18]);
3286
- return [4 /*yield*/, scriptTools.execute($deepFreeze({
3287
- scriptLanguage: currentTemplate.contentLanguage,
3288
- script: preparedContent,
3289
- parameters: parameters,
3290
- }))];
3291
- case 16:
3292
- resultString = _v.sent();
3293
- return [3 /*break*/, 19];
3294
- case 17:
3295
- error_2 = _v.sent();
3296
- if (!(error_2 instanceof Error)) {
3297
- throw error_2;
3298
- }
3299
- if (error_2 instanceof UnexpectedError) {
3300
- throw error_2;
3301
- }
3302
- scriptPipelineExecutionErrors.push(error_2);
3303
- return [3 /*break*/, 18];
3304
- case 18:
3305
- _m = _l.next();
3306
- return [3 /*break*/, 14];
3307
- case 19: return [3 /*break*/, 22];
3308
- case 20:
3309
- e_5_1 = _v.sent();
3310
- e_5 = { error: e_5_1 };
3311
- return [3 /*break*/, 22];
3312
- case 21:
3313
- try {
3314
- if (_m && !_m.done && (_s = _l.return)) _s.call(_l);
3315
- }
3316
- finally { if (e_5) throw e_5.error; }
3317
- return [7 /*endfinally*/];
3318
- case 22:
3319
- if (resultString !== null) {
3320
- return [3 /*break*/, 26];
3321
- }
3322
- if (scriptPipelineExecutionErrors.length === 1) {
3323
- throw scriptPipelineExecutionErrors[0];
3324
- }
3325
- else {
3326
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script execution failed ".concat(scriptPipelineExecutionErrors.length, " times\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block(scriptPipelineExecutionErrors
3327
- .map(function (error) { return '- ' + error.message; })
3328
- .join('\n\n')), "\n "); }));
3329
- }
3330
- case 23:
3331
- if (tools.userInterface === undefined) {
3332
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3333
- }
3334
- return [4 /*yield*/, tools.userInterface.promptDialog($deepFreeze({
3335
- promptTitle: currentTemplate.title,
3336
- promptMessage: replaceParameters(currentTemplate.description || '', parameters),
3337
- defaultValue: replaceParameters(preparedContent, parameters),
3338
- // TODO: [🧠] !! Figure out how to define placeholder in .ptbk.md file
3339
- placeholder: undefined,
3340
- priority: priority,
3341
- }))];
3342
- case 24:
3343
- // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3344
- resultString = _v.sent();
3345
- return [3 /*break*/, 26];
3346
- case 25: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown execution type \"".concat(currentTemplate.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3347
- case 26:
3348
- if (!(!isJokerAttempt && currentTemplate.postprocessingFunctionNames)) return [3 /*break*/, 43];
3349
- _v.label = 27;
3350
- case 27:
3351
- _v.trys.push([27, 41, 42, 43]);
3352
- _o = (e_7 = void 0, __values(currentTemplate.postprocessingFunctionNames)), _p = _o.next();
3353
- _v.label = 28;
3354
- case 28:
3355
- if (!!_p.done) return [3 /*break*/, 40];
3356
- functionName = _p.value;
3357
- // TODO: DRY [1]
3358
- scriptPipelineExecutionErrors = [];
3359
- postprocessingError = null;
3360
- _v.label = 29;
3361
- case 29:
3362
- _v.trys.push([29, 36, 37, 38]);
3363
- _q = (e_6 = void 0, __values(arrayableToArray(tools.script))), _r = _q.next();
3364
- _v.label = 30;
3365
- case 30:
3366
- if (!!_r.done) return [3 /*break*/, 35];
3367
- scriptTools = _r.value;
3368
- _v.label = 31;
3369
- case 31:
3370
- _v.trys.push([31, 33, , 34]);
3371
- return [4 /*yield*/, scriptTools.execute({
3372
- scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
3373
- script: "".concat(functionName, "(resultString)"),
3374
- parameters: {
3375
- resultString: resultString || '',
3376
- // Note: No ...parametersForTemplate, because working with result only
3377
- },
3378
- })];
3379
- case 32:
3380
- resultString = _v.sent();
3381
- postprocessingError = null;
3382
- return [3 /*break*/, 35];
3383
- case 33:
3384
- error_3 = _v.sent();
3385
- if (!(error_3 instanceof Error)) {
3386
- throw error_3;
3387
- }
3388
- if (error_3 instanceof UnexpectedError) {
3389
- throw error_3;
3390
- }
3391
- postprocessingError = error_3;
3392
- scriptPipelineExecutionErrors.push(error_3);
3393
- return [3 /*break*/, 34];
3394
- case 34:
3395
- _r = _q.next();
3396
- return [3 /*break*/, 30];
3397
- case 35: return [3 /*break*/, 38];
3398
- case 36:
3399
- e_6_1 = _v.sent();
3400
- e_6 = { error: e_6_1 };
3401
- return [3 /*break*/, 38];
3402
- case 37:
3403
- try {
3404
- if (_r && !_r.done && (_u = _q.return)) _u.call(_q);
3405
- }
3406
- finally { if (e_6) throw e_6.error; }
3407
- return [7 /*endfinally*/];
3408
- case 38:
3409
- if (postprocessingError) {
3410
- throw postprocessingError;
3411
- }
3412
- _v.label = 39;
3413
- case 39:
3414
- _p = _o.next();
3415
- return [3 /*break*/, 28];
3416
- case 40: return [3 /*break*/, 43];
3417
- case 41:
3418
- e_7_1 = _v.sent();
3419
- e_7 = { error: e_7_1 };
3420
- return [3 /*break*/, 43];
3421
- case 42:
3422
- try {
3423
- if (_p && !_p.done && (_t = _o.return)) _t.call(_o);
3424
- }
3425
- finally { if (e_7) throw e_7.error; }
3426
- return [7 /*endfinally*/];
3427
- case 43:
3428
- // TODO: [💝] Unite object for expecting amount and format
3429
- if (currentTemplate.format) {
3430
- if (currentTemplate.format === 'JSON') {
3431
- if (!isValidJsonString(resultString || '')) {
3432
- // TODO: [🏢] Do more universally via `FormatDefinition`
3433
- try {
3434
- resultString = extractJsonBlock(resultString || '');
3435
- }
3436
- catch (error) {
3437
- keepUnused(error);
3438
- throw new ExpectError(spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block(
3439
- /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); }));
3440
- }
3441
- }
3362
+ function executeAttempts(options) {
3363
+ return __awaiter(this, void 0, void 0, function () {
3364
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
3365
+ return __generator(this, function (_a) {
3366
+ switch (_a.label) {
3367
+ case 0:
3368
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3369
+ maxExecutionAttempts = settings.maxExecutionAttempts;
3370
+ $ongoingTemplateResult = {
3371
+ $result: null,
3372
+ $resultString: null,
3373
+ $expectError: null,
3374
+ $scriptPipelineExecutionErrors: [],
3375
+ };
3376
+ _loop_1 = function (attempt) {
3377
+ var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
3378
+ var e_1, _q, e_3, _r, e_2, _s;
3379
+ return __generator(this, function (_t) {
3380
+ switch (_t.label) {
3381
+ case 0:
3382
+ isJokerAttempt = attempt < 0;
3383
+ jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
3384
+ // TODO: [🧠][🍭] JOKERS, EXPECTATIONS, POSTPROCESSING and FOREACH
3385
+ if (isJokerAttempt && !jokerParameterName) {
3386
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Joker not found in attempt ".concat(attempt, "\n\n ").concat(block(pipelineIdentification), "\n "); }));
3387
+ }
3388
+ $ongoingTemplateResult.$result = null;
3389
+ $ongoingTemplateResult.$resultString = null;
3390
+ $ongoingTemplateResult.$expectError = null;
3391
+ if (isJokerAttempt) {
3392
+ if (parameters[jokerParameterName] === undefined) {
3393
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Joker parameter {".concat(jokerParameterName, "} not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3394
+ // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
3395
+ }
3396
+ else {
3397
+ $ongoingTemplateResult.$resultString = parameters[jokerParameterName];
3398
+ }
3399
+ }
3400
+ _t.label = 1;
3401
+ case 1:
3402
+ _t.trys.push([1, 43, 44, 45]);
3403
+ if (!!isJokerAttempt) return [3 /*break*/, 25];
3404
+ _b = template.templateType;
3405
+ switch (_b) {
3406
+ case 'SIMPLE_TEMPLATE': return [3 /*break*/, 2];
3407
+ case 'PROMPT_TEMPLATE': return [3 /*break*/, 3];
3408
+ case 'SCRIPT_TEMPLATE': return [3 /*break*/, 11];
3409
+ case 'DIALOG_TEMPLATE': return [3 /*break*/, 22];
3410
+ }
3411
+ return [3 /*break*/, 24];
3412
+ case 2:
3413
+ $ongoingTemplateResult.$resultString = replaceParameters(preparedContent, parameters);
3414
+ return [3 /*break*/, 25];
3415
+ case 3:
3416
+ modelRequirements = __assign(__assign({ modelVariant: 'CHAT' }, (preparedPipeline.defaultModelRequirements || {})), (template.modelRequirements || {}));
3417
+ $ongoingTemplateResult.$prompt = {
3418
+ title: template.title,
3419
+ pipelineUrl: "".concat(preparedPipeline.pipelineUrl
3420
+ ? preparedPipeline.pipelineUrl
3421
+ : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(template.name),
3422
+ parameters: parameters,
3423
+ content: preparedContent,
3424
+ modelRequirements: modelRequirements,
3425
+ expectations: __assign(__assign({}, (preparedPipeline.personas.find(function (_a) {
3426
+ var name = _a.name;
3427
+ return name === template.personaName;
3428
+ }) ||
3429
+ {})), template.expectations),
3430
+ format: template.format,
3431
+ postprocessingFunctionNames: template.postprocessingFunctionNames,
3432
+ }; // <- TODO: Not very good type guard
3433
+ _c = modelRequirements.modelVariant;
3434
+ switch (_c) {
3435
+ case 'CHAT': return [3 /*break*/, 4];
3436
+ case 'COMPLETION': return [3 /*break*/, 6];
3437
+ case 'EMBEDDING': return [3 /*break*/, 8];
3438
+ }
3439
+ return [3 /*break*/, 9];
3440
+ case 4:
3441
+ _d = $ongoingTemplateResult;
3442
+ return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3443
+ case 5:
3444
+ _d.$chatResult = _t.sent();
3445
+ // TODO: [🍬] Destroy chatThread
3446
+ $ongoingTemplateResult.$result = $ongoingTemplateResult.$chatResult;
3447
+ $ongoingTemplateResult.$resultString = $ongoingTemplateResult.$chatResult.content;
3448
+ return [3 /*break*/, 10];
3449
+ case 6:
3450
+ _e = $ongoingTemplateResult;
3451
+ return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3452
+ case 7:
3453
+ _e.$completionResult = _t.sent();
3454
+ $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
3455
+ $ongoingTemplateResult.$resultString =
3456
+ $ongoingTemplateResult.$completionResult.content;
3457
+ return [3 /*break*/, 10];
3458
+ case 8: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Embedding model can not be used in pipeline\n\n This should be catched during parsing\n\n ".concat(block(pipelineIdentification), "\n\n "); }));
3459
+ case 9: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown model variant \"".concat(template.modelRequirements.modelVariant, "\"\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3460
+ case 10: return [3 /*break*/, 25];
3461
+ case 11:
3462
+ if (arrayableToArray(tools.script).length === 0) {
3463
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n No script execution tools are available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3464
+ }
3465
+ if (!template.contentLanguage) {
3466
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script language is not defined for SCRIPT TEMPLATE \"".concat(template.name, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3467
+ }
3468
+ _t.label = 12;
3469
+ case 12:
3470
+ _t.trys.push([12, 19, 20, 21]);
3471
+ _f = (e_1 = void 0, __values(arrayableToArray(tools.script))), _g = _f.next();
3472
+ _t.label = 13;
3473
+ case 13:
3474
+ if (!!_g.done) return [3 /*break*/, 18];
3475
+ scriptTools = _g.value;
3476
+ _t.label = 14;
3477
+ case 14:
3478
+ _t.trys.push([14, 16, , 17]);
3479
+ _h = $ongoingTemplateResult;
3480
+ return [4 /*yield*/, scriptTools.execute($deepFreeze({
3481
+ scriptLanguage: template.contentLanguage,
3482
+ script: preparedContent,
3483
+ parameters: parameters,
3484
+ }))];
3485
+ case 15:
3486
+ _h.$resultString = _t.sent();
3487
+ return [3 /*break*/, 18];
3488
+ case 16:
3489
+ error_1 = _t.sent();
3490
+ if (!(error_1 instanceof Error)) {
3491
+ throw error_1;
3492
+ }
3493
+ if (error_1 instanceof UnexpectedError) {
3494
+ throw error_1;
3495
+ }
3496
+ $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_1);
3497
+ return [3 /*break*/, 17];
3498
+ case 17:
3499
+ _g = _f.next();
3500
+ return [3 /*break*/, 13];
3501
+ case 18: return [3 /*break*/, 21];
3502
+ case 19:
3503
+ e_1_1 = _t.sent();
3504
+ e_1 = { error: e_1_1 };
3505
+ return [3 /*break*/, 21];
3506
+ case 20:
3507
+ try {
3508
+ if (_g && !_g.done && (_q = _f.return)) _q.call(_f);
3509
+ }
3510
+ finally { if (e_1) throw e_1.error; }
3511
+ return [7 /*endfinally*/];
3512
+ case 21:
3513
+ if ($ongoingTemplateResult.$resultString !== null) {
3514
+ return [3 /*break*/, 25];
3515
+ }
3516
+ if ($ongoingTemplateResult.$scriptPipelineExecutionErrors.length === 1) {
3517
+ throw $ongoingTemplateResult.$scriptPipelineExecutionErrors[0];
3518
+ }
3519
+ else {
3520
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Script execution failed ".concat($ongoingTemplateResult.$scriptPipelineExecutionErrors.length, "x\n\n ").concat(block(pipelineIdentification), "\n\n ").concat(block($ongoingTemplateResult.$scriptPipelineExecutionErrors
3521
+ .map(function (error) { return '- ' + error.message; })
3522
+ .join('\n\n')), "\n "); }));
3523
+ }
3524
+ case 22:
3525
+ if (tools.userInterface === undefined) {
3526
+ throw new PipelineExecutionError(spaceTrim(function (block) { return "\n User interface tools are not available\n\n ".concat(block(pipelineIdentification), "\n "); }));
3527
+ }
3528
+ // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3529
+ _j = $ongoingTemplateResult;
3530
+ return [4 /*yield*/, tools.userInterface.promptDialog($deepFreeze({
3531
+ promptTitle: template.title,
3532
+ promptMessage: replaceParameters(template.description || '', parameters),
3533
+ defaultValue: replaceParameters(preparedContent, parameters),
3534
+ // TODO: [🧠] !! Figure out how to define placeholder in .ptbk.md file
3535
+ placeholder: undefined,
3536
+ priority: priority,
3537
+ }))];
3538
+ case 23:
3539
+ // TODO: [🌹] When making next attempt for `DIALOG TEMPLATE`, preserve the previous user input
3540
+ _j.$resultString = _t.sent();
3541
+ return [3 /*break*/, 25];
3542
+ case 24: throw new PipelineExecutionError(spaceTrim(function (block) { return "\n Unknown execution type \"".concat(template.templateType, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3543
+ case 25:
3544
+ if (!(!isJokerAttempt && template.postprocessingFunctionNames)) return [3 /*break*/, 42];
3545
+ _t.label = 26;
3546
+ case 26:
3547
+ _t.trys.push([26, 40, 41, 42]);
3548
+ _k = (e_3 = void 0, __values(template.postprocessingFunctionNames)), _l = _k.next();
3549
+ _t.label = 27;
3550
+ case 27:
3551
+ if (!!_l.done) return [3 /*break*/, 39];
3552
+ functionName = _l.value;
3553
+ postprocessingError = null;
3554
+ _t.label = 28;
3555
+ case 28:
3556
+ _t.trys.push([28, 35, 36, 37]);
3557
+ _m = (e_2 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next();
3558
+ _t.label = 29;
3559
+ case 29:
3560
+ if (!!_o.done) return [3 /*break*/, 34];
3561
+ scriptTools = _o.value;
3562
+ _t.label = 30;
3563
+ case 30:
3564
+ _t.trys.push([30, 32, , 33]);
3565
+ _p = $ongoingTemplateResult;
3566
+ return [4 /*yield*/, scriptTools.execute({
3567
+ scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
3568
+ script: "".concat(functionName, "(resultString)"),
3569
+ parameters: {
3570
+ resultString: $ongoingTemplateResult.$resultString || '',
3571
+ // Note: No ...parametersForTemplate, because working with result only
3572
+ },
3573
+ })];
3574
+ case 31:
3575
+ _p.$resultString = _t.sent();
3576
+ postprocessingError = null;
3577
+ return [3 /*break*/, 34];
3578
+ case 32:
3579
+ error_2 = _t.sent();
3580
+ if (!(error_2 instanceof Error)) {
3581
+ throw error_2;
3582
+ }
3583
+ if (error_2 instanceof UnexpectedError) {
3584
+ throw error_2;
3585
+ }
3586
+ postprocessingError = error_2;
3587
+ $ongoingTemplateResult.$scriptPipelineExecutionErrors.push(error_2);
3588
+ return [3 /*break*/, 33];
3589
+ case 33:
3590
+ _o = _m.next();
3591
+ return [3 /*break*/, 29];
3592
+ case 34: return [3 /*break*/, 37];
3593
+ case 35:
3594
+ e_2_1 = _t.sent();
3595
+ e_2 = { error: e_2_1 };
3596
+ return [3 /*break*/, 37];
3597
+ case 36:
3598
+ try {
3599
+ if (_o && !_o.done && (_s = _m.return)) _s.call(_m);
3600
+ }
3601
+ finally { if (e_2) throw e_2.error; }
3602
+ return [7 /*endfinally*/];
3603
+ case 37:
3604
+ if (postprocessingError) {
3605
+ throw postprocessingError;
3606
+ }
3607
+ _t.label = 38;
3608
+ case 38:
3609
+ _l = _k.next();
3610
+ return [3 /*break*/, 27];
3611
+ case 39: return [3 /*break*/, 42];
3612
+ case 40:
3613
+ e_3_1 = _t.sent();
3614
+ e_3 = { error: e_3_1 };
3615
+ return [3 /*break*/, 42];
3616
+ case 41:
3617
+ try {
3618
+ if (_l && !_l.done && (_r = _k.return)) _r.call(_k);
3619
+ }
3620
+ finally { if (e_3) throw e_3.error; }
3621
+ return [7 /*endfinally*/];
3622
+ case 42:
3623
+ // TODO: [💝] Unite object for expecting amount and format
3624
+ if (template.format) {
3625
+ if (template.format === 'JSON') {
3626
+ if (!isValidJsonString($ongoingTemplateResult.$resultString || '')) {
3627
+ // TODO: [🏢] Do more universally via `FormatDefinition`
3628
+ try {
3629
+ $ongoingTemplateResult.$resultString = extractJsonBlock($ongoingTemplateResult.$resultString || '');
3442
3630
  }
3443
- else {
3444
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Unknown format \"".concat(currentTemplate.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3631
+ catch (error) {
3632
+ keepUnused(error);
3633
+ throw new ExpectError(spaceTrim(function (block) { return "\n Expected valid JSON string\n\n ".concat(block(
3634
+ /*<- Note: No need for `pipelineIdentification`, it will be catched and added later */ ''), "\n "); }));
3445
3635
  }
3446
3636
  }
3447
- // TODO: [💝] Unite object for expecting amount and format
3448
- if (currentTemplate.expectations) {
3449
- checkExpectations(currentTemplate.expectations, resultString || '');
3450
- }
3451
- return [2 /*return*/, "break-attempts"];
3452
- case 44:
3453
- error_4 = _v.sent();
3454
- if (!(error_4 instanceof ExpectError)) {
3455
- throw error_4;
3456
- }
3457
- expectError = error_4;
3458
- return [3 /*break*/, 46];
3459
- case 45:
3460
- if (!isJokerAttempt &&
3461
- currentTemplate.templateType === 'PROMPT_TEMPLATE' &&
3462
- prompt
3463
- // <- Note: [2] When some expected parameter is not defined, error will occur in replaceParameters
3464
- // In that case we don’t want to make a report about it because it’s not a llm execution error
3465
- ) {
3466
- // TODO: [🧠] Maybe put other templateTypes into report
3467
- executionReport.promptExecutions.push({
3468
- prompt: __assign({}, prompt),
3469
- result: result || undefined,
3470
- error: expectError === null ? undefined : serializeError(expectError),
3471
- });
3472
- }
3473
- return [7 /*endfinally*/];
3474
- case 46:
3475
- if (expectError !== null && attempt === maxAttempts - 1) {
3476
- throw new PipelineExecutionError(spaceTrim(function (block) { return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block(prompt.content
3477
- .split('\n')
3478
- .map(function (line) { return "> ".concat(line); })
3479
- .join('\n')), "\n\n Last error ").concat((expectError === null || expectError === void 0 ? void 0 : expectError.name) || '', ":\n ").concat(block(((expectError === null || expectError === void 0 ? void 0 : expectError.message) || '')
3637
+ }
3638
+ else {
3639
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Unknown format \"".concat(template.format, "\"\n\n ").concat(block(pipelineIdentification), "\n "); }));
3640
+ }
3641
+ }
3642
+ // TODO: [💝] Unite object for expecting amount and format
3643
+ if (template.expectations) {
3644
+ checkExpectations(template.expectations, $ongoingTemplateResult.$resultString || '');
3645
+ }
3646
+ return [2 /*return*/, "break-attempts"];
3647
+ case 43:
3648
+ error_3 = _t.sent();
3649
+ if (!(error_3 instanceof ExpectError)) {
3650
+ throw error_3;
3651
+ }
3652
+ $ongoingTemplateResult.$expectError = error_3;
3653
+ return [3 /*break*/, 45];
3654
+ case 44:
3655
+ if (!isJokerAttempt &&
3656
+ template.templateType === 'PROMPT_TEMPLATE' &&
3657
+ $ongoingTemplateResult.$prompt
3658
+ // <- Note: [2] When some expected parameter is not defined, error will occur in replaceParameters
3659
+ // In that case we don’t want to make a report about it because it’s not a llm execution error
3660
+ ) {
3661
+ // TODO: [🧠] Maybe put other templateTypes into report
3662
+ $executionReport.promptExecutions.push({
3663
+ prompt: __assign({}, $ongoingTemplateResult.$prompt),
3664
+ result: $ongoingTemplateResult.$result || undefined,
3665
+ error: $ongoingTemplateResult.$expectError === null
3666
+ ? undefined
3667
+ : serializeError($ongoingTemplateResult.$expectError),
3668
+ });
3669
+ }
3670
+ return [7 /*endfinally*/];
3671
+ case 45:
3672
+ if ($ongoingTemplateResult.$expectError !== null && attempt === maxAttempts - 1) {
3673
+ throw new PipelineExecutionError(spaceTrim(function (block) {
3674
+ var _a, _b, _c;
3675
+ return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ").concat(block(pipelineIdentification), "\n\n ---\n The Prompt:\n ").concat(block((((_a = $ongoingTemplateResult.$prompt) === null || _a === void 0 ? void 0 : _a.content) || '')
3676
+ .split('\n')
3677
+ .map(function (line) { return "> ".concat(line); })
3678
+ .join('\n')), "\n\n Last error ").concat(((_b = $ongoingTemplateResult.$expectError) === null || _b === void 0 ? void 0 : _b.name) || '', ":\n ").concat(block((((_c = $ongoingTemplateResult.$expectError) === null || _c === void 0 ? void 0 : _c.message) || '')
3679
+ .split('\n')
3680
+ .map(function (line) { return "> ".concat(line); })
3681
+ .join('\n')), "\n\n Last result:\n ").concat(block($ongoingTemplateResult.$resultString === null
3682
+ ? 'null'
3683
+ : $ongoingTemplateResult.$resultString
3480
3684
  .split('\n')
3481
3685
  .map(function (line) { return "> ".concat(line); })
3482
- .join('\n')), "\n\n Last result:\n ").concat(block(resultString === null
3483
- ? 'null'
3484
- : resultString
3485
- .split('\n')
3486
- .map(function (line) { return "> ".concat(line); })
3487
- .join('\n')), "\n ---\n "); }));
3488
- }
3489
- return [2 /*return*/];
3686
+ .join('\n')), "\n ---\n ");
3687
+ }));
3490
3688
  }
3491
- });
3492
- };
3493
- attempt = -jokerParameterNames.length;
3494
- _h.label = 4;
3495
- case 4:
3496
- if (!(attempt < maxAttempts)) return [3 /*break*/, 7];
3497
- return [5 /*yield**/, _loop_5(attempt)];
3498
- case 5:
3499
- state_2 = _h.sent();
3500
- switch (state_2) {
3501
- case "break-attempts": return [3 /*break*/, 7];
3502
- }
3503
- _h.label = 6;
3504
- case 6:
3505
- attempt++;
3506
- return [3 /*break*/, 4];
3507
- case 7:
3508
- //------------------------------------
3509
- /*
3510
-
3511
-
3512
-
3513
-
3514
-
3515
-
3516
-
3517
-
3518
-
3519
- */
3520
- //------------------------------------
3521
- if (resultString === null) {
3522
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3689
+ return [2 /*return*/];
3523
3690
  }
3524
- if (!(onProgress !== undefined) /* <- [3] */) return [3 /*break*/, 9]; /* <- [3] */
3525
- progress_2 = {
3526
- name: name,
3527
- title: title,
3528
- isStarted: true,
3529
- isDone: true,
3530
- templateType: currentTemplate.templateType,
3531
- parameterName: currentTemplate.resultingParameterName,
3532
- parameterValue: resultString,
3533
- // <- [3]
3534
- };
3535
- if (isReturned) {
3536
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished \uD83C\uDF4E\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress_2, null, 4)
3537
- .split('\n')
3538
- .map(function (line) { return "> ".concat(line); })
3539
- .join('\n')), "\n\n "); }));
3540
- }
3541
- return [4 /*yield*/, onProgress(progress_2)];
3542
- case 8:
3543
- _h.sent();
3544
- _h.label = 9;
3545
- case 9:
3546
- parametersToPass = Object.freeze(__assign(__assign({}, parametersToPass), (_g = {}, _g[currentTemplate.resultingParameterName] = resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */, _g)));
3547
- return [2 /*return*/];
3691
+ });
3692
+ };
3693
+ attempt = -jokerParameterNames.length;
3694
+ _a.label = 1;
3695
+ case 1:
3696
+ if (!(attempt < maxAttempts)) return [3 /*break*/, 4];
3697
+ return [5 /*yield**/, _loop_1(attempt)];
3698
+ case 2:
3699
+ state_1 = _a.sent();
3700
+ switch (state_1) {
3701
+ case "break-attempts": return [3 /*break*/, 4];
3702
+ }
3703
+ _a.label = 3;
3704
+ case 3:
3705
+ attempt++;
3706
+ return [3 /*break*/, 1];
3707
+ case 4:
3708
+ if ($ongoingTemplateResult.$resultString === null) {
3709
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Something went wrong and prompt result is null\n\n ".concat(block(pipelineIdentification), "\n "); }));
3710
+ }
3711
+ return [2 /*return*/, $ongoingTemplateResult.$resultString];
3712
+ }
3713
+ });
3714
+ });
3715
+ }
3716
+ /**
3717
+ * TODO: Break into smaller functions
3718
+ */
3719
+
3720
+ /**
3721
+ * @@@
3722
+ *
3723
+ * @private internal utility of `createPipelineExecutor`
3724
+ */
3725
+ function executeFormatCells(options) {
3726
+ return __awaiter(this, void 0, void 0, function () {
3727
+ var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3728
+ var _this = this;
3729
+ return __generator(this, function (_a) {
3730
+ switch (_a.label) {
3731
+ case 0:
3732
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3733
+ if (template.foreach === undefined) {
3734
+ return [2 /*return*/, /* not await */ executeAttempts(options)];
3735
+ }
3736
+ if (jokerParameterNames.length !== 0) {
3737
+ throw new UnexpectedError(spaceTrim$1(function (block) { return "\n JOKER parameters are not supported together with FOREACH command\n\n [\uD83E\uDDDE\u200D\u2640\uFE0F] This should be prevented in `validatePipeline`\n\n ".concat(block(pipelineIdentification), "\n "); }));
3738
+ }
3739
+ parameterValue = parameters[template.foreach.parameterName] || '';
3740
+ formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
3741
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(template.foreach.formatName);
3742
+ });
3743
+ if (formatDefinition === undefined) {
3744
+ throw new UnexpectedError(
3745
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3746
+ spaceTrim$1(function (block) { return "\n Unsupported format \"".concat(template.foreach.formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
3747
+ .map(function (formatName) { return "- ".concat(formatName); })
3748
+ .join('\n')), "\n\n [\u26F7] This should never happen because format name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3749
+ }
3750
+ subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
3751
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(template.foreach.subformatName);
3752
+ });
3753
+ if (subvalueDefinition === undefined) {
3754
+ throw new UnexpectedError(
3755
+ // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
3756
+ spaceTrim$1(function (block) { return "\n Unsupported subformat name \"".concat(template.foreach.subformatName, "\" for format \"").concat(template.foreach.formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
3757
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
3758
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
3759
+ .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
3760
+ }
3761
+ if (formatDefinition.formatName === 'CSV') {
3762
+ formatSettings = settings.csvSettings;
3763
+ // <- TODO: !!!!!! More universal, make simmilar pattern for other formats for example \n vs \r\n in text
3764
+ }
3765
+ return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
3766
+ var mappedParameters, allSubparameters, subresultString;
3767
+ return __generator(this, function (_a) {
3768
+ switch (_a.label) {
3769
+ case 0:
3770
+ // TODO: !!!!!!! Limit to N concurrent executions
3771
+ // TODO: !!!!!!! Report progress
3772
+ try {
3773
+ mappedParameters = mapAvailableToExpectedParameters({
3774
+ expectedParameters: Object.fromEntries(template.foreach.inputSubparameterNames.map(function (subparameterName) { return [subparameterName, null]; })),
3775
+ availableParameters: subparameters,
3776
+ });
3777
+ }
3778
+ catch (error) {
3779
+ if (!(error instanceof PipelineExecutionError)) {
3780
+ throw error;
3781
+ }
3782
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n ".concat(error.message, "\n\n This is error in FOREACH command\n You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command\n\n ").concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }));
3783
+ }
3784
+ allSubparameters = __assign(__assign({}, parameters), mappedParameters);
3785
+ // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
3786
+ Object.freeze(allSubparameters);
3787
+ return [4 /*yield*/, executeAttempts(__assign(__assign({}, options), { priority: priority + index, parameters: allSubparameters, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Subparameter index: ").concat(index, "\n "); }) }))];
3788
+ case 1:
3789
+ subresultString = _a.sent();
3790
+ return [2 /*return*/, subresultString];
3791
+ }
3792
+ });
3793
+ }); })];
3794
+ case 1:
3795
+ resultString = _a.sent();
3796
+ return [2 /*return*/, resultString];
3797
+ }
3798
+ });
3799
+ });
3800
+ }
3801
+ /**
3802
+ * TODO: !!!!!! Make pipelineIdentification more precise
3803
+ * TODO: !!!!!! How FOREACH execution looks in the report
3804
+ */
3805
+
3806
+ /**
3807
+ * @@@
3808
+ *
3809
+ * @private internal utility of `createPipelineExecutor`
3810
+ */
3811
+ function getContextForTemplate(template) {
3812
+ return __awaiter(this, void 0, void 0, function () {
3813
+ return __generator(this, function (_a) {
3814
+ TODO_USE(template);
3815
+ return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [🏍] Implement */];
3816
+ });
3817
+ });
3818
+ }
3819
+
3820
+ /**
3821
+ * @@@
3822
+ *
3823
+ * @private internal utility of `createPipelineExecutor`
3824
+ */
3825
+ function getKnowledgeForTemplate(options) {
3826
+ return __awaiter(this, void 0, void 0, function () {
3827
+ var preparedPipeline, template;
3828
+ return __generator(this, function (_a) {
3829
+ preparedPipeline = options.preparedPipeline, template = options.template;
3830
+ // TODO: [♨] Implement Better - use real index and keyword search from `template` and {samples}
3831
+ TODO_USE(template);
3832
+ return [2 /*return*/, preparedPipeline.knowledgePieces.map(function (_a) {
3833
+ var content = _a.content;
3834
+ return "- ".concat(content);
3835
+ }).join('\n')];
3836
+ });
3837
+ });
3838
+ }
3839
+
3840
+ /**
3841
+ * @@@
3842
+ *
3843
+ * @private internal utility of `createPipelineExecutor`
3844
+ */
3845
+ function getSamplesForTemplate(template) {
3846
+ return __awaiter(this, void 0, void 0, function () {
3847
+ return __generator(this, function (_a) {
3848
+ // TODO: [♨] Implement Better - use real index and keyword search
3849
+ TODO_USE(template);
3850
+ return [2 /*return*/, RESERVED_PARAMETER_MISSING_VALUE /* <- TODO: [♨] Implement */];
3851
+ });
3852
+ });
3853
+ }
3854
+
3855
+ /**
3856
+ * @@@
3857
+ *
3858
+ * @private internal utility of `createPipelineExecutor`
3859
+ */
3860
+ function getReservedParametersForTemplate(options) {
3861
+ return __awaiter(this, void 0, void 0, function () {
3862
+ var preparedPipeline, template, pipelineIdentification, context, knowledge, samples, currentDate, modelName, reservedParameters, _loop_1, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName;
3863
+ var e_1, _a;
3864
+ return __generator(this, function (_b) {
3865
+ switch (_b.label) {
3866
+ case 0:
3867
+ preparedPipeline = options.preparedPipeline, template = options.template, pipelineIdentification = options.pipelineIdentification;
3868
+ return [4 /*yield*/, getContextForTemplate(template)];
3869
+ case 1:
3870
+ context = _b.sent();
3871
+ return [4 /*yield*/, getKnowledgeForTemplate({ preparedPipeline: preparedPipeline, template: template })];
3872
+ case 2:
3873
+ knowledge = _b.sent();
3874
+ return [4 /*yield*/, getSamplesForTemplate(template)];
3875
+ case 3:
3876
+ samples = _b.sent();
3877
+ currentDate = new Date().toISOString();
3878
+ modelName = RESERVED_PARAMETER_MISSING_VALUE;
3879
+ reservedParameters = {
3880
+ content: RESERVED_PARAMETER_RESTRICTED,
3881
+ context: context,
3882
+ knowledge: knowledge,
3883
+ samples: samples,
3884
+ currentDate: currentDate,
3885
+ modelName: modelName,
3886
+ };
3887
+ _loop_1 = function (parameterName) {
3888
+ if (reservedParameters[parameterName] === undefined) {
3889
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Reserved parameter {".concat(parameterName, "} is not defined\n\n ").concat(block(pipelineIdentification), "\n "); }));
3890
+ }
3891
+ };
3892
+ try {
3893
+ // Note: Doublecheck that ALL reserved parameters are defined:
3894
+ for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
3895
+ parameterName = RESERVED_PARAMETER_NAMES_1_1.value;
3896
+ _loop_1(parameterName);
3897
+ }
3548
3898
  }
3549
- });
3550
- });
3551
- }
3552
- function filterJustOutputParameters() {
3553
- var e_8, _a;
3554
- var outputParameters = {};
3555
- var _loop_6 = function (parameter) {
3556
- if (parametersToPass[parameter.name] === undefined) {
3557
- // [4]
3558
- warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} should be an output parameter, but it was not generated during pipeline execution\n\n ").concat(block(pipelineIdentification), "\n "); })));
3559
- return "continue";
3560
- }
3561
- outputParameters[parameter.name] = parametersToPass[parameter.name] || '';
3562
- };
3563
- try {
3564
- // Note: Filter ONLY output parameters
3565
- for (var _b = __values(preparedPipeline.parameters.filter(function (_a) {
3566
- var isOutput = _a.isOutput;
3567
- return isOutput;
3568
- })), _c = _b.next(); !_c.done; _c = _b.next()) {
3569
- var parameter = _c.value;
3570
- _loop_6(parameter);
3571
- }
3899
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3900
+ finally {
3901
+ try {
3902
+ if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1);
3903
+ }
3904
+ finally { if (e_1) throw e_1.error; }
3905
+ }
3906
+ return [2 /*return*/, reservedParameters];
3572
3907
  }
3573
- catch (e_8_1) { e_8 = { error: e_8_1 }; }
3574
- finally {
3575
- try {
3576
- if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3577
- }
3578
- finally { if (e_8) throw e_8.error; }
3908
+ });
3909
+ });
3910
+ }
3911
+
3912
+ /**
3913
+ * @@@
3914
+ *
3915
+ * @private internal utility of `createPipelineExecutor`
3916
+ */
3917
+ function executeTemplate(options) {
3918
+ return __awaiter(this, void 0, void 0, function () {
3919
+ var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
3920
+ var e_1, _f, _g;
3921
+ return __generator(this, function (_h) {
3922
+ switch (_h.label) {
3923
+ case 0:
3924
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, llmTools = options.llmTools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3925
+ maxExecutionAttempts = settings.maxExecutionAttempts;
3926
+ name = "pipeline-executor-frame-".concat(currentTemplate.name);
3927
+ title = currentTemplate.title;
3928
+ priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
3929
+ return [4 /*yield*/, onProgress({
3930
+ name: name,
3931
+ title: title,
3932
+ isStarted: false,
3933
+ isDone: false,
3934
+ templateType: currentTemplate.templateType,
3935
+ parameterName: currentTemplate.resultingParameterName,
3936
+ parameterValue: null,
3937
+ // <- [🍸]
3938
+ })];
3939
+ case 1:
3940
+ _h.sent();
3941
+ usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
3942
+ dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
3943
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3944
+ if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
3945
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Dependent parameters are not consistent with used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames)
3946
+ .map(function (name) { return "{".concat(name, "}"); })
3947
+ .join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames)
3948
+ .map(function (name) { return "{".concat(name, "}"); })
3949
+ .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3950
+ }
3951
+ _b = (_a = Object).freeze;
3952
+ _c = [{}];
3953
+ return [4 /*yield*/, getReservedParametersForTemplate({
3954
+ preparedPipeline: preparedPipeline,
3955
+ template: currentTemplate,
3956
+ pipelineIdentification: pipelineIdentification,
3957
+ })];
3958
+ case 2:
3959
+ definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
3960
+ definedParameterNames = new Set(Object.keys(definedParameters));
3961
+ parameters = {};
3962
+ _loop_1 = function (parameterName) {
3963
+ // Situation: Parameter is defined and used
3964
+ if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3965
+ parameters[parameterName] = definedParameters[parameterName];
3966
+ }
3967
+ // Situation: Parameter is defined but NOT used
3968
+ else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) ;
3969
+ // Situation: Parameter is NOT defined BUT used
3970
+ else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
3971
+ // Houston, we have a problem
3972
+ // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck
3973
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is NOT defined\n BUT used in template \"").concat(currentTemplate.title || currentTemplate.name, "\"\n\n This should be catched in `validatePipeline`\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
3974
+ }
3975
+ };
3976
+ try {
3977
+ // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
3978
+ // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
3979
+ for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
3980
+ parameterName = _e.value;
3981
+ _loop_1(parameterName);
3982
+ }
3983
+ }
3984
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3985
+ finally {
3986
+ try {
3987
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
3988
+ }
3989
+ finally { if (e_1) throw e_1.error; }
3990
+ }
3991
+ // Note: [👨‍👨‍👧] Now we can freeze `parameters` because we are sure that all and only used parameters are defined and are not going to be changed
3992
+ Object.freeze(parameters);
3993
+ maxAttempts = currentTemplate.templateType === 'DIALOG_TEMPLATE' ? Infinity : maxExecutionAttempts;
3994
+ jokerParameterNames = currentTemplate.jokerParameterNames || [];
3995
+ preparedContent = (currentTemplate.preparedContent || '{content}')
3996
+ .split('{content}')
3997
+ .join(currentTemplate.content);
3998
+ return [4 /*yield*/, executeFormatCells({
3999
+ jokerParameterNames: jokerParameterNames,
4000
+ priority: priority,
4001
+ maxAttempts: maxAttempts,
4002
+ preparedContent: preparedContent,
4003
+ parameters: parameters,
4004
+ template: currentTemplate,
4005
+ preparedPipeline: preparedPipeline,
4006
+ tools: tools,
4007
+ llmTools: llmTools,
4008
+ settings: settings,
4009
+ $executionReport: $executionReport,
4010
+ pipelineIdentification: pipelineIdentification,
4011
+ })];
4012
+ case 3:
4013
+ resultString = _h.sent();
4014
+ return [4 /*yield*/, onProgress({
4015
+ name: name,
4016
+ title: title,
4017
+ isStarted: true,
4018
+ isDone: true,
4019
+ templateType: currentTemplate.templateType,
4020
+ parameterName: currentTemplate.resultingParameterName,
4021
+ parameterValue: resultString,
4022
+ // <- [🍸]
4023
+ })];
4024
+ case 4:
4025
+ _h.sent();
4026
+ return [2 /*return*/, Object.freeze((_g = {},
4027
+ _g[currentTemplate.resultingParameterName] =
4028
+ // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4029
+ resultString,
4030
+ _g))];
3579
4031
  }
3580
- return outputParameters;
4032
+ });
4033
+ });
4034
+ }
4035
+ /**
4036
+ * TODO: [🤹‍♂️]
4037
+ */
4038
+
4039
+ /**
4040
+ * @@@
4041
+ *
4042
+ * @private internal utility of `createPipelineExecutor`
4043
+ */
4044
+ function filterJustOutputParameters(options) {
4045
+ var e_1, _a;
4046
+ var preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, $warnings = options.$warnings, pipelineIdentification = options.pipelineIdentification;
4047
+ var outputParameters = {};
4048
+ var _loop_1 = function (parameter) {
4049
+ if (parametersToPass[parameter.name] === undefined) {
4050
+ // [4]
4051
+ $warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} should be an output parameter, but it was not generated during pipeline execution\n\n ").concat(block(pipelineIdentification), "\n "); })));
4052
+ return "continue";
4053
+ }
4054
+ outputParameters[parameter.name] = parametersToPass[parameter.name] || '';
4055
+ };
4056
+ try {
4057
+ // Note: Filter ONLY output parameters
4058
+ // TODO: [👩🏾‍🤝‍👩🏻] Maybe use here `mapAvailableToExpectedParameters`
4059
+ for (var _b = __values(preparedPipeline.parameters.filter(function (_a) {
4060
+ var isOutput = _a.isOutput;
4061
+ return isOutput;
4062
+ })), _c = _b.next(); !_c.done; _c = _b.next()) {
4063
+ var parameter = _c.value;
4064
+ _loop_1(parameter);
4065
+ }
4066
+ }
4067
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4068
+ finally {
4069
+ try {
4070
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3581
4071
  }
3582
- var errors, warnings, executionReport, isReturned, _a, _b, parameter, e_1_1, _loop_1, _c, _d, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4072
+ finally { if (e_1) throw e_1.error; }
4073
+ }
4074
+ return outputParameters;
4075
+ }
4076
+
4077
+ /**
4078
+ * @@@
4079
+ *
4080
+ * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
4081
+ *
4082
+ * @private internal utility of `createPipelineExecutor`
4083
+ */
4084
+ function executePipeline(options) {
4085
+ return __awaiter(this, void 0, void 0, function () {
4086
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, isVerbose, preparedPipeline, llmTools, errors, warnings, executionReport, isReturned, _a, _b, parameter, e_1_1, _loop_1, _c, _d, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
3583
4087
  var e_1, _e, e_2, _f;
3584
4088
  return __generator(this, function (_g) {
3585
4089
  switch (_g.label) {
3586
4090
  case 0:
4091
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4092
+ maxParallelCount = settings.maxParallelCount, isVerbose = settings.isVerbose;
4093
+ preparedPipeline = options.preparedPipeline;
4094
+ llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
3587
4095
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
3588
4096
  return [4 /*yield*/, preparePipeline(pipeline, {
3589
4097
  llmTools: llmTools,
@@ -3592,6 +4100,7 @@ function createPipelineExecutor(options) {
3592
4100
  })];
3593
4101
  case 1:
3594
4102
  preparedPipeline = _g.sent();
4103
+ setPreparedPipeline(preparedPipeline);
3595
4104
  _g.label = 2;
3596
4105
  case 2:
3597
4106
  errors = [];
@@ -3661,7 +4170,7 @@ function createPipelineExecutor(options) {
3661
4170
  return name === parameterName;
3662
4171
  });
3663
4172
  if (!(parameter === undefined)) return [3 /*break*/, 1];
3664
- warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
4173
+ warnings.push(new PipelineExecutionError(spaceTrim(function (block) { return "\n Extra parameter {".concat(parameterName, "} is being passed which is not part of the pipeline.\n\n ").concat(block(pipelineIdentification), "\n "); })));
3665
4174
  return [3 /*break*/, 4];
3666
4175
  case 1:
3667
4176
  if (!(parameter.isInput === false)) return [3 /*break*/, 4];
@@ -3673,10 +4182,10 @@ function createPipelineExecutor(options) {
3673
4182
  // Note: Wait a short time to prevent race conditions
3674
4183
  _h.sent();
3675
4184
  _h.label = 3;
3676
- case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
4185
+ case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
3677
4186
  isSuccessful: false,
3678
4187
  errors: __spreadArray([
3679
- new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
4188
+ new PipelineExecutionError(spaceTrim(function (block) { return "\n Parameter {".concat(parameter.name, "} is passed as input parameter but it is not input\n\n ").concat(block(pipelineIdentification), "\n "); }))
3680
4189
  ], __read(errors), false).map(serializeError),
3681
4190
  warnings: warnings.map(serializeError),
3682
4191
  executionReport: executionReport,
@@ -3740,7 +4249,7 @@ function createPipelineExecutor(options) {
3740
4249
  case 0:
3741
4250
  if (loopLimit-- < 0) {
3742
4251
  // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
3743
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
4252
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Loop limit reached during resolving parameters pipeline execution\n\n ".concat(block(pipelineIdentification), "\n "); }));
3744
4253
  }
3745
4254
  currentTemplate = unresovedTemplates_1.find(function (template) {
3746
4255
  return template.dependentParameterNames.every(function (name) {
@@ -3750,29 +4259,52 @@ function createPipelineExecutor(options) {
3750
4259
  if (!(!currentTemplate && resolving_1.length === 0)) return [3 /*break*/, 1];
3751
4260
  throw new UnexpectedError(
3752
4261
  // TODO: [🐎] DRY
3753
- spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
4262
+ spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n ".concat(block(pipelineIdentification), "\n\n Can not resolve:\n ").concat(block(unresovedTemplates_1
3754
4263
  .map(function (_a) {
3755
4264
  var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
3756
4265
  return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
3757
4266
  .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
3758
4267
  .join(' and '));
3759
4268
  })
3760
- .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
4269
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameterNames_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
3761
4270
  case 1:
3762
4271
  if (!!currentTemplate) return [3 /*break*/, 3];
3763
- /* [5] */ return [4 /*yield*/, Promise.race(resolving_1)];
4272
+ /* [🤹‍♂️] */ return [4 /*yield*/, Promise.race(resolving_1)];
3764
4273
  case 2:
3765
- /* [5] */ _j.sent();
4274
+ /* [🤹‍♂️] */ _j.sent();
3766
4275
  return [3 /*break*/, 4];
3767
4276
  case 3:
3768
4277
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
3769
- work_1 = executeSingleTemplate(currentTemplate)
3770
- .then(function () {
4278
+ work_1 = executeTemplate({
4279
+ currentTemplate: currentTemplate,
4280
+ preparedPipeline: preparedPipeline,
4281
+ parametersToPass: parametersToPass,
4282
+ tools: tools,
4283
+ llmTools: llmTools,
4284
+ onProgress: function (progress) {
4285
+ if (isReturned) {
4286
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4287
+ .split('\n')
4288
+ .map(function (line) { return "> ".concat(line); })
4289
+ .join('\n')), "\n "); }));
4290
+ }
4291
+ if (onProgress) {
4292
+ onProgress(progress);
4293
+ }
4294
+ },
4295
+ settings: settings,
4296
+ $executionReport: executionReport,
4297
+ pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
4298
+ })
4299
+ .then(function (newParametersToPass) {
4300
+ parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
3771
4301
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
3772
4302
  })
3773
4303
  .then(function () {
3774
4304
  resolving_1 = resolving_1.filter(function (w) { return w !== work_1; });
3775
4305
  });
4306
+ // <- Note: Errors are catched here [3]
4307
+ // TODO: BUT if in multiple templates are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here
3776
4308
  resolving_1.push(work_1);
3777
4309
  _j.label = 4;
3778
4310
  case 4: return [2 /*return*/];
@@ -3799,7 +4331,12 @@ function createPipelineExecutor(options) {
3799
4331
  var result = _a.result;
3800
4332
  return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE;
3801
4333
  })), false));
3802
- outputParameters_1 = filterJustOutputParameters();
4334
+ outputParameters_1 = filterJustOutputParameters({
4335
+ preparedPipeline: preparedPipeline,
4336
+ parametersToPass: parametersToPass,
4337
+ $warnings: warnings,
4338
+ pipelineIdentification: pipelineIdentification,
4339
+ });
3803
4340
  isReturned = true;
3804
4341
  if (!(onProgress !== undefined)) return [3 /*break*/, 27];
3805
4342
  // Note: Wait a short time to prevent race conditions
@@ -3822,7 +4359,12 @@ function createPipelineExecutor(options) {
3822
4359
  var result = _a.result;
3823
4360
  return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE;
3824
4361
  })), false));
3825
- outputParameters = filterJustOutputParameters();
4362
+ outputParameters = filterJustOutputParameters({
4363
+ preparedPipeline: preparedPipeline,
4364
+ parametersToPass: parametersToPass,
4365
+ $warnings: warnings,
4366
+ pipelineIdentification: pipelineIdentification,
4367
+ });
3826
4368
  isReturned = true;
3827
4369
  if (!(onProgress !== undefined)) return [3 /*break*/, 30];
3828
4370
  // Note: Wait a short time to prevent race conditions
@@ -3842,22 +4384,65 @@ function createPipelineExecutor(options) {
3842
4384
  })];
3843
4385
  }
3844
4386
  });
3845
- }); };
3846
- return pipelineExecutor;
4387
+ });
3847
4388
  }
4389
+
3848
4390
  /**
3849
- * TODO: !!! Identify not only pipeline BUT exact template ${block(pipelineIdentification)}
3850
- * TODO: Use isVerbose here (not only pass to `preparePipeline`)
3851
- * TODO: [🧠][🌳] Use here `countTotalUsage` and put preparation and prepared pipiline to report
3852
- * TODO: [🪂] Use maxParallelCount here (not only pass to `preparePipeline`)
3853
- * TODO: [♈] Probbably move expectations from templates to parameters
3854
- * TODO: [🧠] When not meet expectations in DIALOG_TEMPLATE, make some way to tell the user
3855
- * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
3856
- * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
3857
- * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
3858
- * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
3859
- * TODO: [🧠][💷] `assertsExecutionSuccessful` should be the method of `PipelineExecutor` result BUT maybe NOT to preserve pure JSON object
4391
+ * Creates executor function from pipeline and execution tools.
4392
+ *
4393
+ * @returns The executor function
4394
+ * @throws {PipelineLogicError} on logical error in the pipeline
4395
+ * @public exported from `@promptbook/core`
3860
4396
  */
4397
+ function createPipelineExecutor(options) {
4398
+ var _this = this;
4399
+ var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
4400
+ var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f;
4401
+ validatePipeline(pipeline);
4402
+ var pipelineIdentification = (function () {
4403
+ // Note: This is a 😐 implementation of [🚞]
4404
+ var _ = [];
4405
+ if (pipeline.sourceFile !== undefined) {
4406
+ _.push("File: ".concat(pipeline.sourceFile));
4407
+ }
4408
+ if (pipeline.pipelineUrl !== undefined) {
4409
+ _.push("Url: ".concat(pipeline.pipelineUrl));
4410
+ }
4411
+ return _.join('\n');
4412
+ })();
4413
+ var preparedPipeline;
4414
+ if (isPipelinePrepared(pipeline)) {
4415
+ preparedPipeline = pipeline;
4416
+ }
4417
+ else if (isNotPreparedWarningSupressed !== true) {
4418
+ console.warn(spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
4419
+ }
4420
+ var runCount = 0;
4421
+ var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
4422
+ return __generator(this, function (_a) {
4423
+ runCount++;
4424
+ return [2 /*return*/, /* not await */ executePipeline({
4425
+ pipeline: pipeline,
4426
+ preparedPipeline: preparedPipeline,
4427
+ setPreparedPipeline: function (newPreparedPipeline) {
4428
+ preparedPipeline = newPreparedPipeline;
4429
+ },
4430
+ inputParameters: inputParameters,
4431
+ tools: tools,
4432
+ onProgress: onProgress,
4433
+ pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
4434
+ settings: {
4435
+ maxExecutionAttempts: maxExecutionAttempts,
4436
+ maxParallelCount: maxParallelCount,
4437
+ csvSettings: csvSettings,
4438
+ isVerbose: isVerbose,
4439
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4440
+ },
4441
+ })];
4442
+ });
4443
+ }); };
4444
+ return pipelineExecutor;
4445
+ }
3861
4446
 
3862
4447
  /**
3863
4448
  * @@@
@@ -3909,7 +4494,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
3909
4494
  outputParameters = result.outputParameters;
3910
4495
  knowledgePiecesRaw = outputParameters.knowledgePieces;
3911
4496
  knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
3912
- // <- TODO: !!!!! Smarter split and filter out empty pieces
4497
+ // <- TODO:[main] !!!!! Smarter split and filter out empty pieces
3913
4498
  if (isVerbose) {
3914
4499
  console.info('knowledgeTextPieces:', knowledgeTextPieces);
3915
4500
  }
@@ -3967,8 +4552,13 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
3967
4552
  case 6: return [3 /*break*/, 8];
3968
4553
  case 7:
3969
4554
  error_1 = _c.sent();
4555
+ // Note: Here is expected error:
4556
+ // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
4557
+ if (!(error_1 instanceof PipelineExecutionError)) {
4558
+ throw error_1;
4559
+ }
3970
4560
  // TODO: [🟥] Detect browser / node and make it colorfull
3971
- console.error(error_1);
4561
+ console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
3972
4562
  return [3 /*break*/, 8];
3973
4563
  case 8: return [2 /*return*/, {
3974
4564
  name: name,
@@ -3989,7 +4579,7 @@ function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?mayb
3989
4579
  });
3990
4580
  }
3991
4581
  /**
3992
- * TODO: [🐝][🔼] !!! Export via `@promptbook/markdown`
4582
+ * TODO: [🐝][🔼][main] !!! Export via `@promptbook/markdown`
3993
4583
  * TODO: [🪂] Do it in parallel 11:11
3994
4584
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
3995
4585
  */
@@ -4013,7 +4603,7 @@ function prepareKnowledgePieces(knowledgeSources, options) {
4013
4603
  var partialPieces, pieces;
4014
4604
  return __generator(this, function (_a) {
4015
4605
  switch (_a.label) {
4016
- case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4606
+ case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝][main] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
4017
4607
  options)];
4018
4608
  case 1:
4019
4609
  partialPieces = _a.sent();
@@ -4205,7 +4795,7 @@ function preparePersona(personaDescription, options) {
4205
4795
  });
4206
4796
  }
4207
4797
  /**
4208
- * TODO: [🔃] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4798
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4209
4799
  * TODO: [🏢] !! Check validity of `modelName` in pipeline
4210
4800
  * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4211
4801
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
@@ -4254,7 +4844,7 @@ function prepareTemplates(pipeline, options) {
4254
4844
  case 0:
4255
4845
  _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
4256
4846
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
4257
- // TODO: !!!!! Apply samples to each template (if missing and is for the template defined)
4847
+ // TODO:[main] !!!!! Apply samples to each template (if missing and is for the template defined)
4258
4848
  TODO_USE(parameters);
4259
4849
  templatesPrepared = new Array(
4260
4850
  // <- TODO: [🧱] Implement in a functional (not new Class) way
@@ -4286,7 +4876,7 @@ function prepareTemplates(pipeline, options) {
4286
4876
  /**
4287
4877
  * TODO: [🧠] Add context to each template (if missing)
4288
4878
  * TODO: [🧠] What is better name `prepareTemplate` or `prepareTemplateAndParameters`
4289
- * TODO: [♨] !!! Prepare index the samples and maybe templates
4879
+ * TODO: [♨][main] !!! Prepare index the samples and maybe templates
4290
4880
  * TODO: Write tests for `preparePipeline`
4291
4881
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
4292
4882
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
@@ -4458,7 +5048,7 @@ var knowledgeCommandParser = {
4458
5048
  if (sourceContent === '') {
4459
5049
  throw new ParseError("Source is not defined");
4460
5050
  }
4461
- // TODO: !!!! Following checks should be applied every link in the `sourceContent`
5051
+ // TODO:[main] !!!! Following checks should be applied every link in the `sourceContent`
4462
5052
  if (sourceContent.startsWith('http://')) {
4463
5053
  throw new ParseError("Source is not secure");
4464
5054
  }
@@ -4661,7 +5251,7 @@ var templateCommandParser = {
4661
5251
  if (command.templateType === 'KNOWLEDGE') {
4662
5252
  knowledgeCommandParser.$applyToPipelineJson({
4663
5253
  type: 'KNOWLEDGE',
4664
- sourceContent: $templateJson.content, // <- TODO: [🐝] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
5254
+ sourceContent: $templateJson.content, // <- TODO: [🐝][main] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
4665
5255
  }, $pipelineJson);
4666
5256
  $templateJson.isTemplate = false;
4667
5257
  return;
@@ -5011,6 +5601,171 @@ function normalizeTo_SCREAMING_CASE(text) {
5011
5601
  * TODO: [🌺] Use some intermediate util splitWords
5012
5602
  */
5013
5603
 
5604
+ /**
5605
+ * @@@
5606
+ *
5607
+ * @param text @@@
5608
+ * @param _isFirstLetterCapital @@@
5609
+ * @returns @@@
5610
+ * @example 'helloWorld'
5611
+ * @example 'iLovePromptbook'
5612
+ * @public exported from `@promptbook/utils`
5613
+ */
5614
+ function normalizeTo_camelCase(text, _isFirstLetterCapital) {
5615
+ var e_1, _a;
5616
+ if (_isFirstLetterCapital === void 0) { _isFirstLetterCapital = false; }
5617
+ var charType;
5618
+ var lastCharType = null;
5619
+ var normalizedName = '';
5620
+ try {
5621
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5622
+ var char = text_1_1.value;
5623
+ var normalizedChar = void 0;
5624
+ if (/^[a-z]$/.test(char)) {
5625
+ charType = 'LOWERCASE';
5626
+ normalizedChar = char;
5627
+ }
5628
+ else if (/^[A-Z]$/.test(char)) {
5629
+ charType = 'UPPERCASE';
5630
+ normalizedChar = char.toLowerCase();
5631
+ }
5632
+ else if (/^[0-9]$/.test(char)) {
5633
+ charType = 'NUMBER';
5634
+ normalizedChar = char;
5635
+ }
5636
+ else {
5637
+ charType = 'OTHER';
5638
+ normalizedChar = '';
5639
+ }
5640
+ if (!lastCharType) {
5641
+ if (_isFirstLetterCapital) {
5642
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: DRY
5643
+ }
5644
+ }
5645
+ else if (charType !== lastCharType &&
5646
+ !(charType === 'LOWERCASE' && lastCharType === 'UPPERCASE') &&
5647
+ !(lastCharType === 'NUMBER') &&
5648
+ !(charType === 'NUMBER')) {
5649
+ normalizedChar = normalizedChar.toUpperCase(); //TODO: [🌺] DRY
5650
+ }
5651
+ normalizedName += normalizedChar;
5652
+ lastCharType = charType;
5653
+ }
5654
+ }
5655
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5656
+ finally {
5657
+ try {
5658
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5659
+ }
5660
+ finally { if (e_1) throw e_1.error; }
5661
+ }
5662
+ return normalizedName;
5663
+ }
5664
+ /**
5665
+ * TODO: [🌺] Use some intermediate util splitWords
5666
+ */
5667
+
5668
+ /**
5669
+ * Removes quotes from a string
5670
+ *
5671
+ * Tip: This is very usefull for post-processing of the result of the LLM model
5672
+ * Note: This function removes only the same quotes from the beginning and the end of the string
5673
+ * Note: There are two simmilar functions:
5674
+ * - `removeQuotes` which removes only bounding quotes
5675
+ * - `unwrapResult` which removes whole introduce sentence
5676
+ *
5677
+ * @param text optionally quoted text
5678
+ * @returns text without quotes
5679
+ * @public exported from `@promptbook/utils`
5680
+ */
5681
+ function removeQuotes(text) {
5682
+ if (text.startsWith('"') && text.endsWith('"')) {
5683
+ return text.slice(1, -1);
5684
+ }
5685
+ if (text.startsWith('\'') && text.endsWith('\'')) {
5686
+ return text.slice(1, -1);
5687
+ }
5688
+ return text;
5689
+ }
5690
+
5691
+ /**
5692
+ * Function `validateParameterName` will @@@
5693
+ *
5694
+ * @param parameterName @@@
5695
+ * @returns @@@
5696
+ * @throws {ParseError} @@@
5697
+ * @private within the repository
5698
+ */
5699
+ function validateParameterName(parameterName) {
5700
+ var e_1, _a;
5701
+ var rawParameterName = parameterName;
5702
+ try {
5703
+ for (var _b = __values([
5704
+ ['`', '`'],
5705
+ ['{', '}'],
5706
+ ['[', ']'],
5707
+ ['(', ')'],
5708
+ ['<', '>'],
5709
+ ]), _c = _b.next(); !_c.done; _c = _b.next()) {
5710
+ var _d = __read(_c.value, 2), start = _d[0], end = _d[1];
5711
+ if (parameterName.substring(0, 1) === start &&
5712
+ parameterName.substring(parameterName.length - 1, parameterName.length) === end
5713
+ // <- TODO: More universal that 1 character
5714
+ ) {
5715
+ parameterName = parameterName.substring(1, parameterName.length - 1);
5716
+ // <- TODO: More universal that 1 character
5717
+ }
5718
+ }
5719
+ }
5720
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5721
+ finally {
5722
+ try {
5723
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
5724
+ }
5725
+ finally { if (e_1) throw e_1.error; }
5726
+ }
5727
+ // TODO: [🐠] Following try-catch block should be part of common validators logic
5728
+ try {
5729
+ /*
5730
+ Note: We don't need to check for spaces because we are going to normalize the parameter name to camelCase
5731
+ if (parameterName.includes(' ')) {
5732
+ throw new ParseError(`Parameter name cannot contain spaces`);
5733
+ }
5734
+ */
5735
+ if (parameterName.includes('.')) {
5736
+ throw new ParseError("Parameter name cannot contain dots");
5737
+ }
5738
+ if (parameterName.includes('/') || parameterName.includes('\\')) {
5739
+ throw new ParseError("Parameter name cannot contain slashes");
5740
+ }
5741
+ if (parameterName.includes('(') ||
5742
+ parameterName.includes(')') ||
5743
+ parameterName.includes('{') ||
5744
+ parameterName.includes('}') ||
5745
+ parameterName.includes('[') ||
5746
+ parameterName.includes(']')) {
5747
+ throw new ParseError("Parameter name cannot contain braces");
5748
+ }
5749
+ parameterName = removeDiacritics(parameterName);
5750
+ parameterName = removeEmojis(parameterName);
5751
+ parameterName = removeQuotes(parameterName);
5752
+ parameterName = normalizeTo_camelCase(parameterName);
5753
+ if (parameterName === '') {
5754
+ throw new ParseError("Parameter name cannot be empty");
5755
+ }
5756
+ if (RESERVED_PARAMETER_NAMES.includes(parameterName)) {
5757
+ throw new ParseError("{".concat(parameterName, "} is a reserved parameter name"));
5758
+ }
5759
+ }
5760
+ catch (error) {
5761
+ if (!(error instanceof ParseError)) {
5762
+ throw error;
5763
+ }
5764
+ throw new ParseError(spaceTrim$1(function (block) { return "\n ".concat(block(error.message), "\n\n Tried to validate parameter name:\n ").concat(block(rawParameterName), "\n "); }));
5765
+ }
5766
+ return parameterName;
5767
+ }
5768
+
5014
5769
  /**
5015
5770
  * Parses the foreach command
5016
5771
  *
@@ -5040,15 +5795,16 @@ var foreachCommandParser = {
5040
5795
  /**
5041
5796
  * Link to discussion
5042
5797
  */
5043
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5798
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/148',
5044
5799
  /**
5045
5800
  * Example usages of the FOREACH command
5046
5801
  */
5047
5802
  examples: [
5048
- 'FOREACH List Line `{customers}` -> `{customer}`',
5049
- 'FOR List Line `{customers}` -> `{customer}`',
5050
- 'EACH List Line `{customers}` -> `{customer}`',
5051
- // <- TODO: [🍭] !!!!!! More
5803
+ 'FOREACH Text Line `{customers}` -> `{customer}`',
5804
+ 'FOREACH Csv Cell `{customers}` -> `{cell}`',
5805
+ 'FOREACH Csv Row `{customers}` -> `{firstName}`, `{lastName}`, `+{email}`',
5806
+ 'FOR Text Line `{customers}` -> `{customer}`',
5807
+ 'EACH Text Line `{customers}` -> `{customer}`',
5052
5808
  ],
5053
5809
  /**
5054
5810
  * Parses the FOREACH command
@@ -5056,55 +5812,75 @@ var foreachCommandParser = {
5056
5812
  parse: function (input) {
5057
5813
  var args = input.args;
5058
5814
  var formatName = normalizeTo_SCREAMING_CASE(args[0] || '');
5059
- var cellName = normalizeTo_SCREAMING_CASE(args[1] || '');
5060
- var parameterNameWrapped = args[2];
5815
+ var subformatName = normalizeTo_SCREAMING_CASE(args[1] || '');
5816
+ var parameterNameArg = args[2] || '';
5061
5817
  var assignSign = args[3];
5062
- var subparameterNameWrapped = args[4];
5063
- if (![
5064
- 'LIST',
5065
- 'CSV',
5066
- // <- TODO: [🏢] Unhardcode formats
5067
- ].includes(formatName)) {
5068
- console.info({ args: args, formatName: formatName });
5069
- throw new Error("Unsupported format \"".concat(formatName, "\""));
5818
+ var formatDefinition = FORMAT_DEFINITIONS.find(function (formatDefinition) {
5819
+ return __spreadArray([formatDefinition.formatName], __read((formatDefinition.aliases || [])), false).includes(formatName);
5820
+ });
5821
+ if (formatDefinition === undefined) {
5822
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unsupported format \"".concat(formatName, "\"\n\n Available formats:\n ").concat(block(FORMAT_DEFINITIONS.map(function (formatDefinition) { return formatDefinition.formatName; })
5823
+ .map(function (formatName) { return "- ".concat(formatName); })
5824
+ .join('\n')), "\n "); }));
5070
5825
  // <- TODO: [🏢] List all supported format names
5071
5826
  }
5072
- if (![
5073
- 'LINE',
5074
- 'ROW',
5075
- 'COLUMN',
5076
- 'CELL',
5077
- // <- TODO: [🏢] Unhardcode format cells
5078
- ].includes(cellName)) {
5079
- console.info({ args: args, cellName: cellName });
5080
- throw new Error("Format ".concat(formatName, " does not support cell \"").concat(cellName, "\""));
5081
- // <- TODO: [🏢] List all supported cell names for the format
5827
+ var subvalueDefinition = formatDefinition.subvalueDefinitions.find(function (subvalueDefinition) {
5828
+ return __spreadArray([subvalueDefinition.subvalueName], __read((subvalueDefinition.aliases || [])), false).includes(subformatName);
5829
+ });
5830
+ if (subvalueDefinition === undefined) {
5831
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unsupported subformat name \"".concat(subformatName, "\" for format \"").concat(formatName, "\"\n\n Available subformat names for format \"").concat(formatDefinition.formatName, "\":\n ").concat(block(formatDefinition.subvalueDefinitions
5832
+ .map(function (subvalueDefinition) { return subvalueDefinition.subvalueName; })
5833
+ .map(function (subvalueName) { return "- ".concat(subvalueName); })
5834
+ .join('\n')), "\n "); }));
5835
+ // <- TODO: [🏢] List all supported subformat names for the format
5082
5836
  }
5083
5837
  if (assignSign !== '->') {
5084
- console.info({ args: args, assignSign: assignSign });
5085
- throw new Error("FOREACH command must have '->' to assign the value to the parameter");
5086
- }
5087
- // TODO: !!!!!! Replace with propper parameter name validation
5088
- if ((parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1)) !== '{' ||
5089
- (parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length)) !== '}') {
5090
- console.info({ args: args, parameterNameWrapped: parameterNameWrapped }, parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(0, 1), parameterNameWrapped === null || parameterNameWrapped === void 0 ? void 0 : parameterNameWrapped.substring(parameterNameWrapped.length - 1, parameterNameWrapped.length));
5091
- throw new Error("!!!!!! 1 Here will be error (with rules and precise error) from validateParameterName");
5092
- }
5093
- var parameterName = parameterNameWrapped.substring(1, parameterNameWrapped.length - 1);
5094
- // TODO: !!!!!! Replace with propper parameter name validation
5095
- if ((subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(0, 1)) !== '{' ||
5096
- (subparameterNameWrapped === null || subparameterNameWrapped === void 0 ? void 0 : subparameterNameWrapped.substring(subparameterNameWrapped.length - 1, subparameterNameWrapped.length)) !==
5097
- '}') {
5098
- console.info({ args: args, subparameterNameWrapped: subparameterNameWrapped });
5099
- throw new Error("!!!!!! 2 Here will be error (with rules and precise error) from validateParameterName");
5100
- }
5101
- var subparameterName = subparameterNameWrapped.substring(1, subparameterNameWrapped.length - 1);
5838
+ throw new ParseError("FOREACH command must have '->' to assign the value to the parameter");
5839
+ }
5840
+ var parameterName = validateParameterName(parameterNameArg);
5841
+ var outputSubparameterName = null;
5842
+ // TODO: [4] DRY
5843
+ var inputSubparameterNames = args
5844
+ .slice(4)
5845
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5846
+ .filter(function (parameterName) { return !parameterName.includes('+'); })
5847
+ .filter(function (parameterName) { return parameterName !== ''; })
5848
+ .map(validateParameterName);
5849
+ // TODO: [4] DRY
5850
+ var outputSubparameterNames = args
5851
+ .slice(4)
5852
+ .map(function (parameterName) { return parameterName.split(',').join(' ').trim(); })
5853
+ .filter(function (parameterName) { return parameterName.includes('+'); })
5854
+ .map(function (parameterName) { return parameterName.split('+').join(''); })
5855
+ .map(validateParameterName);
5856
+ if (outputSubparameterNames.length === 1) {
5857
+ outputSubparameterName = outputSubparameterNames[0];
5858
+ }
5859
+ else if (outputSubparameterNames.length > 1) {
5860
+ throw new ParseError("FOREACH command can not have more than one output subparameter");
5861
+ }
5862
+ if (inputSubparameterNames.length === 0) {
5863
+ throw new ParseError("FOREACH command must have at least one input subparameter");
5864
+ }
5865
+ if (outputSubparameterName === null) {
5866
+ // TODO: Following code should be unhardcoded from here and moved to the format definition
5867
+ if (formatName === 'CSV' && subformatName === 'CELL') {
5868
+ outputSubparameterName = 'newCell';
5869
+ }
5870
+ else if (formatName === 'TEXT' && subformatName === 'LINE') {
5871
+ outputSubparameterName = 'newLine';
5872
+ }
5873
+ else {
5874
+ throw new ParseError(spaceTrim$1("\n FOREACH ".concat(formatName, " ").concat(subformatName, " must specify output subparameter\n\n Correct example:\n - FOREACH ").concat(formatName, " ").concat(subformatName, " {").concat(parameterName, "} -> {inputSubparameterName1}, {inputSubparameterName2}, +{outputSubparameterName}\n\n ")));
5875
+ }
5876
+ }
5102
5877
  return {
5103
5878
  type: 'FOREACH',
5104
5879
  formatName: formatName,
5105
- cellName: cellName,
5880
+ subformatName: subformatName,
5106
5881
  parameterName: parameterName,
5107
- subparameterName: subparameterName,
5882
+ inputSubparameterNames: inputSubparameterNames,
5883
+ outputSubparameterName: outputSubparameterName,
5108
5884
  };
5109
5885
  },
5110
5886
  /**
@@ -5113,9 +5889,17 @@ var foreachCommandParser = {
5113
5889
  * Note: `$` is used to indicate that this function mutates given `templateJson`
5114
5890
  */
5115
5891
  $applyToTemplateJson: function (command, $templateJson, $pipelineJson) {
5116
- var formatName = command.formatName, cellName = command.cellName, parameterName = command.parameterName, subparameterName = command.subparameterName;
5117
- $templateJson.foreach = { formatName: formatName, cellName: cellName, parameterName: parameterName, subparameterName: subparameterName };
5118
- keepUnused($pipelineJson); // <- TODO: !!!!!! BUT Maybe register subparameter from foreach into parameters of the pipeline
5892
+ var formatName = command.formatName, subformatName = command.subformatName, parameterName = command.parameterName, inputSubparameterNames = command.inputSubparameterNames, outputSubparameterName = command.outputSubparameterName;
5893
+ // TODO: [🍭] Detect double use
5894
+ // TODO: [🍭] Detect usage with JOKER and don't allow it
5895
+ $templateJson.foreach = {
5896
+ formatName: formatName,
5897
+ subformatName: subformatName,
5898
+ parameterName: parameterName,
5899
+ inputSubparameterNames: inputSubparameterNames,
5900
+ outputSubparameterName: outputSubparameterName,
5901
+ };
5902
+ keepUnused($pipelineJson); // <- TODO: [🧠] Maybe register subparameter from foreach into parameters of the pipeline
5119
5903
  // Note: [🍭] FOREACH apply has some sideeffects on different places in codebase
5120
5904
  },
5121
5905
  /**
@@ -5138,8 +5922,7 @@ var foreachCommandParser = {
5138
5922
  },
5139
5923
  };
5140
5924
  /**
5141
- * TODO: !!!!!! Comment console logs
5142
- * TODO: [🍭] !!!!!! Make .ptbk.md file with examples of the FOREACH command and also with wrong parsing and logic
5925
+ * TODO: [🍭] Make .ptbk.md file with examples of the FOREACH with wrong parsing and logic
5143
5926
  */
5144
5927
 
5145
5928
  /**
@@ -5249,12 +6032,11 @@ var jokerCommandParser = {
5249
6032
  */
5250
6033
  parse: function (input) {
5251
6034
  var args = input.args;
5252
- // TODO: !!!!!! Replace with propper parameter name validation
5253
- var parametersMatch = (args.pop() || '').match(/^\{(?<parameterName>[a-z0-9_]+)\}$/im);
5254
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5255
- throw new ParseError("Invalid joker");
6035
+ if (args.length !== 1) {
6036
+ throw new ParseError("JOKE command expects exactly one parameter name");
5256
6037
  }
5257
- var parameterName = parametersMatch.groups.parameterName;
6038
+ var parameterNameArg = args[0] || '';
6039
+ var parameterName = validateParameterName(parameterNameArg);
5258
6040
  return {
5259
6041
  type: 'JOKER',
5260
6042
  parameterName: parameterName,
@@ -5329,6 +6111,9 @@ var modelCommandParser = {
5329
6111
  */
5330
6112
  parse: function (input) {
5331
6113
  var args = input.args, normalized = input.normalized;
6114
+ var availableVariantsMessage = spaceTrim$1(function (block) { return "\n Available variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) {
6115
+ return "- ".concat(variantName).concat(variantName !== 'EMBEDDING' ? '' : ' (Not available in pipeline)');
6116
+ }).join('\n')), "\n "); });
5332
6117
  // TODO: Make this more elegant and dynamically
5333
6118
  if (normalized.startsWith('MODEL_VARIANT')) {
5334
6119
  if (normalized === 'MODEL_VARIANT_CHAT') {
@@ -5344,17 +6129,13 @@ var modelCommandParser = {
5344
6129
  key: 'modelVariant',
5345
6130
  value: 'COMPLETION',
5346
6131
  };
6132
+ // <- Note: [🤖]
5347
6133
  }
5348
6134
  else if (normalized.startsWith('MODEL_VARIANT_EMBED')) {
5349
- return {
5350
- type: 'MODEL',
5351
- key: 'modelVariant',
5352
- value: 'EMBEDDING',
5353
- };
5354
- // <- Note: [🤖]
6135
+ spaceTrim$1(function (block) { return "\n Embedding model can not be used in pipeline\n\n ".concat(block(availableVariantsMessage), "\n "); });
5355
6136
  }
5356
6137
  else {
5357
- throw new ParseError(spaceTrim$1(function (block) { return "\n Unknown model variant in command:\n\n Supported variants are:\n ".concat(block(MODEL_VARIANTS.map(function (variantName) { return "- ".concat(variantName); }).join('\n')), "\n "); }));
6138
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Unknown model variant in command:\n\n ".concat(block(availableVariantsMessage), "\n "); }));
5358
6139
  }
5359
6140
  }
5360
6141
  if (normalized.startsWith('MODEL_NAME')) {
@@ -5479,14 +6260,13 @@ var parameterCommandParser = {
5479
6260
  * Parses the PARAMETER command
5480
6261
  */
5481
6262
  parse: function (input) {
5482
- var normalized = input.normalized, raw = input.raw;
5483
- var parametersMatch = raw.match(/\{(?<parameterName>[a-z0-9_]+)\}[^\S\r\n]*(?<parameterDescription>.*)$/im);
5484
- if (!parametersMatch || !parametersMatch.groups || !parametersMatch.groups.parameterName) {
5485
- throw new ParseError("Invalid parameter");
5486
- }
5487
- var _a = parametersMatch.groups, parameterName = _a.parameterName, parameterDescription = _a.parameterDescription;
5488
- if (parameterDescription && parameterDescription.match(/\{(?<parameterName>[a-z0-9_]+)\}/im)) {
5489
- throw new ParseError("Parameter {".concat(parameterName, "} can not contain another parameter in description"));
6263
+ var normalized = input.normalized, args = input.args, raw = input.raw;
6264
+ var parameterNameRaw = args.shift() || '';
6265
+ var parameterDescriptionRaw = args.join(' ');
6266
+ // <- TODO: When [🥶] fixed, change to:
6267
+ // > const parameterDescriptionRaw = rawArgs.split(parameterNameRaw).join('').trim();
6268
+ if (parameterDescriptionRaw && parameterDescriptionRaw.match(/\{(?<embeddedParameterName>[a-z0-9_]+)\}/im)) {
6269
+ throw new ParseError(spaceTrim$1(function (block) { return "\n Parameter {".concat(parameterNameRaw, "} can not contain another parameter in description\n\n The description:\n ").concat(block(parameterDescriptionRaw), "\n "); }));
5490
6270
  }
5491
6271
  var isInput = normalized.startsWith('INPUT');
5492
6272
  var isOutput = normalized.startsWith('OUTPUT');
@@ -5494,11 +6274,12 @@ var parameterCommandParser = {
5494
6274
  isInput = false;
5495
6275
  isOutput = false;
5496
6276
  }
5497
- // TODO: !!!!!! Add parameter name validation
6277
+ var parameterName = validateParameterName(parameterNameRaw);
6278
+ var parameterDescription = parameterDescriptionRaw.trim() || null;
5498
6279
  return {
5499
6280
  type: 'PARAMETER',
5500
6281
  parameterName: parameterName,
5501
- parameterDescription: parameterDescription.trim() || null,
6282
+ parameterDescription: parameterDescription,
5502
6283
  isInput: isInput,
5503
6284
  isOutput: isOutput,
5504
6285
  };
@@ -5823,6 +6604,7 @@ var promptbookVersionCommandParser = {
5823
6604
  * Note: `$` is used to indicate that this function mutates given `pipelineJson`
5824
6605
  */
5825
6606
  $applyToPipelineJson: function (command, $pipelineJson) {
6607
+ // TODO: Warn if the version is overridden
5826
6608
  $pipelineJson.promptbookVersion = command.promptbookVersion;
5827
6609
  },
5828
6610
  /**
@@ -6305,7 +7087,9 @@ function parseCommand(raw, usagePlace) {
6305
7087
  for (var commandNameSegmentsCount = 0; commandNameSegmentsCount < Math.min(items.length, 3); commandNameSegmentsCount++) {
6306
7088
  var commandNameRaw = items.slice(0, commandNameSegmentsCount + 1).join('_');
6307
7089
  var args = items.slice(commandNameSegmentsCount + 1);
6308
- var rawArgs = raw.substring(commandNameRaw.length).trim();
7090
+ var rawArgs = raw
7091
+ .substring(commandNameRaw.length)
7092
+ .trim();
6309
7093
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6310
7094
  if (command !== null) {
6311
7095
  return command;
@@ -6316,7 +7100,9 @@ function parseCommand(raw, usagePlace) {
6316
7100
  {
6317
7101
  var commandNameRaw = items.slice(-1).join('_');
6318
7102
  var args = items.slice(0, -1); // <- Note: This is probbably not correct
6319
- var rawArgs = raw.substring(0, raw.length - commandNameRaw.length).trim();
7103
+ var rawArgs = raw
7104
+ .substring(0, raw.length - commandNameRaw.length)
7105
+ .trim();
6320
7106
  var command = parseCommandVariant({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args, commandNameRaw: commandNameRaw });
6321
7107
  if (command !== null) {
6322
7108
  return command;
@@ -6456,7 +7242,7 @@ function extractAllListItemsFromMarkdown(markdown) {
6456
7242
  function extractOneBlockFromMarkdown(markdown) {
6457
7243
  var codeBlocks = extractAllBlocksFromMarkdown(markdown);
6458
7244
  if (codeBlocks.length !== 1) {
6459
- throw new ParseError(spaceTrim$1(function (block) { return "\n There should be exactly 1 code block, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
7245
+ throw new ParseError(spaceTrim$1(function (block) { return "\n There should be exactly 1 code block in template, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
6460
7246
  }
6461
7247
  return codeBlocks[0];
6462
7248
  }
@@ -6643,7 +7429,7 @@ function pipelineStringToJsonSync(pipelineString) {
6643
7429
  var $pipelineJson = {
6644
7430
  title: undefined /* <- Note: [🍙] Putting here placeholder to keep `title` on top at final JSON */,
6645
7431
  pipelineUrl: undefined /* <- Note: Putting here placeholder to keep `pipelineUrl` on top at final JSON */,
6646
- promptbookVersion: PROMPTBOOK_VERSION,
7432
+ promptbookVersion: undefined /* <- Note: By default no explicit version */,
6647
7433
  description: undefined /* <- Note: [🍙] Putting here placeholder to keep `description` on top at final JSON */,
6648
7434
  parameters: [],
6649
7435
  templates: [],
@@ -6934,7 +7720,7 @@ function pipelineStringToJsonSync(pipelineString) {
6934
7720
  return $asDeeplyFrozenSerializableJson('pipelineJson', $pipelineJson);
6935
7721
  }
6936
7722
  /**
6937
- * TODO: !!!! Warn if used only sync version
7723
+ * TODO:[main] !!!! Warn if used only sync version
6938
7724
  * TODO: [🚞] Report here line/column of error
6939
7725
  * TODO: Use spaceTrim more effectively
6940
7726
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
@@ -7717,7 +8503,7 @@ function isSerializableAsJson(value) {
7717
8503
  }
7718
8504
  }
7719
8505
  /**
7720
- * TODO: [🧠] !!! In-memory cache of same values to prevent multiple checks
8506
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
7721
8507
  * TODO: [🧠][💺] Can be done this on type-level?
7722
8508
  */
7723
8509