@promptbook/core 0.61.0-2 → 0.61.0-21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (359) hide show
  1. package/esm/index.es.js +1806 -957
  2. package/esm/index.es.js.map +1 -1
  3. package/esm/typings/promptbook-collection/index.d.ts +200 -30
  4. package/esm/typings/src/_packages/core.index.d.ts +7 -2
  5. package/esm/typings/src/_packages/execute-javascript.index.d.ts +1 -1
  6. package/esm/typings/src/_packages/node.index.d.ts +2 -1
  7. package/esm/typings/src/_packages/types.index.d.ts +11 -7
  8. package/esm/typings/src/_packages/utils.index.d.ts +8 -6
  9. package/esm/typings/src/cli/cli-commands/hello.d.ts +3 -0
  10. package/esm/typings/src/cli/cli-commands/make.d.ts +3 -0
  11. package/esm/typings/src/cli/cli-commands/prettify.d.ts +3 -0
  12. package/esm/typings/src/cli/promptbookCli.d.ts +1 -0
  13. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  14. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +10 -10
  15. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -7
  16. package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +2 -2
  17. package/esm/typings/src/commands/KNOWLEDGE/KnowledgeCommand.d.ts +2 -3
  18. package/esm/typings/src/commands/MODEL/ModelCommand.d.ts +2 -2
  19. package/esm/typings/src/commands/PERSONA/PersonaCommand.d.ts +4 -2
  20. package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -0
  21. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +42 -14
  22. package/esm/typings/src/config.d.ts +38 -1
  23. package/esm/typings/src/config.test.d.ts +4 -0
  24. package/esm/typings/src/conversion/pipelineJsonToString.d.ts +1 -1
  25. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +9 -5
  26. package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +5 -3
  27. package/esm/typings/src/conversion/prettify/PrettifyOptions.d.ts +2 -2
  28. package/esm/typings/src/conversion/utils/extractParametersFromPromptTemplate.d.ts +2 -2
  29. package/esm/typings/src/conversion/utils/renameParameter.d.ts +3 -3
  30. package/esm/typings/src/conversion/utils/stringifyPipelineJson.d.ts +13 -0
  31. package/esm/typings/src/conversion/validation/_importPipeline.d.ts +12 -3
  32. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +10 -1
  33. package/esm/typings/src/errors/EnvironmentMismatchError.d.ts +7 -0
  34. package/esm/typings/src/errors/LimitReachedError.d.ts +7 -0
  35. package/esm/typings/src/errors/VersionMismatchError.d.ts +8 -0
  36. package/esm/typings/src/execution/CommonExecutionToolsOptions.d.ts +1 -0
  37. package/esm/typings/src/execution/LlmExecutionTools.d.ts +10 -8
  38. package/esm/typings/src/execution/PipelineExecutor.d.ts +10 -8
  39. package/esm/typings/src/execution/PromptResult.d.ts +28 -58
  40. package/esm/typings/src/execution/PromptResultUsage.d.ts +26 -0
  41. package/esm/typings/src/execution/ScriptExecutionTools.d.ts +5 -4
  42. package/esm/typings/src/execution/UncertainNumber.d.ts +18 -0
  43. package/esm/typings/src/execution/UserInterfaceTools.d.ts +5 -5
  44. package/esm/typings/src/execution/createPipelineExecutor.d.ts +20 -3
  45. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.d.ts +2 -2
  46. package/esm/typings/src/execution/utils/addUsage.d.ts +58 -2
  47. package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +1 -1
  48. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  49. package/esm/typings/src/execution/utils/uncertainNumber.d.ts +1 -1
  50. package/esm/typings/src/execution/utils/usageToWorktime.d.ts +2 -2
  51. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +1 -1
  52. package/esm/typings/src/formats/csv/ListFormatDefinition.d.ts +2 -1
  53. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +2 -1
  54. package/esm/typings/src/formats/list/ListFormatDefinition.d.ts +2 -1
  55. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +1 -0
  56. package/esm/typings/src/knowledge/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +1 -1
  57. package/esm/typings/src/knowledge/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -0
  58. package/esm/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +27 -0
  59. package/esm/typings/src/knowledge/prepare-knowledge/_common/prepareKnowledgePieces.d.ts +18 -0
  60. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +9 -23
  61. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts +3 -0
  62. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +8 -14
  63. package/esm/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +30 -0
  64. package/esm/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +10 -0
  65. package/esm/typings/src/llm-providers/_common/getLlmToolsForTestingAndScriptsAndPlayground.d.ts +11 -0
  66. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +29 -0
  67. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +10 -0
  68. package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +16 -0
  69. package/esm/typings/src/llm-providers/_common/utils/count-total-cost/LlmExecutionToolsWithTotalCost.d.ts +11 -0
  70. package/esm/typings/src/llm-providers/_common/utils/count-total-cost/countTotalCost.d.ts +14 -0
  71. package/esm/typings/src/llm-providers/_common/utils/count-total-cost/limitTotalCost.d.ts +32 -0
  72. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +3 -2
  73. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
  74. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +3 -3
  75. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +3 -0
  76. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +5 -4
  77. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +5 -5
  78. package/esm/typings/src/llm-providers/azure-openai/playground/playground.d.ts +3 -0
  79. package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +3 -0
  80. package/esm/typings/src/llm-providers/langtail/LangtailExecutionToolsOptions.d.ts +1 -1
  81. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +5 -5
  82. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +7 -7
  83. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +12 -8
  84. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +7 -6
  85. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +1 -1
  86. package/esm/typings/src/llm-providers/openai/computeOpenaiUsage.d.ts +1 -1
  87. package/esm/typings/src/llm-providers/openai/computeUsage.d.ts +2 -2
  88. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +3 -2
  89. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +3 -0
  90. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +9 -7
  91. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionToolsOptions.d.ts +1 -1
  92. package/esm/typings/src/personas/preparePersona.d.ts +15 -0
  93. package/esm/typings/src/prepare/PrepareOptions.d.ts +22 -0
  94. package/esm/typings/src/prepare/isPipelinePrepared.d.ts +10 -0
  95. package/esm/typings/src/prepare/preparePipeline.d.ts +18 -0
  96. package/esm/typings/src/prepare/unpreparePipeline.d.ts +9 -0
  97. package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +1 -1
  98. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  99. package/esm/typings/src/scripting/javascript/utils/unknownToString.d.ts +2 -1
  100. package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +25 -0
  101. package/esm/typings/src/storage/_common/PromptbookStorage.test-type.d.ts +5 -0
  102. package/esm/typings/src/storage/files-storage/FilesStorage.d.ts +30 -0
  103. package/esm/typings/src/storage/files-storage/FilesStorageOptions.d.ts +13 -0
  104. package/esm/typings/src/storage/files-storage/utils/nameToSubfolderPath.d.ts +7 -0
  105. package/esm/typings/src/storage/files-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
  106. package/esm/typings/src/storage/local-storage/getLocalStorage.d.ts +9 -0
  107. package/esm/typings/src/storage/local-storage/getSessionStorage.d.ts +9 -0
  108. package/esm/typings/src/storage/memory/MemoryStorage.d.ts +34 -0
  109. package/esm/typings/src/storage/utils/PrefixStorage.d.ts +26 -0
  110. package/esm/typings/src/storage/utils/makePromptbookStorageFromWebStorage.d.ts +11 -0
  111. package/esm/typings/src/types/ModelRequirements.d.ts +71 -11
  112. package/esm/typings/src/types/ModelVariant.d.ts +14 -0
  113. package/esm/typings/src/types/PipelineJson/Expectations.d.ts +6 -2
  114. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +67 -0
  115. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +24 -0
  116. package/esm/typings/src/types/PipelineJson/LlmTemplateJson.d.ts +9 -1
  117. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +38 -0
  118. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +29 -4
  119. package/esm/typings/src/types/PipelineJson/PreparationJson.d.ts +21 -0
  120. package/esm/typings/src/types/PipelineJson/PromptDialogJson.d.ts +5 -0
  121. package/esm/typings/src/types/PipelineJson/PromptTemplateJson.d.ts +5 -1
  122. package/esm/typings/src/types/PipelineJson/PromptTemplateJsonCommon.d.ts +13 -4
  123. package/esm/typings/src/types/PipelineJson/PromptTemplateParameterJson.d.ts +11 -2
  124. package/esm/typings/src/types/PipelineJson/ScriptJson.d.ts +2 -0
  125. package/esm/typings/src/types/PipelineJson/SimpleTemplateJson.d.ts +2 -0
  126. package/esm/typings/src/types/PipelineString.d.ts +1 -1
  127. package/esm/typings/src/types/Prompt.d.ts +54 -10
  128. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +4 -4
  129. package/esm/typings/src/types/execution-report/ExecutionReportStringOptions.d.ts +3 -3
  130. package/esm/typings/src/types/execution-report/countWorkingDuration.d.ts +1 -1
  131. package/esm/typings/src/types/typeAliases.d.ts +114 -15
  132. package/esm/typings/src/utils/FromtoItems.d.ts +3 -3
  133. package/esm/typings/src/utils/currentDate.d.ts +7 -0
  134. package/esm/typings/src/utils/deepClone.d.ts +9 -0
  135. package/esm/typings/src/utils/deepFreeze.d.ts +22 -0
  136. package/esm/typings/src/utils/emojis.d.ts +2 -2
  137. package/esm/typings/src/utils/extractParameters.d.ts +2 -2
  138. package/esm/typings/src/utils/formatNumber.d.ts +1 -1
  139. package/esm/typings/src/utils/isRunningInWhatever.d.ts +3 -0
  140. package/esm/typings/src/utils/markdown/addAutoGeneratedSection.d.ts +3 -3
  141. package/esm/typings/src/utils/markdown/createMarkdownChart.d.ts +7 -10
  142. package/esm/typings/src/utils/markdown/createMarkdownTable.d.ts +1 -1
  143. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.d.ts +1 -0
  144. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +9 -2
  145. package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +1 -1
  146. package/esm/typings/src/utils/markdown/parseMarkdownSection.d.ts +3 -3
  147. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -1
  148. package/esm/typings/src/utils/organization/TODO_USE.d.ts +12 -0
  149. package/esm/typings/src/utils/organization/TODO_any.d.ts +6 -0
  150. package/esm/typings/src/utils/organization/TODO_object.d.ts +6 -0
  151. package/esm/typings/src/utils/organization/TODO_unknown.d.ts +6 -0
  152. package/esm/typings/src/utils/organization/___.d.ts +6 -0
  153. package/esm/typings/src/utils/{just.d.ts → organization/just.d.ts} +5 -1
  154. package/esm/typings/src/utils/organization/keepUnused.d.ts +16 -0
  155. package/esm/typings/src/utils/organization/really_any.d.ts +6 -0
  156. package/esm/typings/src/utils/organization/really_unknown.d.ts +6 -0
  157. package/esm/typings/src/utils/random/randomSeed.d.ts +7 -0
  158. package/{umd/typings/src/execution → esm/typings/src}/utils/replaceParameters.d.ts +2 -4
  159. package/esm/typings/src/utils/replaceParameters.test.d.ts +1 -0
  160. package/esm/typings/src/utils/sets/difference.d.ts +3 -0
  161. package/esm/typings/src/utils/validators/email/isValidEmail.d.ts +2 -1
  162. package/esm/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -1
  163. package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +2 -1
  164. package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +2 -1
  165. package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +2 -1
  166. package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +2 -1
  167. package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +2 -1
  168. package/esm/typings/src/utils/validators/uuid/isValidUuid.d.ts +2 -1
  169. package/package.json +1 -1
  170. package/umd/index.umd.js +1809 -956
  171. package/umd/index.umd.js.map +1 -1
  172. package/umd/typings/promptbook-collection/index.d.ts +200 -30
  173. package/umd/typings/src/_packages/core.index.d.ts +7 -2
  174. package/umd/typings/src/_packages/execute-javascript.index.d.ts +1 -1
  175. package/umd/typings/src/_packages/node.index.d.ts +2 -1
  176. package/umd/typings/src/_packages/types.index.d.ts +11 -7
  177. package/umd/typings/src/_packages/utils.index.d.ts +8 -6
  178. package/umd/typings/src/cli/cli-commands/hello.d.ts +3 -0
  179. package/umd/typings/src/cli/cli-commands/make.d.ts +3 -0
  180. package/umd/typings/src/cli/cli-commands/prettify.d.ts +3 -0
  181. package/umd/typings/src/cli/promptbookCli.d.ts +1 -0
  182. package/umd/typings/src/collection/PipelineCollection.d.ts +1 -1
  183. package/umd/typings/src/collection/SimplePipelineCollection.d.ts +10 -10
  184. package/umd/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -7
  185. package/umd/typings/src/collection/constructors/createCollectionFromUrl.d.ts +2 -2
  186. package/umd/typings/src/commands/KNOWLEDGE/KnowledgeCommand.d.ts +2 -3
  187. package/umd/typings/src/commands/MODEL/ModelCommand.d.ts +2 -2
  188. package/umd/typings/src/commands/PERSONA/PersonaCommand.d.ts +4 -2
  189. package/umd/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -0
  190. package/umd/typings/src/commands/_common/types/CommandParser.d.ts +42 -14
  191. package/umd/typings/src/config.d.ts +38 -1
  192. package/umd/typings/src/config.test.d.ts +4 -0
  193. package/umd/typings/src/conversion/pipelineJsonToString.d.ts +1 -1
  194. package/umd/typings/src/conversion/pipelineStringToJson.d.ts +9 -5
  195. package/umd/typings/src/conversion/pipelineStringToJsonSync.d.ts +5 -3
  196. package/umd/typings/src/conversion/prettify/PrettifyOptions.d.ts +2 -2
  197. package/umd/typings/src/conversion/utils/extractParametersFromPromptTemplate.d.ts +2 -2
  198. package/umd/typings/src/conversion/utils/renameParameter.d.ts +3 -3
  199. package/umd/typings/src/conversion/utils/stringifyPipelineJson.d.ts +13 -0
  200. package/umd/typings/src/conversion/utils/stringifyPipelineJson.test.d.ts +1 -0
  201. package/umd/typings/src/conversion/validation/_importPipeline.d.ts +12 -3
  202. package/umd/typings/src/conversion/validation/validatePipeline.d.ts +10 -1
  203. package/umd/typings/src/errors/EnvironmentMismatchError.d.ts +7 -0
  204. package/umd/typings/src/errors/LimitReachedError.d.ts +7 -0
  205. package/umd/typings/src/errors/VersionMismatchError.d.ts +8 -0
  206. package/umd/typings/src/execution/CommonExecutionToolsOptions.d.ts +1 -0
  207. package/umd/typings/src/execution/LlmExecutionTools.d.ts +10 -8
  208. package/umd/typings/src/execution/PipelineExecutor.d.ts +10 -8
  209. package/umd/typings/src/execution/PromptResult.d.ts +28 -58
  210. package/umd/typings/src/execution/PromptResultUsage.d.ts +26 -0
  211. package/umd/typings/src/execution/ScriptExecutionTools.d.ts +5 -4
  212. package/umd/typings/src/execution/UncertainNumber.d.ts +18 -0
  213. package/umd/typings/src/execution/UserInterfaceTools.d.ts +5 -5
  214. package/umd/typings/src/execution/createPipelineExecutor.d.ts +20 -3
  215. package/umd/typings/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.d.ts +2 -2
  216. package/umd/typings/src/execution/utils/addUsage.d.ts +58 -2
  217. package/umd/typings/src/execution/utils/computeUsageCounts.d.ts +1 -1
  218. package/umd/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  219. package/umd/typings/src/execution/utils/uncertainNumber.d.ts +1 -1
  220. package/umd/typings/src/execution/utils/usageToWorktime.d.ts +2 -2
  221. package/umd/typings/src/formats/_common/FormatDefinition.d.ts +1 -1
  222. package/umd/typings/src/formats/csv/ListFormatDefinition.d.ts +2 -1
  223. package/umd/typings/src/formats/json/JsonFormatDefinition.d.ts +2 -1
  224. package/umd/typings/src/formats/list/ListFormatDefinition.d.ts +2 -1
  225. package/umd/typings/src/formats/xml/XmlFormatDefinition.d.ts +1 -0
  226. package/umd/typings/src/knowledge/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +1 -1
  227. package/umd/typings/src/knowledge/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -0
  228. package/umd/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +27 -0
  229. package/umd/typings/src/knowledge/prepare-knowledge/_common/prepareKnowledgePieces.d.ts +18 -0
  230. package/umd/typings/src/knowledge/prepare-knowledge/_common/prepareKnowledgePieces.test.d.ts +1 -0
  231. package/umd/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +9 -23
  232. package/umd/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts +3 -0
  233. package/umd/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +8 -14
  234. package/umd/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +30 -0
  235. package/umd/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +10 -0
  236. package/umd/typings/src/llm-providers/_common/getLlmToolsForTestingAndScriptsAndPlayground.d.ts +11 -0
  237. package/umd/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +29 -0
  238. package/umd/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +10 -0
  239. package/umd/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +16 -0
  240. package/umd/typings/src/llm-providers/_common/utils/count-total-cost/LlmExecutionToolsWithTotalCost.d.ts +11 -0
  241. package/umd/typings/src/llm-providers/_common/utils/count-total-cost/countTotalCost.d.ts +14 -0
  242. package/umd/typings/src/llm-providers/_common/utils/count-total-cost/limitTotalCost.d.ts +32 -0
  243. package/umd/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +3 -2
  244. package/umd/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
  245. package/umd/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +3 -3
  246. package/umd/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +3 -0
  247. package/umd/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +5 -4
  248. package/umd/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +5 -5
  249. package/umd/typings/src/llm-providers/azure-openai/playground/playground.d.ts +3 -0
  250. package/umd/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +3 -0
  251. package/umd/typings/src/llm-providers/langtail/LangtailExecutionToolsOptions.d.ts +1 -1
  252. package/umd/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +5 -5
  253. package/umd/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +7 -7
  254. package/umd/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +12 -8
  255. package/umd/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +7 -6
  256. package/umd/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +1 -1
  257. package/umd/typings/src/llm-providers/openai/computeOpenaiUsage.d.ts +1 -1
  258. package/umd/typings/src/llm-providers/openai/computeUsage.d.ts +2 -2
  259. package/umd/typings/src/llm-providers/openai/openai-models.d.ts +3 -2
  260. package/umd/typings/src/llm-providers/openai/playground/playground.d.ts +3 -0
  261. package/umd/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +9 -7
  262. package/umd/typings/src/llm-providers/remote/RemoteLlmExecutionToolsOptions.d.ts +1 -1
  263. package/umd/typings/src/personas/preparePersona.d.ts +15 -0
  264. package/umd/typings/src/personas/preparePersona.test.d.ts +1 -0
  265. package/umd/typings/src/prepare/PrepareOptions.d.ts +22 -0
  266. package/umd/typings/src/prepare/isPipelinePrepared.d.ts +10 -0
  267. package/umd/typings/src/prepare/isPipelinePrepared.test.d.ts +1 -0
  268. package/umd/typings/src/prepare/preparePipeline.d.ts +18 -0
  269. package/umd/typings/src/prepare/unpreparePipeline.d.ts +9 -0
  270. package/umd/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +1 -1
  271. package/umd/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  272. package/umd/typings/src/scripting/javascript/utils/unknownToString.d.ts +2 -1
  273. package/umd/typings/src/storage/_common/PromptbookStorage.d.ts +25 -0
  274. package/umd/typings/src/storage/_common/PromptbookStorage.test-type.d.ts +5 -0
  275. package/umd/typings/src/storage/files-storage/FilesStorage.d.ts +30 -0
  276. package/umd/typings/src/storage/files-storage/FilesStorageOptions.d.ts +13 -0
  277. package/umd/typings/src/storage/files-storage/utils/nameToSubfolderPath.d.ts +7 -0
  278. package/umd/typings/src/storage/files-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
  279. package/umd/typings/src/storage/local-storage/getLocalStorage.d.ts +9 -0
  280. package/umd/typings/src/storage/local-storage/getSessionStorage.d.ts +9 -0
  281. package/umd/typings/src/storage/memory/MemoryStorage.d.ts +34 -0
  282. package/umd/typings/src/storage/utils/PrefixStorage.d.ts +26 -0
  283. package/umd/typings/src/storage/utils/makePromptbookStorageFromWebStorage.d.ts +11 -0
  284. package/umd/typings/src/types/ModelRequirements.d.ts +71 -11
  285. package/umd/typings/src/types/ModelVariant.d.ts +14 -0
  286. package/umd/typings/src/types/PipelineJson/Expectations.d.ts +6 -2
  287. package/umd/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +67 -0
  288. package/umd/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +24 -0
  289. package/umd/typings/src/types/PipelineJson/LlmTemplateJson.d.ts +9 -1
  290. package/umd/typings/src/types/PipelineJson/PersonaJson.d.ts +38 -0
  291. package/umd/typings/src/types/PipelineJson/PipelineJson.d.ts +29 -4
  292. package/umd/typings/src/types/PipelineJson/PreparationJson.d.ts +21 -0
  293. package/umd/typings/src/types/PipelineJson/PromptDialogJson.d.ts +5 -0
  294. package/umd/typings/src/types/PipelineJson/PromptTemplateJson.d.ts +5 -1
  295. package/umd/typings/src/types/PipelineJson/PromptTemplateJsonCommon.d.ts +13 -4
  296. package/umd/typings/src/types/PipelineJson/PromptTemplateParameterJson.d.ts +11 -2
  297. package/umd/typings/src/types/PipelineJson/ScriptJson.d.ts +2 -0
  298. package/umd/typings/src/types/PipelineJson/SimpleTemplateJson.d.ts +2 -0
  299. package/umd/typings/src/types/PipelineString.d.ts +1 -1
  300. package/umd/typings/src/types/Prompt.d.ts +54 -10
  301. package/umd/typings/src/types/execution-report/ExecutionReportJson.d.ts +4 -4
  302. package/umd/typings/src/types/execution-report/ExecutionReportStringOptions.d.ts +3 -3
  303. package/umd/typings/src/types/execution-report/countWorkingDuration.d.ts +1 -1
  304. package/umd/typings/src/types/typeAliases.d.ts +114 -15
  305. package/umd/typings/src/utils/FromtoItems.d.ts +3 -3
  306. package/umd/typings/src/utils/currentDate.d.ts +7 -0
  307. package/umd/typings/src/utils/deepClone.d.ts +9 -0
  308. package/umd/typings/src/utils/deepFreeze.d.ts +22 -0
  309. package/umd/typings/src/utils/emojis.d.ts +2 -2
  310. package/umd/typings/src/utils/extractParameters.d.ts +2 -2
  311. package/umd/typings/src/utils/formatNumber.d.ts +1 -1
  312. package/umd/typings/src/utils/isRunningInWhatever.d.ts +3 -0
  313. package/umd/typings/src/utils/markdown/addAutoGeneratedSection.d.ts +3 -3
  314. package/umd/typings/src/utils/markdown/createMarkdownChart.d.ts +7 -10
  315. package/umd/typings/src/utils/markdown/createMarkdownTable.d.ts +1 -1
  316. package/umd/typings/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.d.ts +1 -0
  317. package/umd/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +9 -2
  318. package/umd/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +1 -1
  319. package/umd/typings/src/utils/markdown/parseMarkdownSection.d.ts +3 -3
  320. package/umd/typings/src/utils/normalization/parseKeywords.d.ts +2 -1
  321. package/umd/typings/src/utils/organization/TODO_USE.d.ts +12 -0
  322. package/umd/typings/src/utils/organization/TODO_any.d.ts +6 -0
  323. package/umd/typings/src/utils/organization/TODO_object.d.ts +6 -0
  324. package/umd/typings/src/utils/organization/TODO_unknown.d.ts +6 -0
  325. package/umd/typings/src/utils/organization/___.d.ts +6 -0
  326. package/umd/typings/src/utils/{just.d.ts → organization/just.d.ts} +5 -1
  327. package/umd/typings/src/utils/organization/keepUnused.d.ts +16 -0
  328. package/umd/typings/src/utils/organization/really_any.d.ts +6 -0
  329. package/umd/typings/src/utils/organization/really_unknown.d.ts +6 -0
  330. package/umd/typings/src/utils/random/randomSeed.d.ts +7 -0
  331. package/{esm/typings/src/execution → umd/typings/src}/utils/replaceParameters.d.ts +2 -4
  332. package/umd/typings/src/utils/replaceParameters.test.d.ts +1 -0
  333. package/umd/typings/src/utils/sets/difference.d.ts +3 -0
  334. package/umd/typings/src/utils/validators/email/isValidEmail.d.ts +2 -1
  335. package/umd/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -1
  336. package/umd/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +2 -1
  337. package/umd/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +2 -1
  338. package/umd/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +2 -1
  339. package/umd/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +2 -1
  340. package/umd/typings/src/utils/validators/url/isValidUrl.d.ts +2 -1
  341. package/umd/typings/src/utils/validators/uuid/isValidUuid.d.ts +2 -1
  342. package/esm/typings/src/collection/constructors/justTestFsImport.d.ts +0 -7
  343. package/esm/typings/src/personas/personaToModelRequirements.d.ts +0 -6
  344. package/esm/typings/src/types/Parameters.d.ts +0 -14
  345. package/esm/typings/src/types/PipelineJson/KnowledgeJson.d.ts +0 -6
  346. package/esm/typings/src/types/PipelineJson/MaterialKnowledgePieceJson.d.ts +0 -28
  347. package/umd/typings/src/collection/constructors/justTestFsImport.d.ts +0 -7
  348. package/umd/typings/src/personas/personaToModelRequirements.d.ts +0 -6
  349. package/umd/typings/src/types/Parameters.d.ts +0 -14
  350. package/umd/typings/src/types/PipelineJson/KnowledgeJson.d.ts +0 -6
  351. package/umd/typings/src/types/PipelineJson/MaterialKnowledgePieceJson.d.ts +0 -28
  352. /package/esm/typings/src/_packages/{markdown-utils.d.ts → markdown-utils.index.d.ts} +0 -0
  353. /package/esm/typings/src/{execution/addPromptResultUsage.test.d.ts → conversion/utils/stringifyPipelineJson.test.d.ts} +0 -0
  354. /package/esm/typings/src/{execution/utils/replaceParameters.test.d.ts → knowledge/prepare-knowledge/_common/prepareKnowledgePieces.test.d.ts} +0 -0
  355. /package/{umd/typings/src/execution/addPromptResultUsage.test.d.ts → esm/typings/src/personas/preparePersona.test.d.ts} +0 -0
  356. /package/esm/typings/src/{utils/postprocessing → postprocessing/utils}/extractBlock.d.ts +0 -0
  357. /package/{umd/typings/src/execution/utils/replaceParameters.test.d.ts → esm/typings/src/prepare/isPipelinePrepared.test.d.ts} +0 -0
  358. /package/umd/typings/src/_packages/{markdown-utils.d.ts → markdown-utils.index.d.ts} +0 -0
  359. /package/umd/typings/src/{utils/postprocessing → postprocessing/utils}/extractBlock.d.ts +0 -0
package/umd/index.umd.js CHANGED
@@ -258,7 +258,7 @@
258
258
  /* Note: Not using:> name, */
259
259
  title_1 = promptTemplate.title, description_1 = promptTemplate.description,
260
260
  /* Note: dependentParameterNames, */
261
- jokers = promptTemplate.jokers, blockType = promptTemplate.blockType, content = promptTemplate.content, postprocessing = promptTemplate.postprocessing, expectations = promptTemplate.expectations, expectFormat = promptTemplate.expectFormat, resultingParameterName = promptTemplate.resultingParameterName;
261
+ jokers = promptTemplate.jokerParameterNames, blockType = promptTemplate.blockType, content = promptTemplate.content, postprocessing = promptTemplate.postprocessingFunctionNames, expectations = promptTemplate.expectations, expectFormat = promptTemplate.expectFormat, resultingParameterName = promptTemplate.resultingParameterName;
262
262
  pipelineString += '\n\n';
263
263
  pipelineString += "## ".concat(title_1);
264
264
  if (description_1) {
@@ -392,23 +392,107 @@
392
392
  return parameterString;
393
393
  }
394
394
  /**
395
- * TODO: !!!!! Implement new features and commands into `promptTemplateParameterJsonToString`
395
+ * TODO: !!!! Implement new features and commands into `promptTemplateParameterJsonToString`
396
396
  * TODO: [🧠] Is there a way to auto-detect missing features in pipelineJsonToString
397
397
  * TODO: Escape all
398
398
  */
399
399
 
400
+ /**
401
+ * @@@
402
+ *
403
+ * @returns The same object as the input, but deeply frozen
404
+ *
405
+ * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
406
+ */
407
+ function deepFreeze(objectValue) {
408
+ var e_1, _a;
409
+ var propertyNames = Object.getOwnPropertyNames(objectValue);
410
+ try {
411
+ for (var propertyNames_1 = __values(propertyNames), propertyNames_1_1 = propertyNames_1.next(); !propertyNames_1_1.done; propertyNames_1_1 = propertyNames_1.next()) {
412
+ var propertyName = propertyNames_1_1.value;
413
+ var value = objectValue[propertyName];
414
+ if (value && typeof value === 'object') {
415
+ deepFreeze(value);
416
+ }
417
+ }
418
+ }
419
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
420
+ finally {
421
+ try {
422
+ if (propertyNames_1_1 && !propertyNames_1_1.done && (_a = propertyNames_1.return)) _a.call(propertyNames_1);
423
+ }
424
+ finally { if (e_1) throw e_1.error; }
425
+ }
426
+ return Object.freeze(objectValue);
427
+ }
428
+ /**
429
+ * @@@
430
+ * @@@
431
+ *
432
+ * @returns The same object as the input, but deeply frozen
433
+ *
434
+ * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
435
+ */
436
+ function deepFreezeWithSameType(objectValue) {
437
+ return deepFreeze(objectValue);
438
+ }
439
+ /**
440
+ * TODO: [🔼] Export from `@promptbook/utils`
441
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
442
+ */
443
+
444
+ /**
445
+ * Returns the same value that is passed as argument.
446
+ * No side effects.
447
+ *
448
+ * Note: It can be usefull for:
449
+ *
450
+ * 1) Leveling indentation
451
+ * 2) Putting always-true or always-false conditions without getting eslint errors
452
+ *
453
+ * @param value any values
454
+ * @returns the same values
455
+ * @private within the repository
456
+ */
457
+ function just(value) {
458
+ if (value === undefined) {
459
+ return undefined;
460
+ }
461
+ return value;
462
+ }
463
+
400
464
  /**
401
465
  * The maximum number of iterations for a loops
402
466
  */
403
467
  var LOOP_LIMIT = 1000;
468
+ /**
469
+ * The maximum number of (LLM) tasks running in parallel
470
+ */
471
+ var MAX_PARALLEL_COUNT = 5;
472
+ /**
473
+ * The maximum number of attempts to execute LLM task before giving up
474
+ */
475
+ var MAX_EXECUTION_ATTEMPTS = 3;
404
476
  /**
405
477
  * The names of the parameters that are reserved for special purposes
406
478
  */
407
- var RESERVED_PARAMETER_NAMES = [
479
+ var RESERVED_PARAMETER_NAMES = deepFreeze([
408
480
  'context',
481
+ 'currentDate',
409
482
  // <- TODO: Add more like 'date', 'modelName',...
410
483
  // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
411
- ];
484
+ ]);
485
+ /**
486
+ * Nonce which is used for replacing things in strings
487
+ */
488
+ var REPLACING_NONCE = 'u$k42k%!V2zo34w7Fu#@QUHYPW';
489
+ /*
490
+ TODO: !!! Just testing false-negative detection of [🟡][🟢][🔵][⚪] leak
491
+ */
492
+ // [🟡][🟢][🔵][⚪]
493
+ /**
494
+ * TODO: [🔼] Export all to core
495
+ */
412
496
 
413
497
  /**
414
498
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -611,7 +695,7 @@
611
695
  */
612
696
  function validatePipeline(pipeline) {
613
697
  // TODO: [🧠] Maybe test if promptbook is a promise and make specific error case for that
614
- var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
698
+ var e_1, _a, e_2, _b, e_3, _c, e_4, _d, e_5, _e;
615
699
  if (pipeline.pipelineUrl !== undefined && !isValidPipelineUrl(pipeline.pipelineUrl)) {
616
700
  // <- Note: [🚲]
617
701
  throw new PipelineLogicError("Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\""));
@@ -648,19 +732,19 @@
648
732
  };
649
733
  try {
650
734
  // Note: Check each parameter individually
651
- for (var _e = __values(pipeline.parameters), _f = _e.next(); !_f.done; _f = _e.next()) {
652
- var parameter = _f.value;
735
+ for (var _f = __values(pipeline.parameters), _g = _f.next(); !_g.done; _g = _f.next()) {
736
+ var parameter = _g.value;
653
737
  _loop_1(parameter);
654
738
  }
655
739
  }
656
740
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
657
741
  finally {
658
742
  try {
659
- if (_f && !_f.done && (_a = _e.return)) _a.call(_e);
743
+ if (_g && !_g.done && (_a = _f.return)) _a.call(_f);
660
744
  }
661
745
  finally { if (e_1) throw e_1.error; }
662
746
  }
663
- // Note: Check each template individually
747
+ // Note: All input parameters are defined - so that they can be used as result of some template
664
748
  var definedParameters = new Set(pipeline.parameters.filter(function (_a) {
665
749
  var isInput = _a.isInput;
666
750
  return isInput;
@@ -669,23 +753,27 @@
669
753
  return name;
670
754
  }));
671
755
  try {
672
- for (var _g = __values(pipeline.promptTemplates), _h = _g.next(); !_h.done; _h = _g.next()) {
673
- var template = _h.value;
756
+ // Note: Checking each template individually
757
+ for (var _h = __values(pipeline.promptTemplates), _j = _h.next(); !_j.done; _j = _h.next()) {
758
+ var template = _j.value;
674
759
  if (definedParameters.has(template.resultingParameterName)) {
675
760
  throw new PipelineLogicError("Parameter {".concat(template.resultingParameterName, "} is defined multiple times"));
676
761
  }
762
+ if (RESERVED_PARAMETER_NAMES.includes(template.resultingParameterName)) {
763
+ throw new PipelineLogicError("Parameter name {".concat(template.resultingParameterName, "} is reserved, please use fifferent name"));
764
+ }
677
765
  definedParameters.add(template.resultingParameterName);
678
766
  if (template.blockType === 'PROMPT_TEMPLATE' && template.modelRequirements.modelVariant === undefined) {
679
767
  throw new PipelineLogicError(spaceTrim.spaceTrim("\n\n You must specify MODEL VARIANT in the prompt template \"".concat(template.title, "\"\n\n For example:\n - MODEL VARIANT Chat\n - MODEL NAME `gpt-4-1106-preview`\n\n ")));
680
768
  }
681
- if (template.jokers && template.jokers.length > 0) {
769
+ if (template.jokerParameterNames && template.jokerParameterNames.length > 0) {
682
770
  if (!template.expectFormat &&
683
771
  !template.expectations /* <- TODO: Require at least 1 -> min <- expectation to use jokers */) {
684
772
  throw new PipelineLogicError("Joker parameters are used for {".concat(template.resultingParameterName, "} but no expectations are defined"));
685
773
  }
686
774
  try {
687
- for (var _j = (e_3 = void 0, __values(template.jokers)), _k = _j.next(); !_k.done; _k = _j.next()) {
688
- var joker = _k.value;
775
+ for (var _k = (e_3 = void 0, __values(template.jokerParameterNames)), _l = _k.next(); !_l.done; _l = _k.next()) {
776
+ var joker = _l.value;
689
777
  if (!template.dependentParameterNames.includes(joker)) {
690
778
  throw new PipelineLogicError("Parameter {".concat(joker, "} is used for {").concat(template.resultingParameterName, "} as joker but not in dependentParameterNames"));
691
779
  }
@@ -694,15 +782,15 @@
694
782
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
695
783
  finally {
696
784
  try {
697
- if (_k && !_k.done && (_c = _j.return)) _c.call(_j);
785
+ if (_l && !_l.done && (_c = _k.return)) _c.call(_k);
698
786
  }
699
787
  finally { if (e_3) throw e_3.error; }
700
788
  }
701
789
  }
702
790
  if (template.expectations) {
703
791
  try {
704
- for (var _l = (e_4 = void 0, __values(Object.entries(template.expectations))), _m = _l.next(); !_m.done; _m = _l.next()) {
705
- var _o = __read(_m.value, 2), unit = _o[0], _p = _o[1], min = _p.min, max = _p.max;
792
+ for (var _m = (e_4 = void 0, __values(Object.entries(template.expectations))), _o = _m.next(); !_o.done; _o = _m.next()) {
793
+ var _p = __read(_o.value, 2), unit = _p[0], _q = _p[1], min = _q.min, max = _q.max;
706
794
  if (min !== undefined && max !== undefined && min > max) {
707
795
  throw new PipelineLogicError("Min expectation (=".concat(min, ") of ").concat(unit, " is higher than max expectation (=").concat(max, ")"));
708
796
  }
@@ -717,7 +805,7 @@
717
805
  catch (e_4_1) { e_4 = { error: e_4_1 }; }
718
806
  finally {
719
807
  try {
720
- if (_m && !_m.done && (_d = _l.return)) _d.call(_l);
808
+ if (_o && !_o.done && (_d = _m.return)) _d.call(_m);
721
809
  }
722
810
  finally { if (e_4) throw e_4.error; }
723
811
  }
@@ -727,7 +815,7 @@
727
815
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
728
816
  finally {
729
817
  try {
730
- if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
818
+ if (_j && !_j.done && (_b = _h.return)) _b.call(_h);
731
819
  }
732
820
  finally { if (e_2) throw e_2.error; }
733
821
  }
@@ -741,24 +829,42 @@
741
829
  var name = _a.name;
742
830
  return name;
743
831
  });
832
+ try {
833
+ // Note: All reserved parameters are resolved
834
+ for (var RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
835
+ var reservedParameterName = RESERVED_PARAMETER_NAMES_1_1.value;
836
+ resovedParameters = __spreadArray(__spreadArray([], __read(resovedParameters), false), [reservedParameterName], false);
837
+ }
838
+ }
839
+ catch (e_5_1) { e_5 = { error: e_5_1 }; }
840
+ finally {
841
+ try {
842
+ if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_e = RESERVED_PARAMETER_NAMES_1.return)) _e.call(RESERVED_PARAMETER_NAMES_1);
843
+ }
844
+ finally { if (e_5) throw e_5.error; }
845
+ }
744
846
  var unresovedTemplates = __spreadArray([], __read(pipeline.promptTemplates), false);
847
+ // <- TODO: [🧠][🥜]
745
848
  var loopLimit = LOOP_LIMIT;
746
849
  var _loop_2 = function () {
747
850
  if (loopLimit-- < 0) {
851
+ // Note: Really UnexpectedError not LimitReachedError - this should not happen and be caught below
748
852
  throw new UnexpectedError('Loop limit reached during detection of circular dependencies in `validatePipeline`');
749
853
  }
750
854
  var currentlyResovedTemplates = unresovedTemplates.filter(function (template) {
751
855
  return template.dependentParameterNames.every(function (name) { return resovedParameters.includes(name); });
752
856
  });
753
857
  if (currentlyResovedTemplates.length === 0) {
754
- throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n\n Can not resolve some parameters:\n Either you are using a parameter that is not defined, or there are some circular dependencies.\n\n Can not resolve:\n ".concat(block(unresovedTemplates
858
+ throw new PipelineLogicError(
859
+ // TODO: [🐎] DRY
860
+ spaceTrim.spaceTrim(function (block) { return "\n\n Can not resolve some parameters:\n Either you are using a parameter that is not defined, or there are some circular dependencies.\n\n Can not resolve:\n ".concat(block(unresovedTemplates
755
861
  .map(function (_a) {
756
862
  var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
757
- return "- {".concat(resultingParameterName, "} depends on ").concat(dependentParameterNames
863
+ return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
758
864
  .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
759
- .join(', '));
865
+ .join(' and '));
760
866
  })
761
- .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameters.map(function (name) { return "- {".concat(name, "}"); }).join('\n')), "\n "); }));
867
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameters.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n "); }));
762
868
  }
763
869
  resovedParameters = __spreadArray(__spreadArray([], __read(resovedParameters), false), __read(currentlyResovedTemplates.map(function (_a) {
764
870
  var resultingParameterName = _a.resultingParameterName;
@@ -780,7 +886,16 @@
780
886
  * > * It checks:
781
887
  * > * - it has a valid structure
782
888
  * > * - ...
783
- * > ex port function validatePipeline(promptbook: unknown): asserts promptbook is PipelineJson {
889
+ * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
890
+ */
891
+ /**
892
+ * TODO: [🧠][🐣] !!!! Validate that all samples match expectations
893
+ * TODO: [🧠][🐣] !!!! Validate that knowledge is valid (non-void)
894
+ * TODO: [🧠][🐣] !!!! Validate that persona can be used only with CHAT variant
895
+ * TODO: !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
896
+ * TODO: !!!! Validate that reserved parameter is not used as joker
897
+ * TODO: [🧠] !!! Validationg not only logic itself but imports around - files and websites and rerefenced pipelines exists
898
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
784
899
  */
785
900
 
786
901
  /**
@@ -812,79 +927,100 @@
812
927
  }(Error));
813
928
 
814
929
  /**
815
- * Library of promptbooks that groups together promptbooks for an application.
816
- * This implementation is a very thin wrapper around the Array / Map of promptbooks.
930
+ * Unprepare just strips the preparation data of the pipeline
931
+ */
932
+ function unpreparePipeline(pipeline) {
933
+ var personas = pipeline.personas, knowledgeSources = pipeline.knowledgeSources;
934
+ personas = personas.map(function (persona) { return (__assign(__assign({}, persona), { modelRequirements: undefined, preparationIds: undefined })); });
935
+ knowledgeSources = knowledgeSources.map(function (knowledgeSource) { return (__assign(__assign({}, knowledgeSource), { preparationIds: undefined })); });
936
+ return __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSources, knowledgePieces: [], personas: personas, preparations: [] });
937
+ }
938
+ /**
939
+ * TODO: [🔼] !!! Export via `@promptbook/core`
940
+ * TODO: Write tests for `preparePipeline`
941
+ */
942
+
943
+ /**
944
+ * Library of pipelines that groups together pipelines for an application.
945
+ * This implementation is a very thin wrapper around the Array / Map of pipelines.
817
946
  *
818
947
  * @private use `createCollectionFromJson` instead
819
- * @see https://github.com/webgptorg/promptbook#promptbook-collection
948
+ * @see https://github.com/webgptorg/pipeline#pipeline-collection
820
949
  */
821
950
  var SimplePipelineCollection = /** @class */ (function () {
822
951
  /**
823
- * Constructs a pipeline collection from promptbooks
952
+ * Constructs a pipeline collection from pipelines
824
953
  *
825
- * @param promptbooks !!!
954
+ * @param pipelines @@@
826
955
  *
827
956
  * @private Use instead `createCollectionFromJson`
828
- * Note: During the construction logic of all promptbooks are validated
957
+ * Note: During the construction logic of all pipelines are validated
829
958
  * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
830
959
  */
831
960
  function SimplePipelineCollection() {
832
961
  var e_1, _a;
833
- var promptbooks = [];
962
+ var pipelines = [];
834
963
  for (var _i = 0; _i < arguments.length; _i++) {
835
- promptbooks[_i] = arguments[_i];
964
+ pipelines[_i] = arguments[_i];
836
965
  }
837
966
  this.collection = new Map();
838
967
  try {
839
- for (var promptbooks_1 = __values(promptbooks), promptbooks_1_1 = promptbooks_1.next(); !promptbooks_1_1.done; promptbooks_1_1 = promptbooks_1.next()) {
840
- var promptbook = promptbooks_1_1.value;
841
- if (promptbook.pipelineUrl === undefined) {
842
- throw new ReferenceError$1(spaceTrim.spaceTrim("\n Promptbook with name \"".concat(promptbook.title, "\" does not have defined URL\n\n File:\n ").concat(promptbook.sourceFile || 'Unknown', "\n\n Note: Promptbooks without URLs are called anonymous promptbooks\n They can be used as standalone promptbooks, but they cannot be referenced by other promptbooks\n And also they cannot be used in the pipeline collection\n\n ")));
968
+ for (var pipelines_1 = __values(pipelines), pipelines_1_1 = pipelines_1.next(); !pipelines_1_1.done; pipelines_1_1 = pipelines_1.next()) {
969
+ var pipeline = pipelines_1_1.value;
970
+ // TODO: [👠] DRY
971
+ if (pipeline.pipelineUrl === undefined) {
972
+ throw new ReferenceError$1(spaceTrim.spaceTrim("\n Pipeline with name \"".concat(pipeline.title, "\" does not have defined URL\n\n File:\n ").concat(pipeline.sourceFile || 'Unknown', "\n\n Note: Pipelines without URLs are called anonymous pipelines\n They can be used as standalone pipelines, but they cannot be referenced by other pipelines\n And also they cannot be used in the pipeline collection\n\n ")));
843
973
  }
844
- validatePipeline(promptbook);
974
+ // Note: [🐨]
975
+ validatePipeline(pipeline);
976
+ // TODO: [🦄] DRY
845
977
  // Note: [🦄]
846
- if (this.collection.has(promptbook.pipelineUrl) &&
847
- pipelineJsonToString(promptbook) !== pipelineJsonToString(this.collection.get(promptbook.pipelineUrl))) {
848
- var existing = this.collection.get(promptbook.pipelineUrl);
849
- throw new ReferenceError$1(spaceTrim.spaceTrim("\n Promptbook with URL \"".concat(promptbook.pipelineUrl, "\" is already in the collection\n\n Conflicting files:\n ").concat(existing.sourceFile || 'Unknown', "\n ").concat(promptbook.sourceFile || 'Unknown', "\n\n Note: Promptbooks with the same URL are not allowed\n Only exepction is when the promptbooks are identical\n\n ")));
978
+ if (
979
+ // TODO: [🐽]
980
+ this.collection.has(pipeline.pipelineUrl) &&
981
+ pipelineJsonToString(unpreparePipeline(pipeline)) !==
982
+ pipelineJsonToString(unpreparePipeline(this.collection.get(pipeline.pipelineUrl)))) {
983
+ var existing = this.collection.get(pipeline.pipelineUrl);
984
+ throw new ReferenceError$1(spaceTrim.spaceTrim("\n Pipeline with URL \"".concat(pipeline.pipelineUrl, "\" is already in the collection\n\n Conflicting files:\n ").concat(existing.sourceFile || 'Unknown', "\n ").concat(pipeline.sourceFile || 'Unknown', "\n\n Note: Pipelines with the same URL are not allowed\n Only exepction is when the pipelines are identical\n\n ")));
850
985
  }
851
- this.collection.set(promptbook.pipelineUrl, promptbook);
986
+ // Note: [🧠] Overwrite existing pipeline with the same URL
987
+ this.collection.set(pipeline.pipelineUrl, pipeline);
852
988
  }
853
989
  }
854
990
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
855
991
  finally {
856
992
  try {
857
- if (promptbooks_1_1 && !promptbooks_1_1.done && (_a = promptbooks_1.return)) _a.call(promptbooks_1);
993
+ if (pipelines_1_1 && !pipelines_1_1.done && (_a = pipelines_1.return)) _a.call(pipelines_1);
858
994
  }
859
995
  finally { if (e_1) throw e_1.error; }
860
996
  }
861
997
  }
862
998
  /**
863
- * Gets all promptbooks in the collection
999
+ * Gets all pipelines in the collection
864
1000
  */
865
1001
  SimplePipelineCollection.prototype.listPipelines = function () {
866
1002
  return Array.from(this.collection.keys());
867
1003
  };
868
1004
  /**
869
- * Gets promptbook by its URL
1005
+ * Gets pipeline by its URL
870
1006
  *
871
1007
  * Note: This is not a direct fetching from the URL, but a lookup in the collection
872
1008
  */
873
1009
  SimplePipelineCollection.prototype.getPipelineByUrl = function (url) {
874
1010
  var _this = this;
875
- var promptbook = this.collection.get(url);
876
- if (!promptbook) {
1011
+ var pipeline = this.collection.get(url);
1012
+ if (!pipeline) {
877
1013
  if (this.listPipelines().length === 0) {
878
- throw new NotFoundError(spaceTrim.spaceTrim("\n Promptbook with url \"".concat(url, "\" not found\n\n No promptbooks available\n ")));
1014
+ throw new NotFoundError(spaceTrim.spaceTrim("\n Pipeline with url \"".concat(url, "\" not found\n\n No pipelines available\n ")));
879
1015
  }
880
- throw new NotFoundError(spaceTrim.spaceTrim(function (block) { return "\n Promptbook with url \"".concat(url, "\" not found\n\n Available promptbooks:\n ").concat(block(_this.listPipelines()
1016
+ throw new NotFoundError(spaceTrim.spaceTrim(function (block) { return "\n Pipeline with url \"".concat(url, "\" not found\n\n Available pipelines:\n ").concat(block(_this.listPipelines()
881
1017
  .map(function (pipelineUrl) { return "- ".concat(pipelineUrl); })
882
1018
  .join('\n')), "\n\n "); }));
883
1019
  }
884
- return promptbook;
1020
+ return pipeline;
885
1021
  };
886
1022
  /**
887
- * Checks whether given prompt was defined in any promptbook in the collection
1023
+ * Checks whether given prompt was defined in any pipeline in the collection
888
1024
  */
889
1025
  SimplePipelineCollection.prototype.isResponsibleForPrompt = function (prompt) {
890
1026
  return true;
@@ -1126,7 +1262,199 @@
1126
1262
  // <- [🩻]
1127
1263
  ];
1128
1264
 
1129
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-1",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledge:[{name:"n",title:"N",content:"D",keywords:["E"],index:[{modelName:"mocked-facked",position:[-0.8022132744698469,-0.0007443037013543652,-0.34872812971894707,-0.331923610323706,0.36013380510747606,0.3272238545809909,0.9972619939249201,0.261533967444473,0.969397219635256,0.17310495233925405,0.1658542147196167,0.9463503306650436,-0.11780402348338725,-0.19232799237848175,0.6226110088018051,0.2759831058911306,-0.5311583719379294,0.18853474356656497,-0.4508505143158268,-0.6183896225862329,0.5050055293473035,0.1478100741208901,-0.21453327800030664,0.09724866792207987,0.5336888252898944]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-1",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledge:[{name:"e",title:"E",content:"A",keywords:["L"],index:[{modelName:"mocked-facked",position:[-0.10884747467629419,-0.5326698151631195,-0.9465176322534181,0.7751385851237287,-0.7692699503104152,-0.9601969759421696,0.33722965766570434,0.17842011981786943,0.0276452884638827,-0.5983538271731308,-0.127805978741649,-0.6304775592815672,0.8849987254326983,0.1675220238760482,-0.8943010776373641,-0.9569796049357393,0.24933996044411444,0.8210261943632116,0.7193141705292407,0.06170351501887783,0.26972150191994837,-0.5340726089187089,0.12061502629384036,0.29495956103603005,-0.53082082537217]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-1",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledge:[{name:"s",title:"S",content:"A",keywords:["S"],index:[{modelName:"mocked-facked",position:[-0.916989680936596,0.9339948002315257,0.6004732120236658,0.0028125597650441314,-0.055419157811128716,0.013537996506097105,-0.8796279521347974,0.12223482268345887,0.31637982343954674,0.19555443419711915,-0.1664084533259853,-0.3527797149266507,0.3732921105225979,-0.7943618527999234,0.09240919193726871,0.8207777669058434,0.6464285109148693,0.008174116471551418,-0.933937225718561,-0.790531259317556,-0.04738702314966625,-0.9357754186773284,-0.3074287982247097,-0.4579908642989361,0.9857246614213477]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
1265
+ /**
1266
+ * @@@
1267
+ */
1268
+ function deepClone(objectValue) {
1269
+ return JSON.parse(JSON.stringify(objectValue));
1270
+ /*
1271
+ TODO: [🧠] Is there a better implementation?
1272
+ > const propertyNames = Object.getOwnPropertyNames(objectValue);
1273
+ > for (const propertyName of propertyNames) {
1274
+ > const value = (objectValue as really_any)[propertyName];
1275
+ > if (value && typeof value === 'object') {
1276
+ > deepClone(value);
1277
+ > }
1278
+ > }
1279
+ > return Object.assign({}, objectValue);
1280
+ */
1281
+ }
1282
+ /**
1283
+ * TODO: [🔼] Export from `@promptbook/utils`
1284
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
1285
+ */
1286
+
1287
+ /**
1288
+ * @@@
1289
+ *
1290
+ * TODO: [🔼] Export with addUsage
1291
+ */
1292
+ var ZERO_USAGE = deepFreeze({
1293
+ price: { value: 0 },
1294
+ input: {
1295
+ tokensCount: { value: 0 },
1296
+ charactersCount: { value: 0 },
1297
+ wordsCount: { value: 0 },
1298
+ sentencesCount: { value: 0 },
1299
+ linesCount: { value: 0 },
1300
+ paragraphsCount: { value: 0 },
1301
+ pagesCount: { value: 0 },
1302
+ },
1303
+ output: {
1304
+ tokensCount: { value: 0 },
1305
+ charactersCount: { value: 0 },
1306
+ wordsCount: { value: 0 },
1307
+ sentencesCount: { value: 0 },
1308
+ linesCount: { value: 0 },
1309
+ paragraphsCount: { value: 0 },
1310
+ pagesCount: { value: 0 },
1311
+ },
1312
+ });
1313
+ /**
1314
+ * Function `addUsage` will add multiple usages into one
1315
+ *
1316
+ * Note: If you provide 0 values, it returns ZERO_USAGE
1317
+ */
1318
+ function addUsage() {
1319
+ var usageItems = [];
1320
+ for (var _i = 0; _i < arguments.length; _i++) {
1321
+ usageItems[_i] = arguments[_i];
1322
+ }
1323
+ return usageItems.reduce(function (acc, item) {
1324
+ var e_1, _a, e_2, _b;
1325
+ var _c;
1326
+ acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
1327
+ try {
1328
+ for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
1329
+ var key = _e.value;
1330
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1331
+ //@ts-ignore
1332
+ if (item.input[key]) {
1333
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1334
+ //@ts-ignore
1335
+ acc.input[key].value += item.input[key].value || 0;
1336
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1337
+ //@ts-ignore
1338
+ if (item.input[key].isUncertain) {
1339
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1340
+ //@ts-ignore
1341
+ acc.input[key].isUncertain = true;
1342
+ }
1343
+ }
1344
+ }
1345
+ }
1346
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
1347
+ finally {
1348
+ try {
1349
+ if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
1350
+ }
1351
+ finally { if (e_1) throw e_1.error; }
1352
+ }
1353
+ try {
1354
+ for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
1355
+ var key = _g.value;
1356
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1357
+ //@ts-ignore
1358
+ if (item.output[key]) {
1359
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1360
+ //@ts-ignore
1361
+ acc.output[key].value += item.output[key].value || 0;
1362
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1363
+ //@ts-ignore
1364
+ if (item.output[key].isUncertain) {
1365
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1366
+ //@ts-ignore
1367
+ acc.output[key].isUncertain = true;
1368
+ }
1369
+ }
1370
+ }
1371
+ }
1372
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
1373
+ finally {
1374
+ try {
1375
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
1376
+ }
1377
+ finally { if (e_2) throw e_2.error; }
1378
+ }
1379
+ return acc;
1380
+ }, deepClone(ZERO_USAGE));
1381
+ }
1382
+
1383
+ /**
1384
+ * Async version of Array.forEach
1385
+ *
1386
+ * @param array - Array to iterate over
1387
+ * @param options - Options for the function
1388
+ * @param callbackfunction - Function to call for each item
1389
+ */
1390
+ function forEachAsync(array, options, callbackfunction) {
1391
+ return __awaiter(this, void 0, void 0, function () {
1392
+ var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
1393
+ var e_1, _d;
1394
+ return __generator(this, function (_e) {
1395
+ switch (_e.label) {
1396
+ case 0:
1397
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
1398
+ index = 0;
1399
+ runningTasks = [];
1400
+ tasks = [];
1401
+ _loop_1 = function (item) {
1402
+ var currentIndex, task;
1403
+ return __generator(this, function (_f) {
1404
+ switch (_f.label) {
1405
+ case 0:
1406
+ currentIndex = index++;
1407
+ task = callbackfunction(item, currentIndex, array);
1408
+ tasks.push(task);
1409
+ runningTasks.push(task);
1410
+ /* not await */ Promise.resolve(task).then(function () {
1411
+ runningTasks = runningTasks.filter(function (t) { return t !== task; });
1412
+ });
1413
+ if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
1414
+ return [4 /*yield*/, Promise.race(runningTasks)];
1415
+ case 1:
1416
+ _f.sent();
1417
+ _f.label = 2;
1418
+ case 2: return [2 /*return*/];
1419
+ }
1420
+ });
1421
+ };
1422
+ _e.label = 1;
1423
+ case 1:
1424
+ _e.trys.push([1, 6, 7, 8]);
1425
+ _b = __values(array), _c = _b.next();
1426
+ _e.label = 2;
1427
+ case 2:
1428
+ if (!!_c.done) return [3 /*break*/, 5];
1429
+ item = _c.value;
1430
+ return [5 /*yield**/, _loop_1(item)];
1431
+ case 3:
1432
+ _e.sent();
1433
+ _e.label = 4;
1434
+ case 4:
1435
+ _c = _b.next();
1436
+ return [3 /*break*/, 2];
1437
+ case 5: return [3 /*break*/, 8];
1438
+ case 6:
1439
+ e_1_1 = _e.sent();
1440
+ e_1 = { error: e_1_1 };
1441
+ return [3 /*break*/, 8];
1442
+ case 7:
1443
+ try {
1444
+ if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
1445
+ }
1446
+ finally { if (e_1) throw e_1.error; }
1447
+ return [7 /*endfinally*/];
1448
+ case 8: return [4 /*yield*/, Promise.all(tasks)];
1449
+ case 9:
1450
+ _e.sent();
1451
+ return [2 /*return*/];
1452
+ }
1453
+ });
1454
+ });
1455
+ }
1456
+
1457
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-20",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-20",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-20",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-20",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-20",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {content}",expectations:{words:{min:1,max:8}},dependentParameterNames:["content"],resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-20",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.61.0-20",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",modelRequirements:{modelVariant:"CHAT",modelName:"gpt-4-turbo"},content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n### Option `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Option `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Option `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",expectFormat:"JSON",dependentParameterNames:["availableModelNames","personaDescription"],resultingParameterName:"modelRequirements"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-20",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
1130
1458
 
1131
1459
  var defaultDiacriticsRemovalMap = [
1132
1460
  {
@@ -1453,6 +1781,14 @@
1453
1781
  * Function normalizes title to name which can be used as identifier
1454
1782
  */
1455
1783
  function titleToName(value) {
1784
+ if (value.startsWith('http://') || value.startsWith('https://')) {
1785
+ // TODO: Maybe check against some list unallowed characters
1786
+ return value;
1787
+ }
1788
+ if (value.startsWith('./') || value.startsWith('../')) {
1789
+ // TODO: Maybe check against some list unallowed characters
1790
+ return value;
1791
+ }
1456
1792
  value = removeEmojis(value);
1457
1793
  value = normalizeToKebabCase(value);
1458
1794
  // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
@@ -1499,43 +1835,176 @@
1499
1835
  */
1500
1836
 
1501
1837
  /**
1502
- * This error occurs when some expectation is not met in the execution of the pipeline
1838
+ * Parses the template and returns the list of all parameter names
1503
1839
  *
1504
- * @private Always catched and rethrown as `PipelineExecutionError`
1505
- * Note: This is a kindof subtype of PipelineExecutionError
1840
+ * @param template the template with parameters in {curly} braces
1841
+ * @returns the list of parameter names
1506
1842
  */
1507
- var ExpectError = /** @class */ (function (_super) {
1508
- __extends(ExpectError, _super);
1509
- function ExpectError(message) {
1510
- var _this = _super.call(this, message) || this;
1511
- _this.name = 'ExpectError';
1512
- Object.setPrototypeOf(_this, ExpectError.prototype);
1513
- return _this;
1843
+ function extractParameters(template) {
1844
+ var e_1, _a;
1845
+ var matches = template.matchAll(/{\w+}/g);
1846
+ var parameterNames = new Set();
1847
+ try {
1848
+ for (var matches_1 = __values(matches), matches_1_1 = matches_1.next(); !matches_1_1.done; matches_1_1 = matches_1.next()) {
1849
+ var match = matches_1_1.value;
1850
+ var parameterName = match[0].slice(1, -1);
1851
+ parameterNames.add(parameterName);
1852
+ }
1514
1853
  }
1515
- return ExpectError;
1516
- }(Error));
1854
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
1855
+ finally {
1856
+ try {
1857
+ if (matches_1_1 && !matches_1_1.done && (_a = matches_1.return)) _a.call(matches_1);
1858
+ }
1859
+ finally { if (e_1) throw e_1.error; }
1860
+ }
1861
+ return parameterNames;
1862
+ }
1517
1863
 
1518
1864
  /**
1519
- * Function isValidJsonString will tell you if the string is valid JSON or not
1865
+ * Parses the given script and returns the list of all used variables that are not defined in the script
1866
+ *
1867
+ * @param script from which to extract the variables
1868
+ * @returns the list of variable names
1869
+ * @throws {ParsingError} if the script is invalid
1520
1870
  */
1521
- function isValidJsonString(value /* <-[👨‍⚖️] */) {
1871
+ function extractVariables(script) {
1872
+ var variables = new Set();
1873
+ script = "(()=>{".concat(script, "})()");
1522
1874
  try {
1523
- JSON.parse(value);
1524
- return true;
1875
+ for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
1876
+ try {
1877
+ eval(script);
1878
+ }
1879
+ catch (error) {
1880
+ if (!(error instanceof ReferenceError)) {
1881
+ throw error;
1882
+ }
1883
+ var undefinedName = error.message.split(' ')[0];
1884
+ /*
1885
+ Note: Parsing the error
1886
+ [ReferenceError: thing is not defined]
1887
+ */
1888
+ if (!undefinedName) {
1889
+ throw error;
1890
+ }
1891
+ if (script.includes(undefinedName + '(')) {
1892
+ script = "const ".concat(undefinedName, " = ()=>'';") + script;
1893
+ }
1894
+ else {
1895
+ variables.add(undefinedName);
1896
+ script = "const ".concat(undefinedName, " = '';") + script;
1897
+ }
1898
+ }
1525
1899
  }
1526
1900
  catch (error) {
1527
1901
  if (!(error instanceof Error)) {
1528
1902
  throw error;
1529
1903
  }
1530
- if (error.message.includes('Unexpected token')) {
1531
- return false;
1532
- }
1533
- return false;
1904
+ throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Can not extract variables from the script\n\n ".concat(block(error.name), ": ").concat(block(error.message), "\n "); }));
1534
1905
  }
1906
+ return variables;
1535
1907
  }
1908
+ /**
1909
+ * TODO: [🔣] Support for multiple languages - python, java,...
1910
+ */
1536
1911
 
1537
1912
  /**
1538
- * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1913
+ * Parses the prompt template and returns the set of all used parameters
1914
+ *
1915
+ * @param promptTemplate the template with used parameters
1916
+ * @returns the set of parameter names
1917
+ * @throws {ParsingError} if the script is invalid
1918
+ */
1919
+ function extractParametersFromPromptTemplate(promptTemplate) {
1920
+ var e_1, _a, e_2, _b, e_3, _c;
1921
+ var title = promptTemplate.title, description = promptTemplate.description, blockType = promptTemplate.blockType, content = promptTemplate.content, jokerParameterNames = promptTemplate.jokerParameterNames;
1922
+ var parameterNames = new Set();
1923
+ try {
1924
+ for (var _d = __values(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameters(title)), false), __read(extractParameters(description || '')), false), __read(extractParameters(content)), false)), _e = _d.next(); !_e.done; _e = _d.next()) {
1925
+ var parameterName = _e.value;
1926
+ parameterNames.add(parameterName);
1927
+ }
1928
+ }
1929
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
1930
+ finally {
1931
+ try {
1932
+ if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
1933
+ }
1934
+ finally { if (e_1) throw e_1.error; }
1935
+ }
1936
+ if (blockType === 'SCRIPT') {
1937
+ try {
1938
+ for (var _f = __values(extractVariables(content)), _g = _f.next(); !_g.done; _g = _f.next()) {
1939
+ var parameterName = _g.value;
1940
+ parameterNames.add(parameterName);
1941
+ }
1942
+ }
1943
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
1944
+ finally {
1945
+ try {
1946
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
1947
+ }
1948
+ finally { if (e_2) throw e_2.error; }
1949
+ }
1950
+ }
1951
+ try {
1952
+ for (var _h = __values(jokerParameterNames || []), _j = _h.next(); !_j.done; _j = _h.next()) {
1953
+ var jokerName = _j.value;
1954
+ parameterNames.add(jokerName);
1955
+ }
1956
+ }
1957
+ catch (e_3_1) { e_3 = { error: e_3_1 }; }
1958
+ finally {
1959
+ try {
1960
+ if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
1961
+ }
1962
+ finally { if (e_3) throw e_3.error; }
1963
+ }
1964
+ return parameterNames;
1965
+ }
1966
+ /**
1967
+ * TODO: [🔣] If script require contentLanguage
1968
+ */
1969
+
1970
+ /**
1971
+ * This error occurs when some expectation is not met in the execution of the pipeline
1972
+ *
1973
+ * @private Always catched and rethrown as `PipelineExecutionError`
1974
+ * Note: This is a kindof subtype of PipelineExecutionError
1975
+ */
1976
+ var ExpectError = /** @class */ (function (_super) {
1977
+ __extends(ExpectError, _super);
1978
+ function ExpectError(message) {
1979
+ var _this = _super.call(this, message) || this;
1980
+ _this.name = 'ExpectError';
1981
+ Object.setPrototypeOf(_this, ExpectError.prototype);
1982
+ return _this;
1983
+ }
1984
+ return ExpectError;
1985
+ }(Error));
1986
+
1987
+ /**
1988
+ * Function isValidJsonString will tell you if the string is valid JSON or not
1989
+ */
1990
+ function isValidJsonString(value /* <-[👨‍⚖️] */) {
1991
+ try {
1992
+ JSON.parse(value);
1993
+ return true;
1994
+ }
1995
+ catch (error) {
1996
+ if (!(error instanceof Error)) {
1997
+ throw error;
1998
+ }
1999
+ if (error.message.includes('Unexpected token')) {
2000
+ return false;
2001
+ }
2002
+ return false;
2003
+ }
2004
+ }
2005
+
2006
+ /**
2007
+ * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1539
2008
  *
1540
2009
  * @private Internal utility of `joinLlmExecutionTools`
1541
2010
  */
@@ -1664,7 +2133,7 @@
1664
2133
  throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n All execution tools failed:\n\n ".concat(block(errors.map(function (error) { return "- ".concat(error.name || 'Error', ": ").concat(error.message); }).join('\n')), "\n\n "); }));
1665
2134
  }
1666
2135
  else {
1667
- throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n No execution tools available for model variant \"".concat(prompt.modelRequirements.modelVariant, "\".\n\n tl;dr\n\n You have provided no LLM Execution Tools that support model variant \"").concat(prompt.modelRequirements.modelVariant, ":\n ").concat(block(_this.llmExecutionTools
2136
+ throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.llmExecutionTools
1668
2137
  .map(function (tools) { return "- ".concat(tools.title, " ").concat(tools.description || ''); })
1669
2138
  .join('\n')), "\n\n "); }));
1670
2139
  }
@@ -1674,7 +2143,7 @@
1674
2143
  };
1675
2144
  /**
1676
2145
  * List all available models that can be used
1677
- * This liost is a combination of all available models from all execution tools
2146
+ * This lists is a combination of all available models from all execution tools
1678
2147
  */
1679
2148
  MultipleLlmExecutionTools.prototype.listModels = function () {
1680
2149
  return __awaiter(this, void 0, void 0, function () {
@@ -1720,6 +2189,8 @@
1720
2189
  }());
1721
2190
  /**
1722
2191
  * TODO: [🧠][🎛] Aggregating multiple models - have result not only from one first aviable model BUT all of them
2192
+ * TODO: [🏖] If no llmTools have for example not defined `callCompletionModel` this will still return object with defined `callCompletionModel` which just throws `PipelineExecutionError`, make it undefined instead
2193
+ * Look how `countTotalUsage` (and `cacheLlmTools`) implements it
1723
2194
  */
1724
2195
 
1725
2196
  /**
@@ -1741,7 +2212,7 @@
1741
2212
  llmExecutionTools[_i] = arguments[_i];
1742
2213
  }
1743
2214
  if (llmExecutionTools.length === 0) {
1744
- var warningMessage = spaceTrim__default["default"]("\n You have provided no LLM Execution Tools.\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ");
2215
+ var warningMessage = spaceTrim__default["default"]("\n You have not provided any `LlmExecutionTools`\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ");
1745
2216
  // TODO: [🟥] Detect browser / node and make it colorfull
1746
2217
  console.warn(warningMessage);
1747
2218
  /*
@@ -1768,6 +2239,31 @@
1768
2239
  return new (MultipleLlmExecutionTools.bind.apply(MultipleLlmExecutionTools, __spreadArray([void 0], __read(llmExecutionTools), false)))();
1769
2240
  }
1770
2241
 
2242
+ /**
2243
+ * Determine if the pipeline is fully prepared
2244
+ */
2245
+ function isPipelinePrepared(pipeline) {
2246
+ // Note: Ignoring `pipeline.preparations` @@@
2247
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2248
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2249
+ console.log('!!!!', 'Not all personas have modelRequirements');
2250
+ return false;
2251
+ }
2252
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2253
+ console.log('!!!!', 'Not all knowledgeSources have preparationIds');
2254
+ return false;
2255
+ }
2256
+ // TODO: !!!!! Is context in each template
2257
+ // TODO: !!!!! Are samples prepared
2258
+ // TODO: !!!!! Are templates prepared
2259
+ return true;
2260
+ }
2261
+ /**
2262
+ * TODO: [🐠] Maybe base this on `makeValidator`
2263
+ * TODO: [🔼] Export via core or utils
2264
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2265
+ */
2266
+
1771
2267
  /**
1772
2268
  * Takes an item or an array of items and returns an array of items
1773
2269
  *
@@ -1788,102 +2284,173 @@
1788
2284
  }
1789
2285
 
1790
2286
  /**
1791
- * The version of the Promptbook library
2287
+ * Just marks a place of place where should be something implemented
2288
+ * No side effects.
2289
+ *
2290
+ * Note: It can be usefull suppressing eslint errors of unused variables
2291
+ *
2292
+ * @param value any values
2293
+ * @returns void
2294
+ * @private within the repository
1792
2295
  */
1793
- var PROMPTBOOK_VERSION = '0.61.0-1';
1794
- // TODO: !!!! List here all the versions and annotate + put into script
2296
+ function TODO_USE() {
2297
+ var value = [];
2298
+ for (var _i = 0; _i < arguments.length; _i++) {
2299
+ value[_i] = arguments[_i];
2300
+ }
2301
+ }
1795
2302
 
1796
2303
  /**
1797
- * Function `addUsage` will add multiple usages into one
2304
+ * This error type indicates that some limit was reached
2305
+ */
2306
+ var LimitReachedError = /** @class */ (function (_super) {
2307
+ __extends(LimitReachedError, _super);
2308
+ function LimitReachedError(message) {
2309
+ var _this = _super.call(this, message) || this;
2310
+ _this.name = 'LimitReachedError';
2311
+ Object.setPrototypeOf(_this, LimitReachedError.prototype);
2312
+ return _this;
2313
+ }
2314
+ return LimitReachedError;
2315
+ }(Error));
2316
+
2317
+ /**
2318
+ * Replaces parameters in template with values from parameters object
1798
2319
  *
1799
- * Note: If you provide 0 values, it returns void usage
2320
+ * @param template the template with parameters in {curly} braces
2321
+ * @param parameters the object with parameters
2322
+ * @returns the template with replaced parameters
2323
+ * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened
1800
2324
  */
1801
- function addUsage() {
1802
- var usageItems = [];
1803
- for (var _i = 0; _i < arguments.length; _i++) {
1804
- usageItems[_i] = arguments[_i];
2325
+ function replaceParameters(template, parameters) {
2326
+ var replacedTemplate = template;
2327
+ var match;
2328
+ var loopLimit = LOOP_LIMIT;
2329
+ var _loop_1 = function () {
2330
+ if (loopLimit-- < 0) {
2331
+ throw new LimitReachedError('Loop limit reached during parameters replacement in `replaceParameters`');
2332
+ }
2333
+ var precol = match.groups.precol;
2334
+ var parameterName = match.groups.parameterName;
2335
+ if (parameterName === '') {
2336
+ return "continue";
2337
+ }
2338
+ if (parameterName.indexOf('{') !== -1 || parameterName.indexOf('}') !== -1) {
2339
+ throw new PipelineExecutionError('Parameter is already opened or not closed');
2340
+ }
2341
+ if (parameters[parameterName] === undefined) {
2342
+ throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2343
+ }
2344
+ var parameterValue = parameters[parameterName];
2345
+ if (parameterValue === undefined) {
2346
+ throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2347
+ }
2348
+ parameterValue = parameterValue.toString();
2349
+ if (parameterValue.includes('\n') && /^\s*\W{0,3}\s*$/.test(precol)) {
2350
+ parameterValue = parameterValue
2351
+ .split('\n')
2352
+ .map(function (line, index) { return (index === 0 ? line : "".concat(precol).concat(line)); })
2353
+ .join('\n');
2354
+ }
2355
+ replacedTemplate =
2356
+ replacedTemplate.substring(0, match.index + precol.length) +
2357
+ parameterValue +
2358
+ replacedTemplate.substring(match.index + precol.length + parameterName.length + 2);
2359
+ };
2360
+ while ((match = /^(?<precol>.*){(?<parameterName>\w+)}(.*)/m /* <- Not global */
2361
+ .exec(replacedTemplate))) {
2362
+ _loop_1();
1805
2363
  }
1806
- var initialStructure = {
1807
- price: { value: 0 },
1808
- input: {
1809
- tokensCount: { value: 0 },
1810
- charactersCount: { value: 0 },
1811
- wordsCount: { value: 0 },
1812
- sentencesCount: { value: 0 },
1813
- linesCount: { value: 0 },
1814
- paragraphsCount: { value: 0 },
1815
- pagesCount: { value: 0 },
1816
- },
1817
- output: {
1818
- tokensCount: { value: 0 },
1819
- charactersCount: { value: 0 },
1820
- wordsCount: { value: 0 },
1821
- sentencesCount: { value: 0 },
1822
- linesCount: { value: 0 },
1823
- paragraphsCount: { value: 0 },
1824
- pagesCount: { value: 0 },
1825
- },
2364
+ // [💫] Check if there are parameters that are not closed properly
2365
+ if (/{\w+$/.test(replacedTemplate)) {
2366
+ throw new PipelineExecutionError('Parameter is not closed');
2367
+ }
2368
+ // [💫] Check if there are parameters that are not opened properly
2369
+ if (/^\w+}/.test(replacedTemplate)) {
2370
+ throw new PipelineExecutionError('Parameter is not opened');
2371
+ }
2372
+ return replacedTemplate;
2373
+ }
2374
+
2375
+ /**
2376
+ * Create difference set of two sets.
2377
+ *
2378
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2379
+ */
2380
+ function difference(a, b, isEqual) {
2381
+ var e_1, _a;
2382
+ if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
2383
+ var diff = new Set();
2384
+ var _loop_1 = function (itemA) {
2385
+ if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
2386
+ diff.add(itemA);
2387
+ }
1826
2388
  };
1827
- return usageItems.reduce(function (acc, item) {
1828
- var e_1, _a, e_2, _b;
1829
- var _c;
1830
- acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
2389
+ try {
2390
+ for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
2391
+ var itemA = _c.value;
2392
+ _loop_1(itemA);
2393
+ }
2394
+ }
2395
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2396
+ finally {
1831
2397
  try {
1832
- for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
1833
- var key = _e.value;
1834
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1835
- //@ts-ignore
1836
- if (item.input[key]) {
1837
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1838
- //@ts-ignore
1839
- acc.input[key].value += item.input[key].value || 0;
1840
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1841
- //@ts-ignore
1842
- if (item.input[key].isUncertain) {
1843
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1844
- //@ts-ignore
1845
- acc.input[key].isUncertain = true;
1846
- }
1847
- }
1848
- }
2398
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
1849
2399
  }
1850
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
1851
- finally {
2400
+ finally { if (e_1) throw e_1.error; }
2401
+ }
2402
+ return diff;
2403
+ }
2404
+ /**
2405
+ * TODO: [🧠][💯] Maybe also implement symmetricDifference
2406
+ */
2407
+
2408
+ /**
2409
+ * Creates a new set with all elements that are present in either set
2410
+ *
2411
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2412
+ */
2413
+ function union() {
2414
+ var e_1, _a, e_2, _b;
2415
+ var sets = [];
2416
+ for (var _i = 0; _i < arguments.length; _i++) {
2417
+ sets[_i] = arguments[_i];
2418
+ }
2419
+ var union = new Set();
2420
+ try {
2421
+ for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
2422
+ var set = sets_1_1.value;
1852
2423
  try {
1853
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2424
+ for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
2425
+ var item = _d.value;
2426
+ union.add(item);
2427
+ }
1854
2428
  }
1855
- finally { if (e_1) throw e_1.error; }
1856
- }
1857
- try {
1858
- for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
1859
- var key = _g.value;
1860
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1861
- //@ts-ignore
1862
- if (item.output[key]) {
1863
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1864
- //@ts-ignore
1865
- acc.output[key].value += item.output[key].value || 0;
1866
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1867
- //@ts-ignore
1868
- if (item.output[key].isUncertain) {
1869
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1870
- //@ts-ignore
1871
- acc.output[key].isUncertain = true;
1872
- }
2429
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2430
+ finally {
2431
+ try {
2432
+ if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
1873
2433
  }
2434
+ finally { if (e_2) throw e_2.error; }
1874
2435
  }
1875
2436
  }
1876
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
1877
- finally {
1878
- try {
1879
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
1880
- }
1881
- finally { if (e_2) throw e_2.error; }
2437
+ }
2438
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2439
+ finally {
2440
+ try {
2441
+ if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
1882
2442
  }
1883
- return acc;
1884
- }, initialStructure);
2443
+ finally { if (e_1) throw e_1.error; }
2444
+ }
2445
+ return union;
1885
2446
  }
1886
2447
 
2448
+ /**
2449
+ * The version of the Promptbook library
2450
+ */
2451
+ var PROMPTBOOK_VERSION = '0.61.0-20';
2452
+ // TODO: !!!! List here all the versions and annotate + put into script
2453
+
1887
2454
  /**
1888
2455
  * Counts number of characters in the text
1889
2456
  */
@@ -2015,66 +2582,6 @@
2015
2582
  * TODO: [💝] Unite object for expecting amount and format
2016
2583
  */
2017
2584
 
2018
- /**
2019
- * Replaces parameters in template with values from parameters object
2020
- *
2021
- * @param template the template with parameters in {curly} braces
2022
- * @param parameters the object with parameters
2023
- * @returns the template with replaced parameters
2024
- * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened
2025
- *
2026
- * @private within the createPipelineExecutor
2027
- */
2028
- function replaceParameters(template, parameters) {
2029
- var replacedTemplate = template;
2030
- var match;
2031
- var loopLimit = LOOP_LIMIT;
2032
- var _loop_1 = function () {
2033
- if (loopLimit-- < 0) {
2034
- throw new UnexpectedError('Loop limit reached during parameters replacement in `replaceParameters`');
2035
- }
2036
- var precol = match.groups.precol;
2037
- var parameterName = match.groups.parameterName;
2038
- if (parameterName === '') {
2039
- return "continue";
2040
- }
2041
- if (parameterName.indexOf('{') !== -1 || parameterName.indexOf('}') !== -1) {
2042
- throw new PipelineExecutionError('Parameter is already opened or not closed');
2043
- }
2044
- if (parameters[parameterName] === undefined) {
2045
- throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2046
- }
2047
- var parameterValue = parameters[parameterName];
2048
- if (parameterValue === undefined) {
2049
- throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2050
- }
2051
- parameterValue = parameterValue.toString();
2052
- if (parameterValue.includes('\n') && /^\s*\W{0,3}\s*$/.test(precol)) {
2053
- parameterValue = parameterValue
2054
- .split('\n')
2055
- .map(function (line, index) { return (index === 0 ? line : "".concat(precol).concat(line)); })
2056
- .join('\n');
2057
- }
2058
- replacedTemplate =
2059
- replacedTemplate.substring(0, match.index + precol.length) +
2060
- parameterValue +
2061
- replacedTemplate.substring(match.index + precol.length + parameterName.length + 2);
2062
- };
2063
- while ((match = /^(?<precol>.*){(?<parameterName>\w+)}(.*)/m /* <- Not global */
2064
- .exec(replacedTemplate))) {
2065
- _loop_1();
2066
- }
2067
- // [💫] Check if there are parameters that are not closed properly
2068
- if (/{\w+$/.test(replacedTemplate)) {
2069
- throw new PipelineExecutionError('Parameter is not closed');
2070
- }
2071
- // [💫] Check if there are parameters that are not opened properly
2072
- if (/^\w+}/.test(replacedTemplate)) {
2073
- throw new PipelineExecutionError('Parameter is not opened');
2074
- }
2075
- return replacedTemplate;
2076
- }
2077
-
2078
2585
  /**
2079
2586
  * Creates executor function from pipeline and execution tools.
2080
2587
  *
@@ -2083,21 +2590,73 @@
2083
2590
  */
2084
2591
  function createPipelineExecutor(options) {
2085
2592
  var _this = this;
2086
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
2087
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? 3 : _b;
2088
- // TODO: !!!!! Implement new commands
2089
- validatePipeline(pipeline);
2090
- // TODO: !!!!! Do here materialization of still unmaterialized or dynamic knowledge
2091
- // TODO: !!!!! Do here personaToModelRequirements
2593
+ var rawPipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
2594
+ var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.isVerbose, isVerbose = _d === void 0 ? false : _d;
2595
+ validatePipeline(rawPipeline);
2092
2596
  var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
2597
+ var pipeline;
2598
+ if (isPipelinePrepared(rawPipeline)) {
2599
+ pipeline = rawPipeline;
2600
+ }
2601
+ else {
2602
+ console.warn(spaceTrim.spaceTrim("\n Pipeline ".concat(rawPipeline.pipelineUrl || rawPipeline.sourceFile || rawPipeline.title, " is not prepared\n\n It will be prepared ad-hoc before the first execution\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n ")));
2603
+ }
2093
2604
  var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
2605
+ function getContextForTemplate(// <- TODO: [🧠][🥜]
2606
+ template) {
2607
+ return __awaiter(this, void 0, void 0, function () {
2608
+ return __generator(this, function (_a) {
2609
+ // TODO: !!!!!! Implement Better - use real index and keyword search
2610
+ TODO_USE(template);
2611
+ return [2 /*return*/, pipeline.knowledgePieces.map(function (_a) {
2612
+ var content = _a.content;
2613
+ return "- ".concat(content);
2614
+ }).join('\n')];
2615
+ });
2616
+ });
2617
+ }
2618
+ function getReservedParametersForTemplate(template) {
2619
+ return __awaiter(this, void 0, void 0, function () {
2620
+ var context, currentDate, reservedParameters, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName;
2621
+ var e_3, _a;
2622
+ return __generator(this, function (_b) {
2623
+ switch (_b.label) {
2624
+ case 0: return [4 /*yield*/, getContextForTemplate(template)];
2625
+ case 1:
2626
+ context = _b.sent();
2627
+ currentDate = new Date().toISOString();
2628
+ reservedParameters = {
2629
+ context: context,
2630
+ currentDate: currentDate,
2631
+ };
2632
+ try {
2633
+ // Note: Doublecheck that ALL reserved parameters are defined:
2634
+ for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
2635
+ parameterName = RESERVED_PARAMETER_NAMES_1_1.value;
2636
+ if (reservedParameters[parameterName] === undefined) {
2637
+ throw new UnexpectedError("Reserved parameter {".concat(parameterName, "} is not defined"));
2638
+ }
2639
+ }
2640
+ }
2641
+ catch (e_3_1) { e_3 = { error: e_3_1 }; }
2642
+ finally {
2643
+ try {
2644
+ if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1);
2645
+ }
2646
+ finally { if (e_3) throw e_3.error; }
2647
+ }
2648
+ return [2 /*return*/, reservedParameters];
2649
+ }
2650
+ });
2651
+ });
2652
+ }
2094
2653
  function executeSingleTemplate(currentTemplate) {
2095
2654
  return __awaiter(this, void 0, void 0, function () {
2096
- var name, title, priority, prompt, chatResult, completionResult, embeddingResult, result, resultString, expectError, scriptPipelineExecutionErrors, maxAttempts, jokers, attempt, isJokerAttempt, joker, _a, _b, _c, _d, scriptTools, error_2, e_2_1, _e, _f, functionName, postprocessingError, _g, _h, scriptTools, error_3, e_3_1, e_4_1, error_4;
2097
- var e_2, _j, e_4, _k, e_3, _l, _m;
2655
+ var name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _d, _e, parameterName, prompt, chatResult, completionResult, embeddingResult, result, resultString, expectError, scriptPipelineExecutionErrors, maxAttempts, jokerParameterNames, attempt, isJokerAttempt, jokerParameterName, _f, _g, _h, _j, scriptTools, error_2, e_4_1, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, error_3, e_5_1, e_6_1, error_4;
2656
+ var e_7, _p, e_4, _q, e_6, _r, e_5, _s, _t;
2098
2657
  var _this = this;
2099
- return __generator(this, function (_o) {
2100
- switch (_o.label) {
2658
+ return __generator(this, function (_u) {
2659
+ switch (_u.label) {
2101
2660
  case 0:
2102
2661
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
2103
2662
  title = currentTemplate.title;
@@ -2114,61 +2673,108 @@
2114
2673
  // <- [3]
2115
2674
  })];
2116
2675
  case 1:
2117
- _o.sent();
2118
- _o.label = 2;
2676
+ _u.sent();
2677
+ _u.label = 2;
2119
2678
  case 2:
2679
+ usedParameterNames = extractParametersFromPromptTemplate(currentTemplate);
2680
+ dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
2681
+ if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
2682
+ throw new UnexpectedError(spaceTrim.spaceTrim("\n Dependent parameters are not consistent used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames).join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames).join(', '), "\n\n ")));
2683
+ }
2684
+ _b = (_a = Object).freeze;
2685
+ _c = [{}];
2686
+ return [4 /*yield*/, getReservedParametersForTemplate(currentTemplate)];
2687
+ case 3:
2688
+ definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_u.sent())])), parametersToPass])]);
2689
+ definedParameterNames = new Set(Object.keys(definedParameters));
2690
+ parameters = {};
2691
+ try {
2692
+ // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
2693
+ for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
2694
+ parameterName = _e.value;
2695
+ // Situation: Parameter is defined and used
2696
+ if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
2697
+ parameters[parameterName] = definedParameters[parameterName];
2698
+ }
2699
+ // Situation: Parameter is defined but NOT used
2700
+ else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) {
2701
+ // Do not pass this parameter to prompt
2702
+ }
2703
+ // Situation: Parameter is NOT defined BUT used
2704
+ else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
2705
+ // Houston, we have a problem
2706
+ // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck
2707
+ throw new UnexpectedError(spaceTrim.spaceTrim("\n Parameter {".concat(parameterName, "} is NOT defined\n BUT used in template \"").concat(currentTemplate.title || currentTemplate.name, "\"\n\n This should be catched in `validatePipeline`\n\n ")));
2708
+ }
2709
+ }
2710
+ }
2711
+ catch (e_7_1) { e_7 = { error: e_7_1 }; }
2712
+ finally {
2713
+ try {
2714
+ if (_e && !_e.done && (_p = _d.return)) _p.call(_d);
2715
+ }
2716
+ finally { if (e_7) throw e_7.error; }
2717
+ }
2718
+ // Note: Now we can freeze `parameters` because we are sure that all and only used parameters are defined
2719
+ Object.freeze(parameters);
2120
2720
  result = null;
2121
2721
  resultString = null;
2122
2722
  expectError = null;
2123
2723
  maxAttempts = currentTemplate.blockType === 'PROMPT_DIALOG' ? Infinity : maxExecutionAttempts;
2124
- jokers = currentTemplate.jokers || [];
2125
- attempt = -jokers.length;
2126
- _o.label = 3;
2127
- case 3:
2128
- if (!(attempt < maxAttempts)) return [3 /*break*/, 51];
2724
+ jokerParameterNames = currentTemplate.jokerParameterNames || [];
2725
+ attempt = -jokerParameterNames.length;
2726
+ _u.label = 4;
2727
+ case 4:
2728
+ if (!(attempt < maxAttempts)) return [3 /*break*/, 52];
2129
2729
  isJokerAttempt = attempt < 0;
2130
- joker = jokers[jokers.length + attempt];
2131
- if (isJokerAttempt && !joker) {
2730
+ jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
2731
+ if (isJokerAttempt && !jokerParameterName) {
2132
2732
  throw new UnexpectedError("Joker not found in attempt ".concat(attempt));
2133
2733
  }
2134
2734
  result = null;
2135
2735
  resultString = null;
2136
2736
  expectError = null;
2137
2737
  if (isJokerAttempt) {
2138
- if (typeof parametersToPass[joker] === 'undefined') {
2139
- throw new PipelineExecutionError("Joker parameter {".concat(joker, "} not defined"));
2738
+ if (parameters[jokerParameterName] === undefined) {
2739
+ throw new PipelineExecutionError("Joker parameter {".concat(jokerParameterName, "} not defined"));
2740
+ // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
2741
+ }
2742
+ else {
2743
+ resultString = parameters[jokerParameterName];
2140
2744
  }
2141
- resultString = parametersToPass[joker];
2142
- }
2143
- _o.label = 4;
2144
- case 4:
2145
- _o.trys.push([4, 47, 48, 49]);
2146
- if (!!isJokerAttempt) return [3 /*break*/, 29];
2147
- _a = currentTemplate.blockType;
2148
- switch (_a) {
2149
- case 'SIMPLE_TEMPLATE': return [3 /*break*/, 5];
2150
- case 'PROMPT_TEMPLATE': return [3 /*break*/, 6];
2151
- case 'SCRIPT': return [3 /*break*/, 15];
2152
- case 'PROMPT_DIALOG': return [3 /*break*/, 26];
2153
2745
  }
2154
- return [3 /*break*/, 28];
2746
+ _u.label = 5;
2155
2747
  case 5:
2156
- resultString = replaceParameters(currentTemplate.content, parametersToPass);
2748
+ _u.trys.push([5, 48, 49, 50]);
2749
+ if (!!isJokerAttempt) return [3 /*break*/, 30];
2750
+ _f = currentTemplate.blockType;
2751
+ switch (_f) {
2752
+ case 'SIMPLE_TEMPLATE': return [3 /*break*/, 6];
2753
+ case 'PROMPT_TEMPLATE': return [3 /*break*/, 7];
2754
+ case 'SCRIPT': return [3 /*break*/, 16];
2755
+ case 'PROMPT_DIALOG': return [3 /*break*/, 27];
2756
+ }
2157
2757
  return [3 /*break*/, 29];
2158
2758
  case 6:
2759
+ resultString = replaceParameters(currentTemplate.content, parameters);
2760
+ return [3 /*break*/, 30];
2761
+ case 7:
2159
2762
  prompt = {
2160
2763
  title: currentTemplate.title,
2161
2764
  pipelineUrl: "".concat(pipeline.pipelineUrl
2162
2765
  ? pipeline.pipelineUrl
2163
2766
  : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(currentTemplate.name),
2164
- parameters: parametersToPass,
2165
- content: replaceParameters(currentTemplate.content, parametersToPass) /* <- [2] */,
2767
+ parameters: parameters,
2768
+ content: currentTemplate.content,
2166
2769
  modelRequirements: currentTemplate.modelRequirements,
2167
- expectations: currentTemplate.expectations,
2770
+ expectations: __assign(__assign({}, (pipeline.personas.find(function (_a) {
2771
+ var name = _a.name;
2772
+ return name === currentTemplate.personaName;
2773
+ }) || {})), currentTemplate.expectations),
2168
2774
  expectFormat: currentTemplate.expectFormat,
2169
- postprocessing: (currentTemplate.postprocessing || []).map(function (functionName) { return function (result) { return __awaiter(_this, void 0, void 0, function () {
2170
- var errors, _a, _b, scriptTools, error_5, e_5_1;
2171
- var e_5, _c;
2775
+ postprocessing: (currentTemplate.postprocessingFunctionNames || []).map(function (functionName) { return function (result) { return __awaiter(_this, void 0, void 0, function () {
2776
+ var errors, _a, _b, scriptTools, error_5, e_8_1;
2777
+ var e_8, _c;
2172
2778
  return __generator(this, function (_d) {
2173
2779
  switch (_d.label) {
2174
2780
  case 0:
@@ -2189,7 +2795,7 @@
2189
2795
  script: "".concat(functionName, "(result)"),
2190
2796
  parameters: {
2191
2797
  result: result || '',
2192
- // Note: No ...parametersToPass, because working with result only
2798
+ // Note: No ...parametersForTemplate, because working with result only
2193
2799
  },
2194
2800
  })];
2195
2801
  case 4: return [2 /*return*/, _d.sent()];
@@ -2198,6 +2804,9 @@
2198
2804
  if (!(error_5 instanceof Error)) {
2199
2805
  throw error_5;
2200
2806
  }
2807
+ if (error_5 instanceof UnexpectedError) {
2808
+ throw error_5;
2809
+ }
2201
2810
  errors.push(error_5);
2202
2811
  return [3 /*break*/, 6];
2203
2812
  case 6:
@@ -2205,14 +2814,14 @@
2205
2814
  return [3 /*break*/, 2];
2206
2815
  case 7: return [3 /*break*/, 10];
2207
2816
  case 8:
2208
- e_5_1 = _d.sent();
2209
- e_5 = { error: e_5_1 };
2817
+ e_8_1 = _d.sent();
2818
+ e_8 = { error: e_8_1 };
2210
2819
  return [3 /*break*/, 10];
2211
2820
  case 9:
2212
2821
  try {
2213
2822
  if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2214
2823
  }
2215
- finally { if (e_5) throw e_5.error; }
2824
+ finally { if (e_8) throw e_8.error; }
2216
2825
  return [7 /*endfinally*/];
2217
2826
  case 10:
2218
2827
  if (errors.length === 0) {
@@ -2228,35 +2837,35 @@
2228
2837
  });
2229
2838
  }); }; }),
2230
2839
  };
2231
- _b = currentTemplate.modelRequirements.modelVariant;
2232
- switch (_b) {
2233
- case 'CHAT': return [3 /*break*/, 7];
2234
- case 'COMPLETION': return [3 /*break*/, 9];
2235
- case 'EMBEDDING': return [3 /*break*/, 11];
2840
+ _g = currentTemplate.modelRequirements.modelVariant;
2841
+ switch (_g) {
2842
+ case 'CHAT': return [3 /*break*/, 8];
2843
+ case 'COMPLETION': return [3 /*break*/, 10];
2844
+ case 'EMBEDDING': return [3 /*break*/, 12];
2236
2845
  }
2237
- return [3 /*break*/, 13];
2238
- case 7: return [4 /*yield*/, llmTools.callChatModel(prompt)];
2239
- case 8:
2240
- chatResult = _o.sent();
2846
+ return [3 /*break*/, 14];
2847
+ case 8: return [4 /*yield*/, llmTools.callChatModel(deepFreeze(prompt))];
2848
+ case 9:
2849
+ chatResult = _u.sent();
2241
2850
  // TODO: [🍬] Destroy chatThread
2242
2851
  result = chatResult;
2243
2852
  resultString = chatResult.content;
2244
- return [3 /*break*/, 14];
2245
- case 9: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
2246
- case 10:
2247
- completionResult = _o.sent();
2853
+ return [3 /*break*/, 15];
2854
+ case 10: return [4 /*yield*/, llmTools.callCompletionModel(deepFreeze(prompt))];
2855
+ case 11:
2856
+ completionResult = _u.sent();
2248
2857
  result = completionResult;
2249
2858
  resultString = completionResult.content;
2250
- return [3 /*break*/, 14];
2251
- case 11: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
2252
- case 12:
2253
- embeddingResult = _o.sent();
2859
+ return [3 /*break*/, 15];
2860
+ case 12: return [4 /*yield*/, llmTools.callEmbeddingModel(deepFreeze(prompt))];
2861
+ case 13:
2862
+ embeddingResult = _u.sent();
2254
2863
  result = embeddingResult;
2255
2864
  resultString = embeddingResult.content.join(',');
2256
- return [3 /*break*/, 14];
2257
- case 13: throw new PipelineExecutionError("Unknown model variant \"".concat(currentTemplate.modelRequirements.modelVariant, "\""));
2258
- case 14: return [3 /*break*/, 29];
2259
- case 15:
2865
+ return [3 /*break*/, 15];
2866
+ case 14: throw new PipelineExecutionError("Unknown model variant \"".concat(currentTemplate.modelRequirements.modelVariant, "\""));
2867
+ case 15: return [3 /*break*/, 30];
2868
+ case 16:
2260
2869
  if (arrayableToArray(tools.script).length === 0) {
2261
2870
  throw new PipelineExecutionError('No script execution tools are available');
2262
2871
  }
@@ -2265,49 +2874,52 @@
2265
2874
  }
2266
2875
  // TODO: DRY [1]
2267
2876
  scriptPipelineExecutionErrors = [];
2268
- _o.label = 16;
2269
- case 16:
2270
- _o.trys.push([16, 23, 24, 25]);
2271
- _c = (e_2 = void 0, __values(arrayableToArray(tools.script))), _d = _c.next();
2272
- _o.label = 17;
2877
+ _u.label = 17;
2273
2878
  case 17:
2274
- if (!!_d.done) return [3 /*break*/, 22];
2275
- scriptTools = _d.value;
2276
- _o.label = 18;
2879
+ _u.trys.push([17, 24, 25, 26]);
2880
+ _h = (e_4 = void 0, __values(arrayableToArray(tools.script))), _j = _h.next();
2881
+ _u.label = 18;
2277
2882
  case 18:
2278
- _o.trys.push([18, 20, , 21]);
2279
- return [4 /*yield*/, scriptTools.execute({
2883
+ if (!!_j.done) return [3 /*break*/, 23];
2884
+ scriptTools = _j.value;
2885
+ _u.label = 19;
2886
+ case 19:
2887
+ _u.trys.push([19, 21, , 22]);
2888
+ return [4 /*yield*/, scriptTools.execute(deepFreeze({
2280
2889
  scriptLanguage: currentTemplate.contentLanguage,
2281
2890
  script: currentTemplate.content,
2282
- parameters: parametersToPass,
2283
- })];
2284
- case 19:
2285
- resultString = _o.sent();
2286
- return [3 /*break*/, 22];
2891
+ parameters: parameters,
2892
+ }))];
2287
2893
  case 20:
2288
- error_2 = _o.sent();
2894
+ resultString = _u.sent();
2895
+ return [3 /*break*/, 23];
2896
+ case 21:
2897
+ error_2 = _u.sent();
2289
2898
  if (!(error_2 instanceof Error)) {
2290
2899
  throw error_2;
2291
2900
  }
2901
+ if (error_2 instanceof UnexpectedError) {
2902
+ throw error_2;
2903
+ }
2292
2904
  scriptPipelineExecutionErrors.push(error_2);
2293
- return [3 /*break*/, 21];
2294
- case 21:
2295
- _d = _c.next();
2296
- return [3 /*break*/, 17];
2297
- case 22: return [3 /*break*/, 25];
2298
- case 23:
2299
- e_2_1 = _o.sent();
2300
- e_2 = { error: e_2_1 };
2301
- return [3 /*break*/, 25];
2905
+ return [3 /*break*/, 22];
2906
+ case 22:
2907
+ _j = _h.next();
2908
+ return [3 /*break*/, 18];
2909
+ case 23: return [3 /*break*/, 26];
2302
2910
  case 24:
2911
+ e_4_1 = _u.sent();
2912
+ e_4 = { error: e_4_1 };
2913
+ return [3 /*break*/, 26];
2914
+ case 25:
2303
2915
  try {
2304
- if (_d && !_d.done && (_j = _c.return)) _j.call(_c);
2916
+ if (_j && !_j.done && (_q = _h.return)) _q.call(_h);
2305
2917
  }
2306
- finally { if (e_2) throw e_2.error; }
2918
+ finally { if (e_4) throw e_4.error; }
2307
2919
  return [7 /*endfinally*/];
2308
- case 25:
2920
+ case 26:
2309
2921
  if (resultString !== null) {
2310
- return [3 /*break*/, 29];
2922
+ return [3 /*break*/, 30];
2311
2923
  }
2312
2924
  if (scriptPipelineExecutionErrors.length === 1) {
2313
2925
  throw scriptPipelineExecutionErrors[0];
@@ -2317,101 +2929,104 @@
2317
2929
  .map(function (error) { return '- ' + error.message; })
2318
2930
  .join('\n\n')), "\n "); }));
2319
2931
  }
2320
- case 26:
2932
+ case 27:
2321
2933
  if (tools.userInterface === undefined) {
2322
2934
  throw new PipelineExecutionError('User interface tools are not available');
2323
2935
  }
2324
- return [4 /*yield*/, tools.userInterface.promptDialog({
2936
+ return [4 /*yield*/, tools.userInterface.promptDialog(deepFreeze({
2325
2937
  promptTitle: currentTemplate.title,
2326
- promptMessage: replaceParameters(currentTemplate.description || '', parametersToPass),
2327
- defaultValue: replaceParameters(currentTemplate.content, parametersToPass),
2938
+ promptMessage: replaceParameters(currentTemplate.description || '', parameters),
2939
+ defaultValue: replaceParameters(currentTemplate.content, parameters),
2328
2940
  // TODO: [🧠] !! Figure out how to define placeholder in .ptbk.md file
2329
2941
  placeholder: undefined,
2330
2942
  priority: priority,
2331
- })];
2332
- case 27:
2943
+ }))];
2944
+ case 28:
2333
2945
  // TODO: [🌹] When making next attempt for `PROMPT DIALOG`, preserve the previous user input
2334
- resultString = _o.sent();
2335
- return [3 /*break*/, 29];
2336
- case 28: throw new PipelineExecutionError("Unknown execution type \"".concat(currentTemplate.blockType, "\""));
2337
- case 29:
2338
- if (!(!isJokerAttempt && currentTemplate.postprocessing)) return [3 /*break*/, 46];
2339
- _o.label = 30;
2946
+ resultString = _u.sent();
2947
+ return [3 /*break*/, 30];
2948
+ case 29: throw new PipelineExecutionError("Unknown execution type \"".concat(currentTemplate.blockType, "\""));
2340
2949
  case 30:
2341
- _o.trys.push([30, 44, 45, 46]);
2342
- _e = (e_4 = void 0, __values(currentTemplate.postprocessing)), _f = _e.next();
2343
- _o.label = 31;
2950
+ if (!(!isJokerAttempt && currentTemplate.postprocessingFunctionNames)) return [3 /*break*/, 47];
2951
+ _u.label = 31;
2344
2952
  case 31:
2345
- if (!!_f.done) return [3 /*break*/, 43];
2346
- functionName = _f.value;
2953
+ _u.trys.push([31, 45, 46, 47]);
2954
+ _k = (e_6 = void 0, __values(currentTemplate.postprocessingFunctionNames)), _l = _k.next();
2955
+ _u.label = 32;
2956
+ case 32:
2957
+ if (!!_l.done) return [3 /*break*/, 44];
2958
+ functionName = _l.value;
2347
2959
  // TODO: DRY [1]
2348
2960
  scriptPipelineExecutionErrors = [];
2349
2961
  postprocessingError = null;
2350
- _o.label = 32;
2351
- case 32:
2352
- _o.trys.push([32, 39, 40, 41]);
2353
- _g = (e_3 = void 0, __values(arrayableToArray(tools.script))), _h = _g.next();
2354
- _o.label = 33;
2962
+ _u.label = 33;
2355
2963
  case 33:
2356
- if (!!_h.done) return [3 /*break*/, 38];
2357
- scriptTools = _h.value;
2358
- _o.label = 34;
2964
+ _u.trys.push([33, 40, 41, 42]);
2965
+ _m = (e_5 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next();
2966
+ _u.label = 34;
2359
2967
  case 34:
2360
- _o.trys.push([34, 36, , 37]);
2968
+ if (!!_o.done) return [3 /*break*/, 39];
2969
+ scriptTools = _o.value;
2970
+ _u.label = 35;
2971
+ case 35:
2972
+ _u.trys.push([35, 37, , 38]);
2361
2973
  return [4 /*yield*/, scriptTools.execute({
2362
2974
  scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
2363
2975
  script: "".concat(functionName, "(resultString)"),
2364
2976
  parameters: {
2365
2977
  resultString: resultString || '',
2366
- // Note: No ...parametersToPass, because working with result only
2978
+ // Note: No ...parametersForTemplate, because working with result only
2367
2979
  },
2368
2980
  })];
2369
- case 35:
2370
- resultString = _o.sent();
2371
- postprocessingError = null;
2372
- return [3 /*break*/, 38];
2373
2981
  case 36:
2374
- error_3 = _o.sent();
2982
+ resultString = _u.sent();
2983
+ postprocessingError = null;
2984
+ return [3 /*break*/, 39];
2985
+ case 37:
2986
+ error_3 = _u.sent();
2375
2987
  if (!(error_3 instanceof Error)) {
2376
2988
  throw error_3;
2377
2989
  }
2990
+ if (error_3 instanceof UnexpectedError) {
2991
+ throw error_3;
2992
+ }
2378
2993
  postprocessingError = error_3;
2379
2994
  scriptPipelineExecutionErrors.push(error_3);
2380
- return [3 /*break*/, 37];
2381
- case 37:
2382
- _h = _g.next();
2383
- return [3 /*break*/, 33];
2384
- case 38: return [3 /*break*/, 41];
2385
- case 39:
2386
- e_3_1 = _o.sent();
2387
- e_3 = { error: e_3_1 };
2388
- return [3 /*break*/, 41];
2995
+ return [3 /*break*/, 38];
2996
+ case 38:
2997
+ _o = _m.next();
2998
+ return [3 /*break*/, 34];
2999
+ case 39: return [3 /*break*/, 42];
2389
3000
  case 40:
3001
+ e_5_1 = _u.sent();
3002
+ e_5 = { error: e_5_1 };
3003
+ return [3 /*break*/, 42];
3004
+ case 41:
2390
3005
  try {
2391
- if (_h && !_h.done && (_l = _g.return)) _l.call(_g);
3006
+ if (_o && !_o.done && (_s = _m.return)) _s.call(_m);
2392
3007
  }
2393
- finally { if (e_3) throw e_3.error; }
3008
+ finally { if (e_5) throw e_5.error; }
2394
3009
  return [7 /*endfinally*/];
2395
- case 41:
3010
+ case 42:
2396
3011
  if (postprocessingError) {
2397
3012
  throw postprocessingError;
2398
3013
  }
2399
- _o.label = 42;
2400
- case 42:
2401
- _f = _e.next();
2402
- return [3 /*break*/, 31];
2403
- case 43: return [3 /*break*/, 46];
2404
- case 44:
2405
- e_4_1 = _o.sent();
2406
- e_4 = { error: e_4_1 };
2407
- return [3 /*break*/, 46];
3014
+ _u.label = 43;
3015
+ case 43:
3016
+ _l = _k.next();
3017
+ return [3 /*break*/, 32];
3018
+ case 44: return [3 /*break*/, 47];
2408
3019
  case 45:
3020
+ e_6_1 = _u.sent();
3021
+ e_6 = { error: e_6_1 };
3022
+ return [3 /*break*/, 47];
3023
+ case 46:
2409
3024
  try {
2410
- if (_f && !_f.done && (_k = _e.return)) _k.call(_e);
3025
+ if (_l && !_l.done && (_r = _k.return)) _r.call(_k);
2411
3026
  }
2412
- finally { if (e_4) throw e_4.error; }
3027
+ finally { if (e_6) throw e_6.error; }
2413
3028
  return [7 /*endfinally*/];
2414
- case 46:
3029
+ case 47:
2415
3030
  // TODO: [💝] Unite object for expecting amount and format
2416
3031
  if (currentTemplate.expectFormat) {
2417
3032
  if (currentTemplate.expectFormat === 'JSON') {
@@ -2424,15 +3039,18 @@
2424
3039
  if (currentTemplate.expectations) {
2425
3040
  checkExpectations(currentTemplate.expectations, resultString || '');
2426
3041
  }
2427
- return [3 /*break*/, 51];
2428
- case 47:
2429
- error_4 = _o.sent();
3042
+ return [3 /*break*/, 52];
3043
+ case 48:
3044
+ error_4 = _u.sent();
2430
3045
  if (!(error_4 instanceof ExpectError)) {
2431
3046
  throw error_4;
2432
3047
  }
3048
+ if (error_4 instanceof UnexpectedError) {
3049
+ throw error_4;
3050
+ }
2433
3051
  expectError = error_4;
2434
- return [3 /*break*/, 49];
2435
- case 48:
3052
+ return [3 /*break*/, 50];
3053
+ case 49:
2436
3054
  if (!isJokerAttempt &&
2437
3055
  currentTemplate.blockType === 'PROMPT_TEMPLATE' &&
2438
3056
  prompt
@@ -2454,15 +3072,15 @@
2454
3072
  });
2455
3073
  }
2456
3074
  return [7 /*endfinally*/];
2457
- case 49:
3075
+ case 50:
2458
3076
  if (expectError !== null && attempt === maxAttempts - 1) {
2459
3077
  throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ---\n Last error ").concat((expectError === null || expectError === void 0 ? void 0 : expectError.name) || '', ":\n ").concat(block((expectError === null || expectError === void 0 ? void 0 : expectError.message) || ''), "\n\n Last result:\n ").concat(resultString, "\n ---\n "); }));
2460
3078
  }
2461
- _o.label = 50;
2462
- case 50:
2463
- attempt++;
2464
- return [3 /*break*/, 3];
3079
+ _u.label = 51;
2465
3080
  case 51:
3081
+ attempt++;
3082
+ return [3 /*break*/, 4];
3083
+ case 52:
2466
3084
  if (resultString === null) {
2467
3085
  throw new UnexpectedError('Something went wrong and prompt result is null');
2468
3086
  }
@@ -2478,18 +3096,54 @@
2478
3096
  // <- [3]
2479
3097
  });
2480
3098
  }
2481
- parametersToPass = __assign(__assign({}, parametersToPass), (_m = {}, _m[currentTemplate.resultingParameterName] = resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */, _m));
3099
+ parametersToPass = Object.freeze(__assign(__assign({}, parametersToPass), (_t = {}, _t[currentTemplate.resultingParameterName] = resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */, _t)));
2482
3100
  return [2 /*return*/];
2483
3101
  }
2484
3102
  });
2485
3103
  });
2486
3104
  }
2487
- var parametersToPass, executionReport, resovedParameters_1, unresovedTemplates, resolving_1, loopLimit, _loop_1, error_1, usage_1, _a, _b, parameter, usage;
2488
- var e_1, _c;
2489
- return __generator(this, function (_d) {
2490
- switch (_d.label) {
3105
+ function filterJustOutputParameters() {
3106
+ var e_9, _a;
3107
+ var outputParameters = {};
3108
+ try {
3109
+ // Note: Filter ONLY output parameters
3110
+ for (var _b = __values(pipeline.parameters.filter(function (_a) {
3111
+ var isOutput = _a.isOutput;
3112
+ return isOutput;
3113
+ })), _c = _b.next(); !_c.done; _c = _b.next()) {
3114
+ var parameter = _c.value;
3115
+ if (parametersToPass[parameter.name] === undefined) {
3116
+ // [4]
3117
+ errors.push(new PipelineExecutionError("Parameter {".concat(parameter.name, "} is required as an output parameter but not set in the pipeline")));
3118
+ continue;
3119
+ }
3120
+ outputParameters[parameter.name] = parametersToPass[parameter.name] || '';
3121
+ }
3122
+ }
3123
+ catch (e_9_1) { e_9 = { error: e_9_1 }; }
3124
+ finally {
3125
+ try {
3126
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
3127
+ }
3128
+ finally { if (e_9) throw e_9.error; }
3129
+ }
3130
+ return outputParameters;
3131
+ }
3132
+ var executionReport, _a, _b, parameter, errors, _loop_1, _c, _d, parameterName, state_1, parametersToPass, resovedParameters_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
3133
+ var e_1, _e, e_2, _f;
3134
+ return __generator(this, function (_g) {
3135
+ switch (_g.label) {
2491
3136
  case 0:
2492
- parametersToPass = inputParameters;
3137
+ if (!(pipeline === undefined)) return [3 /*break*/, 2];
3138
+ return [4 /*yield*/, preparePipeline(rawPipeline, {
3139
+ llmTools: llmTools,
3140
+ isVerbose: isVerbose,
3141
+ maxParallelCount: maxParallelCount,
3142
+ })];
3143
+ case 1:
3144
+ pipeline = _g.sent();
3145
+ _g.label = 2;
3146
+ case 2:
2493
3147
  executionReport = {
2494
3148
  pipelineUrl: pipeline.pipelineUrl,
2495
3149
  title: pipeline.title,
@@ -2498,9 +3152,76 @@
2498
3152
  description: pipeline.description,
2499
3153
  promptExecutions: [],
2500
3154
  };
2501
- _d.label = 1;
2502
- case 1:
2503
- _d.trys.push([1, 6, , 7]);
3155
+ try {
3156
+ // Note: Check that all input input parameters are defined
3157
+ for (_a = __values(pipeline.parameters.filter(function (_a) {
3158
+ var isInput = _a.isInput;
3159
+ return isInput;
3160
+ })), _b = _a.next(); !_b.done; _b = _a.next()) {
3161
+ parameter = _b.value;
3162
+ if (inputParameters[parameter.name] === undefined) {
3163
+ return [2 /*return*/, deepFreezeWithSameType({
3164
+ isSuccessful: false,
3165
+ errors: [
3166
+ new PipelineExecutionError("Parameter {".concat(parameter.name, "} is required as an input parameter")),
3167
+ // <- TODO: !!!!! Test this error
3168
+ ],
3169
+ executionReport: executionReport,
3170
+ outputParameters: {},
3171
+ usage: ZERO_USAGE,
3172
+ })];
3173
+ }
3174
+ }
3175
+ }
3176
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3177
+ finally {
3178
+ try {
3179
+ if (_b && !_b.done && (_e = _a.return)) _e.call(_a);
3180
+ }
3181
+ finally { if (e_1) throw e_1.error; }
3182
+ }
3183
+ errors = [];
3184
+ _loop_1 = function (parameterName) {
3185
+ var parameter = pipeline.parameters.find(function (_a) {
3186
+ var name = _a.name;
3187
+ return name === parameterName;
3188
+ });
3189
+ if (parameter === undefined) {
3190
+ errors.push(new PipelineExecutionError("Extra parameter {".concat(parameterName, "} is passed as input parameter")));
3191
+ }
3192
+ else if (parameter.isInput === false) {
3193
+ return { value: deepFreezeWithSameType({
3194
+ isSuccessful: false,
3195
+ errors: [
3196
+ new PipelineExecutionError("Parameter {".concat(parameter.name, "} is passed as input parameter but is not input")),
3197
+ // <- TODO: !!!!! Test this error
3198
+ ],
3199
+ executionReport: executionReport,
3200
+ outputParameters: {},
3201
+ usage: ZERO_USAGE,
3202
+ }) };
3203
+ }
3204
+ };
3205
+ try {
3206
+ // Note: Check that no extra input parameters are passed
3207
+ for (_c = __values(Object.keys(inputParameters)), _d = _c.next(); !_d.done; _d = _c.next()) {
3208
+ parameterName = _d.value;
3209
+ state_1 = _loop_1(parameterName);
3210
+ if (typeof state_1 === "object")
3211
+ return [2 /*return*/, state_1.value];
3212
+ }
3213
+ }
3214
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
3215
+ finally {
3216
+ try {
3217
+ if (_d && !_d.done && (_f = _c.return)) _f.call(_c);
3218
+ }
3219
+ finally { if (e_2) throw e_2.error; }
3220
+ }
3221
+ parametersToPass = inputParameters;
3222
+ _g.label = 3;
3223
+ case 3:
3224
+ _g.trys.push([3, 8, , 9]);
2504
3225
  resovedParameters_1 = pipeline.parameters
2505
3226
  .filter(function (_a) {
2506
3227
  var isInput = _a.isInput;
@@ -2510,30 +3231,40 @@
2510
3231
  var name = _a.name;
2511
3232
  return name;
2512
3233
  });
2513
- unresovedTemplates = __spreadArray([], __read(pipeline.promptTemplates), false);
3234
+ unresovedTemplates_1 = __spreadArray([], __read(pipeline.promptTemplates), false);
2514
3235
  resolving_1 = [];
2515
3236
  loopLimit = LOOP_LIMIT;
2516
- _loop_1 = function () {
3237
+ _loop_2 = function () {
2517
3238
  var currentTemplate, work_1;
2518
- return __generator(this, function (_e) {
2519
- switch (_e.label) {
3239
+ return __generator(this, function (_h) {
3240
+ switch (_h.label) {
2520
3241
  case 0:
2521
3242
  if (loopLimit-- < 0) {
3243
+ // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
2522
3244
  throw new UnexpectedError('Loop limit reached during resolving parameters pipeline execution');
2523
3245
  }
2524
- currentTemplate = unresovedTemplates.find(function (template) {
3246
+ currentTemplate = unresovedTemplates_1.find(function (template) {
2525
3247
  return template.dependentParameterNames.every(function (name) { return resovedParameters_1.includes(name); });
2526
3248
  });
2527
3249
  if (!(!currentTemplate && resolving_1.length === 0)) return [3 /*break*/, 1];
2528
- throw new UnexpectedError(spaceTrim.spaceTrim("\n Can not resolve some parameters\n\n Note: This should be catched during validatePipeline\n "));
3250
+ throw new UnexpectedError(
3251
+ // TODO: [🐎] DRY
3252
+ spaceTrim.spaceTrim(function (block) { return "\n Can not resolve some parameters:\n\n Can not resolve:\n ".concat(block(unresovedTemplates_1
3253
+ .map(function (_a) {
3254
+ var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
3255
+ return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
3256
+ .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
3257
+ .join(' and '));
3258
+ })
3259
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameters_1.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n\n Note: This should be catched in `validatePipeline`\n "); }));
2529
3260
  case 1:
2530
3261
  if (!!currentTemplate) return [3 /*break*/, 3];
2531
3262
  /* [5] */ return [4 /*yield*/, Promise.race(resolving_1)];
2532
3263
  case 2:
2533
- /* [5] */ _e.sent();
3264
+ /* [5] */ _h.sent();
2534
3265
  return [3 /*break*/, 4];
2535
3266
  case 3:
2536
- unresovedTemplates = unresovedTemplates.filter(function (template) { return template !== currentTemplate; });
3267
+ unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
2537
3268
  work_1 = executeSingleTemplate(currentTemplate)
2538
3269
  .then(function () {
2539
3270
  resovedParameters_1 = __spreadArray(__spreadArray([], __read(resovedParameters_1), false), [currentTemplate.resultingParameterName], false);
@@ -2542,120 +3273,113 @@
2542
3273
  resolving_1 = resolving_1.filter(function (w) { return w !== work_1; });
2543
3274
  });
2544
3275
  resolving_1.push(work_1);
2545
- _e.label = 4;
3276
+ _h.label = 4;
2546
3277
  case 4: return [2 /*return*/];
2547
3278
  }
2548
3279
  });
2549
3280
  };
2550
- _d.label = 2;
2551
- case 2:
2552
- if (!(unresovedTemplates.length > 0)) return [3 /*break*/, 4];
2553
- return [5 /*yield**/, _loop_1()];
2554
- case 3:
2555
- _d.sent();
2556
- return [3 /*break*/, 2];
2557
- case 4: return [4 /*yield*/, Promise.all(resolving_1)];
3281
+ _g.label = 4;
3282
+ case 4:
3283
+ if (!(unresovedTemplates_1.length > 0)) return [3 /*break*/, 6];
3284
+ return [5 /*yield**/, _loop_2()];
2558
3285
  case 5:
2559
- _d.sent();
2560
- return [3 /*break*/, 7];
2561
- case 6:
2562
- error_1 = _d.sent();
3286
+ _g.sent();
3287
+ return [3 /*break*/, 4];
3288
+ case 6: return [4 /*yield*/, Promise.all(resolving_1)];
3289
+ case 7:
3290
+ _g.sent();
3291
+ return [3 /*break*/, 9];
3292
+ case 8:
3293
+ error_1 = _g.sent();
2563
3294
  if (!(error_1 instanceof Error)) {
2564
3295
  throw error_1;
2565
3296
  }
2566
3297
  usage_1 = addUsage.apply(void 0, __spreadArray([], __read(executionReport.promptExecutions.map(function (_a) {
2567
3298
  var result = _a.result;
2568
- return (result === null || result === void 0 ? void 0 : result.usage) || addUsage();
3299
+ return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE;
2569
3300
  })), false));
2570
- return [2 /*return*/, {
3301
+ outputParameters_1 = filterJustOutputParameters();
3302
+ return [2 /*return*/, deepFreezeWithSameType({
2571
3303
  isSuccessful: false,
2572
- errors: [error_1],
3304
+ errors: __spreadArray([error_1], __read(errors), false),
2573
3305
  usage: usage_1,
2574
3306
  executionReport: executionReport,
2575
- outputParameters: parametersToPass,
2576
- }];
2577
- case 7:
2578
- try {
2579
- // Note: Filter ONLY output parameters
2580
- for (_a = __values(pipeline.parameters), _b = _a.next(); !_b.done; _b = _a.next()) {
2581
- parameter = _b.value;
2582
- if (parameter.isOutput) {
2583
- continue;
2584
- }
2585
- delete parametersToPass[parameter.name];
2586
- }
2587
- }
2588
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2589
- finally {
2590
- try {
2591
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2592
- }
2593
- finally { if (e_1) throw e_1.error; }
2594
- }
3307
+ outputParameters: outputParameters_1,
3308
+ })];
3309
+ case 9:
2595
3310
  usage = addUsage.apply(void 0, __spreadArray([], __read(executionReport.promptExecutions.map(function (_a) {
2596
3311
  var result = _a.result;
2597
- return (result === null || result === void 0 ? void 0 : result.usage) || addUsage();
3312
+ return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE;
2598
3313
  })), false));
2599
- return [2 /*return*/, {
3314
+ outputParameters = filterJustOutputParameters();
3315
+ return [2 /*return*/, deepFreezeWithSameType({
2600
3316
  isSuccessful: true,
2601
- errors: [],
3317
+ errors: errors,
2602
3318
  usage: usage,
2603
3319
  executionReport: executionReport,
2604
- outputParameters: parametersToPass,
2605
- }];
3320
+ outputParameters: outputParameters,
3321
+ })];
2606
3322
  }
2607
3323
  });
2608
3324
  }); };
2609
3325
  return pipelineExecutor;
2610
3326
  }
2611
3327
  /**
3328
+ * TODO: Use isVerbose here (not only pass to `preparePipeline`)
3329
+ * TODO: [🪂] Use maxParallelCount here (not only pass to `preparePipeline`)
3330
+ * TODO: [♈] Probbably move expectations from templates to parameters
2612
3331
  * TODO: [🧠] When not meet expectations in PROMPT_DIALOG, make some way to tell the user
2613
3332
  * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
2614
3333
  * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
2615
3334
  * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
3335
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
3336
+ * TODO: [💷] !!!! `assertsExecutionSuccessful` should be the method of `PipelineExecutor` result
2616
3337
  */
2617
3338
 
2618
- function prepareKnowledgeFromMarkdown(options) {
3339
+ /**
3340
+ * @@@
3341
+ */
3342
+ function prepareKnowledgeFromMarkdown(content /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
2619
3343
  return __awaiter(this, void 0, void 0, function () {
2620
- var content, llmTools, _a, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _b, prepareTitleExecutor, _c, prepareKeywordsExecutor, _d, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
2621
- var _e, _f, _g;
3344
+ var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
3345
+ var _f, _g, _h;
2622
3346
  var _this = this;
2623
- return __generator(this, function (_h) {
2624
- switch (_h.label) {
3347
+ return __generator(this, function (_j) {
3348
+ switch (_j.label) {
2625
3349
  case 0:
2626
- content = options.content, llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
3350
+ llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
3351
+ TODO_USE(maxParallelCount); // <- [🪂]
2627
3352
  collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
2628
- _b = createPipelineExecutor;
2629
- _e = {};
2630
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
2631
- case 1:
2632
- prepareKnowledgeFromMarkdownExecutor = _b.apply(void 0, [(_e.pipeline = _h.sent(),
2633
- _e.tools = {
2634
- llm: llmTools,
2635
- },
2636
- _e)]);
2637
3353
  _c = createPipelineExecutor;
2638
3354
  _f = {};
2639
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
2640
- case 2:
2641
- prepareTitleExecutor = _c.apply(void 0, [(_f.pipeline = _h.sent(),
3355
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
3356
+ case 1:
3357
+ prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
2642
3358
  _f.tools = {
2643
3359
  llm: llmTools,
2644
3360
  },
2645
3361
  _f)]);
2646
3362
  _d = createPipelineExecutor;
2647
3363
  _g = {};
2648
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
2649
- case 3:
2650
- prepareKeywordsExecutor = _d.apply(void 0, [(_g.pipeline = _h.sent(),
3364
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
3365
+ case 2:
3366
+ prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
2651
3367
  _g.tools = {
2652
3368
  llm: llmTools,
2653
3369
  },
2654
3370
  _g)]);
3371
+ _e = createPipelineExecutor;
3372
+ _h = {};
3373
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
3374
+ case 3:
3375
+ prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
3376
+ _h.tools = {
3377
+ llm: llmTools,
3378
+ },
3379
+ _h)]);
2655
3380
  return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ content: content })];
2656
3381
  case 4:
2657
- result = _h.sent();
2658
- // TODO: [0] !!! Aggeregate usage
3382
+ result = _j.sent();
2659
3383
  assertsExecutionSuccessful(result);
2660
3384
  outputParameters = result.outputParameters;
2661
3385
  knowledgeRaw = outputParameters.knowledge;
@@ -2664,9 +3388,9 @@
2664
3388
  console.info('knowledgeTextPieces:', knowledgeTextPieces);
2665
3389
  }
2666
3390
  return [4 /*yield*/, Promise.all(
2667
- // TODO: !!! Do not send all at once but in chunks
3391
+ // TODO: [🪂] !! Do not send all at once but in chunks
2668
3392
  knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
2669
- var name, title, content, keywords, index, sources, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
3393
+ var name, title, content, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
2670
3394
  return __generator(this, function (_c) {
2671
3395
  switch (_c.label) {
2672
3396
  case 0:
@@ -2675,12 +3399,6 @@
2675
3399
  content = spaceTrim__default["default"](knowledgeTextPiece);
2676
3400
  keywords = [];
2677
3401
  index = [];
2678
- sources = [
2679
- {
2680
- title: 'Markdown document' /* <- TODO: !!! Unhardcode */,
2681
- href: '#' /* <- TODO: !!! Unhardcode */,
2682
- },
2683
- ];
2684
3402
  _c.label = 1;
2685
3403
  case 1:
2686
3404
  _c.trys.push([1, 7, , 8]);
@@ -2715,7 +3433,6 @@
2715
3433
  })];
2716
3434
  case 5:
2717
3435
  embeddingResult = _c.sent();
2718
- // TODO: [0] !!! Aggeregate usage embeddingResult.usage
2719
3436
  index.push({
2720
3437
  modelName: embeddingResult.modelName,
2721
3438
  position: embeddingResult.content,
@@ -2733,21 +3450,403 @@
2733
3450
  content: content,
2734
3451
  keywords: keywords,
2735
3452
  index: index,
2736
- sources: sources,
3453
+ // <- TODO: [☀] sources,
2737
3454
  }];
2738
3455
  }
2739
3456
  });
2740
3457
  }); }))];
2741
3458
  case 5:
2742
- knowledge = _h.sent();
3459
+ knowledge = _j.sent();
2743
3460
  return [2 /*return*/, knowledge];
2744
3461
  }
2745
3462
  });
2746
- });
2747
- }
3463
+ });
3464
+ }
3465
+ /**
3466
+ * TODO: [🐝][🔼] !!! Export via `@promptbook/markdown`
3467
+ * TODO: [🪂] Do it in parallel 11:11
3468
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
3469
+ */
3470
+
3471
+ /**
3472
+ * Prepares the knowle
3473
+ *
3474
+ * @see https://github.com/webgptorg/promptbook/discussions/41
3475
+ */
3476
+ function prepareKnowledgePieces(knowledgeSources, options) {
3477
+ return __awaiter(this, void 0, void 0, function () {
3478
+ var _a, maxParallelCount, knowledgePrepared;
3479
+ var _this = this;
3480
+ return __generator(this, function (_b) {
3481
+ switch (_b.label) {
3482
+ case 0:
3483
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
3484
+ knowledgePrepared = [];
3485
+ return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
3486
+ var partialPieces, pieces;
3487
+ return __generator(this, function (_a) {
3488
+ switch (_a.label) {
3489
+ case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.source, // <- TODO: [🐝] !!! Unhardcode markdown, detect which type it is
3490
+ options)];
3491
+ case 1:
3492
+ partialPieces = _a.sent();
3493
+ pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
3494
+ {
3495
+ name: knowledgeSource.name,
3496
+ // line, column <- TODO: [☀]
3497
+ // <- TODO: [❎]
3498
+ },
3499
+ ] })); });
3500
+ knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
3501
+ return [2 /*return*/];
3502
+ }
3503
+ });
3504
+ }); })];
3505
+ case 1:
3506
+ _b.sent();
3507
+ return [2 /*return*/, knowledgePrepared];
3508
+ }
3509
+ });
3510
+ });
3511
+ }
3512
+ /*
3513
+ TODO: [🧊] This is how it can look in future
3514
+ > type PrepareKnowledgeKnowledge = {
3515
+ > /**
3516
+ > * Unprepared knowledge
3517
+ > * /
3518
+ > readonly knowledgeSources: Array<KnowledgeSourceJson>;
3519
+ > };
3520
+ >
3521
+ > export async function prepareKnowledgePieces(
3522
+ > knowledge: PrepareKnowledgeKnowledge,
3523
+ > options: PrepareOptions,
3524
+ > ):
3525
+ */
3526
+ /**
3527
+ * TODO: [🐝][🔼] !!! Export via `@promptbook/core`
3528
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
3529
+ * Put `knowledgePieces` into `PrepareKnowledgeOptions`
3530
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
3531
+ * TODO: [🧠][❎] Do here propper M:N mapping
3532
+ * [x] One source can make multiple pieces
3533
+ * [ ] One piece can have multiple sources
3534
+ */
3535
+
3536
+ /**
3537
+ * Prepares the persona for the pipeline
3538
+ *
3539
+ * @see https://github.com/webgptorg/promptbook/discussions/22
3540
+ */
3541
+ function preparePersona(personaDescription, options) {
3542
+ return __awaiter(this, void 0, void 0, function () {
3543
+ var llmTools, _a, isVerbose, collection, preparePersonaExecutor, _b, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
3544
+ var _c;
3545
+ return __generator(this, function (_d) {
3546
+ switch (_d.label) {
3547
+ case 0:
3548
+ llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
3549
+ collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
3550
+ _b = createPipelineExecutor;
3551
+ _c = {};
3552
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
3553
+ case 1:
3554
+ preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
3555
+ _c.tools = {
3556
+ llm: llmTools,
3557
+ },
3558
+ _c)]);
3559
+ return [4 /*yield*/, llmTools.listModels()];
3560
+ case 2:
3561
+ availableModels = _d.sent();
3562
+ availableModelNames = availableModels
3563
+ .filter(function (_a) {
3564
+ var modelVariant = _a.modelVariant;
3565
+ return modelVariant === 'CHAT';
3566
+ })
3567
+ .map(function (_a) {
3568
+ var modelName = _a.modelName;
3569
+ return modelName;
3570
+ })
3571
+ .join(',');
3572
+ return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
3573
+ case 3:
3574
+ result = _d.sent();
3575
+ assertsExecutionSuccessful(result);
3576
+ outputParameters = result.outputParameters;
3577
+ modelRequirementsRaw = outputParameters.modelRequirements;
3578
+ modelRequirements = JSON.parse(modelRequirementsRaw);
3579
+ if (isVerbose) {
3580
+ console.info("PERSONA ".concat(personaDescription), modelRequirements);
3581
+ }
3582
+ modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
3583
+ return [2 /*return*/, {
3584
+ modelVariant: 'CHAT',
3585
+ modelName: modelName,
3586
+ systemMessage: systemMessage,
3587
+ temperature: temperature,
3588
+ }];
3589
+ }
3590
+ });
3591
+ });
3592
+ }
3593
+ /**
3594
+ * TODO: [🔼] !!! Export via `@promptbook/core`
3595
+ * TODO: [🏢] !! Check validity of `modelName` in pipeline
3596
+ * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
3597
+ * TODO: [🏢] !! Check validity of `temperature` in pipeline
3598
+ */
3599
+
3600
+ /**
3601
+ * Prepare pipeline from string (markdown) format to JSON format
3602
+ *
3603
+ * Note: This function does not validate logic of the pipeline
3604
+ * Note: This function acts as part of compilation process
3605
+ */
3606
+ function preparePipeline(pipeline, options) {
3607
+ return __awaiter(this, void 0, void 0, function () {
3608
+ var _a, maxParallelCount,
3609
+ /*
3610
+ <- TODO: [🧠][0] `promptbookVersion` */
3611
+ knowledgeSources /*
3612
+ <- TODO: [🧊] `knowledgePieces` */, personas /*
3613
+ <- TODO: [🧊] `preparations` */, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared;
3614
+ var _this = this;
3615
+ return __generator(this, function (_b) {
3616
+ switch (_b.label) {
3617
+ case 0:
3618
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
3619
+ knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
3620
+ currentPreparation = {
3621
+ id: 1,
3622
+ // TODO: [🍥]> date: $currentDate(),
3623
+ promptbookVersion: PROMPTBOOK_VERSION,
3624
+ modelUsage: ZERO_USAGE,
3625
+ };
3626
+ preparations = [
3627
+ // ...preparations
3628
+ // <- TODO: [🧊]
3629
+ currentPreparation,
3630
+ ];
3631
+ preparedPersonas = new Array(personas.length);
3632
+ return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () {
3633
+ var modelRequirements, preparedPersona;
3634
+ return __generator(this, function (_a) {
3635
+ switch (_a.label) {
3636
+ case 0: return [4 /*yield*/, preparePersona(persona.description, options)];
3637
+ case 1:
3638
+ modelRequirements = _a.sent();
3639
+ preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] });
3640
+ preparedPersonas[index] = preparedPersona;
3641
+ return [2 /*return*/];
3642
+ }
3643
+ });
3644
+ }); })];
3645
+ case 1:
3646
+ _b.sent();
3647
+ knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
3648
+ return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, options)];
3649
+ case 2:
3650
+ partialknowledgePiecesPrepared = _b.sent();
3651
+ knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
3652
+ // ----- /Knowledge preparation -----
3653
+ // TODO: !!!!! Add context to each template (if missing)
3654
+ // TODO: !!!!! Apply samples to each template (if missing)
3655
+ return [2 /*return*/, __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations })];
3656
+ }
3657
+ });
3658
+ });
3659
+ }
3660
+ /**
3661
+ * TODO: !!!!! Index the samples and maybe templates
3662
+ * TODO: [🔼] !!! Export via `@promptbook/core`
3663
+ * TODO: Write tests for `preparePipeline`
3664
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
3665
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
3666
+ * TODO: [🎐] !!!! Use here countTotalUsage
3667
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
3668
+ */
3669
+
3670
+ /**
3671
+ * Tests if given string is valid URL.
3672
+ *
3673
+ * Note: This does not check if the file exists only if the path is valid
3674
+ */
3675
+ function isValidFilePath(filePath) {
3676
+ if (typeof filePath !== 'string') {
3677
+ return false;
3678
+ }
3679
+ var filePathSlashes = filePath.split('\\').join('/');
3680
+ // Absolute Unix path: /hello.txt
3681
+ if (/^(\/)/i.test(filePathSlashes)) {
3682
+ return true;
3683
+ }
3684
+ // Absolute Windows path: /hello.txt
3685
+ if (/^([A-Z]{1,2}:\/?)\//i.test(filePathSlashes)) {
3686
+ return true;
3687
+ }
3688
+ // Relative path: ./hello.txt
3689
+ if (/^(\.\.?\/)+/i.test(filePathSlashes)) {
3690
+ return true;
3691
+ }
3692
+ return false;
3693
+ }
3694
+
3695
+ /**
3696
+ * Parses the knowledge command
3697
+ *
3698
+ * @see ./KNOWLEDGE-README.md for more details
3699
+ * @private within the commands folder
3700
+ */
3701
+ var knowledgeCommandParser = {
3702
+ /**
3703
+ * Name of the command
3704
+ */
3705
+ name: 'KNOWLEDGE',
3706
+ /**
3707
+ * BOILERPLATE command can be used in:
3708
+ */
3709
+ usagePlaces: ['PIPELINE_HEAD'],
3710
+ /**
3711
+ * Description of the KNOWLEDGE command
3712
+ */
3713
+ description: "Tells promptbook which external knowledge to use",
3714
+ /**
3715
+ * Link to discussion
3716
+ */
3717
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/41',
3718
+ /**
3719
+ * Example usages of the KNOWLEDGE command
3720
+ */
3721
+ examples: [
3722
+ 'KNOWLEDGE https://www.pavolhejny.com/',
3723
+ 'KNOWLEDGE ./hejny-cv.txt',
3724
+ 'KNOWLEDGE ./hejny-cv.md',
3725
+ 'KNOWLEDGE ./hejny-cv.pdf',
3726
+ 'KNOWLEDGE ./hejny-cv.docx',
3727
+ ],
3728
+ /**
3729
+ * Parses the KNOWLEDGE command
3730
+ */
3731
+ parse: function (input) {
3732
+ var args = input.args;
3733
+ var source = args[0];
3734
+ if (source === undefined) {
3735
+ throw new ParsingError("Source is not defined");
3736
+ }
3737
+ if (source.startsWith('http://')) {
3738
+ throw new ParsingError("Source is not secure");
3739
+ }
3740
+ if (!(isValidFilePath(source) || isValidUrl(source))) {
3741
+ throw new ParsingError("Source not valid");
3742
+ }
3743
+ if (source.startsWith('../') || source.startsWith('/') || /^[A-Z]:[\\/]+/i.test(source)) {
3744
+ throw new ParsingError("Source cannot be outside of the .ptbk.md folder");
3745
+ }
3746
+ return {
3747
+ type: 'KNOWLEDGE',
3748
+ source: source,
3749
+ };
3750
+ },
3751
+ /**
3752
+ * Note: Prototype of [🍧] (remove this comment after full implementation)
3753
+ */
3754
+ applyToPipelineJson: function (personaCommand, subjects) {
3755
+ var source = personaCommand.source;
3756
+ var pipelineJson = subjects.pipelineJson;
3757
+ var name = titleToName(source);
3758
+ pipelineJson.knowledgeSources.push({
3759
+ name: name,
3760
+ source: source,
3761
+ });
3762
+ },
3763
+ };
3764
+
2748
3765
  /**
2749
- * 11:11
3766
+ * Parses the persona command
3767
+ *
3768
+ * @see ./PERSONA-README.md for more details
3769
+ * @private within the commands folder
2750
3770
  */
3771
+ var personaCommandParser = {
3772
+ /**
3773
+ * Name of the command
3774
+ */
3775
+ name: 'PERSONA',
3776
+ /**
3777
+ * Aliases for the PERSONA command
3778
+ */
3779
+ aliasNames: ['PERSON'],
3780
+ /**
3781
+ * PERSONA command can be used in:
3782
+ */
3783
+ usagePlaces: ['PIPELINE_HEAD', 'PIPELINE_TEMPLATE'],
3784
+ /**
3785
+ * Description of the PERSONA command
3786
+ */
3787
+ description: "Persona command is used to specify who the system is, it will be transformed into system message, top_t,...",
3788
+ /**
3789
+ * Link to discussion
3790
+ */
3791
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/22',
3792
+ /**
3793
+ * Example usages of the PERSONA command
3794
+ */
3795
+ examples: ['PERSONA Jane, skilled copywriter', 'PERSONA Joe, male 28 years old, programmer'],
3796
+ /**
3797
+ * Parses the PERSONA command
3798
+ */
3799
+ parse: function (input) {
3800
+ var rawArgs = input.rawArgs;
3801
+ var _a = __read(rawArgs.split(/[,;:]/, 2), 2), personaNameRaw = _a[0], personaDescriptionRaw = _a[1];
3802
+ var personaName = (personaNameRaw || '').trim();
3803
+ if (personaName === '') {
3804
+ throw new ParsingError("You must set name for the persona");
3805
+ }
3806
+ var personaDescription = (personaDescriptionRaw || '').trim();
3807
+ if (personaDescription === '') {
3808
+ personaDescription = null;
3809
+ }
3810
+ return {
3811
+ type: 'PERSONA',
3812
+ personaName: personaName,
3813
+ personaDescription: personaDescription,
3814
+ };
3815
+ },
3816
+ /**
3817
+ * Note: Prototype of [🍧] (remove this comment after full implementation)
3818
+ */
3819
+ applyToPipelineJson: function (personaCommand, subjects) {
3820
+ var personaName = personaCommand.personaName, personaDescription = personaCommand.personaDescription;
3821
+ var pipelineJson = subjects.pipelineJson, templateJson = subjects.templateJson;
3822
+ if (templateJson !== null) {
3823
+ if (templateJson.blockType !== 'PROMPT_TEMPLATE') {
3824
+ throw new ParsingError("PERSONA command can be used only in PROMPT_TEMPLATE block");
3825
+ }
3826
+ templateJson.personaName = personaName;
3827
+ }
3828
+ var persona = pipelineJson.personas.find(function (persona) { return persona.name === personaName; });
3829
+ if (persona === undefined) {
3830
+ pipelineJson.personas.push({
3831
+ name: personaName,
3832
+ description: personaDescription || '',
3833
+ });
3834
+ return;
3835
+ }
3836
+ if (persona.description === personaDescription) {
3837
+ return;
3838
+ }
3839
+ if (personaDescription === null) {
3840
+ return;
3841
+ }
3842
+ if (persona.description === '') {
3843
+ persona.description = personaDescription;
3844
+ return;
3845
+ }
3846
+ console.warn(spaceTrim__default["default"]("\n\n Persona \"".concat(personaName, "\" is defined multiple times with different description:\n\n First definition:\n ").concat(persona.description, "\n\n Second definition:\n ").concat(personaDescription, "\n\n ")));
3847
+ persona.description += spaceTrim__default["default"]('\n\n' + personaDescription);
3848
+ },
3849
+ };
2751
3850
 
2752
3851
  /**
2753
3852
  * Removes Markdown formatting tags from a string.
@@ -2867,7 +3966,7 @@
2867
3966
  /**
2868
3967
  * Link to discussion
2869
3968
  */
2870
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/64',
3969
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/64',
2871
3970
  /**
2872
3971
  * Example usages of the BLOCK command
2873
3972
  */
@@ -2887,7 +3986,7 @@
2887
3986
  'Knowledge BLOCK',
2888
3987
  // 'Knowledge', // <- Note: [⛱] For execution blocks which are also separate commands shortcut does not work
2889
3988
  //---
2890
- /* TODO: !!!! Not implemented block types will be in examples in future -> */
3989
+ /* Note: Not implemented block types will be in examples in future -> */
2891
3990
  'Instrument BLOCK',
2892
3991
  // 'Instrument', // <- Note: [⛱]
2893
3992
  'Action BLOCK',
@@ -2895,6 +3994,7 @@
2895
3994
  //---
2896
3995
  /* <- TODO: [🧠] Maybe dynamic */
2897
3996
  ],
3997
+ // TODO: [♓️] order: -10 /* <- Note: Putting before other commands */
2898
3998
  /**
2899
3999
  * Parses the BLOCK command
2900
4000
  */
@@ -2903,19 +4003,19 @@
2903
4003
  normalized = normalized.split('EXAMPLE').join('SAMPLE');
2904
4004
  var blockTypes = BlockTypes.filter(function (blockType) { return normalized.includes(blockType); });
2905
4005
  if (blockTypes.length !== 1) {
2906
- // console.log('!!!', { blockType });
2907
4006
  throw new ParsingError(spaceTrim__default["default"](function (block) { return "\n Unknown block type in BLOCK command\n\n Supported block types are:\n ".concat(block(BlockTypes.join(', ')), "\n "); }));
2908
4007
  }
2909
- // TODO: !!!! Not supported yet
4008
+ var blockType = blockTypes[0];
2910
4009
  return {
2911
4010
  type: 'BLOCK',
2912
- blockType: blockTypes[0],
4011
+ blockType: blockType,
2913
4012
  };
2914
4013
  },
2915
4014
  };
2916
4015
 
2917
4016
  /**
2918
4017
  * Units of text measurement
4018
+ * @see https://github.com/webgptorg/promptbook/discussions/30
2919
4019
  */
2920
4020
  var EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'];
2921
4021
  /**
@@ -3011,7 +4111,7 @@
3011
4111
  /**
3012
4112
  * Link to discussion
3013
4113
  */
3014
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/30',
4114
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/30',
3015
4115
  /**
3016
4116
  * Example usages of the EXPECT command
3017
4117
  */
@@ -3129,7 +4229,7 @@
3129
4229
  /**
3130
4230
  * Link to discussion
3131
4231
  */
3132
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/66',
4232
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/66',
3133
4233
  /**
3134
4234
  * Example usages of the JOKER command
3135
4235
  */
@@ -3152,94 +4252,11 @@
3152
4252
  };
3153
4253
 
3154
4254
  /**
3155
- * Tests if given string is valid URL.
3156
- *
3157
- * Note: This does not check if the file exists only if the path is valid
3158
- */
3159
- function isValidFilePath(filePath) {
3160
- if (typeof filePath !== 'string') {
3161
- return false;
3162
- }
3163
- var filePathSlashes = filePath.split('\\').join('/');
3164
- // Absolute Unix path: /hello.txt
3165
- if (/^(\/)/i.test(filePathSlashes)) {
3166
- return true;
3167
- }
3168
- // Absolute Windows path: /hello.txt
3169
- if (/^([A-Z]{1,2}:\/?)\//i.test(filePathSlashes)) {
3170
- return true;
3171
- }
3172
- // Relative path: ./hello.txt
3173
- if (/^(\.\.?\/)+/i.test(filePathSlashes)) {
3174
- return true;
3175
- }
3176
- return false;
3177
- }
3178
-
3179
- /**
3180
- * Parses the knowledge command
4255
+ * @@@
3181
4256
  *
3182
- * @see ./KNOWLEDGE-README.md for more details
3183
- * @private within the commands folder
4257
+ * @private for `ModelVariant` and `modelCommandParser`
3184
4258
  */
3185
- var knowledgeCommandParser = {
3186
- /**
3187
- * Name of the command
3188
- */
3189
- name: 'KNOWLEDGE',
3190
- /**
3191
- * BOILERPLATE command can be used in:
3192
- */
3193
- usagePlaces: ['PIPELINE_HEAD'],
3194
- /**
3195
- * Description of the KNOWLEDGE command
3196
- */
3197
- description: "Tells promptbook which external knowledge to use",
3198
- /**
3199
- * Link to discussion
3200
- */
3201
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/41',
3202
- /**
3203
- * Example usages of the KNOWLEDGE command
3204
- */
3205
- examples: [
3206
- 'KNOWLEDGE https://www.pavolhejny.com/',
3207
- 'KNOWLEDGE ./hejny-cv.txt',
3208
- 'KNOWLEDGE ./hejny-cv.md',
3209
- 'KNOWLEDGE ./hejny-cv.pdf',
3210
- 'KNOWLEDGE ./hejny-cv.docx',
3211
- ],
3212
- /**
3213
- * Parses the KNOWLEDGE command
3214
- */
3215
- parse: function (input) {
3216
- var args = input.args;
3217
- var source = args[0];
3218
- if (source === undefined) {
3219
- throw new ParsingError("Source is not defined");
3220
- }
3221
- if (source.startsWith('http://')) {
3222
- throw new ParsingError("Source is not secure");
3223
- }
3224
- if (!(isValidFilePath(source) || isValidUrl(source))) {
3225
- throw new ParsingError("Source not valid");
3226
- }
3227
- if (source.startsWith('../') || source.startsWith('/') || /^[A-Z]:[\\/]+/i.test(source)) {
3228
- throw new ParsingError("Source cannot be outside of the .ptbk.md folder");
3229
- }
3230
- return {
3231
- type: 'KNOWLEDGE',
3232
- source: source,
3233
- };
3234
- },
3235
- };
3236
-
3237
4259
  var MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
3238
- /**
3239
- * TODO: Maybe figure out better word than "variant"
3240
- * TODO: Add here more requirement options like max context size, max tokens, etc.
3241
- * TODO: [👙][🧠] Just selecting gpt3 or gpt4 level of model
3242
- */
3243
4260
 
3244
4261
  /**
3245
4262
  * Parses the model command
@@ -3255,7 +4272,11 @@
3255
4272
  /**
3256
4273
  * BOILERPLATE command can be used in:
3257
4274
  */
3258
- usagePlaces: ['PIPELINE_HEAD', 'PIPELINE_TEMPLATE'],
4275
+ usagePlaces: [
4276
+ 'PIPELINE_HEAD',
4277
+ // <- TODO: [🧠][❔] Should there be possibility to set MODEL for entire pipeline?
4278
+ 'PIPELINE_TEMPLATE',
4279
+ ],
3259
4280
  /**
3260
4281
  * Description of the MODEL command
3261
4282
  */
@@ -3263,7 +4284,7 @@
3263
4284
  /**
3264
4285
  * Link to discussion
3265
4286
  */
3266
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/67',
4287
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/67',
3267
4288
  /**
3268
4289
  * Example usages of the MODEL command
3269
4290
  */
@@ -3343,7 +4364,7 @@
3343
4364
  /**
3344
4365
  * Link to discussion
3345
4366
  */
3346
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/68',
4367
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/68',
3347
4368
  /**
3348
4369
  * Example usages of the PARAMETER command
3349
4370
  */
@@ -3377,59 +4398,6 @@
3377
4398
  },
3378
4399
  };
3379
4400
 
3380
- /**
3381
- * Parses the persona command
3382
- *
3383
- * @see ./PERSONA-README.md for more details
3384
- * @private within the commands folder
3385
- */
3386
- var personaCommandParser = {
3387
- /**
3388
- * Name of the command
3389
- */
3390
- name: 'PERSONA',
3391
- /**
3392
- * Aliases for the PERSONA command
3393
- */
3394
- aliasNames: ['PERSON'],
3395
- /**
3396
- * PERSONA command can be used in:
3397
- */
3398
- usagePlaces: ['PIPELINE_HEAD', 'PIPELINE_TEMPLATE'],
3399
- /**
3400
- * Description of the PERSONA command
3401
- */
3402
- description: "Persona command is used to specify who the system is, it will be transformed into system message, top_t,...",
3403
- /**
3404
- * Link to discussion
3405
- */
3406
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/22',
3407
- /**
3408
- * Example usages of the PERSONA command
3409
- */
3410
- examples: ['PERSONA Jane, skilled copywriter', 'PERSONA Joe, male 28 years old, programmer'],
3411
- /**
3412
- * Parses the PERSONA command
3413
- */
3414
- parse: function (input) {
3415
- var rawArgs = input.rawArgs;
3416
- var _a = __read(rawArgs.split(/[,;:]/, 2), 2), personaNameRaw = _a[0], personaDescriptionRaw = _a[1];
3417
- var personaName = (personaNameRaw || '').trim();
3418
- if (personaName === '') {
3419
- throw new ParsingError("You must set name for the persona");
3420
- }
3421
- var personaDescription = (personaDescriptionRaw || '').trim();
3422
- if (personaDescription === '') {
3423
- personaDescription = null;
3424
- }
3425
- return {
3426
- type: 'PERSONA',
3427
- personaName: personaName,
3428
- personaDescription: personaDescription,
3429
- };
3430
- },
3431
- };
3432
-
3433
4401
  function isValidJavascriptName(javascriptName) {
3434
4402
  if (typeof javascriptName !== 'string') {
3435
4403
  return false;
@@ -3460,7 +4428,7 @@
3460
4428
  /**
3461
4429
  * Link to discussion
3462
4430
  */
3463
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/31',
4431
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/31',
3464
4432
  /**
3465
4433
  * Example usages of the POSTPROCESS command
3466
4434
  */
@@ -3515,7 +4483,7 @@
3515
4483
  /**
3516
4484
  * Link to discussion
3517
4485
  */
3518
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/69',
4486
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/69',
3519
4487
  /**
3520
4488
  * Example usages of the PROMPTBOOK_VERSION command
3521
4489
  */
@@ -3568,7 +4536,7 @@
3568
4536
  /**
3569
4537
  * Link to discussion
3570
4538
  */
3571
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/70',
4539
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/70',
3572
4540
  /**
3573
4541
  * Example usages of the URL command
3574
4542
  */
@@ -3616,22 +4584,6 @@
3616
4584
  },
3617
4585
  };
3618
4586
 
3619
- /**
3620
- * Returns the same value that is passed as argument.
3621
- * No side effects.
3622
- *
3623
- * Note: It can be usefull for leveling indentation
3624
- *
3625
- * @param value any values
3626
- * @returns the same values
3627
- */
3628
- function just(value) {
3629
- if (value === undefined) {
3630
- return undefined;
3631
- }
3632
- return value;
3633
- }
3634
-
3635
4587
  /**
3636
4588
  * Parses the action command
3637
4589
  *
@@ -3654,7 +4606,7 @@
3654
4606
  /**
3655
4607
  * Link to discussion
3656
4608
  */
3657
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/72',
4609
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/72',
3658
4610
  /**
3659
4611
  * Example usages of the ACTION command
3660
4612
  */
@@ -3663,7 +4615,8 @@
3663
4615
  * Parses the ACTION command
3664
4616
  */
3665
4617
  parse: function (input) {
3666
- input.args;
4618
+ var args = input.args;
4619
+ TODO_USE(args);
3667
4620
  return {
3668
4621
  type: 'ACTION',
3669
4622
  };
@@ -3692,7 +4645,7 @@
3692
4645
  /**
3693
4646
  * Link to discussion
3694
4647
  */
3695
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/71',
4648
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/71',
3696
4649
  /**
3697
4650
  * Example usages of the INSTRUMENT command
3698
4651
  */
@@ -3701,7 +4654,8 @@
3701
4654
  * Parses the INSTRUMENT command
3702
4655
  */
3703
4656
  parse: function (input) {
3704
- input.args;
4657
+ var args = input.args;
4658
+ TODO_USE(args);
3705
4659
  return {
3706
4660
  type: 'INSTRUMENT',
3707
4661
  };
@@ -3734,7 +4688,7 @@
3734
4688
  /**
3735
4689
  * Link to discussion
3736
4690
  */
3737
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
4691
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
3738
4692
  /**
3739
4693
  * Example usages of the BOILERPLATE command
3740
4694
  */
@@ -3759,6 +4713,7 @@
3759
4713
  };
3760
4714
  /**
3761
4715
  * TODO: [💐] Implement BOILERPLATE command into `pipelineStringToJsonSync` function
4716
+ * Note: [⚪] This should never be in any released package
3762
4717
  */
3763
4718
 
3764
4719
  /**
@@ -3777,7 +4732,7 @@
3777
4732
  actionCommandParser,
3778
4733
  instrumentCommandParser,
3779
4734
  personaCommandParser,
3780
- boilerplateCommandParser, // <- TODO: !!!! Only in development, remove in production
4735
+ boilerplateCommandParser, // <- TODO: !! Only in development, remove in production
3781
4736
  ];
3782
4737
 
3783
4738
  /**
@@ -3863,18 +4818,18 @@
3863
4818
  }));
3864
4819
  }
3865
4820
  /**
3866
- * !!!
4821
+ * @@@
3867
4822
  */
3868
4823
  function getSupportedCommandsMessage() {
3869
4824
  return COMMANDS.flatMap(function (_a) {
3870
- var name = _a.name, aliasNames = _a.aliasNames, description = _a.description, discussionUrl = _a.discussionUrl;
4825
+ var name = _a.name, aliasNames = _a.aliasNames, description = _a.description, documentationUrl = _a.documentationUrl;
3871
4826
  return __spreadArray([
3872
- "- **".concat(name, "** ").concat(description, ", see [discussion](").concat(discussionUrl, ")")
4827
+ "- **".concat(name, "** ").concat(description, ", see [discussion](").concat(documentationUrl, ")")
3873
4828
  ], __read((aliasNames || []).map(function (aliasName) { return " - **".concat(aliasName, "** Alias for **").concat(name, "**"); })), false);
3874
4829
  }).join('\n');
3875
4830
  }
3876
4831
  /**
3877
- * !!!
4832
+ * @@@
3878
4833
  */
3879
4834
  function parseCommandVariant(input) {
3880
4835
  var e_1, _a;
@@ -3883,7 +4838,6 @@
3883
4838
  var _loop_1 = function (commandParser) {
3884
4839
  var name_1 = commandParser.name, aliasNames = commandParser.aliasNames, deprecatedNames = commandParser.deprecatedNames, parse = commandParser.parse;
3885
4840
  var names = __spreadArray(__spreadArray([name_1], __read((aliasNames || [])), false), __read((deprecatedNames || [])), false);
3886
- // console.log('!!!', { commandName, names });
3887
4841
  if (names.includes(commandName)) {
3888
4842
  try {
3889
4843
  return { value: parse({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args }) };
@@ -3993,25 +4947,42 @@
3993
4947
  var e_1, _a;
3994
4948
  var codeBlocks = [];
3995
4949
  var lines = markdown.split('\n');
4950
+ // Note: [0] Ensure that the last block notated by gt > will be closed
4951
+ lines.push('');
3996
4952
  var currentCodeBlock = null;
3997
4953
  try {
3998
4954
  for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) {
3999
4955
  var line = lines_1_1.value;
4956
+ if (line.startsWith('> ') || line === '>') {
4957
+ if (currentCodeBlock === null) {
4958
+ currentCodeBlock = { blockNotation: '>', language: null, content: '' };
4959
+ } /* not else */
4960
+ if (currentCodeBlock.blockNotation === '>') {
4961
+ if (currentCodeBlock.content !== '') {
4962
+ currentCodeBlock.content += '\n';
4963
+ }
4964
+ currentCodeBlock.content += line.slice(2);
4965
+ }
4966
+ }
4967
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) {
4968
+ codeBlocks.push(currentCodeBlock);
4969
+ currentCodeBlock = null;
4970
+ }
4971
+ /* not else */
4000
4972
  if (line.startsWith('```')) {
4001
4973
  var language = line.slice(3).trim() || null;
4002
4974
  if (currentCodeBlock === null) {
4003
- currentCodeBlock = { language: language, content: '' };
4975
+ currentCodeBlock = { blockNotation: '```', language: language, content: '' };
4004
4976
  }
4005
4977
  else {
4006
4978
  if (language !== null) {
4007
- // [🌻]
4008
- throw new Error("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
4979
+ throw new ParsingError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
4009
4980
  }
4010
4981
  codeBlocks.push(currentCodeBlock);
4011
4982
  currentCodeBlock = null;
4012
4983
  }
4013
4984
  }
4014
- else if (currentCodeBlock !== null) {
4985
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') {
4015
4986
  if (currentCodeBlock.content !== '') {
4016
4987
  currentCodeBlock.content += '\n';
4017
4988
  }
@@ -4027,11 +4998,13 @@
4027
4998
  finally { if (e_1) throw e_1.error; }
4028
4999
  }
4029
5000
  if (currentCodeBlock !== null) {
4030
- // [🌻]
4031
- throw new Error("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
5001
+ throw new ParsingError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
4032
5002
  }
4033
5003
  return codeBlocks;
4034
5004
  }
5005
+ /**
5006
+ * TODO: Maybe name for `blockNotation` instead of '```' and '>'
5007
+ */
4035
5008
 
4036
5009
  /**
4037
5010
  * Extracts exactly ONE code block from markdown.
@@ -4049,13 +5022,12 @@
4049
5022
  function extractOneBlockFromMarkdown(markdown) {
4050
5023
  var codeBlocks = extractAllBlocksFromMarkdown(markdown);
4051
5024
  if (codeBlocks.length !== 1) {
4052
- // TODO: Report more specific place where the error happened
4053
- throw new Error(/* <- [🌻] */ 'There should be exactly one code block in the markdown');
5025
+ throw new ParsingError(spaceTrim__default["default"](function (block) { return "\n There should be exactly 1 code block, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
4054
5026
  }
4055
5027
  return codeBlocks[0];
4056
5028
  }
4057
5029
  /***
4058
- * TODO: [🍓][🌻] !!! Decide of this is internal util, external util OR validator/postprocessor
5030
+ * TODO: [🍓][🌻] Decide of this is internal util, external util OR validator/postprocessor
4059
5031
  */
4060
5032
 
4061
5033
  /**
@@ -4065,13 +5037,13 @@
4065
5037
  var _a, _b;
4066
5038
  var lines = value.split('\n');
4067
5039
  if (!lines[0].startsWith('#')) {
4068
- throw new Error('Markdown section must start with heading');
5040
+ throw new ParsingError('Markdown section must start with heading');
4069
5041
  }
4070
5042
  var title = lines[0].replace(/^#+\s*/, '');
4071
5043
  var level = (_b = (_a = lines[0].match(/^#+/)) === null || _a === void 0 ? void 0 : _a[0].length) !== null && _b !== void 0 ? _b : 0;
4072
5044
  var content = spaceTrim__default["default"](lines.slice(1).join('\n'));
4073
5045
  if (level < 1 || level > 6) {
4074
- throw new Error('Markdown section must have heading level between 1 and 6');
5046
+ throw new ParsingError('Markdown section must have heading level between 1 and 6');
4075
5047
  }
4076
5048
  return { title: title, level: level, content: content };
4077
5049
  }
@@ -4208,203 +5180,14 @@
4208
5180
  }
4209
5181
 
4210
5182
  /**
4211
- * Create difference set of two sets.
4212
- *
4213
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
4214
- */
4215
- function difference(a, b, isEqual) {
4216
- var e_1, _a;
4217
- if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
4218
- var diff = new Set();
4219
- var _loop_1 = function (itemA) {
4220
- if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
4221
- diff.add(itemA);
4222
- }
4223
- };
4224
- try {
4225
- for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
4226
- var itemA = _c.value;
4227
- _loop_1(itemA);
4228
- }
4229
- }
4230
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4231
- finally {
4232
- try {
4233
- if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
4234
- }
4235
- finally { if (e_1) throw e_1.error; }
4236
- }
4237
- return diff;
4238
- }
4239
-
4240
- /**
4241
- * Creates a new set with all elements that are present in either set
4242
- *
4243
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
4244
- */
4245
- function union() {
4246
- var e_1, _a, e_2, _b;
4247
- var sets = [];
4248
- for (var _i = 0; _i < arguments.length; _i++) {
4249
- sets[_i] = arguments[_i];
4250
- }
4251
- var union = new Set();
4252
- try {
4253
- for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
4254
- var set = sets_1_1.value;
4255
- try {
4256
- for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
4257
- var item = _d.value;
4258
- union.add(item);
4259
- }
4260
- }
4261
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
4262
- finally {
4263
- try {
4264
- if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
4265
- }
4266
- finally { if (e_2) throw e_2.error; }
4267
- }
4268
- }
4269
- }
4270
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4271
- finally {
4272
- try {
4273
- if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
4274
- }
4275
- finally { if (e_1) throw e_1.error; }
4276
- }
4277
- return union;
4278
- }
4279
-
4280
- /**
4281
- * Parses the template and returns the list of all parameter names
4282
- *
4283
- * @param template the template with parameters in {curly} braces
4284
- * @returns the list of parameter names
4285
- */
4286
- function extractParameters(template) {
4287
- var e_1, _a;
4288
- var matches = template.matchAll(/{\w+}/g);
4289
- var parameterNames = new Set();
4290
- try {
4291
- for (var matches_1 = __values(matches), matches_1_1 = matches_1.next(); !matches_1_1.done; matches_1_1 = matches_1.next()) {
4292
- var match = matches_1_1.value;
4293
- var parameterName = match[0].slice(1, -1);
4294
- parameterNames.add(parameterName);
4295
- }
4296
- }
4297
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4298
- finally {
4299
- try {
4300
- if (matches_1_1 && !matches_1_1.done && (_a = matches_1.return)) _a.call(matches_1);
4301
- }
4302
- finally { if (e_1) throw e_1.error; }
4303
- }
4304
- return parameterNames;
4305
- }
4306
-
4307
- /**
4308
- * Parses the given script and returns the list of all used variables that are not defined in the script
4309
- *
4310
- * @param script from which to extract the variables
4311
- * @returns the list of variable names
4312
- * @throws {ParsingError} if the script is invalid
4313
- */
4314
- function extractVariables(script) {
4315
- var variables = new Set();
4316
- script = "(()=>{".concat(script, "})()");
4317
- try {
4318
- for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
4319
- try {
4320
- eval(script);
4321
- }
4322
- catch (error) {
4323
- if (!(error instanceof ReferenceError)) {
4324
- throw error;
4325
- }
4326
- var undefinedName = error.message.split(' ')[0];
4327
- /*
4328
- Note: Parsing the error
4329
- [ReferenceError: thing is not defined]
4330
- */
4331
- if (!undefinedName) {
4332
- throw error;
4333
- }
4334
- if (script.includes(undefinedName + '(')) {
4335
- script = "const ".concat(undefinedName, " = ()=>'';") + script;
4336
- }
4337
- else {
4338
- variables.add(undefinedName);
4339
- script = "const ".concat(undefinedName, " = '';") + script;
4340
- }
4341
- }
4342
- }
4343
- catch (error) {
4344
- if (!(error instanceof Error)) {
4345
- throw error;
4346
- }
4347
- throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Can not extract variables from the script\n\n ".concat(block(error.name), ": ").concat(block(error.message), "\n "); }));
4348
- }
4349
- return variables;
4350
- }
4351
- /**
4352
- * TODO: [🔣] Support for multiple languages - python, java,...
4353
- */
4354
-
4355
- /**
4356
- * Parses the prompt template and returns the set of all used parameters
4357
- *
4358
- * @param promptTemplate the template with used parameters
4359
- * @returns the set of parameter names
4360
- * @throws {ParsingError} if the script is invalid
4361
- */
4362
- function extractParametersFromPromptTemplate(promptTemplate) {
4363
- var e_1, _a, e_2, _b;
4364
- var parameterNames = new Set();
4365
- try {
4366
- for (var _c = __values(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameters(promptTemplate.title)), false), __read(extractParameters(promptTemplate.description || '')), false), __read(extractParameters(promptTemplate.content)), false)), _d = _c.next(); !_d.done; _d = _c.next()) {
4367
- var parameterName = _d.value;
4368
- parameterNames.add(parameterName);
4369
- }
4370
- }
4371
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4372
- finally {
4373
- try {
4374
- if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
4375
- }
4376
- finally { if (e_1) throw e_1.error; }
4377
- }
4378
- if (promptTemplate.blockType === 'SCRIPT') {
4379
- try {
4380
- for (var _e = __values(extractVariables(promptTemplate.content)), _f = _e.next(); !_f.done; _f = _e.next()) {
4381
- var parameterName = _f.value;
4382
- parameterNames.add(parameterName);
4383
- }
4384
- }
4385
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
4386
- finally {
4387
- try {
4388
- if (_f && !_f.done && (_b = _e.return)) _b.call(_e);
4389
- }
4390
- finally { if (e_2) throw e_2.error; }
4391
- }
4392
- }
4393
- return parameterNames;
4394
- }
4395
- /**
4396
- * TODO: [🔣] If script require contentLanguage
4397
- */
4398
-
4399
- /**
4400
- * Compile promptbook from string (markdown) format to JSON format synchronously
5183
+ * Compile pipeline from string (markdown) format to JSON format synchronously
4401
5184
  *
4402
- * Note: There are two similar functions:
5185
+ * Note: There are 3 similar functions:
4403
5186
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
4404
5187
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
5188
+ * - `preparePipeline` - just one step in the compilation process
4405
5189
  *
4406
5190
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
4407
- * @param options - Options and tools for the compilation
4408
5191
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
4409
5192
  * @throws {ParsingError} if the promptbook string is not valid
4410
5193
  *
@@ -4420,7 +5203,10 @@
4420
5203
  description: undefined /* <- Note: Putting here placeholder to keep `description` on top at final JSON */,
4421
5204
  parameters: [],
4422
5205
  promptTemplates: [],
4423
- knowledge: [],
5206
+ knowledgeSources: [],
5207
+ knowledgePieces: [],
5208
+ personas: [],
5209
+ preparations: [],
4424
5210
  };
4425
5211
  // =============================================================
4426
5212
  // Note: 1️⃣ Parsing of the markdown into object
@@ -4450,7 +5236,7 @@
4450
5236
  existingParameter.description &&
4451
5237
  existingParameter.description !== parameterDescription &&
4452
5238
  parameterDescription) {
4453
- throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is defined multiple times with different description.\n\n First definition:\n ").concat(block(existingParameter.description || '[undefined]'), "\n\n Second definition:\n ").concat(block(parameterDescription || '[undefined]'), "\n "); }));
5239
+ throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is defined multiple times with different description:\n\n First definition:\n ").concat(block(existingParameter.description || '[undefined]'), "\n\n Second definition:\n ").concat(block(parameterDescription || '[undefined]'), "\n "); }));
4454
5240
  }
4455
5241
  if (existingParameter) {
4456
5242
  if (parameterDescription) {
@@ -4469,11 +5255,12 @@
4469
5255
  // =============================================================
4470
5256
  // Note: 3️⃣ Process pipeline head
4471
5257
  pipelineJson.title = pipelineHead.title;
4472
- // TODO: [1] DRY description
5258
+ // TODO: [🎾][1] DRY description
4473
5259
  var description = pipelineHead.content;
4474
- // Note: Remove codeblocks - TODO: Maybe put this into util (exported from `@promptbool/utils`)
5260
+ // Note: Remove codeblocks - TODO: [🎾] Make util removeAllBlocksFromMarkdown (exported from `@promptbool/utils`)
4475
5261
  description = description.split(/^```.*^```/gms).join('');
4476
- //Note: Remove lists and return statement - TODO: Maybe put this into util (exported from `@promptbool/utils`)
5262
+ description = description.split(/^>.*$/gm).join('');
5263
+ //Note: Remove lists and return statement - TODO: [🎾] Make util (exported from `@promptbool/utils`)
4477
5264
  description = description.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join('');
4478
5265
  description = spaceTrim.spaceTrim(description);
4479
5266
  if (description === '') {
@@ -4501,7 +5288,7 @@
4501
5288
  pipelineJson.pipelineUrl = command.pipelineUrl.href;
4502
5289
  break;
4503
5290
  case 'KNOWLEDGE':
4504
- console.error(new NotYetImplementedError('Knowledge is not implemented yet'));
5291
+ knowledgeCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: null });
4505
5292
  break;
4506
5293
  case 'ACTION':
4507
5294
  console.error(new NotYetImplementedError('Actions are not implemented yet'));
@@ -4510,7 +5297,8 @@
4510
5297
  console.error(new NotYetImplementedError('Instruments are not implemented yet'));
4511
5298
  break;
4512
5299
  case 'PERSONA':
4513
- console.error(new NotYetImplementedError('Personas are not implemented yet'));
5300
+ personaCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: null });
5301
+ // <- Note: Prototype of [🍧] (remove this comment after full implementation)
4514
5302
  break;
4515
5303
  case 'BOILERPLATE':
4516
5304
  throw new ParsingError('BOILERPLATE command is only for testing purposes and should not be used in the .ptbk.md file'); // <- TODO: [🚞]
@@ -4533,29 +5321,83 @@
4533
5321
  // TODO: Parse prompt template description (the content out of the codeblock and lists)
4534
5322
  var templateModelRequirements = __assign({}, defaultModelRequirements);
4535
5323
  var listItems_3 = extractAllListItemsFromMarkdown(section.content);
4536
- var dependentParameterNames = new Set();
4537
- var blockType = 'PROMPT_TEMPLATE';
4538
- var jokers = [];
4539
- var postprocessing = [];
4540
- var expectAmount = {};
4541
- var expectFormat = undefined;
4542
- var isBlockTypeChanged = false;
5324
+ var lastLine = section.content.split('\n').pop();
5325
+ var resultingParameterNameMatch = /^->\s*\{(?<resultingParamName>[a-z0-9_]+)\}/im.exec(lastLine);
5326
+ var resultingParameterName = null;
5327
+ if (resultingParameterNameMatch &&
5328
+ resultingParameterNameMatch.groups !== undefined &&
5329
+ resultingParameterNameMatch.groups.resultingParamName !== undefined) {
5330
+ resultingParameterName = resultingParameterNameMatch.groups.resultingParamName;
5331
+ }
5332
+ var expectResultingParameterName = function () {
5333
+ if (resultingParameterName !== null) {
5334
+ return resultingParameterName;
5335
+ }
5336
+ throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Template section must end with -> {parameterName}\n\n Invalid section:\n ".concat(block(
5337
+ // TODO: Show code of invalid sections each time + DRY
5338
+ section.content
5339
+ .split('\n')
5340
+ .map(function (line) { return " | ".concat(line); } /* <- TODO: [🚞] */)
5341
+ .join('\n')), "\n "); }));
5342
+ };
5343
+ var _e = extractOneBlockFromMarkdown(section.content), language = _e.language, content = _e.content;
5344
+ // TODO: [🎾][1] DRY description
5345
+ var description_1 = section.content;
5346
+ // Note: Remove codeblocks - TODO: [🎾]
5347
+ description_1 = description_1.split(/^```.*^```/gms).join('');
5348
+ description_1 = description_1.split(/^>.*$/gm).join('');
5349
+ //Note: Remove lists and return statement - TODO: [🎾]
5350
+ description_1 = description_1.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join('');
5351
+ description_1 = spaceTrim.spaceTrim(description_1);
5352
+ if (description_1 === '') {
5353
+ description_1 = undefined;
5354
+ }
5355
+ var templateJson = {
5356
+ blockType: 'PROMPT_TEMPLATE',
5357
+ name: titleToName(section.title),
5358
+ title: section.title,
5359
+ description: description_1,
5360
+ modelRequirements: templateModelRequirements,
5361
+ content: content,
5362
+ };
5363
+ /**
5364
+ * This is nessesary because block type can be
5365
+ * - Set zero times, so anticipate 'PROMPT_TEMPLATE'
5366
+ * - Set one time
5367
+ * - Set more times - throw error
5368
+ *
5369
+ * Note: [2]
5370
+ */
5371
+ var isBlockTypeSet = false;
4543
5372
  try {
4544
5373
  for (var listItems_2 = (e_3 = void 0, __values(listItems_3)), listItems_2_1 = listItems_2.next(); !listItems_2_1.done; listItems_2_1 = listItems_2.next()) {
4545
5374
  var listItem = listItems_2_1.value;
4546
5375
  var command = parseCommand(listItem, 'PIPELINE_TEMPLATE');
5376
+ // TODO [🍧][♓️] List commands and before apply order them
4547
5377
  switch (command.type) {
4548
5378
  // TODO: [🍧] Use here applyToPipelineJson and remove switch statement
4549
5379
  case 'BLOCK':
4550
- if (isBlockTypeChanged) {
5380
+ if (isBlockTypeSet) {
4551
5381
  throw new ParsingError('Block type is already defined in the prompt template. It can be defined only once.');
4552
5382
  }
4553
5383
  if (command.blockType === 'SAMPLE') {
4554
- console.error(new NotYetImplementedError('Block type SAMPLE is not implemented yet'));
5384
+ expectResultingParameterName();
5385
+ var parameter = pipelineJson.parameters.find(function (param) { return param.name === resultingParameterName; });
5386
+ if (parameter === undefined) {
5387
+ throw new UnexpectedError("Can not find parameter {".concat(resultingParameterName, "} to assign sample value"));
5388
+ }
5389
+ parameter.sampleValues = parameter.sampleValues || [];
5390
+ parameter.sampleValues.push(content);
4555
5391
  return "continue-templates";
4556
5392
  }
4557
5393
  if (command.blockType === 'KNOWLEDGE') {
4558
- console.error(new NotYetImplementedError('Knowledge is not implemented yet'));
5394
+ knowledgeCommandParser.applyToPipelineJson({
5395
+ type: 'KNOWLEDGE',
5396
+ source: content, // <- TODO: [🐝] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
5397
+ }, {
5398
+ pipelineJson: pipelineJson,
5399
+ templateJson: templateJson,
5400
+ });
4559
5401
  return "continue-templates";
4560
5402
  }
4561
5403
  if (command.blockType === 'ACTION') {
@@ -4566,35 +5408,37 @@
4566
5408
  console.error(new NotYetImplementedError('Instruments are not implemented yet'));
4567
5409
  return "continue-templates";
4568
5410
  }
4569
- blockType = command.blockType;
4570
- isBlockTypeChanged = true;
5411
+ expectResultingParameterName();
5412
+ templateJson.blockType = command.blockType;
5413
+ isBlockTypeSet = true; //<- Note: [2]
4571
5414
  break;
4572
5415
  case 'EXPECT_AMOUNT':
4573
5416
  // eslint-disable-next-line no-case-declarations
4574
5417
  var unit = command.unit.toLowerCase();
4575
- expectAmount[unit] = expectAmount[unit] || {};
5418
+ templateJson.expectations = templateJson.expectations || {};
5419
+ templateJson.expectations[unit] = templateJson.expectations[unit] || {};
4576
5420
  if (command.sign === 'MINIMUM' || command.sign === 'EXACTLY') {
4577
- if (expectAmount[unit].min !== undefined) {
4578
- throw new ParsingError("Already defined minumum ".concat(expectAmount[unit].min, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
5421
+ if (templateJson.expectations[unit].min !== undefined) {
5422
+ throw new ParsingError("Already defined minumum ".concat(templateJson.expectations[unit].min, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
4579
5423
  }
4580
- expectAmount[unit].min = command.amount;
5424
+ templateJson.expectations[unit].min = command.amount;
4581
5425
  } /* not else */
4582
5426
  if (command.sign === 'MAXIMUM' || command.sign === 'EXACTLY') {
4583
- if (expectAmount[unit].max !== undefined) {
4584
- throw new ParsingError("Already defined maximum ".concat(expectAmount[unit].max, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
5427
+ if (templateJson.expectations[unit].max !== undefined) {
5428
+ throw new ParsingError("Already defined maximum ".concat(templateJson.expectations[unit].max, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
4585
5429
  }
4586
- expectAmount[unit].max = command.amount;
5430
+ templateJson.expectations[unit].max = command.amount;
4587
5431
  }
4588
5432
  break;
4589
5433
  case 'EXPECT_FORMAT':
4590
- if (expectFormat !== undefined && command.format !== expectFormat) {
4591
- throw new ParsingError("Expect format is already defined to \"".concat(expectFormat, "\". Now you try to redefine it by \"").concat(command.format, "\"."));
5434
+ if (templateJson.expectFormat !== undefined && command.format !== templateJson.expectFormat) {
5435
+ throw new ParsingError(spaceTrim.spaceTrim("\n Expect format is already defined to \"".concat(templateJson.expectFormat, "\".\n Now you try to redefine it by \"").concat(command.format, "\".\n ")));
4592
5436
  }
4593
- expectFormat = command.format;
5437
+ templateJson.expectFormat = command.format;
4594
5438
  break;
4595
5439
  case 'JOKER':
4596
- jokers.push(command.parameterName);
4597
- dependentParameterNames.add(command.parameterName);
5440
+ templateJson.jokerParameterNames = templateJson.jokerParameterNames || [];
5441
+ templateJson.jokerParameterNames.push(command.parameterName);
4598
5442
  break;
4599
5443
  case 'MODEL':
4600
5444
  templateModelRequirements[command.key] = command.value;
@@ -4604,19 +5448,24 @@
4604
5448
  defineParam(command);
4605
5449
  break;
4606
5450
  case 'POSTPROCESS':
4607
- postprocessing.push(command.functionName);
5451
+ templateJson.postprocessingFunctionNames = templateJson.postprocessingFunctionNames || [];
5452
+ templateJson.postprocessingFunctionNames.push(command.functionName);
4608
5453
  break;
4609
5454
  case 'KNOWLEDGE':
4610
- console.error(new NotYetImplementedError('Knowledge is not implemented yet'));
5455
+ // TODO: [👙] The knowledge is maybe relevant for just this template
5456
+ knowledgeCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: templateJson });
4611
5457
  break;
4612
5458
  case 'ACTION':
5459
+ // TODO: [👙] The action is maybe relevant for just this template
4613
5460
  console.error(new NotYetImplementedError('Actions are not implemented yet'));
4614
5461
  break;
4615
5462
  case 'INSTRUMENT':
5463
+ // TODO: [👙] The instrument is maybe relevant for just this template
4616
5464
  console.error(new NotYetImplementedError('Instruments are not implemented yet'));
4617
5465
  break;
4618
5466
  case 'PERSONA':
4619
- console.error(new NotYetImplementedError('Personas are not implemented yet'));
5467
+ personaCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: templateJson });
5468
+ // <- Note: Prototype of [🍧] (remove this comment after full implementation)
4620
5469
  break;
4621
5470
  case 'BOILERPLATE':
4622
5471
  console.error(new ParsingError('BOILERPLATE command is only for testing purposes and should not be used in the .ptbk.md file'));
@@ -4634,70 +5483,29 @@
4634
5483
  }
4635
5484
  finally { if (e_3) throw e_3.error; }
4636
5485
  }
4637
- var _e = extractOneBlockFromMarkdown(section.content), language = _e.language, content = _e.content;
4638
- if (blockType === 'SCRIPT') {
5486
+ // TODO: [🍧] Should be done in BLOCK command
5487
+ if (templateJson.blockType === 'SCRIPT') {
4639
5488
  if (!language) {
4640
5489
  throw new ParsingError('You must specify the language of the script in the prompt template');
4641
5490
  }
4642
- else if (!SUPPORTED_SCRIPT_LANGUAGES.includes(language)) {
5491
+ if (!SUPPORTED_SCRIPT_LANGUAGES.includes(language)) {
4643
5492
  throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Script language ".concat(language, " is not supported.\n\n Supported languages are:\n ").concat(block(SUPPORTED_SCRIPT_LANGUAGES.join(', ')), "\n\n "); }));
4644
5493
  }
5494
+ templateJson.contentLanguage = language;
4645
5495
  }
4646
- var lastLine = section.content.split('\n').pop();
4647
- var match = /^->\s*\{(?<resultingParamName>[a-z0-9_]+)\}/im.exec(lastLine);
4648
- if (!match || match.groups === undefined || match.groups.resultingParamName === undefined) {
4649
- throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Each section must end with -> {parameterName}\n\n Invalid section:\n ".concat(block(
4650
- // TODO: Show code of invalid sections each time + DRY
4651
- section.content
4652
- .split('\n')
4653
- .map(function (line) { return " | ".concat(line); } /* <- TODO: [🚞] */)
4654
- .join('\n')), "\n "); }));
4655
- }
4656
- var resultingParameterName = match.groups.resultingParamName;
4657
- // TODO: [1] DRY description
4658
- var description_1 = section.content;
4659
- // Note: Remove codeblocks
4660
- description_1 = description_1.split(/^```.*^```/gms).join('');
4661
- //Note: Remove lists and return statement
4662
- description_1 = description_1.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join('');
4663
- description_1 = spaceTrim.spaceTrim(description_1);
4664
- if (description_1 === '') {
4665
- description_1 = undefined;
4666
- }
4667
- if (Object.keys(jokers).length === 0) {
4668
- jokers = undefined;
4669
- }
4670
- if (Object.keys(expectAmount).length === 0) {
4671
- expectAmount = undefined;
4672
- }
4673
- if (Object.keys(postprocessing).length === 0) {
4674
- postprocessing = undefined;
4675
- }
4676
- dependentParameterNames = union(dependentParameterNames, extractParametersFromPromptTemplate(__assign(__assign({}, section), { description: description_1, blockType: blockType, content: content })));
5496
+ // TODO: [🍧][❔] Should be done in BLOCK command
4677
5497
  if (templateModelRequirements.modelVariant === undefined) {
4678
5498
  templateModelRequirements.modelVariant = 'CHAT';
4679
5499
  }
4680
- dependentParameterNames = difference(dependentParameterNames, new Set(RESERVED_PARAMETER_NAMES));
4681
- var template = {
4682
- name: titleToName(section.title),
4683
- title: section.title,
4684
- description: description_1,
4685
- dependentParameterNames: Array.from(dependentParameterNames),
4686
- blockType: blockType,
4687
- jokers: jokers,
4688
- postprocessing: postprocessing,
4689
- expectations: expectAmount,
4690
- expectFormat: expectFormat,
4691
- modelRequirements: templateModelRequirements,
4692
- contentLanguage: blockType === 'SCRIPT' ? language : undefined,
4693
- content: content,
4694
- resultingParameterName: resultingParameterName,
4695
- };
4696
- if (blockType !== 'PROMPT_TEMPLATE') {
4697
- delete template.modelRequirements;
5500
+ templateJson.dependentParameterNames = Array.from(extractParametersFromPromptTemplate(templateJson));
5501
+ // TODO: [🍧][❔] Remove this condition - modelRequirements should be put here via BLOCK command not removed when PROMPT_TEMPLATE
5502
+ if (templateJson.blockType !== 'PROMPT_TEMPLATE') {
5503
+ delete templateJson.modelRequirements;
4698
5504
  }
5505
+ // TODO: [🍧] Make this better - for example each command parser can call and apply this
5506
+ templateJson.resultingParameterName = expectResultingParameterName( /* <- Note: This is once more redundant */);
4699
5507
  // TODO: [🍧] What actually about preparation and pushing the block into `promptTemplates`
4700
- pipelineJson.promptTemplates.push(template /* <- !!! */);
5508
+ pipelineJson.promptTemplates.push(templateJson);
4701
5509
  };
4702
5510
  try {
4703
5511
  // =============================================================
@@ -4764,14 +5572,17 @@
4764
5572
  * TODO: Use spaceTrim more effectively
4765
5573
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
4766
5574
  * TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
5575
+ * TODO: [♈] Probbably move expectations from templates to parameters
5576
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
4767
5577
  */
4768
5578
 
4769
5579
  /**
4770
- * Compile promptbook from string (markdown) format to JSON format
5580
+ * Compile pipeline from string (markdown) format to JSON format
4771
5581
  *
4772
- * Note: There are two similar functions:
5582
+ * Note: There are 3 similar functions:
4773
5583
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
4774
5584
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
5585
+ * - `preparePipeline` - just one step in the compilation process
4775
5586
  *
4776
5587
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
4777
5588
  * @param options - Options and tools for the compilation
@@ -4782,22 +5593,18 @@
4782
5593
  * Note: This function acts as compilation process
4783
5594
  */
4784
5595
  function pipelineStringToJson(pipelineString, options) {
4785
- if (options === void 0) { options = {}; }
5596
+ if (options === void 0) { options = { llmTools: null }; }
4786
5597
  return __awaiter(this, void 0, void 0, function () {
4787
- var llmTools, pipelineJson, knowledge;
5598
+ var llmTools, pipelineJson;
4788
5599
  return __generator(this, function (_a) {
4789
5600
  switch (_a.label) {
4790
5601
  case 0:
4791
5602
  llmTools = options.llmTools;
4792
5603
  pipelineJson = pipelineStringToJsonSync(pipelineString);
4793
- if (!llmTools) return [3 /*break*/, 2];
4794
- return [4 /*yield*/, prepareKnowledgeFromMarkdown({
4795
- content: 'Roses are red, violets are blue, programmers use Promptbook, users too',
4796
- llmTools: llmTools,
4797
- })];
5604
+ if (!(llmTools !== null)) return [3 /*break*/, 2];
5605
+ return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
4798
5606
  case 1:
4799
- knowledge = _a.sent();
4800
- pipelineJson = __assign(__assign({}, pipelineJson), { knowledge: __spreadArray(__spreadArray([], __read((pipelineJson.knowledge || [])), false), __read(knowledge), false) });
5607
+ pipelineJson = _a.sent();
4801
5608
  _a.label = 2;
4802
5609
  case 2: return [2 /*return*/, pipelineJson];
4803
5610
  }
@@ -4806,12 +5613,13 @@
4806
5613
  }
4807
5614
  /**
4808
5615
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
5616
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
4809
5617
  */
4810
5618
 
4811
5619
  /**
4812
5620
  * Add or modify an auto-generated section in a markdown file
4813
5621
  *
4814
- * @private within the package
5622
+ * @private within the repository
4815
5623
  */
4816
5624
  function addAutoGeneratedSection(content, options) {
4817
5625
  var sectionName = options.sectionName, sectionContent = options.sectionContent;
@@ -4823,7 +5631,10 @@
4823
5631
  }
4824
5632
  var placeForSection = removeContentComments(content).match(/^##.*$/im);
4825
5633
  if (!placeForSection) {
4826
- throw new Error("No place where to put the section <!--".concat(sectionName, "-->"));
5634
+ throw new ParsingError(
5635
+ // <- [🧠] Maybe something better than `ParsingError`
5636
+ "No place where to put the section <!--".concat(sectionName, "-->"));
5637
+ // <- [🚞]
4827
5638
  }
4828
5639
  var _a = __read(placeForSection, 1), heading = _a[0];
4829
5640
  return content.replace(heading, "<!--".concat(sectionName, "-->\n").concat(warningLine, "\n").concat(sectionContent, "\n<!--/").concat(sectionName, "-->\n\n").concat(heading));
@@ -4980,6 +5791,27 @@
4980
5791
  * TODO: [🕌] When more than 2 functionalities, split into separate functions
4981
5792
  */
4982
5793
 
5794
+ /**
5795
+ * Stringify the PipelineJson with proper formatting
5796
+ *
5797
+ * Note: [0] It can be used for more JSON types like whole collection of pipelines, single knowledge piece, etc.
5798
+ * Note: In contrast to JSON.stringify, this function ensures that **embedding index** is on single line
5799
+ */
5800
+ function stringifyPipelineJson(pipeline) {
5801
+ var pipelineJsonStringified = JSON.stringify(pipeline, null, 4);
5802
+ for (var i = 0; i < LOOP_LIMIT; i++) {
5803
+ pipelineJsonStringified = pipelineJsonStringified.replace(/(-?0\.\d+),[\n\s]+(-?0\.\d+)/gms, "$1".concat(REPLACING_NONCE, "$2"));
5804
+ }
5805
+ pipelineJsonStringified = pipelineJsonStringified.split(REPLACING_NONCE).join(', ');
5806
+ pipelineJsonStringified += '\n';
5807
+ return pipelineJsonStringified;
5808
+ }
5809
+ /**
5810
+ * TODO: !!!! Not Working propperly @see https://promptbook.studio/samples/mixed-knowledge.ptbk.md
5811
+ * TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
5812
+ * TODO: [🧠] Maybe more elegant solution than replacing via regex
5813
+ */
5814
+
4983
5815
  /**
4984
5816
  * This error indicates that the pipeline collection cannot be propperly loaded
4985
5817
  */
@@ -4994,6 +5826,20 @@
4994
5826
  return CollectionError;
4995
5827
  }(Error));
4996
5828
 
5829
+ /**
5830
+ * This error type indicates that the version of the pipeline is not matching the expected version
5831
+ */
5832
+ var VersionMismatchError = /** @class */ (function (_super) {
5833
+ __extends(VersionMismatchError, _super);
5834
+ function VersionMismatchError(message, expectedVersion) {
5835
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n This error indicates that there is error with compatibility\n\n Expected version:\n ").concat(expectedVersion, "\n\n Current version:\n ").concat(PROMPTBOOK_VERSION, "\n\n "); })) || this;
5836
+ _this.name = 'UnexpectedError';
5837
+ Object.setPrototypeOf(_this, VersionMismatchError.prototype);
5838
+ return _this;
5839
+ }
5840
+ return VersionMismatchError;
5841
+ }(Error));
5842
+
4997
5843
  /**
4998
5844
  * Pretty print an embedding vector for logging
4999
5845
  */
@@ -5081,6 +5927,9 @@
5081
5927
  };
5082
5928
  return SimplePromptInterfaceTools;
5083
5929
  }());
5930
+ /**
5931
+ * Note: [🔵] This code should never be published outside of `@promptbook/browser`
5932
+ */
5084
5933
 
5085
5934
  /**
5086
5935
  * Default options for generating an execution report string
@@ -5093,7 +5942,7 @@
5093
5942
  /**
5094
5943
  * Format either small or big number
5095
5944
  *
5096
- * @private within the package
5945
+ * @private within the repository
5097
5946
  */
5098
5947
  function formatNumber(value) {
5099
5948
  if (value === 0) {
@@ -5113,7 +5962,7 @@
5113
5962
  /**
5114
5963
  * Create a markdown table from a 2D array of strings
5115
5964
  *
5116
- * @private within the package
5965
+ * @private within the repository
5117
5966
  */
5118
5967
  function createMarkdownTable(table) {
5119
5968
  var columnWidths = table.reduce(function (widths, row) {
@@ -5141,7 +5990,7 @@
5141
5990
  /**
5142
5991
  * Function createMarkdownChart will draw a chart in markdown from ⬛+🟦 tiles
5143
5992
  *
5144
- * @private within the package
5993
+ * @private within the repository
5145
5994
  */
5146
5995
  function createMarkdownChart(options) {
5147
5996
  var e_1, _a;
@@ -5194,7 +6043,7 @@
5194
6043
  /**
5195
6044
  * Count the duration of working time
5196
6045
  *
5197
- * @private within the package
6046
+ * @private within the repository
5198
6047
  */
5199
6048
  function countWorkingDuration(items) {
5200
6049
  var e_1, _a;
@@ -5406,6 +6255,7 @@
5406
6255
  exports.ReferenceError = ReferenceError$1;
5407
6256
  exports.SimplePromptInterfaceTools = SimplePromptInterfaceTools;
5408
6257
  exports.UnexpectedError = UnexpectedError;
6258
+ exports.VersionMismatchError = VersionMismatchError;
5409
6259
  exports.addUsage = addUsage;
5410
6260
  exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
5411
6261
  exports.checkExpectations = checkExpectations;
@@ -5423,7 +6273,10 @@
5423
6273
  exports.pipelineStringToJson = pipelineStringToJson;
5424
6274
  exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
5425
6275
  exports.prepareKnowledgeFromMarkdown = prepareKnowledgeFromMarkdown;
6276
+ exports.preparePipeline = preparePipeline;
5426
6277
  exports.prettifyPipelineString = prettifyPipelineString;
6278
+ exports.stringifyPipelineJson = stringifyPipelineJson;
6279
+ exports.unpreparePipeline = unpreparePipeline;
5427
6280
  exports.usageToWorktime = usageToWorktime;
5428
6281
  exports.validatePipeline = validatePipeline;
5429
6282