@promptbook/core 0.61.0-2 → 0.61.0-20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (359) hide show
  1. package/esm/index.es.js +1737 -942
  2. package/esm/index.es.js.map +1 -1
  3. package/esm/typings/promptbook-collection/index.d.ts +200 -30
  4. package/esm/typings/src/_packages/core.index.d.ts +7 -2
  5. package/esm/typings/src/_packages/execute-javascript.index.d.ts +1 -1
  6. package/esm/typings/src/_packages/node.index.d.ts +2 -1
  7. package/esm/typings/src/_packages/types.index.d.ts +11 -7
  8. package/esm/typings/src/_packages/utils.index.d.ts +8 -6
  9. package/esm/typings/src/cli/cli-commands/hello.d.ts +3 -0
  10. package/esm/typings/src/cli/cli-commands/make.d.ts +3 -0
  11. package/esm/typings/src/cli/cli-commands/prettify.d.ts +3 -0
  12. package/esm/typings/src/cli/promptbookCli.d.ts +1 -0
  13. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  14. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +10 -10
  15. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -7
  16. package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +2 -2
  17. package/esm/typings/src/commands/KNOWLEDGE/KnowledgeCommand.d.ts +2 -3
  18. package/esm/typings/src/commands/MODEL/ModelCommand.d.ts +2 -2
  19. package/esm/typings/src/commands/PERSONA/PersonaCommand.d.ts +4 -2
  20. package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -0
  21. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +42 -14
  22. package/esm/typings/src/config.d.ts +38 -1
  23. package/esm/typings/src/config.test.d.ts +4 -0
  24. package/esm/typings/src/conversion/pipelineJsonToString.d.ts +1 -1
  25. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +9 -5
  26. package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +5 -3
  27. package/esm/typings/src/conversion/prettify/PrettifyOptions.d.ts +2 -2
  28. package/esm/typings/src/conversion/utils/extractParametersFromPromptTemplate.d.ts +2 -2
  29. package/esm/typings/src/conversion/utils/renameParameter.d.ts +3 -3
  30. package/esm/typings/src/conversion/utils/stringifyPipelineJson.d.ts +13 -0
  31. package/esm/typings/src/conversion/validation/_importPipeline.d.ts +12 -3
  32. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +10 -1
  33. package/esm/typings/src/errors/EnvironmentMismatchError.d.ts +7 -0
  34. package/esm/typings/src/errors/LimitReachedError.d.ts +7 -0
  35. package/esm/typings/src/errors/VersionMismatchError.d.ts +8 -0
  36. package/esm/typings/src/execution/CommonExecutionToolsOptions.d.ts +1 -0
  37. package/esm/typings/src/execution/LlmExecutionTools.d.ts +10 -8
  38. package/esm/typings/src/execution/PipelineExecutor.d.ts +9 -8
  39. package/esm/typings/src/execution/PromptResult.d.ts +28 -58
  40. package/esm/typings/src/execution/PromptResultUsage.d.ts +26 -0
  41. package/esm/typings/src/execution/ScriptExecutionTools.d.ts +5 -4
  42. package/esm/typings/src/execution/UncertainNumber.d.ts +18 -0
  43. package/esm/typings/src/execution/UserInterfaceTools.d.ts +5 -5
  44. package/esm/typings/src/execution/createPipelineExecutor.d.ts +19 -3
  45. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.d.ts +2 -2
  46. package/esm/typings/src/execution/utils/addUsage.d.ts +58 -2
  47. package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +1 -1
  48. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  49. package/esm/typings/src/execution/utils/uncertainNumber.d.ts +1 -1
  50. package/esm/typings/src/execution/utils/usageToWorktime.d.ts +2 -2
  51. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +1 -1
  52. package/esm/typings/src/formats/csv/ListFormatDefinition.d.ts +2 -1
  53. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +2 -1
  54. package/esm/typings/src/formats/list/ListFormatDefinition.d.ts +2 -1
  55. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +1 -0
  56. package/esm/typings/src/knowledge/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +1 -1
  57. package/esm/typings/src/knowledge/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -0
  58. package/esm/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +27 -0
  59. package/esm/typings/src/knowledge/prepare-knowledge/_common/prepareKnowledgePieces.d.ts +18 -0
  60. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +9 -23
  61. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts +3 -0
  62. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +8 -14
  63. package/esm/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +30 -0
  64. package/esm/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +10 -0
  65. package/esm/typings/src/llm-providers/_common/getLlmToolsForTestingAndScriptsAndPlayground.d.ts +11 -0
  66. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +29 -0
  67. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +10 -0
  68. package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +16 -0
  69. package/esm/typings/src/llm-providers/_common/utils/count-total-cost/LlmExecutionToolsWithTotalCost.d.ts +11 -0
  70. package/esm/typings/src/llm-providers/_common/utils/count-total-cost/countTotalCost.d.ts +14 -0
  71. package/esm/typings/src/llm-providers/_common/utils/count-total-cost/limitTotalCost.d.ts +32 -0
  72. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +3 -2
  73. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
  74. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +3 -3
  75. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +3 -0
  76. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +5 -4
  77. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +5 -5
  78. package/esm/typings/src/llm-providers/azure-openai/playground/playground.d.ts +3 -0
  79. package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +3 -0
  80. package/esm/typings/src/llm-providers/langtail/LangtailExecutionToolsOptions.d.ts +1 -1
  81. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +5 -5
  82. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +7 -7
  83. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +12 -8
  84. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +7 -6
  85. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +1 -1
  86. package/esm/typings/src/llm-providers/openai/computeOpenaiUsage.d.ts +1 -1
  87. package/esm/typings/src/llm-providers/openai/computeUsage.d.ts +2 -2
  88. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +3 -2
  89. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +3 -0
  90. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +9 -7
  91. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionToolsOptions.d.ts +1 -1
  92. package/esm/typings/src/personas/preparePersona.d.ts +15 -0
  93. package/esm/typings/src/prepare/PrepareOptions.d.ts +22 -0
  94. package/esm/typings/src/prepare/isPipelinePrepared.d.ts +10 -0
  95. package/esm/typings/src/prepare/preparePipeline.d.ts +18 -0
  96. package/esm/typings/src/prepare/unpreparePipeline.d.ts +9 -0
  97. package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +1 -1
  98. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  99. package/esm/typings/src/scripting/javascript/utils/unknownToString.d.ts +2 -1
  100. package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +25 -0
  101. package/esm/typings/src/storage/_common/PromptbookStorage.test-type.d.ts +5 -0
  102. package/esm/typings/src/storage/files-storage/FilesStorage.d.ts +30 -0
  103. package/esm/typings/src/storage/files-storage/FilesStorageOptions.d.ts +13 -0
  104. package/esm/typings/src/storage/files-storage/utils/nameToSubfolderPath.d.ts +7 -0
  105. package/esm/typings/src/storage/files-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
  106. package/esm/typings/src/storage/local-storage/getLocalStorage.d.ts +9 -0
  107. package/esm/typings/src/storage/local-storage/getSessionStorage.d.ts +9 -0
  108. package/esm/typings/src/storage/memory/MemoryStorage.d.ts +34 -0
  109. package/esm/typings/src/storage/utils/PrefixStorage.d.ts +26 -0
  110. package/esm/typings/src/storage/utils/makePromptbookStorageFromWebStorage.d.ts +11 -0
  111. package/esm/typings/src/types/ModelRequirements.d.ts +71 -11
  112. package/esm/typings/src/types/ModelVariant.d.ts +14 -0
  113. package/esm/typings/src/types/PipelineJson/Expectations.d.ts +6 -2
  114. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +67 -0
  115. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +24 -0
  116. package/esm/typings/src/types/PipelineJson/LlmTemplateJson.d.ts +9 -1
  117. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +38 -0
  118. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +29 -4
  119. package/esm/typings/src/types/PipelineJson/PreparationJson.d.ts +21 -0
  120. package/esm/typings/src/types/PipelineJson/PromptDialogJson.d.ts +5 -0
  121. package/esm/typings/src/types/PipelineJson/PromptTemplateJson.d.ts +5 -1
  122. package/esm/typings/src/types/PipelineJson/PromptTemplateJsonCommon.d.ts +13 -4
  123. package/esm/typings/src/types/PipelineJson/PromptTemplateParameterJson.d.ts +11 -2
  124. package/esm/typings/src/types/PipelineJson/ScriptJson.d.ts +2 -0
  125. package/esm/typings/src/types/PipelineJson/SimpleTemplateJson.d.ts +2 -0
  126. package/esm/typings/src/types/PipelineString.d.ts +1 -1
  127. package/esm/typings/src/types/Prompt.d.ts +54 -10
  128. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +4 -4
  129. package/esm/typings/src/types/execution-report/ExecutionReportStringOptions.d.ts +3 -3
  130. package/esm/typings/src/types/execution-report/countWorkingDuration.d.ts +1 -1
  131. package/esm/typings/src/types/typeAliases.d.ts +114 -15
  132. package/esm/typings/src/utils/FromtoItems.d.ts +3 -3
  133. package/esm/typings/src/utils/currentDate.d.ts +7 -0
  134. package/esm/typings/src/utils/deepClone.d.ts +9 -0
  135. package/esm/typings/src/utils/deepFreeze.d.ts +22 -0
  136. package/esm/typings/src/utils/emojis.d.ts +2 -2
  137. package/esm/typings/src/utils/extractParameters.d.ts +2 -2
  138. package/esm/typings/src/utils/formatNumber.d.ts +1 -1
  139. package/esm/typings/src/utils/isRunningInWhatever.d.ts +3 -0
  140. package/esm/typings/src/utils/markdown/addAutoGeneratedSection.d.ts +3 -3
  141. package/esm/typings/src/utils/markdown/createMarkdownChart.d.ts +7 -10
  142. package/esm/typings/src/utils/markdown/createMarkdownTable.d.ts +1 -1
  143. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.d.ts +1 -0
  144. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +9 -2
  145. package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +1 -1
  146. package/esm/typings/src/utils/markdown/parseMarkdownSection.d.ts +3 -3
  147. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -1
  148. package/esm/typings/src/utils/organization/TODO_USE.d.ts +12 -0
  149. package/esm/typings/src/utils/organization/TODO_any.d.ts +6 -0
  150. package/esm/typings/src/utils/organization/TODO_object.d.ts +6 -0
  151. package/esm/typings/src/utils/organization/TODO_unknown.d.ts +6 -0
  152. package/esm/typings/src/utils/organization/___.d.ts +6 -0
  153. package/esm/typings/src/utils/{just.d.ts → organization/just.d.ts} +5 -1
  154. package/esm/typings/src/utils/organization/keepUnused.d.ts +16 -0
  155. package/esm/typings/src/utils/organization/really_any.d.ts +6 -0
  156. package/esm/typings/src/utils/organization/really_unknown.d.ts +6 -0
  157. package/esm/typings/src/utils/random/randomSeed.d.ts +7 -0
  158. package/{umd/typings/src/execution → esm/typings/src}/utils/replaceParameters.d.ts +2 -4
  159. package/esm/typings/src/utils/replaceParameters.test.d.ts +1 -0
  160. package/esm/typings/src/utils/sets/difference.d.ts +3 -0
  161. package/esm/typings/src/utils/validators/email/isValidEmail.d.ts +2 -1
  162. package/esm/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -1
  163. package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +2 -1
  164. package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +2 -1
  165. package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +2 -1
  166. package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +2 -1
  167. package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +2 -1
  168. package/esm/typings/src/utils/validators/uuid/isValidUuid.d.ts +2 -1
  169. package/package.json +1 -1
  170. package/umd/index.umd.js +1740 -941
  171. package/umd/index.umd.js.map +1 -1
  172. package/umd/typings/promptbook-collection/index.d.ts +200 -30
  173. package/umd/typings/src/_packages/core.index.d.ts +7 -2
  174. package/umd/typings/src/_packages/execute-javascript.index.d.ts +1 -1
  175. package/umd/typings/src/_packages/node.index.d.ts +2 -1
  176. package/umd/typings/src/_packages/types.index.d.ts +11 -7
  177. package/umd/typings/src/_packages/utils.index.d.ts +8 -6
  178. package/umd/typings/src/cli/cli-commands/hello.d.ts +3 -0
  179. package/umd/typings/src/cli/cli-commands/make.d.ts +3 -0
  180. package/umd/typings/src/cli/cli-commands/prettify.d.ts +3 -0
  181. package/umd/typings/src/cli/promptbookCli.d.ts +1 -0
  182. package/umd/typings/src/collection/PipelineCollection.d.ts +1 -1
  183. package/umd/typings/src/collection/SimplePipelineCollection.d.ts +10 -10
  184. package/umd/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -7
  185. package/umd/typings/src/collection/constructors/createCollectionFromUrl.d.ts +2 -2
  186. package/umd/typings/src/commands/KNOWLEDGE/KnowledgeCommand.d.ts +2 -3
  187. package/umd/typings/src/commands/MODEL/ModelCommand.d.ts +2 -2
  188. package/umd/typings/src/commands/PERSONA/PersonaCommand.d.ts +4 -2
  189. package/umd/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -0
  190. package/umd/typings/src/commands/_common/types/CommandParser.d.ts +42 -14
  191. package/umd/typings/src/config.d.ts +38 -1
  192. package/umd/typings/src/config.test.d.ts +4 -0
  193. package/umd/typings/src/conversion/pipelineJsonToString.d.ts +1 -1
  194. package/umd/typings/src/conversion/pipelineStringToJson.d.ts +9 -5
  195. package/umd/typings/src/conversion/pipelineStringToJsonSync.d.ts +5 -3
  196. package/umd/typings/src/conversion/prettify/PrettifyOptions.d.ts +2 -2
  197. package/umd/typings/src/conversion/utils/extractParametersFromPromptTemplate.d.ts +2 -2
  198. package/umd/typings/src/conversion/utils/renameParameter.d.ts +3 -3
  199. package/umd/typings/src/conversion/utils/stringifyPipelineJson.d.ts +13 -0
  200. package/umd/typings/src/conversion/utils/stringifyPipelineJson.test.d.ts +1 -0
  201. package/umd/typings/src/conversion/validation/_importPipeline.d.ts +12 -3
  202. package/umd/typings/src/conversion/validation/validatePipeline.d.ts +10 -1
  203. package/umd/typings/src/errors/EnvironmentMismatchError.d.ts +7 -0
  204. package/umd/typings/src/errors/LimitReachedError.d.ts +7 -0
  205. package/umd/typings/src/errors/VersionMismatchError.d.ts +8 -0
  206. package/umd/typings/src/execution/CommonExecutionToolsOptions.d.ts +1 -0
  207. package/umd/typings/src/execution/LlmExecutionTools.d.ts +10 -8
  208. package/umd/typings/src/execution/PipelineExecutor.d.ts +9 -8
  209. package/umd/typings/src/execution/PromptResult.d.ts +28 -58
  210. package/umd/typings/src/execution/PromptResultUsage.d.ts +26 -0
  211. package/umd/typings/src/execution/ScriptExecutionTools.d.ts +5 -4
  212. package/umd/typings/src/execution/UncertainNumber.d.ts +18 -0
  213. package/umd/typings/src/execution/UserInterfaceTools.d.ts +5 -5
  214. package/umd/typings/src/execution/createPipelineExecutor.d.ts +19 -3
  215. package/umd/typings/src/execution/translation/automatic-translate/automatic-translators/TranslatorOptions.d.ts +2 -2
  216. package/umd/typings/src/execution/utils/addUsage.d.ts +58 -2
  217. package/umd/typings/src/execution/utils/computeUsageCounts.d.ts +1 -1
  218. package/umd/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  219. package/umd/typings/src/execution/utils/uncertainNumber.d.ts +1 -1
  220. package/umd/typings/src/execution/utils/usageToWorktime.d.ts +2 -2
  221. package/umd/typings/src/formats/_common/FormatDefinition.d.ts +1 -1
  222. package/umd/typings/src/formats/csv/ListFormatDefinition.d.ts +2 -1
  223. package/umd/typings/src/formats/json/JsonFormatDefinition.d.ts +2 -1
  224. package/umd/typings/src/formats/list/ListFormatDefinition.d.ts +2 -1
  225. package/umd/typings/src/formats/xml/XmlFormatDefinition.d.ts +1 -0
  226. package/umd/typings/src/knowledge/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +1 -1
  227. package/umd/typings/src/knowledge/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -0
  228. package/umd/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +27 -0
  229. package/umd/typings/src/knowledge/prepare-knowledge/_common/prepareKnowledgePieces.d.ts +18 -0
  230. package/umd/typings/src/knowledge/prepare-knowledge/_common/prepareKnowledgePieces.test.d.ts +1 -0
  231. package/umd/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +9 -23
  232. package/umd/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts +3 -0
  233. package/umd/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +8 -14
  234. package/umd/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +30 -0
  235. package/umd/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +10 -0
  236. package/umd/typings/src/llm-providers/_common/getLlmToolsForTestingAndScriptsAndPlayground.d.ts +11 -0
  237. package/umd/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +29 -0
  238. package/umd/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +10 -0
  239. package/umd/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +16 -0
  240. package/umd/typings/src/llm-providers/_common/utils/count-total-cost/LlmExecutionToolsWithTotalCost.d.ts +11 -0
  241. package/umd/typings/src/llm-providers/_common/utils/count-total-cost/countTotalCost.d.ts +14 -0
  242. package/umd/typings/src/llm-providers/_common/utils/count-total-cost/limitTotalCost.d.ts +32 -0
  243. package/umd/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +3 -2
  244. package/umd/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
  245. package/umd/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +3 -3
  246. package/umd/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +3 -0
  247. package/umd/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +5 -4
  248. package/umd/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +5 -5
  249. package/umd/typings/src/llm-providers/azure-openai/playground/playground.d.ts +3 -0
  250. package/umd/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +3 -0
  251. package/umd/typings/src/llm-providers/langtail/LangtailExecutionToolsOptions.d.ts +1 -1
  252. package/umd/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +5 -5
  253. package/umd/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +7 -7
  254. package/umd/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +12 -8
  255. package/umd/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +7 -6
  256. package/umd/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +1 -1
  257. package/umd/typings/src/llm-providers/openai/computeOpenaiUsage.d.ts +1 -1
  258. package/umd/typings/src/llm-providers/openai/computeUsage.d.ts +2 -2
  259. package/umd/typings/src/llm-providers/openai/openai-models.d.ts +3 -2
  260. package/umd/typings/src/llm-providers/openai/playground/playground.d.ts +3 -0
  261. package/umd/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +9 -7
  262. package/umd/typings/src/llm-providers/remote/RemoteLlmExecutionToolsOptions.d.ts +1 -1
  263. package/umd/typings/src/personas/preparePersona.d.ts +15 -0
  264. package/umd/typings/src/personas/preparePersona.test.d.ts +1 -0
  265. package/umd/typings/src/prepare/PrepareOptions.d.ts +22 -0
  266. package/umd/typings/src/prepare/isPipelinePrepared.d.ts +10 -0
  267. package/umd/typings/src/prepare/isPipelinePrepared.test.d.ts +1 -0
  268. package/umd/typings/src/prepare/preparePipeline.d.ts +18 -0
  269. package/umd/typings/src/prepare/unpreparePipeline.d.ts +9 -0
  270. package/umd/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +1 -1
  271. package/umd/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  272. package/umd/typings/src/scripting/javascript/utils/unknownToString.d.ts +2 -1
  273. package/umd/typings/src/storage/_common/PromptbookStorage.d.ts +25 -0
  274. package/umd/typings/src/storage/_common/PromptbookStorage.test-type.d.ts +5 -0
  275. package/umd/typings/src/storage/files-storage/FilesStorage.d.ts +30 -0
  276. package/umd/typings/src/storage/files-storage/FilesStorageOptions.d.ts +13 -0
  277. package/umd/typings/src/storage/files-storage/utils/nameToSubfolderPath.d.ts +7 -0
  278. package/umd/typings/src/storage/files-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
  279. package/umd/typings/src/storage/local-storage/getLocalStorage.d.ts +9 -0
  280. package/umd/typings/src/storage/local-storage/getSessionStorage.d.ts +9 -0
  281. package/umd/typings/src/storage/memory/MemoryStorage.d.ts +34 -0
  282. package/umd/typings/src/storage/utils/PrefixStorage.d.ts +26 -0
  283. package/umd/typings/src/storage/utils/makePromptbookStorageFromWebStorage.d.ts +11 -0
  284. package/umd/typings/src/types/ModelRequirements.d.ts +71 -11
  285. package/umd/typings/src/types/ModelVariant.d.ts +14 -0
  286. package/umd/typings/src/types/PipelineJson/Expectations.d.ts +6 -2
  287. package/umd/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +67 -0
  288. package/umd/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +24 -0
  289. package/umd/typings/src/types/PipelineJson/LlmTemplateJson.d.ts +9 -1
  290. package/umd/typings/src/types/PipelineJson/PersonaJson.d.ts +38 -0
  291. package/umd/typings/src/types/PipelineJson/PipelineJson.d.ts +29 -4
  292. package/umd/typings/src/types/PipelineJson/PreparationJson.d.ts +21 -0
  293. package/umd/typings/src/types/PipelineJson/PromptDialogJson.d.ts +5 -0
  294. package/umd/typings/src/types/PipelineJson/PromptTemplateJson.d.ts +5 -1
  295. package/umd/typings/src/types/PipelineJson/PromptTemplateJsonCommon.d.ts +13 -4
  296. package/umd/typings/src/types/PipelineJson/PromptTemplateParameterJson.d.ts +11 -2
  297. package/umd/typings/src/types/PipelineJson/ScriptJson.d.ts +2 -0
  298. package/umd/typings/src/types/PipelineJson/SimpleTemplateJson.d.ts +2 -0
  299. package/umd/typings/src/types/PipelineString.d.ts +1 -1
  300. package/umd/typings/src/types/Prompt.d.ts +54 -10
  301. package/umd/typings/src/types/execution-report/ExecutionReportJson.d.ts +4 -4
  302. package/umd/typings/src/types/execution-report/ExecutionReportStringOptions.d.ts +3 -3
  303. package/umd/typings/src/types/execution-report/countWorkingDuration.d.ts +1 -1
  304. package/umd/typings/src/types/typeAliases.d.ts +114 -15
  305. package/umd/typings/src/utils/FromtoItems.d.ts +3 -3
  306. package/umd/typings/src/utils/currentDate.d.ts +7 -0
  307. package/umd/typings/src/utils/deepClone.d.ts +9 -0
  308. package/umd/typings/src/utils/deepFreeze.d.ts +22 -0
  309. package/umd/typings/src/utils/emojis.d.ts +2 -2
  310. package/umd/typings/src/utils/extractParameters.d.ts +2 -2
  311. package/umd/typings/src/utils/formatNumber.d.ts +1 -1
  312. package/umd/typings/src/utils/isRunningInWhatever.d.ts +3 -0
  313. package/umd/typings/src/utils/markdown/addAutoGeneratedSection.d.ts +3 -3
  314. package/umd/typings/src/utils/markdown/createMarkdownChart.d.ts +7 -10
  315. package/umd/typings/src/utils/markdown/createMarkdownTable.d.ts +1 -1
  316. package/umd/typings/src/utils/markdown/extractAllBlocksFromMarkdown-real.test.d.ts +1 -0
  317. package/umd/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +9 -2
  318. package/umd/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +1 -1
  319. package/umd/typings/src/utils/markdown/parseMarkdownSection.d.ts +3 -3
  320. package/umd/typings/src/utils/normalization/parseKeywords.d.ts +2 -1
  321. package/umd/typings/src/utils/organization/TODO_USE.d.ts +12 -0
  322. package/umd/typings/src/utils/organization/TODO_any.d.ts +6 -0
  323. package/umd/typings/src/utils/organization/TODO_object.d.ts +6 -0
  324. package/umd/typings/src/utils/organization/TODO_unknown.d.ts +6 -0
  325. package/umd/typings/src/utils/organization/___.d.ts +6 -0
  326. package/umd/typings/src/utils/{just.d.ts → organization/just.d.ts} +5 -1
  327. package/umd/typings/src/utils/organization/keepUnused.d.ts +16 -0
  328. package/umd/typings/src/utils/organization/really_any.d.ts +6 -0
  329. package/umd/typings/src/utils/organization/really_unknown.d.ts +6 -0
  330. package/umd/typings/src/utils/random/randomSeed.d.ts +7 -0
  331. package/{esm/typings/src/execution → umd/typings/src}/utils/replaceParameters.d.ts +2 -4
  332. package/umd/typings/src/utils/replaceParameters.test.d.ts +1 -0
  333. package/umd/typings/src/utils/sets/difference.d.ts +3 -0
  334. package/umd/typings/src/utils/validators/email/isValidEmail.d.ts +2 -1
  335. package/umd/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -1
  336. package/umd/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +2 -1
  337. package/umd/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +2 -1
  338. package/umd/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +2 -1
  339. package/umd/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +2 -1
  340. package/umd/typings/src/utils/validators/url/isValidUrl.d.ts +2 -1
  341. package/umd/typings/src/utils/validators/uuid/isValidUuid.d.ts +2 -1
  342. package/esm/typings/src/collection/constructors/justTestFsImport.d.ts +0 -7
  343. package/esm/typings/src/personas/personaToModelRequirements.d.ts +0 -6
  344. package/esm/typings/src/types/Parameters.d.ts +0 -14
  345. package/esm/typings/src/types/PipelineJson/KnowledgeJson.d.ts +0 -6
  346. package/esm/typings/src/types/PipelineJson/MaterialKnowledgePieceJson.d.ts +0 -28
  347. package/umd/typings/src/collection/constructors/justTestFsImport.d.ts +0 -7
  348. package/umd/typings/src/personas/personaToModelRequirements.d.ts +0 -6
  349. package/umd/typings/src/types/Parameters.d.ts +0 -14
  350. package/umd/typings/src/types/PipelineJson/KnowledgeJson.d.ts +0 -6
  351. package/umd/typings/src/types/PipelineJson/MaterialKnowledgePieceJson.d.ts +0 -28
  352. /package/esm/typings/src/_packages/{markdown-utils.d.ts → markdown-utils.index.d.ts} +0 -0
  353. /package/esm/typings/src/{execution/addPromptResultUsage.test.d.ts → conversion/utils/stringifyPipelineJson.test.d.ts} +0 -0
  354. /package/esm/typings/src/{execution/utils/replaceParameters.test.d.ts → knowledge/prepare-knowledge/_common/prepareKnowledgePieces.test.d.ts} +0 -0
  355. /package/{umd/typings/src/execution/addPromptResultUsage.test.d.ts → esm/typings/src/personas/preparePersona.test.d.ts} +0 -0
  356. /package/esm/typings/src/{utils/postprocessing → postprocessing/utils}/extractBlock.d.ts +0 -0
  357. /package/{umd/typings/src/execution/utils/replaceParameters.test.d.ts → esm/typings/src/prepare/isPipelinePrepared.test.d.ts} +0 -0
  358. /package/umd/typings/src/_packages/{markdown-utils.d.ts → markdown-utils.index.d.ts} +0 -0
  359. /package/umd/typings/src/{utils/postprocessing → postprocessing/utils}/extractBlock.d.ts +0 -0
package/umd/index.umd.js CHANGED
@@ -258,7 +258,7 @@
258
258
  /* Note: Not using:> name, */
259
259
  title_1 = promptTemplate.title, description_1 = promptTemplate.description,
260
260
  /* Note: dependentParameterNames, */
261
- jokers = promptTemplate.jokers, blockType = promptTemplate.blockType, content = promptTemplate.content, postprocessing = promptTemplate.postprocessing, expectations = promptTemplate.expectations, expectFormat = promptTemplate.expectFormat, resultingParameterName = promptTemplate.resultingParameterName;
261
+ jokers = promptTemplate.jokerParameterNames, blockType = promptTemplate.blockType, content = promptTemplate.content, postprocessing = promptTemplate.postprocessingFunctionNames, expectations = promptTemplate.expectations, expectFormat = promptTemplate.expectFormat, resultingParameterName = promptTemplate.resultingParameterName;
262
262
  pipelineString += '\n\n';
263
263
  pipelineString += "## ".concat(title_1);
264
264
  if (description_1) {
@@ -392,23 +392,107 @@
392
392
  return parameterString;
393
393
  }
394
394
  /**
395
- * TODO: !!!!! Implement new features and commands into `promptTemplateParameterJsonToString`
395
+ * TODO: !!!! Implement new features and commands into `promptTemplateParameterJsonToString`
396
396
  * TODO: [🧠] Is there a way to auto-detect missing features in pipelineJsonToString
397
397
  * TODO: Escape all
398
398
  */
399
399
 
400
+ /**
401
+ * @@@
402
+ *
403
+ * @returns The same object as the input, but deeply frozen
404
+ *
405
+ * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
406
+ */
407
+ function deepFreeze(objectValue) {
408
+ var e_1, _a;
409
+ var propertyNames = Object.getOwnPropertyNames(objectValue);
410
+ try {
411
+ for (var propertyNames_1 = __values(propertyNames), propertyNames_1_1 = propertyNames_1.next(); !propertyNames_1_1.done; propertyNames_1_1 = propertyNames_1.next()) {
412
+ var propertyName = propertyNames_1_1.value;
413
+ var value = objectValue[propertyName];
414
+ if (value && typeof value === 'object') {
415
+ deepFreeze(value);
416
+ }
417
+ }
418
+ }
419
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
420
+ finally {
421
+ try {
422
+ if (propertyNames_1_1 && !propertyNames_1_1.done && (_a = propertyNames_1.return)) _a.call(propertyNames_1);
423
+ }
424
+ finally { if (e_1) throw e_1.error; }
425
+ }
426
+ return Object.freeze(objectValue);
427
+ }
428
+ /**
429
+ * @@@
430
+ * @@@
431
+ *
432
+ * @returns The same object as the input, but deeply frozen
433
+ *
434
+ * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
435
+ */
436
+ function deepFreezeWithSameType(objectValue) {
437
+ return deepFreeze(objectValue);
438
+ }
439
+ /**
440
+ * TODO: [🔼] Export from `@promptbook/utils`
441
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
442
+ */
443
+
444
+ /**
445
+ * Returns the same value that is passed as argument.
446
+ * No side effects.
447
+ *
448
+ * Note: It can be usefull for:
449
+ *
450
+ * 1) Leveling indentation
451
+ * 2) Putting always-true or always-false conditions without getting eslint errors
452
+ *
453
+ * @param value any values
454
+ * @returns the same values
455
+ * @private within the repository
456
+ */
457
+ function just(value) {
458
+ if (value === undefined) {
459
+ return undefined;
460
+ }
461
+ return value;
462
+ }
463
+
400
464
  /**
401
465
  * The maximum number of iterations for a loops
402
466
  */
403
467
  var LOOP_LIMIT = 1000;
468
+ /**
469
+ * The maximum number of (LLM) tasks running in parallel
470
+ */
471
+ var MAX_PARALLEL_COUNT = 5;
472
+ /**
473
+ * The maximum number of attempts to execute LLM task before giving up
474
+ */
475
+ var MAX_EXECUTION_ATTEMPTS = 3;
404
476
  /**
405
477
  * The names of the parameters that are reserved for special purposes
406
478
  */
407
- var RESERVED_PARAMETER_NAMES = [
479
+ var RESERVED_PARAMETER_NAMES = deepFreeze([
408
480
  'context',
481
+ 'currentDate',
409
482
  // <- TODO: Add more like 'date', 'modelName',...
410
483
  // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
411
- ];
484
+ ]);
485
+ /**
486
+ * Nonce which is used for replacing things in strings
487
+ */
488
+ var REPLACING_NONCE = 'u$k42k%!V2zo34w7Fu#@QUHYPW';
489
+ /*
490
+ TODO: !!! Just testing false-negative detection of [🟡][🟢][🔵][⚪] leak
491
+ */
492
+ // [🟡][🟢][🔵][⚪]
493
+ /**
494
+ * TODO: [🔼] Export all to core
495
+ */
412
496
 
413
497
  /**
414
498
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -611,7 +695,7 @@
611
695
  */
612
696
  function validatePipeline(pipeline) {
613
697
  // TODO: [🧠] Maybe test if promptbook is a promise and make specific error case for that
614
- var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
698
+ var e_1, _a, e_2, _b, e_3, _c, e_4, _d, e_5, _e;
615
699
  if (pipeline.pipelineUrl !== undefined && !isValidPipelineUrl(pipeline.pipelineUrl)) {
616
700
  // <- Note: [🚲]
617
701
  throw new PipelineLogicError("Invalid promptbook URL \"".concat(pipeline.pipelineUrl, "\""));
@@ -648,19 +732,19 @@
648
732
  };
649
733
  try {
650
734
  // Note: Check each parameter individually
651
- for (var _e = __values(pipeline.parameters), _f = _e.next(); !_f.done; _f = _e.next()) {
652
- var parameter = _f.value;
735
+ for (var _f = __values(pipeline.parameters), _g = _f.next(); !_g.done; _g = _f.next()) {
736
+ var parameter = _g.value;
653
737
  _loop_1(parameter);
654
738
  }
655
739
  }
656
740
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
657
741
  finally {
658
742
  try {
659
- if (_f && !_f.done && (_a = _e.return)) _a.call(_e);
743
+ if (_g && !_g.done && (_a = _f.return)) _a.call(_f);
660
744
  }
661
745
  finally { if (e_1) throw e_1.error; }
662
746
  }
663
- // Note: Check each template individually
747
+ // Note: All input parameters are defined - so that they can be used as result of some template
664
748
  var definedParameters = new Set(pipeline.parameters.filter(function (_a) {
665
749
  var isInput = _a.isInput;
666
750
  return isInput;
@@ -669,23 +753,27 @@
669
753
  return name;
670
754
  }));
671
755
  try {
672
- for (var _g = __values(pipeline.promptTemplates), _h = _g.next(); !_h.done; _h = _g.next()) {
673
- var template = _h.value;
756
+ // Note: Checking each template individually
757
+ for (var _h = __values(pipeline.promptTemplates), _j = _h.next(); !_j.done; _j = _h.next()) {
758
+ var template = _j.value;
674
759
  if (definedParameters.has(template.resultingParameterName)) {
675
760
  throw new PipelineLogicError("Parameter {".concat(template.resultingParameterName, "} is defined multiple times"));
676
761
  }
762
+ if (RESERVED_PARAMETER_NAMES.includes(template.resultingParameterName)) {
763
+ throw new PipelineLogicError("Parameter name {".concat(template.resultingParameterName, "} is reserved, please use fifferent name"));
764
+ }
677
765
  definedParameters.add(template.resultingParameterName);
678
766
  if (template.blockType === 'PROMPT_TEMPLATE' && template.modelRequirements.modelVariant === undefined) {
679
767
  throw new PipelineLogicError(spaceTrim.spaceTrim("\n\n You must specify MODEL VARIANT in the prompt template \"".concat(template.title, "\"\n\n For example:\n - MODEL VARIANT Chat\n - MODEL NAME `gpt-4-1106-preview`\n\n ")));
680
768
  }
681
- if (template.jokers && template.jokers.length > 0) {
769
+ if (template.jokerParameterNames && template.jokerParameterNames.length > 0) {
682
770
  if (!template.expectFormat &&
683
771
  !template.expectations /* <- TODO: Require at least 1 -> min <- expectation to use jokers */) {
684
772
  throw new PipelineLogicError("Joker parameters are used for {".concat(template.resultingParameterName, "} but no expectations are defined"));
685
773
  }
686
774
  try {
687
- for (var _j = (e_3 = void 0, __values(template.jokers)), _k = _j.next(); !_k.done; _k = _j.next()) {
688
- var joker = _k.value;
775
+ for (var _k = (e_3 = void 0, __values(template.jokerParameterNames)), _l = _k.next(); !_l.done; _l = _k.next()) {
776
+ var joker = _l.value;
689
777
  if (!template.dependentParameterNames.includes(joker)) {
690
778
  throw new PipelineLogicError("Parameter {".concat(joker, "} is used for {").concat(template.resultingParameterName, "} as joker but not in dependentParameterNames"));
691
779
  }
@@ -694,15 +782,15 @@
694
782
  catch (e_3_1) { e_3 = { error: e_3_1 }; }
695
783
  finally {
696
784
  try {
697
- if (_k && !_k.done && (_c = _j.return)) _c.call(_j);
785
+ if (_l && !_l.done && (_c = _k.return)) _c.call(_k);
698
786
  }
699
787
  finally { if (e_3) throw e_3.error; }
700
788
  }
701
789
  }
702
790
  if (template.expectations) {
703
791
  try {
704
- for (var _l = (e_4 = void 0, __values(Object.entries(template.expectations))), _m = _l.next(); !_m.done; _m = _l.next()) {
705
- var _o = __read(_m.value, 2), unit = _o[0], _p = _o[1], min = _p.min, max = _p.max;
792
+ for (var _m = (e_4 = void 0, __values(Object.entries(template.expectations))), _o = _m.next(); !_o.done; _o = _m.next()) {
793
+ var _p = __read(_o.value, 2), unit = _p[0], _q = _p[1], min = _q.min, max = _q.max;
706
794
  if (min !== undefined && max !== undefined && min > max) {
707
795
  throw new PipelineLogicError("Min expectation (=".concat(min, ") of ").concat(unit, " is higher than max expectation (=").concat(max, ")"));
708
796
  }
@@ -717,7 +805,7 @@
717
805
  catch (e_4_1) { e_4 = { error: e_4_1 }; }
718
806
  finally {
719
807
  try {
720
- if (_m && !_m.done && (_d = _l.return)) _d.call(_l);
808
+ if (_o && !_o.done && (_d = _m.return)) _d.call(_m);
721
809
  }
722
810
  finally { if (e_4) throw e_4.error; }
723
811
  }
@@ -727,7 +815,7 @@
727
815
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
728
816
  finally {
729
817
  try {
730
- if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
818
+ if (_j && !_j.done && (_b = _h.return)) _b.call(_h);
731
819
  }
732
820
  finally { if (e_2) throw e_2.error; }
733
821
  }
@@ -741,10 +829,26 @@
741
829
  var name = _a.name;
742
830
  return name;
743
831
  });
832
+ try {
833
+ // Note: All reserved parameters are resolved
834
+ for (var RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
835
+ var reservedParameterName = RESERVED_PARAMETER_NAMES_1_1.value;
836
+ resovedParameters = __spreadArray(__spreadArray([], __read(resovedParameters), false), [reservedParameterName], false);
837
+ }
838
+ }
839
+ catch (e_5_1) { e_5 = { error: e_5_1 }; }
840
+ finally {
841
+ try {
842
+ if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_e = RESERVED_PARAMETER_NAMES_1.return)) _e.call(RESERVED_PARAMETER_NAMES_1);
843
+ }
844
+ finally { if (e_5) throw e_5.error; }
845
+ }
744
846
  var unresovedTemplates = __spreadArray([], __read(pipeline.promptTemplates), false);
847
+ // <- TODO: [🧠][🥜]
745
848
  var loopLimit = LOOP_LIMIT;
746
849
  var _loop_2 = function () {
747
850
  if (loopLimit-- < 0) {
851
+ // Note: Really UnexpectedError not LimitReachedError - this should not happen and be caught below
748
852
  throw new UnexpectedError('Loop limit reached during detection of circular dependencies in `validatePipeline`');
749
853
  }
750
854
  var currentlyResovedTemplates = unresovedTemplates.filter(function (template) {
@@ -754,11 +858,11 @@
754
858
  throw new PipelineLogicError(spaceTrim.spaceTrim(function (block) { return "\n\n Can not resolve some parameters:\n Either you are using a parameter that is not defined, or there are some circular dependencies.\n\n Can not resolve:\n ".concat(block(unresovedTemplates
755
859
  .map(function (_a) {
756
860
  var resultingParameterName = _a.resultingParameterName, dependentParameterNames = _a.dependentParameterNames;
757
- return "- {".concat(resultingParameterName, "} depends on ").concat(dependentParameterNames
861
+ return "- Parameter {".concat(resultingParameterName, "} which depends on ").concat(dependentParameterNames
758
862
  .map(function (dependentParameterName) { return "{".concat(dependentParameterName, "}"); })
759
- .join(', '));
863
+ .join(' and '));
760
864
  })
761
- .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameters.map(function (name) { return "- {".concat(name, "}"); }).join('\n')), "\n "); }));
865
+ .join('\n')), "\n\n Resolved:\n ").concat(block(resovedParameters.map(function (name) { return "- Parameter {".concat(name, "}"); }).join('\n')), "\n "); }));
762
866
  }
763
867
  resovedParameters = __spreadArray(__spreadArray([], __read(resovedParameters), false), __read(currentlyResovedTemplates.map(function (_a) {
764
868
  var resultingParameterName = _a.resultingParameterName;
@@ -780,7 +884,16 @@
780
884
  * > * It checks:
781
885
  * > * - it has a valid structure
782
886
  * > * - ...
783
- * > ex port function validatePipeline(promptbook: unknown): asserts promptbook is PipelineJson {
887
+ * > ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
888
+ */
889
+ /**
890
+ * TODO: [🧠][🐣] !!!! Validate that all samples match expectations
891
+ * TODO: [🧠][🐣] !!!! Validate that knowledge is valid (non-void)
892
+ * TODO: [🧠][🐣] !!!! Validate that persona can be used only with CHAT variant
893
+ * TODO: !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
894
+ * TODO: !!!! Validate that reserved parameter is not used as joker
895
+ * TODO: [🧠] !!! Validationg not only logic itself but imports around - files and websites and rerefenced pipelines exists
896
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
784
897
  */
785
898
 
786
899
  /**
@@ -812,79 +925,100 @@
812
925
  }(Error));
813
926
 
814
927
  /**
815
- * Library of promptbooks that groups together promptbooks for an application.
816
- * This implementation is a very thin wrapper around the Array / Map of promptbooks.
928
+ * Unprepare just strips the preparation data of the pipeline
929
+ */
930
+ function unpreparePipeline(pipeline) {
931
+ var personas = pipeline.personas, knowledgeSources = pipeline.knowledgeSources;
932
+ personas = personas.map(function (persona) { return (__assign(__assign({}, persona), { modelRequirements: undefined, preparationIds: undefined })); });
933
+ knowledgeSources = knowledgeSources.map(function (knowledgeSource) { return (__assign(__assign({}, knowledgeSource), { preparationIds: undefined })); });
934
+ return __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSources, knowledgePieces: [], personas: personas, preparations: [] });
935
+ }
936
+ /**
937
+ * TODO: [🔼] !!! Export via `@promptbook/core`
938
+ * TODO: Write tests for `preparePipeline`
939
+ */
940
+
941
+ /**
942
+ * Library of pipelines that groups together pipelines for an application.
943
+ * This implementation is a very thin wrapper around the Array / Map of pipelines.
817
944
  *
818
945
  * @private use `createCollectionFromJson` instead
819
- * @see https://github.com/webgptorg/promptbook#promptbook-collection
946
+ * @see https://github.com/webgptorg/pipeline#pipeline-collection
820
947
  */
821
948
  var SimplePipelineCollection = /** @class */ (function () {
822
949
  /**
823
- * Constructs a pipeline collection from promptbooks
950
+ * Constructs a pipeline collection from pipelines
824
951
  *
825
- * @param promptbooks !!!
952
+ * @param pipelines @@@
826
953
  *
827
954
  * @private Use instead `createCollectionFromJson`
828
- * Note: During the construction logic of all promptbooks are validated
955
+ * Note: During the construction logic of all pipelines are validated
829
956
  * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
830
957
  */
831
958
  function SimplePipelineCollection() {
832
959
  var e_1, _a;
833
- var promptbooks = [];
960
+ var pipelines = [];
834
961
  for (var _i = 0; _i < arguments.length; _i++) {
835
- promptbooks[_i] = arguments[_i];
962
+ pipelines[_i] = arguments[_i];
836
963
  }
837
964
  this.collection = new Map();
838
965
  try {
839
- for (var promptbooks_1 = __values(promptbooks), promptbooks_1_1 = promptbooks_1.next(); !promptbooks_1_1.done; promptbooks_1_1 = promptbooks_1.next()) {
840
- var promptbook = promptbooks_1_1.value;
841
- if (promptbook.pipelineUrl === undefined) {
842
- throw new ReferenceError$1(spaceTrim.spaceTrim("\n Promptbook with name \"".concat(promptbook.title, "\" does not have defined URL\n\n File:\n ").concat(promptbook.sourceFile || 'Unknown', "\n\n Note: Promptbooks without URLs are called anonymous promptbooks\n They can be used as standalone promptbooks, but they cannot be referenced by other promptbooks\n And also they cannot be used in the pipeline collection\n\n ")));
966
+ for (var pipelines_1 = __values(pipelines), pipelines_1_1 = pipelines_1.next(); !pipelines_1_1.done; pipelines_1_1 = pipelines_1.next()) {
967
+ var pipeline = pipelines_1_1.value;
968
+ // TODO: [👠] DRY
969
+ if (pipeline.pipelineUrl === undefined) {
970
+ throw new ReferenceError$1(spaceTrim.spaceTrim("\n Pipeline with name \"".concat(pipeline.title, "\" does not have defined URL\n\n File:\n ").concat(pipeline.sourceFile || 'Unknown', "\n\n Note: Pipelines without URLs are called anonymous pipelines\n They can be used as standalone pipelines, but they cannot be referenced by other pipelines\n And also they cannot be used in the pipeline collection\n\n ")));
843
971
  }
844
- validatePipeline(promptbook);
972
+ // Note: [🐨]
973
+ validatePipeline(pipeline);
974
+ // TODO: [🦄] DRY
845
975
  // Note: [🦄]
846
- if (this.collection.has(promptbook.pipelineUrl) &&
847
- pipelineJsonToString(promptbook) !== pipelineJsonToString(this.collection.get(promptbook.pipelineUrl))) {
848
- var existing = this.collection.get(promptbook.pipelineUrl);
849
- throw new ReferenceError$1(spaceTrim.spaceTrim("\n Promptbook with URL \"".concat(promptbook.pipelineUrl, "\" is already in the collection\n\n Conflicting files:\n ").concat(existing.sourceFile || 'Unknown', "\n ").concat(promptbook.sourceFile || 'Unknown', "\n\n Note: Promptbooks with the same URL are not allowed\n Only exepction is when the promptbooks are identical\n\n ")));
976
+ if (
977
+ // TODO: [🐽]
978
+ this.collection.has(pipeline.pipelineUrl) &&
979
+ pipelineJsonToString(unpreparePipeline(pipeline)) !==
980
+ pipelineJsonToString(unpreparePipeline(this.collection.get(pipeline.pipelineUrl)))) {
981
+ var existing = this.collection.get(pipeline.pipelineUrl);
982
+ throw new ReferenceError$1(spaceTrim.spaceTrim("\n Pipeline with URL \"".concat(pipeline.pipelineUrl, "\" is already in the collection\n\n Conflicting files:\n ").concat(existing.sourceFile || 'Unknown', "\n ").concat(pipeline.sourceFile || 'Unknown', "\n\n Note: Pipelines with the same URL are not allowed\n Only exepction is when the pipelines are identical\n\n ")));
850
983
  }
851
- this.collection.set(promptbook.pipelineUrl, promptbook);
984
+ // Note: [🧠] Overwrite existing pipeline with the same URL
985
+ this.collection.set(pipeline.pipelineUrl, pipeline);
852
986
  }
853
987
  }
854
988
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
855
989
  finally {
856
990
  try {
857
- if (promptbooks_1_1 && !promptbooks_1_1.done && (_a = promptbooks_1.return)) _a.call(promptbooks_1);
991
+ if (pipelines_1_1 && !pipelines_1_1.done && (_a = pipelines_1.return)) _a.call(pipelines_1);
858
992
  }
859
993
  finally { if (e_1) throw e_1.error; }
860
994
  }
861
995
  }
862
996
  /**
863
- * Gets all promptbooks in the collection
997
+ * Gets all pipelines in the collection
864
998
  */
865
999
  SimplePipelineCollection.prototype.listPipelines = function () {
866
1000
  return Array.from(this.collection.keys());
867
1001
  };
868
1002
  /**
869
- * Gets promptbook by its URL
1003
+ * Gets pipeline by its URL
870
1004
  *
871
1005
  * Note: This is not a direct fetching from the URL, but a lookup in the collection
872
1006
  */
873
1007
  SimplePipelineCollection.prototype.getPipelineByUrl = function (url) {
874
1008
  var _this = this;
875
- var promptbook = this.collection.get(url);
876
- if (!promptbook) {
1009
+ var pipeline = this.collection.get(url);
1010
+ if (!pipeline) {
877
1011
  if (this.listPipelines().length === 0) {
878
- throw new NotFoundError(spaceTrim.spaceTrim("\n Promptbook with url \"".concat(url, "\" not found\n\n No promptbooks available\n ")));
1012
+ throw new NotFoundError(spaceTrim.spaceTrim("\n Pipeline with url \"".concat(url, "\" not found\n\n No pipelines available\n ")));
879
1013
  }
880
- throw new NotFoundError(spaceTrim.spaceTrim(function (block) { return "\n Promptbook with url \"".concat(url, "\" not found\n\n Available promptbooks:\n ").concat(block(_this.listPipelines()
1014
+ throw new NotFoundError(spaceTrim.spaceTrim(function (block) { return "\n Pipeline with url \"".concat(url, "\" not found\n\n Available pipelines:\n ").concat(block(_this.listPipelines()
881
1015
  .map(function (pipelineUrl) { return "- ".concat(pipelineUrl); })
882
1016
  .join('\n')), "\n\n "); }));
883
1017
  }
884
- return promptbook;
1018
+ return pipeline;
885
1019
  };
886
1020
  /**
887
- * Checks whether given prompt was defined in any promptbook in the collection
1021
+ * Checks whether given prompt was defined in any pipeline in the collection
888
1022
  */
889
1023
  SimplePipelineCollection.prototype.isResponsibleForPrompt = function (prompt) {
890
1024
  return true;
@@ -1126,7 +1260,199 @@
1126
1260
  // <- [🩻]
1127
1261
  ];
1128
1262
 
1129
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-1",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledge:[{name:"n",title:"N",content:"D",keywords:["E"],index:[{modelName:"mocked-facked",position:[-0.8022132744698469,-0.0007443037013543652,-0.34872812971894707,-0.331923610323706,0.36013380510747606,0.3272238545809909,0.9972619939249201,0.261533967444473,0.969397219635256,0.17310495233925405,0.1658542147196167,0.9463503306650436,-0.11780402348338725,-0.19232799237848175,0.6226110088018051,0.2759831058911306,-0.5311583719379294,0.18853474356656497,-0.4508505143158268,-0.6183896225862329,0.5050055293473035,0.1478100741208901,-0.21453327800030664,0.09724866792207987,0.5336888252898944]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-1",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledge:[{name:"e",title:"E",content:"A",keywords:["L"],index:[{modelName:"mocked-facked",position:[-0.10884747467629419,-0.5326698151631195,-0.9465176322534181,0.7751385851237287,-0.7692699503104152,-0.9601969759421696,0.33722965766570434,0.17842011981786943,0.0276452884638827,-0.5983538271731308,-0.127805978741649,-0.6304775592815672,0.8849987254326983,0.1675220238760482,-0.8943010776373641,-0.9569796049357393,0.24933996044411444,0.8210261943632116,0.7193141705292407,0.06170351501887783,0.26972150191994837,-0.5340726089187089,0.12061502629384036,0.29495956103603005,-0.53082082537217]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-1",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledge:[{name:"s",title:"S",content:"A",keywords:["S"],index:[{modelName:"mocked-facked",position:[-0.916989680936596,0.9339948002315257,0.6004732120236658,0.0028125597650441314,-0.055419157811128716,0.013537996506097105,-0.8796279521347974,0.12223482268345887,0.31637982343954674,0.19555443419711915,-0.1664084533259853,-0.3527797149266507,0.3732921105225979,-0.7943618527999234,0.09240919193726871,0.8207777669058434,0.6464285109148693,0.008174116471551418,-0.933937225718561,-0.790531259317556,-0.04738702314966625,-0.9357754186773284,-0.3074287982247097,-0.4579908642989361,0.9857246614213477]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
1263
+ /**
1264
+ * @@@
1265
+ */
1266
+ function deepClone(objectValue) {
1267
+ return JSON.parse(JSON.stringify(objectValue));
1268
+ /*
1269
+ TODO: [🧠] Is there a better implementation?
1270
+ > const propertyNames = Object.getOwnPropertyNames(objectValue);
1271
+ > for (const propertyName of propertyNames) {
1272
+ > const value = (objectValue as really_any)[propertyName];
1273
+ > if (value && typeof value === 'object') {
1274
+ > deepClone(value);
1275
+ > }
1276
+ > }
1277
+ > return Object.assign({}, objectValue);
1278
+ */
1279
+ }
1280
+ /**
1281
+ * TODO: [🔼] Export from `@promptbook/utils`
1282
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
1283
+ */
1284
+
1285
+ /**
1286
+ * @@@
1287
+ *
1288
+ * TODO: [🔼] Export with addUsage
1289
+ */
1290
+ var ZERO_USAGE = deepFreeze({
1291
+ price: { value: 0 },
1292
+ input: {
1293
+ tokensCount: { value: 0 },
1294
+ charactersCount: { value: 0 },
1295
+ wordsCount: { value: 0 },
1296
+ sentencesCount: { value: 0 },
1297
+ linesCount: { value: 0 },
1298
+ paragraphsCount: { value: 0 },
1299
+ pagesCount: { value: 0 },
1300
+ },
1301
+ output: {
1302
+ tokensCount: { value: 0 },
1303
+ charactersCount: { value: 0 },
1304
+ wordsCount: { value: 0 },
1305
+ sentencesCount: { value: 0 },
1306
+ linesCount: { value: 0 },
1307
+ paragraphsCount: { value: 0 },
1308
+ pagesCount: { value: 0 },
1309
+ },
1310
+ });
1311
+ /**
1312
+ * Function `addUsage` will add multiple usages into one
1313
+ *
1314
+ * Note: If you provide 0 values, it returns ZERO_USAGE
1315
+ */
1316
+ function addUsage() {
1317
+ var usageItems = [];
1318
+ for (var _i = 0; _i < arguments.length; _i++) {
1319
+ usageItems[_i] = arguments[_i];
1320
+ }
1321
+ return usageItems.reduce(function (acc, item) {
1322
+ var e_1, _a, e_2, _b;
1323
+ var _c;
1324
+ acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
1325
+ try {
1326
+ for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
1327
+ var key = _e.value;
1328
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1329
+ //@ts-ignore
1330
+ if (item.input[key]) {
1331
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1332
+ //@ts-ignore
1333
+ acc.input[key].value += item.input[key].value || 0;
1334
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1335
+ //@ts-ignore
1336
+ if (item.input[key].isUncertain) {
1337
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1338
+ //@ts-ignore
1339
+ acc.input[key].isUncertain = true;
1340
+ }
1341
+ }
1342
+ }
1343
+ }
1344
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
1345
+ finally {
1346
+ try {
1347
+ if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
1348
+ }
1349
+ finally { if (e_1) throw e_1.error; }
1350
+ }
1351
+ try {
1352
+ for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
1353
+ var key = _g.value;
1354
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1355
+ //@ts-ignore
1356
+ if (item.output[key]) {
1357
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1358
+ //@ts-ignore
1359
+ acc.output[key].value += item.output[key].value || 0;
1360
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1361
+ //@ts-ignore
1362
+ if (item.output[key].isUncertain) {
1363
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1364
+ //@ts-ignore
1365
+ acc.output[key].isUncertain = true;
1366
+ }
1367
+ }
1368
+ }
1369
+ }
1370
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
1371
+ finally {
1372
+ try {
1373
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
1374
+ }
1375
+ finally { if (e_2) throw e_2.error; }
1376
+ }
1377
+ return acc;
1378
+ }, deepClone(ZERO_USAGE));
1379
+ }
1380
+
1381
+ /**
1382
+ * Async version of Array.forEach
1383
+ *
1384
+ * @param array - Array to iterate over
1385
+ * @param options - Options for the function
1386
+ * @param callbackfunction - Function to call for each item
1387
+ */
1388
+ function forEachAsync(array, options, callbackfunction) {
1389
+ return __awaiter(this, void 0, void 0, function () {
1390
+ var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
1391
+ var e_1, _d;
1392
+ return __generator(this, function (_e) {
1393
+ switch (_e.label) {
1394
+ case 0:
1395
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
1396
+ index = 0;
1397
+ runningTasks = [];
1398
+ tasks = [];
1399
+ _loop_1 = function (item) {
1400
+ var currentIndex, task;
1401
+ return __generator(this, function (_f) {
1402
+ switch (_f.label) {
1403
+ case 0:
1404
+ currentIndex = index++;
1405
+ task = callbackfunction(item, currentIndex, array);
1406
+ tasks.push(task);
1407
+ runningTasks.push(task);
1408
+ /* not await */ Promise.resolve(task).then(function () {
1409
+ runningTasks = runningTasks.filter(function (t) { return t !== task; });
1410
+ });
1411
+ if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
1412
+ return [4 /*yield*/, Promise.race(runningTasks)];
1413
+ case 1:
1414
+ _f.sent();
1415
+ _f.label = 2;
1416
+ case 2: return [2 /*return*/];
1417
+ }
1418
+ });
1419
+ };
1420
+ _e.label = 1;
1421
+ case 1:
1422
+ _e.trys.push([1, 6, 7, 8]);
1423
+ _b = __values(array), _c = _b.next();
1424
+ _e.label = 2;
1425
+ case 2:
1426
+ if (!!_c.done) return [3 /*break*/, 5];
1427
+ item = _c.value;
1428
+ return [5 /*yield**/, _loop_1(item)];
1429
+ case 3:
1430
+ _e.sent();
1431
+ _e.label = 4;
1432
+ case 4:
1433
+ _c = _b.next();
1434
+ return [3 /*break*/, 2];
1435
+ case 5: return [3 /*break*/, 8];
1436
+ case 6:
1437
+ e_1_1 = _e.sent();
1438
+ e_1 = { error: e_1_1 };
1439
+ return [3 /*break*/, 8];
1440
+ case 7:
1441
+ try {
1442
+ if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
1443
+ }
1444
+ finally { if (e_1) throw e_1.error; }
1445
+ return [7 /*endfinally*/];
1446
+ case 8: return [4 /*yield*/, Promise.all(tasks)];
1447
+ case 9:
1448
+ _e.sent();
1449
+ return [2 /*return*/];
1450
+ }
1451
+ });
1452
+ });
1453
+ }
1454
+
1455
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-19",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-19",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-19",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-19",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-19",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {content}",expectations:{words:{min:1,max:8}},dependentParameterNames:["content"],resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-19",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.61.0-19",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",modelRequirements:{modelVariant:"CHAT",modelName:"gpt-4-turbo"},content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n### Option `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Option `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Option `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",expectFormat:"JSON",dependentParameterNames:["availableModelNames","personaDescription"],resultingParameterName:"modelRequirements"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-19",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
1130
1456
 
1131
1457
  var defaultDiacriticsRemovalMap = [
1132
1458
  {
@@ -1453,6 +1779,14 @@
1453
1779
  * Function normalizes title to name which can be used as identifier
1454
1780
  */
1455
1781
  function titleToName(value) {
1782
+ if (value.startsWith('http://') || value.startsWith('https://')) {
1783
+ // TODO: Maybe check against some list unallowed characters
1784
+ return value;
1785
+ }
1786
+ if (value.startsWith('./') || value.startsWith('../')) {
1787
+ // TODO: Maybe check against some list unallowed characters
1788
+ return value;
1789
+ }
1456
1790
  value = removeEmojis(value);
1457
1791
  value = normalizeToKebabCase(value);
1458
1792
  // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
@@ -1499,43 +1833,176 @@
1499
1833
  */
1500
1834
 
1501
1835
  /**
1502
- * This error occurs when some expectation is not met in the execution of the pipeline
1836
+ * Parses the template and returns the list of all parameter names
1503
1837
  *
1504
- * @private Always catched and rethrown as `PipelineExecutionError`
1505
- * Note: This is a kindof subtype of PipelineExecutionError
1838
+ * @param template the template with parameters in {curly} braces
1839
+ * @returns the list of parameter names
1506
1840
  */
1507
- var ExpectError = /** @class */ (function (_super) {
1508
- __extends(ExpectError, _super);
1509
- function ExpectError(message) {
1510
- var _this = _super.call(this, message) || this;
1511
- _this.name = 'ExpectError';
1512
- Object.setPrototypeOf(_this, ExpectError.prototype);
1513
- return _this;
1841
+ function extractParameters(template) {
1842
+ var e_1, _a;
1843
+ var matches = template.matchAll(/{\w+}/g);
1844
+ var parameterNames = new Set();
1845
+ try {
1846
+ for (var matches_1 = __values(matches), matches_1_1 = matches_1.next(); !matches_1_1.done; matches_1_1 = matches_1.next()) {
1847
+ var match = matches_1_1.value;
1848
+ var parameterName = match[0].slice(1, -1);
1849
+ parameterNames.add(parameterName);
1850
+ }
1514
1851
  }
1515
- return ExpectError;
1516
- }(Error));
1852
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
1853
+ finally {
1854
+ try {
1855
+ if (matches_1_1 && !matches_1_1.done && (_a = matches_1.return)) _a.call(matches_1);
1856
+ }
1857
+ finally { if (e_1) throw e_1.error; }
1858
+ }
1859
+ return parameterNames;
1860
+ }
1517
1861
 
1518
1862
  /**
1519
- * Function isValidJsonString will tell you if the string is valid JSON or not
1863
+ * Parses the given script and returns the list of all used variables that are not defined in the script
1864
+ *
1865
+ * @param script from which to extract the variables
1866
+ * @returns the list of variable names
1867
+ * @throws {ParsingError} if the script is invalid
1520
1868
  */
1521
- function isValidJsonString(value /* <-[👨‍⚖️] */) {
1869
+ function extractVariables(script) {
1870
+ var variables = new Set();
1871
+ script = "(()=>{".concat(script, "})()");
1522
1872
  try {
1523
- JSON.parse(value);
1524
- return true;
1873
+ for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
1874
+ try {
1875
+ eval(script);
1876
+ }
1877
+ catch (error) {
1878
+ if (!(error instanceof ReferenceError)) {
1879
+ throw error;
1880
+ }
1881
+ var undefinedName = error.message.split(' ')[0];
1882
+ /*
1883
+ Note: Parsing the error
1884
+ [ReferenceError: thing is not defined]
1885
+ */
1886
+ if (!undefinedName) {
1887
+ throw error;
1888
+ }
1889
+ if (script.includes(undefinedName + '(')) {
1890
+ script = "const ".concat(undefinedName, " = ()=>'';") + script;
1891
+ }
1892
+ else {
1893
+ variables.add(undefinedName);
1894
+ script = "const ".concat(undefinedName, " = '';") + script;
1895
+ }
1896
+ }
1525
1897
  }
1526
1898
  catch (error) {
1527
1899
  if (!(error instanceof Error)) {
1528
1900
  throw error;
1529
1901
  }
1530
- if (error.message.includes('Unexpected token')) {
1531
- return false;
1532
- }
1533
- return false;
1902
+ throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Can not extract variables from the script\n\n ".concat(block(error.name), ": ").concat(block(error.message), "\n "); }));
1534
1903
  }
1904
+ return variables;
1535
1905
  }
1906
+ /**
1907
+ * TODO: [🔣] Support for multiple languages - python, java,...
1908
+ */
1536
1909
 
1537
1910
  /**
1538
- * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1911
+ * Parses the prompt template and returns the set of all used parameters
1912
+ *
1913
+ * @param promptTemplate the template with used parameters
1914
+ * @returns the set of parameter names
1915
+ * @throws {ParsingError} if the script is invalid
1916
+ */
1917
+ function extractParametersFromPromptTemplate(promptTemplate) {
1918
+ var e_1, _a, e_2, _b, e_3, _c;
1919
+ var title = promptTemplate.title, description = promptTemplate.description, blockType = promptTemplate.blockType, content = promptTemplate.content, jokerParameterNames = promptTemplate.jokerParameterNames;
1920
+ var parameterNames = new Set();
1921
+ try {
1922
+ for (var _d = __values(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameters(title)), false), __read(extractParameters(description || '')), false), __read(extractParameters(content)), false)), _e = _d.next(); !_e.done; _e = _d.next()) {
1923
+ var parameterName = _e.value;
1924
+ parameterNames.add(parameterName);
1925
+ }
1926
+ }
1927
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
1928
+ finally {
1929
+ try {
1930
+ if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
1931
+ }
1932
+ finally { if (e_1) throw e_1.error; }
1933
+ }
1934
+ if (blockType === 'SCRIPT') {
1935
+ try {
1936
+ for (var _f = __values(extractVariables(content)), _g = _f.next(); !_g.done; _g = _f.next()) {
1937
+ var parameterName = _g.value;
1938
+ parameterNames.add(parameterName);
1939
+ }
1940
+ }
1941
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
1942
+ finally {
1943
+ try {
1944
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
1945
+ }
1946
+ finally { if (e_2) throw e_2.error; }
1947
+ }
1948
+ }
1949
+ try {
1950
+ for (var _h = __values(jokerParameterNames || []), _j = _h.next(); !_j.done; _j = _h.next()) {
1951
+ var jokerName = _j.value;
1952
+ parameterNames.add(jokerName);
1953
+ }
1954
+ }
1955
+ catch (e_3_1) { e_3 = { error: e_3_1 }; }
1956
+ finally {
1957
+ try {
1958
+ if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
1959
+ }
1960
+ finally { if (e_3) throw e_3.error; }
1961
+ }
1962
+ return parameterNames;
1963
+ }
1964
+ /**
1965
+ * TODO: [🔣] If script require contentLanguage
1966
+ */
1967
+
1968
+ /**
1969
+ * This error occurs when some expectation is not met in the execution of the pipeline
1970
+ *
1971
+ * @private Always catched and rethrown as `PipelineExecutionError`
1972
+ * Note: This is a kindof subtype of PipelineExecutionError
1973
+ */
1974
+ var ExpectError = /** @class */ (function (_super) {
1975
+ __extends(ExpectError, _super);
1976
+ function ExpectError(message) {
1977
+ var _this = _super.call(this, message) || this;
1978
+ _this.name = 'ExpectError';
1979
+ Object.setPrototypeOf(_this, ExpectError.prototype);
1980
+ return _this;
1981
+ }
1982
+ return ExpectError;
1983
+ }(Error));
1984
+
1985
+ /**
1986
+ * Function isValidJsonString will tell you if the string is valid JSON or not
1987
+ */
1988
+ function isValidJsonString(value /* <-[👨‍⚖️] */) {
1989
+ try {
1990
+ JSON.parse(value);
1991
+ return true;
1992
+ }
1993
+ catch (error) {
1994
+ if (!(error instanceof Error)) {
1995
+ throw error;
1996
+ }
1997
+ if (error.message.includes('Unexpected token')) {
1998
+ return false;
1999
+ }
2000
+ return false;
2001
+ }
2002
+ }
2003
+
2004
+ /**
2005
+ * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1539
2006
  *
1540
2007
  * @private Internal utility of `joinLlmExecutionTools`
1541
2008
  */
@@ -1664,7 +2131,7 @@
1664
2131
  throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n All execution tools failed:\n\n ".concat(block(errors.map(function (error) { return "- ".concat(error.name || 'Error', ": ").concat(error.message); }).join('\n')), "\n\n "); }));
1665
2132
  }
1666
2133
  else {
1667
- throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n No execution tools available for model variant \"".concat(prompt.modelRequirements.modelVariant, "\".\n\n tl;dr\n\n You have provided no LLM Execution Tools that support model variant \"").concat(prompt.modelRequirements.modelVariant, ":\n ").concat(block(_this.llmExecutionTools
2134
+ throw new PipelineExecutionError(spaceTrim__default["default"](function (block) { return "\n You have not provided any `LlmExecutionTools` that support model variant \"".concat(prompt.modelRequirements.modelVariant, "\n\n Available `LlmExecutionTools`:\n ").concat(block(_this.llmExecutionTools
1668
2135
  .map(function (tools) { return "- ".concat(tools.title, " ").concat(tools.description || ''); })
1669
2136
  .join('\n')), "\n\n "); }));
1670
2137
  }
@@ -1674,7 +2141,7 @@
1674
2141
  };
1675
2142
  /**
1676
2143
  * List all available models that can be used
1677
- * This liost is a combination of all available models from all execution tools
2144
+ * This lists is a combination of all available models from all execution tools
1678
2145
  */
1679
2146
  MultipleLlmExecutionTools.prototype.listModels = function () {
1680
2147
  return __awaiter(this, void 0, void 0, function () {
@@ -1720,6 +2187,8 @@
1720
2187
  }());
1721
2188
  /**
1722
2189
  * TODO: [🧠][🎛] Aggregating multiple models - have result not only from one first aviable model BUT all of them
2190
+ * TODO: [🏖] If no llmTools have for example not defined `callCompletionModel` this will still return object with defined `callCompletionModel` which just throws `PipelineExecutionError`, make it undefined instead
2191
+ * Look how `countTotalUsage` (and `cacheLlmTools`) implements it
1723
2192
  */
1724
2193
 
1725
2194
  /**
@@ -1741,7 +2210,7 @@
1741
2210
  llmExecutionTools[_i] = arguments[_i];
1742
2211
  }
1743
2212
  if (llmExecutionTools.length === 0) {
1744
- var warningMessage = spaceTrim__default["default"]("\n You have provided no LLM Execution Tools.\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ");
2213
+ var warningMessage = spaceTrim__default["default"]("\n You have not provided any `LlmExecutionTools`\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ");
1745
2214
  // TODO: [🟥] Detect browser / node and make it colorfull
1746
2215
  console.warn(warningMessage);
1747
2216
  /*
@@ -1768,6 +2237,31 @@
1768
2237
  return new (MultipleLlmExecutionTools.bind.apply(MultipleLlmExecutionTools, __spreadArray([void 0], __read(llmExecutionTools), false)))();
1769
2238
  }
1770
2239
 
2240
+ /**
2241
+ * Determine if the pipeline is fully prepared
2242
+ */
2243
+ function isPipelinePrepared(pipeline) {
2244
+ // Note: Ignoring `pipeline.preparations` @@@
2245
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2246
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2247
+ console.log('!!!!', 'Not all personas have modelRequirements');
2248
+ return false;
2249
+ }
2250
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2251
+ console.log('!!!!', 'Not all knowledgeSources have preparationIds');
2252
+ return false;
2253
+ }
2254
+ // TODO: !!!!! Is context in each template
2255
+ // TODO: !!!!! Are samples prepared
2256
+ // TODO: !!!!! Are templates prepared
2257
+ return true;
2258
+ }
2259
+ /**
2260
+ * TODO: [🐠] Maybe base this on `makeValidator`
2261
+ * TODO: [🔼] Export via core or utils
2262
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2263
+ */
2264
+
1771
2265
  /**
1772
2266
  * Takes an item or an array of items and returns an array of items
1773
2267
  *
@@ -1788,102 +2282,173 @@
1788
2282
  }
1789
2283
 
1790
2284
  /**
1791
- * The version of the Promptbook library
2285
+ * Just marks a place of place where should be something implemented
2286
+ * No side effects.
2287
+ *
2288
+ * Note: It can be usefull suppressing eslint errors of unused variables
2289
+ *
2290
+ * @param value any values
2291
+ * @returns void
2292
+ * @private within the repository
1792
2293
  */
1793
- var PROMPTBOOK_VERSION = '0.61.0-1';
1794
- // TODO: !!!! List here all the versions and annotate + put into script
2294
+ function TODO_USE() {
2295
+ var value = [];
2296
+ for (var _i = 0; _i < arguments.length; _i++) {
2297
+ value[_i] = arguments[_i];
2298
+ }
2299
+ }
1795
2300
 
1796
2301
  /**
1797
- * Function `addUsage` will add multiple usages into one
2302
+ * This error type indicates that some limit was reached
2303
+ */
2304
+ var LimitReachedError = /** @class */ (function (_super) {
2305
+ __extends(LimitReachedError, _super);
2306
+ function LimitReachedError(message) {
2307
+ var _this = _super.call(this, message) || this;
2308
+ _this.name = 'LimitReachedError';
2309
+ Object.setPrototypeOf(_this, LimitReachedError.prototype);
2310
+ return _this;
2311
+ }
2312
+ return LimitReachedError;
2313
+ }(Error));
2314
+
2315
+ /**
2316
+ * Replaces parameters in template with values from parameters object
1798
2317
  *
1799
- * Note: If you provide 0 values, it returns void usage
2318
+ * @param template the template with parameters in {curly} braces
2319
+ * @param parameters the object with parameters
2320
+ * @returns the template with replaced parameters
2321
+ * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened
1800
2322
  */
1801
- function addUsage() {
1802
- var usageItems = [];
1803
- for (var _i = 0; _i < arguments.length; _i++) {
1804
- usageItems[_i] = arguments[_i];
2323
+ function replaceParameters(template, parameters) {
2324
+ var replacedTemplate = template;
2325
+ var match;
2326
+ var loopLimit = LOOP_LIMIT;
2327
+ var _loop_1 = function () {
2328
+ if (loopLimit-- < 0) {
2329
+ throw new LimitReachedError('Loop limit reached during parameters replacement in `replaceParameters`');
2330
+ }
2331
+ var precol = match.groups.precol;
2332
+ var parameterName = match.groups.parameterName;
2333
+ if (parameterName === '') {
2334
+ return "continue";
2335
+ }
2336
+ if (parameterName.indexOf('{') !== -1 || parameterName.indexOf('}') !== -1) {
2337
+ throw new PipelineExecutionError('Parameter is already opened or not closed');
2338
+ }
2339
+ if (parameters[parameterName] === undefined) {
2340
+ throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2341
+ }
2342
+ var parameterValue = parameters[parameterName];
2343
+ if (parameterValue === undefined) {
2344
+ throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2345
+ }
2346
+ parameterValue = parameterValue.toString();
2347
+ if (parameterValue.includes('\n') && /^\s*\W{0,3}\s*$/.test(precol)) {
2348
+ parameterValue = parameterValue
2349
+ .split('\n')
2350
+ .map(function (line, index) { return (index === 0 ? line : "".concat(precol).concat(line)); })
2351
+ .join('\n');
2352
+ }
2353
+ replacedTemplate =
2354
+ replacedTemplate.substring(0, match.index + precol.length) +
2355
+ parameterValue +
2356
+ replacedTemplate.substring(match.index + precol.length + parameterName.length + 2);
2357
+ };
2358
+ while ((match = /^(?<precol>.*){(?<parameterName>\w+)}(.*)/m /* <- Not global */
2359
+ .exec(replacedTemplate))) {
2360
+ _loop_1();
1805
2361
  }
1806
- var initialStructure = {
1807
- price: { value: 0 },
1808
- input: {
1809
- tokensCount: { value: 0 },
1810
- charactersCount: { value: 0 },
1811
- wordsCount: { value: 0 },
1812
- sentencesCount: { value: 0 },
1813
- linesCount: { value: 0 },
1814
- paragraphsCount: { value: 0 },
1815
- pagesCount: { value: 0 },
1816
- },
1817
- output: {
1818
- tokensCount: { value: 0 },
1819
- charactersCount: { value: 0 },
1820
- wordsCount: { value: 0 },
1821
- sentencesCount: { value: 0 },
1822
- linesCount: { value: 0 },
1823
- paragraphsCount: { value: 0 },
1824
- pagesCount: { value: 0 },
1825
- },
2362
+ // [💫] Check if there are parameters that are not closed properly
2363
+ if (/{\w+$/.test(replacedTemplate)) {
2364
+ throw new PipelineExecutionError('Parameter is not closed');
2365
+ }
2366
+ // [💫] Check if there are parameters that are not opened properly
2367
+ if (/^\w+}/.test(replacedTemplate)) {
2368
+ throw new PipelineExecutionError('Parameter is not opened');
2369
+ }
2370
+ return replacedTemplate;
2371
+ }
2372
+
2373
+ /**
2374
+ * Create difference set of two sets.
2375
+ *
2376
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2377
+ */
2378
+ function difference(a, b, isEqual) {
2379
+ var e_1, _a;
2380
+ if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
2381
+ var diff = new Set();
2382
+ var _loop_1 = function (itemA) {
2383
+ if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
2384
+ diff.add(itemA);
2385
+ }
1826
2386
  };
1827
- return usageItems.reduce(function (acc, item) {
1828
- var e_1, _a, e_2, _b;
1829
- var _c;
1830
- acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
2387
+ try {
2388
+ for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
2389
+ var itemA = _c.value;
2390
+ _loop_1(itemA);
2391
+ }
2392
+ }
2393
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2394
+ finally {
1831
2395
  try {
1832
- for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
1833
- var key = _e.value;
1834
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1835
- //@ts-ignore
1836
- if (item.input[key]) {
1837
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1838
- //@ts-ignore
1839
- acc.input[key].value += item.input[key].value || 0;
1840
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1841
- //@ts-ignore
1842
- if (item.input[key].isUncertain) {
1843
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1844
- //@ts-ignore
1845
- acc.input[key].isUncertain = true;
1846
- }
1847
- }
1848
- }
2396
+ if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
1849
2397
  }
1850
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
1851
- finally {
2398
+ finally { if (e_1) throw e_1.error; }
2399
+ }
2400
+ return diff;
2401
+ }
2402
+ /**
2403
+ * TODO: [🧠][💯] Maybe also implement symmetricDifference
2404
+ */
2405
+
2406
+ /**
2407
+ * Creates a new set with all elements that are present in either set
2408
+ *
2409
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
2410
+ */
2411
+ function union() {
2412
+ var e_1, _a, e_2, _b;
2413
+ var sets = [];
2414
+ for (var _i = 0; _i < arguments.length; _i++) {
2415
+ sets[_i] = arguments[_i];
2416
+ }
2417
+ var union = new Set();
2418
+ try {
2419
+ for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
2420
+ var set = sets_1_1.value;
1852
2421
  try {
1853
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2422
+ for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
2423
+ var item = _d.value;
2424
+ union.add(item);
2425
+ }
1854
2426
  }
1855
- finally { if (e_1) throw e_1.error; }
1856
- }
1857
- try {
1858
- for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
1859
- var key = _g.value;
1860
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1861
- //@ts-ignore
1862
- if (item.output[key]) {
1863
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1864
- //@ts-ignore
1865
- acc.output[key].value += item.output[key].value || 0;
1866
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1867
- //@ts-ignore
1868
- if (item.output[key].isUncertain) {
1869
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
1870
- //@ts-ignore
1871
- acc.output[key].isUncertain = true;
1872
- }
2427
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2428
+ finally {
2429
+ try {
2430
+ if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
1873
2431
  }
2432
+ finally { if (e_2) throw e_2.error; }
1874
2433
  }
1875
2434
  }
1876
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
1877
- finally {
1878
- try {
1879
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
1880
- }
1881
- finally { if (e_2) throw e_2.error; }
2435
+ }
2436
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2437
+ finally {
2438
+ try {
2439
+ if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
1882
2440
  }
1883
- return acc;
1884
- }, initialStructure);
2441
+ finally { if (e_1) throw e_1.error; }
2442
+ }
2443
+ return union;
1885
2444
  }
1886
2445
 
2446
+ /**
2447
+ * The version of the Promptbook library
2448
+ */
2449
+ var PROMPTBOOK_VERSION = '0.61.0-19';
2450
+ // TODO: !!!! List here all the versions and annotate + put into script
2451
+
1887
2452
  /**
1888
2453
  * Counts number of characters in the text
1889
2454
  */
@@ -2015,66 +2580,6 @@
2015
2580
  * TODO: [💝] Unite object for expecting amount and format
2016
2581
  */
2017
2582
 
2018
- /**
2019
- * Replaces parameters in template with values from parameters object
2020
- *
2021
- * @param template the template with parameters in {curly} braces
2022
- * @param parameters the object with parameters
2023
- * @returns the template with replaced parameters
2024
- * @throws {PipelineExecutionError} if parameter is not defined, not closed, or not opened
2025
- *
2026
- * @private within the createPipelineExecutor
2027
- */
2028
- function replaceParameters(template, parameters) {
2029
- var replacedTemplate = template;
2030
- var match;
2031
- var loopLimit = LOOP_LIMIT;
2032
- var _loop_1 = function () {
2033
- if (loopLimit-- < 0) {
2034
- throw new UnexpectedError('Loop limit reached during parameters replacement in `replaceParameters`');
2035
- }
2036
- var precol = match.groups.precol;
2037
- var parameterName = match.groups.parameterName;
2038
- if (parameterName === '') {
2039
- return "continue";
2040
- }
2041
- if (parameterName.indexOf('{') !== -1 || parameterName.indexOf('}') !== -1) {
2042
- throw new PipelineExecutionError('Parameter is already opened or not closed');
2043
- }
2044
- if (parameters[parameterName] === undefined) {
2045
- throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2046
- }
2047
- var parameterValue = parameters[parameterName];
2048
- if (parameterValue === undefined) {
2049
- throw new PipelineExecutionError("Parameter {".concat(parameterName, "} is not defined"));
2050
- }
2051
- parameterValue = parameterValue.toString();
2052
- if (parameterValue.includes('\n') && /^\s*\W{0,3}\s*$/.test(precol)) {
2053
- parameterValue = parameterValue
2054
- .split('\n')
2055
- .map(function (line, index) { return (index === 0 ? line : "".concat(precol).concat(line)); })
2056
- .join('\n');
2057
- }
2058
- replacedTemplate =
2059
- replacedTemplate.substring(0, match.index + precol.length) +
2060
- parameterValue +
2061
- replacedTemplate.substring(match.index + precol.length + parameterName.length + 2);
2062
- };
2063
- while ((match = /^(?<precol>.*){(?<parameterName>\w+)}(.*)/m /* <- Not global */
2064
- .exec(replacedTemplate))) {
2065
- _loop_1();
2066
- }
2067
- // [💫] Check if there are parameters that are not closed properly
2068
- if (/{\w+$/.test(replacedTemplate)) {
2069
- throw new PipelineExecutionError('Parameter is not closed');
2070
- }
2071
- // [💫] Check if there are parameters that are not opened properly
2072
- if (/^\w+}/.test(replacedTemplate)) {
2073
- throw new PipelineExecutionError('Parameter is not opened');
2074
- }
2075
- return replacedTemplate;
2076
- }
2077
-
2078
2583
  /**
2079
2584
  * Creates executor function from pipeline and execution tools.
2080
2585
  *
@@ -2083,21 +2588,73 @@
2083
2588
  */
2084
2589
  function createPipelineExecutor(options) {
2085
2590
  var _this = this;
2086
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
2087
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? 3 : _b;
2088
- // TODO: !!!!! Implement new commands
2089
- validatePipeline(pipeline);
2090
- // TODO: !!!!! Do here materialization of still unmaterialized or dynamic knowledge
2091
- // TODO: !!!!! Do here personaToModelRequirements
2591
+ var rawPipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
2592
+ var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.isVerbose, isVerbose = _d === void 0 ? false : _d;
2593
+ validatePipeline(rawPipeline);
2092
2594
  var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
2595
+ var pipeline;
2596
+ if (isPipelinePrepared(rawPipeline)) {
2597
+ pipeline = rawPipeline;
2598
+ }
2599
+ else {
2600
+ console.warn(spaceTrim.spaceTrim("\n Pipeline ".concat(rawPipeline.pipelineUrl || rawPipeline.sourceFile || rawPipeline.title, " is not prepared\n\n It will be prepared ad-hoc before the first execution\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n ")));
2601
+ }
2093
2602
  var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
2603
+ function getContextForTemplate(// <- TODO: [🧠][🥜]
2604
+ template) {
2605
+ return __awaiter(this, void 0, void 0, function () {
2606
+ return __generator(this, function (_a) {
2607
+ // TODO: !!!!!! Implement Better - use real index and keyword search
2608
+ TODO_USE(template);
2609
+ return [2 /*return*/, pipeline.knowledgePieces.map(function (_a) {
2610
+ var content = _a.content;
2611
+ return "- ".concat(content);
2612
+ }).join('\n')];
2613
+ });
2614
+ });
2615
+ }
2616
+ function getReservedParametersForTemplate(template) {
2617
+ return __awaiter(this, void 0, void 0, function () {
2618
+ var context, currentDate, reservedParameters, RESERVED_PARAMETER_NAMES_1, RESERVED_PARAMETER_NAMES_1_1, parameterName;
2619
+ var e_3, _a;
2620
+ return __generator(this, function (_b) {
2621
+ switch (_b.label) {
2622
+ case 0: return [4 /*yield*/, getContextForTemplate(template)];
2623
+ case 1:
2624
+ context = _b.sent();
2625
+ currentDate = new Date().toISOString();
2626
+ reservedParameters = {
2627
+ context: context,
2628
+ currentDate: currentDate,
2629
+ };
2630
+ try {
2631
+ // Note: Doublecheck that ALL reserved parameters are defined:
2632
+ for (RESERVED_PARAMETER_NAMES_1 = __values(RESERVED_PARAMETER_NAMES), RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next(); !RESERVED_PARAMETER_NAMES_1_1.done; RESERVED_PARAMETER_NAMES_1_1 = RESERVED_PARAMETER_NAMES_1.next()) {
2633
+ parameterName = RESERVED_PARAMETER_NAMES_1_1.value;
2634
+ if (reservedParameters[parameterName] === undefined) {
2635
+ throw new UnexpectedError("Reserved parameter {".concat(parameterName, "} is not defined"));
2636
+ }
2637
+ }
2638
+ }
2639
+ catch (e_3_1) { e_3 = { error: e_3_1 }; }
2640
+ finally {
2641
+ try {
2642
+ if (RESERVED_PARAMETER_NAMES_1_1 && !RESERVED_PARAMETER_NAMES_1_1.done && (_a = RESERVED_PARAMETER_NAMES_1.return)) _a.call(RESERVED_PARAMETER_NAMES_1);
2643
+ }
2644
+ finally { if (e_3) throw e_3.error; }
2645
+ }
2646
+ return [2 /*return*/, reservedParameters];
2647
+ }
2648
+ });
2649
+ });
2650
+ }
2094
2651
  function executeSingleTemplate(currentTemplate) {
2095
2652
  return __awaiter(this, void 0, void 0, function () {
2096
- var name, title, priority, prompt, chatResult, completionResult, embeddingResult, result, resultString, expectError, scriptPipelineExecutionErrors, maxAttempts, jokers, attempt, isJokerAttempt, joker, _a, _b, _c, _d, scriptTools, error_2, e_2_1, _e, _f, functionName, postprocessingError, _g, _h, scriptTools, error_3, e_3_1, e_4_1, error_4;
2097
- var e_2, _j, e_4, _k, e_3, _l, _m;
2653
+ var name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _d, _e, parameterName, prompt, chatResult, completionResult, embeddingResult, result, resultString, expectError, scriptPipelineExecutionErrors, maxAttempts, jokerParameterNames, attempt, isJokerAttempt, jokerParameterName, _f, _g, _h, _j, scriptTools, error_2, e_4_1, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, error_3, e_5_1, e_6_1, error_4;
2654
+ var e_7, _p, e_4, _q, e_6, _r, e_5, _s, _t;
2098
2655
  var _this = this;
2099
- return __generator(this, function (_o) {
2100
- switch (_o.label) {
2656
+ return __generator(this, function (_u) {
2657
+ switch (_u.label) {
2101
2658
  case 0:
2102
2659
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
2103
2660
  title = currentTemplate.title;
@@ -2114,61 +2671,108 @@
2114
2671
  // <- [3]
2115
2672
  })];
2116
2673
  case 1:
2117
- _o.sent();
2118
- _o.label = 2;
2674
+ _u.sent();
2675
+ _u.label = 2;
2119
2676
  case 2:
2677
+ usedParameterNames = extractParametersFromPromptTemplate(currentTemplate);
2678
+ dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
2679
+ if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
2680
+ throw new UnexpectedError(spaceTrim.spaceTrim("\n Dependent parameters are not consistent used parameters:\n\n Dependent parameters:\n ".concat(Array.from(dependentParameterNames).join(', '), "\n\n Used parameters:\n ").concat(Array.from(usedParameterNames).join(', '), "\n\n ")));
2681
+ }
2682
+ _b = (_a = Object).freeze;
2683
+ _c = [{}];
2684
+ return [4 /*yield*/, getReservedParametersForTemplate(currentTemplate)];
2685
+ case 3:
2686
+ definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_u.sent())])), parametersToPass])]);
2687
+ definedParameterNames = new Set(Object.keys(definedParameters));
2688
+ parameters = {};
2689
+ try {
2690
+ // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
2691
+ for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
2692
+ parameterName = _e.value;
2693
+ // Situation: Parameter is defined and used
2694
+ if (definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
2695
+ parameters[parameterName] = definedParameters[parameterName];
2696
+ }
2697
+ // Situation: Parameter is defined but NOT used
2698
+ else if (definedParameterNames.has(parameterName) && !usedParameterNames.has(parameterName)) {
2699
+ // Do not pass this parameter to prompt
2700
+ }
2701
+ // Situation: Parameter is NOT defined BUT used
2702
+ else if (!definedParameterNames.has(parameterName) && usedParameterNames.has(parameterName)) {
2703
+ // Houston, we have a problem
2704
+ // Note: Checking part is also done in `validatePipeline`, but it’s good to doublecheck
2705
+ throw new UnexpectedError(spaceTrim.spaceTrim("\n Parameter {".concat(parameterName, "} is NOT defined\n BUT used in template \"").concat(currentTemplate.title || currentTemplate.name, "\"\n\n This should be catched in `validatePipeline`\n\n ")));
2706
+ }
2707
+ }
2708
+ }
2709
+ catch (e_7_1) { e_7 = { error: e_7_1 }; }
2710
+ finally {
2711
+ try {
2712
+ if (_e && !_e.done && (_p = _d.return)) _p.call(_d);
2713
+ }
2714
+ finally { if (e_7) throw e_7.error; }
2715
+ }
2716
+ // Note: Now we can freeze `parameters` because we are sure that all and only used parameters are defined
2717
+ Object.freeze(parameters);
2120
2718
  result = null;
2121
2719
  resultString = null;
2122
2720
  expectError = null;
2123
2721
  maxAttempts = currentTemplate.blockType === 'PROMPT_DIALOG' ? Infinity : maxExecutionAttempts;
2124
- jokers = currentTemplate.jokers || [];
2125
- attempt = -jokers.length;
2126
- _o.label = 3;
2127
- case 3:
2128
- if (!(attempt < maxAttempts)) return [3 /*break*/, 51];
2722
+ jokerParameterNames = currentTemplate.jokerParameterNames || [];
2723
+ attempt = -jokerParameterNames.length;
2724
+ _u.label = 4;
2725
+ case 4:
2726
+ if (!(attempt < maxAttempts)) return [3 /*break*/, 52];
2129
2727
  isJokerAttempt = attempt < 0;
2130
- joker = jokers[jokers.length + attempt];
2131
- if (isJokerAttempt && !joker) {
2728
+ jokerParameterName = jokerParameterNames[jokerParameterNames.length + attempt];
2729
+ if (isJokerAttempt && !jokerParameterName) {
2132
2730
  throw new UnexpectedError("Joker not found in attempt ".concat(attempt));
2133
2731
  }
2134
2732
  result = null;
2135
2733
  resultString = null;
2136
2734
  expectError = null;
2137
2735
  if (isJokerAttempt) {
2138
- if (typeof parametersToPass[joker] === 'undefined') {
2139
- throw new PipelineExecutionError("Joker parameter {".concat(joker, "} not defined"));
2736
+ if (parameters[jokerParameterName] === undefined) {
2737
+ throw new PipelineExecutionError("Joker parameter {".concat(jokerParameterName, "} not defined"));
2738
+ // <- TODO: This is maybe `PipelineLogicError` which should be detected in `validatePipeline` and here just thrown as `UnexpectedError`
2739
+ }
2740
+ else {
2741
+ resultString = parameters[jokerParameterName];
2140
2742
  }
2141
- resultString = parametersToPass[joker];
2142
- }
2143
- _o.label = 4;
2144
- case 4:
2145
- _o.trys.push([4, 47, 48, 49]);
2146
- if (!!isJokerAttempt) return [3 /*break*/, 29];
2147
- _a = currentTemplate.blockType;
2148
- switch (_a) {
2149
- case 'SIMPLE_TEMPLATE': return [3 /*break*/, 5];
2150
- case 'PROMPT_TEMPLATE': return [3 /*break*/, 6];
2151
- case 'SCRIPT': return [3 /*break*/, 15];
2152
- case 'PROMPT_DIALOG': return [3 /*break*/, 26];
2153
2743
  }
2154
- return [3 /*break*/, 28];
2744
+ _u.label = 5;
2155
2745
  case 5:
2156
- resultString = replaceParameters(currentTemplate.content, parametersToPass);
2746
+ _u.trys.push([5, 48, 49, 50]);
2747
+ if (!!isJokerAttempt) return [3 /*break*/, 30];
2748
+ _f = currentTemplate.blockType;
2749
+ switch (_f) {
2750
+ case 'SIMPLE_TEMPLATE': return [3 /*break*/, 6];
2751
+ case 'PROMPT_TEMPLATE': return [3 /*break*/, 7];
2752
+ case 'SCRIPT': return [3 /*break*/, 16];
2753
+ case 'PROMPT_DIALOG': return [3 /*break*/, 27];
2754
+ }
2157
2755
  return [3 /*break*/, 29];
2158
2756
  case 6:
2757
+ resultString = replaceParameters(currentTemplate.content, parameters);
2758
+ return [3 /*break*/, 30];
2759
+ case 7:
2159
2760
  prompt = {
2160
2761
  title: currentTemplate.title,
2161
2762
  pipelineUrl: "".concat(pipeline.pipelineUrl
2162
2763
  ? pipeline.pipelineUrl
2163
2764
  : 'anonymous' /* <- TODO: [🧠] How to deal with anonymous pipelines, do here some auto-url like SHA-256 based ad-hoc identifier? */, "#").concat(currentTemplate.name),
2164
- parameters: parametersToPass,
2165
- content: replaceParameters(currentTemplate.content, parametersToPass) /* <- [2] */,
2765
+ parameters: parameters,
2766
+ content: currentTemplate.content,
2166
2767
  modelRequirements: currentTemplate.modelRequirements,
2167
- expectations: currentTemplate.expectations,
2768
+ expectations: __assign(__assign({}, (pipeline.personas.find(function (_a) {
2769
+ var name = _a.name;
2770
+ return name === currentTemplate.personaName;
2771
+ }) || {})), currentTemplate.expectations),
2168
2772
  expectFormat: currentTemplate.expectFormat,
2169
- postprocessing: (currentTemplate.postprocessing || []).map(function (functionName) { return function (result) { return __awaiter(_this, void 0, void 0, function () {
2170
- var errors, _a, _b, scriptTools, error_5, e_5_1;
2171
- var e_5, _c;
2773
+ postprocessing: (currentTemplate.postprocessingFunctionNames || []).map(function (functionName) { return function (result) { return __awaiter(_this, void 0, void 0, function () {
2774
+ var errors, _a, _b, scriptTools, error_5, e_8_1;
2775
+ var e_8, _c;
2172
2776
  return __generator(this, function (_d) {
2173
2777
  switch (_d.label) {
2174
2778
  case 0:
@@ -2189,7 +2793,7 @@
2189
2793
  script: "".concat(functionName, "(result)"),
2190
2794
  parameters: {
2191
2795
  result: result || '',
2192
- // Note: No ...parametersToPass, because working with result only
2796
+ // Note: No ...parametersForTemplate, because working with result only
2193
2797
  },
2194
2798
  })];
2195
2799
  case 4: return [2 /*return*/, _d.sent()];
@@ -2198,6 +2802,9 @@
2198
2802
  if (!(error_5 instanceof Error)) {
2199
2803
  throw error_5;
2200
2804
  }
2805
+ if (error_5 instanceof UnexpectedError) {
2806
+ throw error_5;
2807
+ }
2201
2808
  errors.push(error_5);
2202
2809
  return [3 /*break*/, 6];
2203
2810
  case 6:
@@ -2205,14 +2812,14 @@
2205
2812
  return [3 /*break*/, 2];
2206
2813
  case 7: return [3 /*break*/, 10];
2207
2814
  case 8:
2208
- e_5_1 = _d.sent();
2209
- e_5 = { error: e_5_1 };
2815
+ e_8_1 = _d.sent();
2816
+ e_8 = { error: e_8_1 };
2210
2817
  return [3 /*break*/, 10];
2211
2818
  case 9:
2212
2819
  try {
2213
2820
  if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2214
2821
  }
2215
- finally { if (e_5) throw e_5.error; }
2822
+ finally { if (e_8) throw e_8.error; }
2216
2823
  return [7 /*endfinally*/];
2217
2824
  case 10:
2218
2825
  if (errors.length === 0) {
@@ -2228,35 +2835,35 @@
2228
2835
  });
2229
2836
  }); }; }),
2230
2837
  };
2231
- _b = currentTemplate.modelRequirements.modelVariant;
2232
- switch (_b) {
2233
- case 'CHAT': return [3 /*break*/, 7];
2234
- case 'COMPLETION': return [3 /*break*/, 9];
2235
- case 'EMBEDDING': return [3 /*break*/, 11];
2838
+ _g = currentTemplate.modelRequirements.modelVariant;
2839
+ switch (_g) {
2840
+ case 'CHAT': return [3 /*break*/, 8];
2841
+ case 'COMPLETION': return [3 /*break*/, 10];
2842
+ case 'EMBEDDING': return [3 /*break*/, 12];
2236
2843
  }
2237
- return [3 /*break*/, 13];
2238
- case 7: return [4 /*yield*/, llmTools.callChatModel(prompt)];
2239
- case 8:
2240
- chatResult = _o.sent();
2844
+ return [3 /*break*/, 14];
2845
+ case 8: return [4 /*yield*/, llmTools.callChatModel(deepFreeze(prompt))];
2846
+ case 9:
2847
+ chatResult = _u.sent();
2241
2848
  // TODO: [🍬] Destroy chatThread
2242
2849
  result = chatResult;
2243
2850
  resultString = chatResult.content;
2244
- return [3 /*break*/, 14];
2245
- case 9: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
2246
- case 10:
2247
- completionResult = _o.sent();
2851
+ return [3 /*break*/, 15];
2852
+ case 10: return [4 /*yield*/, llmTools.callCompletionModel(deepFreeze(prompt))];
2853
+ case 11:
2854
+ completionResult = _u.sent();
2248
2855
  result = completionResult;
2249
2856
  resultString = completionResult.content;
2250
- return [3 /*break*/, 14];
2251
- case 11: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
2252
- case 12:
2253
- embeddingResult = _o.sent();
2857
+ return [3 /*break*/, 15];
2858
+ case 12: return [4 /*yield*/, llmTools.callEmbeddingModel(deepFreeze(prompt))];
2859
+ case 13:
2860
+ embeddingResult = _u.sent();
2254
2861
  result = embeddingResult;
2255
2862
  resultString = embeddingResult.content.join(',');
2256
- return [3 /*break*/, 14];
2257
- case 13: throw new PipelineExecutionError("Unknown model variant \"".concat(currentTemplate.modelRequirements.modelVariant, "\""));
2258
- case 14: return [3 /*break*/, 29];
2259
- case 15:
2863
+ return [3 /*break*/, 15];
2864
+ case 14: throw new PipelineExecutionError("Unknown model variant \"".concat(currentTemplate.modelRequirements.modelVariant, "\""));
2865
+ case 15: return [3 /*break*/, 30];
2866
+ case 16:
2260
2867
  if (arrayableToArray(tools.script).length === 0) {
2261
2868
  throw new PipelineExecutionError('No script execution tools are available');
2262
2869
  }
@@ -2265,49 +2872,52 @@
2265
2872
  }
2266
2873
  // TODO: DRY [1]
2267
2874
  scriptPipelineExecutionErrors = [];
2268
- _o.label = 16;
2269
- case 16:
2270
- _o.trys.push([16, 23, 24, 25]);
2271
- _c = (e_2 = void 0, __values(arrayableToArray(tools.script))), _d = _c.next();
2272
- _o.label = 17;
2875
+ _u.label = 17;
2273
2876
  case 17:
2274
- if (!!_d.done) return [3 /*break*/, 22];
2275
- scriptTools = _d.value;
2276
- _o.label = 18;
2877
+ _u.trys.push([17, 24, 25, 26]);
2878
+ _h = (e_4 = void 0, __values(arrayableToArray(tools.script))), _j = _h.next();
2879
+ _u.label = 18;
2277
2880
  case 18:
2278
- _o.trys.push([18, 20, , 21]);
2279
- return [4 /*yield*/, scriptTools.execute({
2881
+ if (!!_j.done) return [3 /*break*/, 23];
2882
+ scriptTools = _j.value;
2883
+ _u.label = 19;
2884
+ case 19:
2885
+ _u.trys.push([19, 21, , 22]);
2886
+ return [4 /*yield*/, scriptTools.execute(deepFreeze({
2280
2887
  scriptLanguage: currentTemplate.contentLanguage,
2281
2888
  script: currentTemplate.content,
2282
- parameters: parametersToPass,
2283
- })];
2284
- case 19:
2285
- resultString = _o.sent();
2286
- return [3 /*break*/, 22];
2889
+ parameters: parameters,
2890
+ }))];
2287
2891
  case 20:
2288
- error_2 = _o.sent();
2892
+ resultString = _u.sent();
2893
+ return [3 /*break*/, 23];
2894
+ case 21:
2895
+ error_2 = _u.sent();
2289
2896
  if (!(error_2 instanceof Error)) {
2290
2897
  throw error_2;
2291
2898
  }
2899
+ if (error_2 instanceof UnexpectedError) {
2900
+ throw error_2;
2901
+ }
2292
2902
  scriptPipelineExecutionErrors.push(error_2);
2293
- return [3 /*break*/, 21];
2294
- case 21:
2295
- _d = _c.next();
2296
- return [3 /*break*/, 17];
2297
- case 22: return [3 /*break*/, 25];
2298
- case 23:
2299
- e_2_1 = _o.sent();
2300
- e_2 = { error: e_2_1 };
2301
- return [3 /*break*/, 25];
2903
+ return [3 /*break*/, 22];
2904
+ case 22:
2905
+ _j = _h.next();
2906
+ return [3 /*break*/, 18];
2907
+ case 23: return [3 /*break*/, 26];
2302
2908
  case 24:
2909
+ e_4_1 = _u.sent();
2910
+ e_4 = { error: e_4_1 };
2911
+ return [3 /*break*/, 26];
2912
+ case 25:
2303
2913
  try {
2304
- if (_d && !_d.done && (_j = _c.return)) _j.call(_c);
2914
+ if (_j && !_j.done && (_q = _h.return)) _q.call(_h);
2305
2915
  }
2306
- finally { if (e_2) throw e_2.error; }
2916
+ finally { if (e_4) throw e_4.error; }
2307
2917
  return [7 /*endfinally*/];
2308
- case 25:
2918
+ case 26:
2309
2919
  if (resultString !== null) {
2310
- return [3 /*break*/, 29];
2920
+ return [3 /*break*/, 30];
2311
2921
  }
2312
2922
  if (scriptPipelineExecutionErrors.length === 1) {
2313
2923
  throw scriptPipelineExecutionErrors[0];
@@ -2317,101 +2927,104 @@
2317
2927
  .map(function (error) { return '- ' + error.message; })
2318
2928
  .join('\n\n')), "\n "); }));
2319
2929
  }
2320
- case 26:
2930
+ case 27:
2321
2931
  if (tools.userInterface === undefined) {
2322
2932
  throw new PipelineExecutionError('User interface tools are not available');
2323
2933
  }
2324
- return [4 /*yield*/, tools.userInterface.promptDialog({
2934
+ return [4 /*yield*/, tools.userInterface.promptDialog(deepFreeze({
2325
2935
  promptTitle: currentTemplate.title,
2326
- promptMessage: replaceParameters(currentTemplate.description || '', parametersToPass),
2327
- defaultValue: replaceParameters(currentTemplate.content, parametersToPass),
2936
+ promptMessage: replaceParameters(currentTemplate.description || '', parameters),
2937
+ defaultValue: replaceParameters(currentTemplate.content, parameters),
2328
2938
  // TODO: [🧠] !! Figure out how to define placeholder in .ptbk.md file
2329
2939
  placeholder: undefined,
2330
2940
  priority: priority,
2331
- })];
2332
- case 27:
2941
+ }))];
2942
+ case 28:
2333
2943
  // TODO: [🌹] When making next attempt for `PROMPT DIALOG`, preserve the previous user input
2334
- resultString = _o.sent();
2335
- return [3 /*break*/, 29];
2336
- case 28: throw new PipelineExecutionError("Unknown execution type \"".concat(currentTemplate.blockType, "\""));
2337
- case 29:
2338
- if (!(!isJokerAttempt && currentTemplate.postprocessing)) return [3 /*break*/, 46];
2339
- _o.label = 30;
2944
+ resultString = _u.sent();
2945
+ return [3 /*break*/, 30];
2946
+ case 29: throw new PipelineExecutionError("Unknown execution type \"".concat(currentTemplate.blockType, "\""));
2340
2947
  case 30:
2341
- _o.trys.push([30, 44, 45, 46]);
2342
- _e = (e_4 = void 0, __values(currentTemplate.postprocessing)), _f = _e.next();
2343
- _o.label = 31;
2948
+ if (!(!isJokerAttempt && currentTemplate.postprocessingFunctionNames)) return [3 /*break*/, 47];
2949
+ _u.label = 31;
2344
2950
  case 31:
2345
- if (!!_f.done) return [3 /*break*/, 43];
2346
- functionName = _f.value;
2951
+ _u.trys.push([31, 45, 46, 47]);
2952
+ _k = (e_6 = void 0, __values(currentTemplate.postprocessingFunctionNames)), _l = _k.next();
2953
+ _u.label = 32;
2954
+ case 32:
2955
+ if (!!_l.done) return [3 /*break*/, 44];
2956
+ functionName = _l.value;
2347
2957
  // TODO: DRY [1]
2348
2958
  scriptPipelineExecutionErrors = [];
2349
2959
  postprocessingError = null;
2350
- _o.label = 32;
2351
- case 32:
2352
- _o.trys.push([32, 39, 40, 41]);
2353
- _g = (e_3 = void 0, __values(arrayableToArray(tools.script))), _h = _g.next();
2354
- _o.label = 33;
2960
+ _u.label = 33;
2355
2961
  case 33:
2356
- if (!!_h.done) return [3 /*break*/, 38];
2357
- scriptTools = _h.value;
2358
- _o.label = 34;
2962
+ _u.trys.push([33, 40, 41, 42]);
2963
+ _m = (e_5 = void 0, __values(arrayableToArray(tools.script))), _o = _m.next();
2964
+ _u.label = 34;
2359
2965
  case 34:
2360
- _o.trys.push([34, 36, , 37]);
2966
+ if (!!_o.done) return [3 /*break*/, 39];
2967
+ scriptTools = _o.value;
2968
+ _u.label = 35;
2969
+ case 35:
2970
+ _u.trys.push([35, 37, , 38]);
2361
2971
  return [4 /*yield*/, scriptTools.execute({
2362
2972
  scriptLanguage: "javascript" /* <- TODO: Try it in each languages; In future allow postprocessing with arbitrary combination of languages to combine */,
2363
2973
  script: "".concat(functionName, "(resultString)"),
2364
2974
  parameters: {
2365
2975
  resultString: resultString || '',
2366
- // Note: No ...parametersToPass, because working with result only
2976
+ // Note: No ...parametersForTemplate, because working with result only
2367
2977
  },
2368
2978
  })];
2369
- case 35:
2370
- resultString = _o.sent();
2371
- postprocessingError = null;
2372
- return [3 /*break*/, 38];
2373
2979
  case 36:
2374
- error_3 = _o.sent();
2980
+ resultString = _u.sent();
2981
+ postprocessingError = null;
2982
+ return [3 /*break*/, 39];
2983
+ case 37:
2984
+ error_3 = _u.sent();
2375
2985
  if (!(error_3 instanceof Error)) {
2376
2986
  throw error_3;
2377
2987
  }
2988
+ if (error_3 instanceof UnexpectedError) {
2989
+ throw error_3;
2990
+ }
2378
2991
  postprocessingError = error_3;
2379
2992
  scriptPipelineExecutionErrors.push(error_3);
2380
- return [3 /*break*/, 37];
2381
- case 37:
2382
- _h = _g.next();
2383
- return [3 /*break*/, 33];
2384
- case 38: return [3 /*break*/, 41];
2385
- case 39:
2386
- e_3_1 = _o.sent();
2387
- e_3 = { error: e_3_1 };
2388
- return [3 /*break*/, 41];
2993
+ return [3 /*break*/, 38];
2994
+ case 38:
2995
+ _o = _m.next();
2996
+ return [3 /*break*/, 34];
2997
+ case 39: return [3 /*break*/, 42];
2389
2998
  case 40:
2999
+ e_5_1 = _u.sent();
3000
+ e_5 = { error: e_5_1 };
3001
+ return [3 /*break*/, 42];
3002
+ case 41:
2390
3003
  try {
2391
- if (_h && !_h.done && (_l = _g.return)) _l.call(_g);
3004
+ if (_o && !_o.done && (_s = _m.return)) _s.call(_m);
2392
3005
  }
2393
- finally { if (e_3) throw e_3.error; }
3006
+ finally { if (e_5) throw e_5.error; }
2394
3007
  return [7 /*endfinally*/];
2395
- case 41:
3008
+ case 42:
2396
3009
  if (postprocessingError) {
2397
3010
  throw postprocessingError;
2398
3011
  }
2399
- _o.label = 42;
2400
- case 42:
2401
- _f = _e.next();
2402
- return [3 /*break*/, 31];
2403
- case 43: return [3 /*break*/, 46];
2404
- case 44:
2405
- e_4_1 = _o.sent();
2406
- e_4 = { error: e_4_1 };
2407
- return [3 /*break*/, 46];
3012
+ _u.label = 43;
3013
+ case 43:
3014
+ _l = _k.next();
3015
+ return [3 /*break*/, 32];
3016
+ case 44: return [3 /*break*/, 47];
2408
3017
  case 45:
3018
+ e_6_1 = _u.sent();
3019
+ e_6 = { error: e_6_1 };
3020
+ return [3 /*break*/, 47];
3021
+ case 46:
2409
3022
  try {
2410
- if (_f && !_f.done && (_k = _e.return)) _k.call(_e);
3023
+ if (_l && !_l.done && (_r = _k.return)) _r.call(_k);
2411
3024
  }
2412
- finally { if (e_4) throw e_4.error; }
3025
+ finally { if (e_6) throw e_6.error; }
2413
3026
  return [7 /*endfinally*/];
2414
- case 46:
3027
+ case 47:
2415
3028
  // TODO: [💝] Unite object for expecting amount and format
2416
3029
  if (currentTemplate.expectFormat) {
2417
3030
  if (currentTemplate.expectFormat === 'JSON') {
@@ -2424,15 +3037,18 @@
2424
3037
  if (currentTemplate.expectations) {
2425
3038
  checkExpectations(currentTemplate.expectations, resultString || '');
2426
3039
  }
2427
- return [3 /*break*/, 51];
2428
- case 47:
2429
- error_4 = _o.sent();
3040
+ return [3 /*break*/, 52];
3041
+ case 48:
3042
+ error_4 = _u.sent();
2430
3043
  if (!(error_4 instanceof ExpectError)) {
2431
3044
  throw error_4;
2432
3045
  }
3046
+ if (error_4 instanceof UnexpectedError) {
3047
+ throw error_4;
3048
+ }
2433
3049
  expectError = error_4;
2434
- return [3 /*break*/, 49];
2435
- case 48:
3050
+ return [3 /*break*/, 50];
3051
+ case 49:
2436
3052
  if (!isJokerAttempt &&
2437
3053
  currentTemplate.blockType === 'PROMPT_TEMPLATE' &&
2438
3054
  prompt
@@ -2454,15 +3070,15 @@
2454
3070
  });
2455
3071
  }
2456
3072
  return [7 /*endfinally*/];
2457
- case 49:
3073
+ case 50:
2458
3074
  if (expectError !== null && attempt === maxAttempts - 1) {
2459
3075
  throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n LLM execution failed ".concat(maxExecutionAttempts, "x\n\n ---\n Last error ").concat((expectError === null || expectError === void 0 ? void 0 : expectError.name) || '', ":\n ").concat(block((expectError === null || expectError === void 0 ? void 0 : expectError.message) || ''), "\n\n Last result:\n ").concat(resultString, "\n ---\n "); }));
2460
3076
  }
2461
- _o.label = 50;
2462
- case 50:
2463
- attempt++;
2464
- return [3 /*break*/, 3];
3077
+ _u.label = 51;
2465
3078
  case 51:
3079
+ attempt++;
3080
+ return [3 /*break*/, 4];
3081
+ case 52:
2466
3082
  if (resultString === null) {
2467
3083
  throw new UnexpectedError('Something went wrong and prompt result is null');
2468
3084
  }
@@ -2478,18 +3094,27 @@
2478
3094
  // <- [3]
2479
3095
  });
2480
3096
  }
2481
- parametersToPass = __assign(__assign({}, parametersToPass), (_m = {}, _m[currentTemplate.resultingParameterName] = resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */, _m));
3097
+ parametersToPass = Object.freeze(__assign(__assign({}, parametersToPass), (_t = {}, _t[currentTemplate.resultingParameterName] = resultString /* <- Note: Not need to detect parameter collision here because pipeline checks logic consistency during construction */, _t)));
2482
3098
  return [2 /*return*/];
2483
3099
  }
2484
3100
  });
2485
3101
  });
2486
3102
  }
2487
- var parametersToPass, executionReport, resovedParameters_1, unresovedTemplates, resolving_1, loopLimit, _loop_1, error_1, usage_1, _a, _b, parameter, usage;
2488
- var e_1, _c;
2489
- return __generator(this, function (_d) {
2490
- switch (_d.label) {
3103
+ var executionReport, _a, _b, parameter, parametersToPass, resovedParameters_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_1, error_1, usage_1, outputParameters, errors, _c, _d, parameter, usage;
3104
+ var e_1, _e, e_2, _f;
3105
+ return __generator(this, function (_g) {
3106
+ switch (_g.label) {
2491
3107
  case 0:
2492
- parametersToPass = inputParameters;
3108
+ if (!(pipeline === undefined)) return [3 /*break*/, 2];
3109
+ return [4 /*yield*/, preparePipeline(rawPipeline, {
3110
+ llmTools: llmTools,
3111
+ isVerbose: isVerbose,
3112
+ maxParallelCount: maxParallelCount,
3113
+ })];
3114
+ case 1:
3115
+ pipeline = _g.sent();
3116
+ _g.label = 2;
3117
+ case 2:
2493
3118
  executionReport = {
2494
3119
  pipelineUrl: pipeline.pipelineUrl,
2495
3120
  title: pipeline.title,
@@ -2498,9 +3123,35 @@
2498
3123
  description: pipeline.description,
2499
3124
  promptExecutions: [],
2500
3125
  };
2501
- _d.label = 1;
2502
- case 1:
2503
- _d.trys.push([1, 6, , 7]);
3126
+ try {
3127
+ // Note: Check that all input input parameters are defined
3128
+ for (_a = __values(pipeline.parameters), _b = _a.next(); !_b.done; _b = _a.next()) {
3129
+ parameter = _b.value;
3130
+ if (parameter.isInput && inputParameters[parameter.name] === undefined) {
3131
+ return [2 /*return*/, deepFreezeWithSameType({
3132
+ isSuccessful: false,
3133
+ errors: [
3134
+ new PipelineExecutionError("Parameter {".concat(parameter.name, "} is required as an input parameter")),
3135
+ // <- TODO: !!!!! Test this error
3136
+ ],
3137
+ executionReport: executionReport,
3138
+ outputParameters: {},
3139
+ usage: ZERO_USAGE,
3140
+ })];
3141
+ }
3142
+ }
3143
+ }
3144
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
3145
+ finally {
3146
+ try {
3147
+ if (_b && !_b.done && (_e = _a.return)) _e.call(_a);
3148
+ }
3149
+ finally { if (e_1) throw e_1.error; }
3150
+ }
3151
+ parametersToPass = inputParameters;
3152
+ _g.label = 3;
3153
+ case 3:
3154
+ _g.trys.push([3, 8, , 9]);
2504
3155
  resovedParameters_1 = pipeline.parameters
2505
3156
  .filter(function (_a) {
2506
3157
  var isInput = _a.isInput;
@@ -2510,30 +3161,36 @@
2510
3161
  var name = _a.name;
2511
3162
  return name;
2512
3163
  });
2513
- unresovedTemplates = __spreadArray([], __read(pipeline.promptTemplates), false);
3164
+ unresovedTemplates_1 = __spreadArray([], __read(pipeline.promptTemplates), false);
2514
3165
  resolving_1 = [];
2515
3166
  loopLimit = LOOP_LIMIT;
2516
3167
  _loop_1 = function () {
2517
3168
  var currentTemplate, work_1;
2518
- return __generator(this, function (_e) {
2519
- switch (_e.label) {
3169
+ return __generator(this, function (_h) {
3170
+ switch (_h.label) {
2520
3171
  case 0:
2521
3172
  if (loopLimit-- < 0) {
3173
+ // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
2522
3174
  throw new UnexpectedError('Loop limit reached during resolving parameters pipeline execution');
2523
3175
  }
2524
- currentTemplate = unresovedTemplates.find(function (template) {
3176
+ currentTemplate = unresovedTemplates_1.find(function (template) {
2525
3177
  return template.dependentParameterNames.every(function (name) { return resovedParameters_1.includes(name); });
2526
3178
  });
2527
3179
  if (!(!currentTemplate && resolving_1.length === 0)) return [3 /*break*/, 1];
2528
- throw new UnexpectedError(spaceTrim.spaceTrim("\n Can not resolve some parameters\n\n Note: This should be catched during validatePipeline\n "));
3180
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not resolve some parameters\n\n Note: This should be catched in `validatePipeline`\n\n\n Unresolved parameters:\n ".concat(block(unresovedTemplates_1
3181
+ .map(function (_a) {
3182
+ var resultingParameterName = _a.resultingParameterName;
3183
+ return "- {".concat(resultingParameterName, "}");
3184
+ })
3185
+ .join('\n')), "\n\n Resolved parameters:\n ").concat(block(resovedParameters_1.map(function (name) { return "- {".concat(name, "}"); }).join('\n')), "\n "); }));
2529
3186
  case 1:
2530
3187
  if (!!currentTemplate) return [3 /*break*/, 3];
2531
3188
  /* [5] */ return [4 /*yield*/, Promise.race(resolving_1)];
2532
3189
  case 2:
2533
- /* [5] */ _e.sent();
3190
+ /* [5] */ _h.sent();
2534
3191
  return [3 /*break*/, 4];
2535
3192
  case 3:
2536
- unresovedTemplates = unresovedTemplates.filter(function (template) { return template !== currentTemplate; });
3193
+ unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
2537
3194
  work_1 = executeSingleTemplate(currentTemplate)
2538
3195
  .then(function () {
2539
3196
  resovedParameters_1 = __spreadArray(__spreadArray([], __read(resovedParameters_1), false), [currentTemplate.resultingParameterName], false);
@@ -2542,120 +3199,133 @@
2542
3199
  resolving_1 = resolving_1.filter(function (w) { return w !== work_1; });
2543
3200
  });
2544
3201
  resolving_1.push(work_1);
2545
- _e.label = 4;
3202
+ _h.label = 4;
2546
3203
  case 4: return [2 /*return*/];
2547
3204
  }
2548
3205
  });
2549
3206
  };
2550
- _d.label = 2;
2551
- case 2:
2552
- if (!(unresovedTemplates.length > 0)) return [3 /*break*/, 4];
3207
+ _g.label = 4;
3208
+ case 4:
3209
+ if (!(unresovedTemplates_1.length > 0)) return [3 /*break*/, 6];
2553
3210
  return [5 /*yield**/, _loop_1()];
2554
- case 3:
2555
- _d.sent();
2556
- return [3 /*break*/, 2];
2557
- case 4: return [4 /*yield*/, Promise.all(resolving_1)];
2558
3211
  case 5:
2559
- _d.sent();
2560
- return [3 /*break*/, 7];
2561
- case 6:
2562
- error_1 = _d.sent();
3212
+ _g.sent();
3213
+ return [3 /*break*/, 4];
3214
+ case 6: return [4 /*yield*/, Promise.all(resolving_1)];
3215
+ case 7:
3216
+ _g.sent();
3217
+ return [3 /*break*/, 9];
3218
+ case 8:
3219
+ error_1 = _g.sent();
2563
3220
  if (!(error_1 instanceof Error)) {
2564
3221
  throw error_1;
2565
3222
  }
2566
3223
  usage_1 = addUsage.apply(void 0, __spreadArray([], __read(executionReport.promptExecutions.map(function (_a) {
2567
3224
  var result = _a.result;
2568
- return (result === null || result === void 0 ? void 0 : result.usage) || addUsage();
3225
+ return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE;
2569
3226
  })), false));
2570
- return [2 /*return*/, {
3227
+ return [2 /*return*/, deepFreezeWithSameType({
2571
3228
  isSuccessful: false,
2572
3229
  errors: [error_1],
2573
3230
  usage: usage_1,
2574
3231
  executionReport: executionReport,
2575
3232
  outputParameters: parametersToPass,
2576
- }];
2577
- case 7:
3233
+ })];
3234
+ case 9:
3235
+ outputParameters = {};
3236
+ errors = [];
2578
3237
  try {
2579
3238
  // Note: Filter ONLY output parameters
2580
- for (_a = __values(pipeline.parameters), _b = _a.next(); !_b.done; _b = _a.next()) {
2581
- parameter = _b.value;
2582
- if (parameter.isOutput) {
3239
+ for (_c = __values(pipeline.parameters.filter(function (_a) {
3240
+ var isOutput = _a.isOutput;
3241
+ return isOutput;
3242
+ })), _d = _c.next(); !_d.done; _d = _c.next()) {
3243
+ parameter = _d.value;
3244
+ if (parametersToPass[parameter.name] === undefined) {
3245
+ errors.push(new PipelineExecutionError("Parameter {".concat(parameter.name, "} is required as an output parameter but not set in the pipeline")));
2583
3246
  continue;
2584
3247
  }
2585
- delete parametersToPass[parameter.name];
3248
+ outputParameters[parameter.name] = parametersToPass[parameter.name] || '';
2586
3249
  }
2587
3250
  }
2588
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
3251
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2589
3252
  finally {
2590
3253
  try {
2591
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
3254
+ if (_d && !_d.done && (_f = _c.return)) _f.call(_c);
2592
3255
  }
2593
- finally { if (e_1) throw e_1.error; }
3256
+ finally { if (e_2) throw e_2.error; }
2594
3257
  }
2595
3258
  usage = addUsage.apply(void 0, __spreadArray([], __read(executionReport.promptExecutions.map(function (_a) {
2596
3259
  var result = _a.result;
2597
- return (result === null || result === void 0 ? void 0 : result.usage) || addUsage();
3260
+ return (result === null || result === void 0 ? void 0 : result.usage) || ZERO_USAGE;
2598
3261
  })), false));
2599
- return [2 /*return*/, {
3262
+ return [2 /*return*/, deepFreezeWithSameType({
2600
3263
  isSuccessful: true,
2601
- errors: [],
3264
+ errors: errors,
2602
3265
  usage: usage,
2603
3266
  executionReport: executionReport,
2604
- outputParameters: parametersToPass,
2605
- }];
3267
+ outputParameters: outputParameters,
3268
+ })];
2606
3269
  }
2607
3270
  });
2608
3271
  }); };
2609
3272
  return pipelineExecutor;
2610
3273
  }
2611
3274
  /**
3275
+ * TODO: Use isVerbose here (not only pass to `preparePipeline`)
3276
+ * TODO: [🪂] Use maxParallelCount here (not only pass to `preparePipeline`)
3277
+ * TODO: [♈] Probbably move expectations from templates to parameters
2612
3278
  * TODO: [🧠] When not meet expectations in PROMPT_DIALOG, make some way to tell the user
2613
3279
  * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
2614
3280
  * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
2615
3281
  * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
3282
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
2616
3283
  */
2617
3284
 
2618
- function prepareKnowledgeFromMarkdown(options) {
3285
+ /**
3286
+ * @@@
3287
+ */
3288
+ function prepareKnowledgeFromMarkdown(content /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
2619
3289
  return __awaiter(this, void 0, void 0, function () {
2620
- var content, llmTools, _a, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _b, prepareTitleExecutor, _c, prepareKeywordsExecutor, _d, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
2621
- var _e, _f, _g;
3290
+ var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
3291
+ var _f, _g, _h;
2622
3292
  var _this = this;
2623
- return __generator(this, function (_h) {
2624
- switch (_h.label) {
3293
+ return __generator(this, function (_j) {
3294
+ switch (_j.label) {
2625
3295
  case 0:
2626
- content = options.content, llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
3296
+ llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
3297
+ TODO_USE(maxParallelCount); // <- [🪂]
2627
3298
  collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
2628
- _b = createPipelineExecutor;
2629
- _e = {};
2630
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
2631
- case 1:
2632
- prepareKnowledgeFromMarkdownExecutor = _b.apply(void 0, [(_e.pipeline = _h.sent(),
2633
- _e.tools = {
2634
- llm: llmTools,
2635
- },
2636
- _e)]);
2637
3299
  _c = createPipelineExecutor;
2638
3300
  _f = {};
2639
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
2640
- case 2:
2641
- prepareTitleExecutor = _c.apply(void 0, [(_f.pipeline = _h.sent(),
3301
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
3302
+ case 1:
3303
+ prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
2642
3304
  _f.tools = {
2643
3305
  llm: llmTools,
2644
3306
  },
2645
3307
  _f)]);
2646
3308
  _d = createPipelineExecutor;
2647
3309
  _g = {};
2648
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
2649
- case 3:
2650
- prepareKeywordsExecutor = _d.apply(void 0, [(_g.pipeline = _h.sent(),
3310
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
3311
+ case 2:
3312
+ prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
2651
3313
  _g.tools = {
2652
3314
  llm: llmTools,
2653
3315
  },
2654
3316
  _g)]);
3317
+ _e = createPipelineExecutor;
3318
+ _h = {};
3319
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
3320
+ case 3:
3321
+ prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
3322
+ _h.tools = {
3323
+ llm: llmTools,
3324
+ },
3325
+ _h)]);
2655
3326
  return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ content: content })];
2656
3327
  case 4:
2657
- result = _h.sent();
2658
- // TODO: [0] !!! Aggeregate usage
3328
+ result = _j.sent();
2659
3329
  assertsExecutionSuccessful(result);
2660
3330
  outputParameters = result.outputParameters;
2661
3331
  knowledgeRaw = outputParameters.knowledge;
@@ -2664,9 +3334,9 @@
2664
3334
  console.info('knowledgeTextPieces:', knowledgeTextPieces);
2665
3335
  }
2666
3336
  return [4 /*yield*/, Promise.all(
2667
- // TODO: !!! Do not send all at once but in chunks
3337
+ // TODO: [🪂] !! Do not send all at once but in chunks
2668
3338
  knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
2669
- var name, title, content, keywords, index, sources, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
3339
+ var name, title, content, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
2670
3340
  return __generator(this, function (_c) {
2671
3341
  switch (_c.label) {
2672
3342
  case 0:
@@ -2675,12 +3345,6 @@
2675
3345
  content = spaceTrim__default["default"](knowledgeTextPiece);
2676
3346
  keywords = [];
2677
3347
  index = [];
2678
- sources = [
2679
- {
2680
- title: 'Markdown document' /* <- TODO: !!! Unhardcode */,
2681
- href: '#' /* <- TODO: !!! Unhardcode */,
2682
- },
2683
- ];
2684
3348
  _c.label = 1;
2685
3349
  case 1:
2686
3350
  _c.trys.push([1, 7, , 8]);
@@ -2715,7 +3379,6 @@
2715
3379
  })];
2716
3380
  case 5:
2717
3381
  embeddingResult = _c.sent();
2718
- // TODO: [0] !!! Aggeregate usage embeddingResult.usage
2719
3382
  index.push({
2720
3383
  modelName: embeddingResult.modelName,
2721
3384
  position: embeddingResult.content,
@@ -2733,21 +3396,403 @@
2733
3396
  content: content,
2734
3397
  keywords: keywords,
2735
3398
  index: index,
2736
- sources: sources,
3399
+ // <- TODO: [☀] sources,
2737
3400
  }];
2738
3401
  }
2739
3402
  });
2740
3403
  }); }))];
2741
3404
  case 5:
2742
- knowledge = _h.sent();
3405
+ knowledge = _j.sent();
2743
3406
  return [2 /*return*/, knowledge];
2744
3407
  }
2745
3408
  });
2746
- });
2747
- }
3409
+ });
3410
+ }
3411
+ /**
3412
+ * TODO: [🐝][🔼] !!! Export via `@promptbook/markdown`
3413
+ * TODO: [🪂] Do it in parallel 11:11
3414
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
3415
+ */
3416
+
3417
+ /**
3418
+ * Prepares the knowle
3419
+ *
3420
+ * @see https://github.com/webgptorg/promptbook/discussions/41
3421
+ */
3422
+ function prepareKnowledgePieces(knowledgeSources, options) {
3423
+ return __awaiter(this, void 0, void 0, function () {
3424
+ var _a, maxParallelCount, knowledgePrepared;
3425
+ var _this = this;
3426
+ return __generator(this, function (_b) {
3427
+ switch (_b.label) {
3428
+ case 0:
3429
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
3430
+ knowledgePrepared = [];
3431
+ return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
3432
+ var partialPieces, pieces;
3433
+ return __generator(this, function (_a) {
3434
+ switch (_a.label) {
3435
+ case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.source, // <- TODO: [🐝] !!! Unhardcode markdown, detect which type it is
3436
+ options)];
3437
+ case 1:
3438
+ partialPieces = _a.sent();
3439
+ pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
3440
+ {
3441
+ name: knowledgeSource.name,
3442
+ // line, column <- TODO: [☀]
3443
+ // <- TODO: [❎]
3444
+ },
3445
+ ] })); });
3446
+ knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
3447
+ return [2 /*return*/];
3448
+ }
3449
+ });
3450
+ }); })];
3451
+ case 1:
3452
+ _b.sent();
3453
+ return [2 /*return*/, knowledgePrepared];
3454
+ }
3455
+ });
3456
+ });
3457
+ }
3458
+ /*
3459
+ TODO: [🧊] This is how it can look in future
3460
+ > type PrepareKnowledgeKnowledge = {
3461
+ > /**
3462
+ > * Unprepared knowledge
3463
+ > * /
3464
+ > readonly knowledgeSources: Array<KnowledgeSourceJson>;
3465
+ > };
3466
+ >
3467
+ > export async function prepareKnowledgePieces(
3468
+ > knowledge: PrepareKnowledgeKnowledge,
3469
+ > options: PrepareOptions,
3470
+ > ):
3471
+ */
3472
+ /**
3473
+ * TODO: [🐝][🔼] !!! Export via `@promptbook/core`
3474
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
3475
+ * Put `knowledgePieces` into `PrepareKnowledgeOptions`
3476
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
3477
+ * TODO: [🧠][❎] Do here propper M:N mapping
3478
+ * [x] One source can make multiple pieces
3479
+ * [ ] One piece can have multiple sources
3480
+ */
3481
+
3482
+ /**
3483
+ * Prepares the persona for the pipeline
3484
+ *
3485
+ * @see https://github.com/webgptorg/promptbook/discussions/22
3486
+ */
3487
+ function preparePersona(personaDescription, options) {
3488
+ return __awaiter(this, void 0, void 0, function () {
3489
+ var llmTools, _a, isVerbose, collection, preparePersonaExecutor, _b, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
3490
+ var _c;
3491
+ return __generator(this, function (_d) {
3492
+ switch (_d.label) {
3493
+ case 0:
3494
+ llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
3495
+ collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
3496
+ _b = createPipelineExecutor;
3497
+ _c = {};
3498
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
3499
+ case 1:
3500
+ preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
3501
+ _c.tools = {
3502
+ llm: llmTools,
3503
+ },
3504
+ _c)]);
3505
+ return [4 /*yield*/, llmTools.listModels()];
3506
+ case 2:
3507
+ availableModels = _d.sent();
3508
+ availableModelNames = availableModels
3509
+ .filter(function (_a) {
3510
+ var modelVariant = _a.modelVariant;
3511
+ return modelVariant === 'CHAT';
3512
+ })
3513
+ .map(function (_a) {
3514
+ var modelName = _a.modelName;
3515
+ return modelName;
3516
+ })
3517
+ .join(',');
3518
+ return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
3519
+ case 3:
3520
+ result = _d.sent();
3521
+ assertsExecutionSuccessful(result);
3522
+ outputParameters = result.outputParameters;
3523
+ modelRequirementsRaw = outputParameters.modelRequirements;
3524
+ modelRequirements = JSON.parse(modelRequirementsRaw);
3525
+ if (isVerbose) {
3526
+ console.info("PERSONA ".concat(personaDescription), modelRequirements);
3527
+ }
3528
+ modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
3529
+ return [2 /*return*/, {
3530
+ modelVariant: 'CHAT',
3531
+ modelName: modelName,
3532
+ systemMessage: systemMessage,
3533
+ temperature: temperature,
3534
+ }];
3535
+ }
3536
+ });
3537
+ });
3538
+ }
3539
+ /**
3540
+ * TODO: [🔼] !!! Export via `@promptbook/core`
3541
+ * TODO: [🏢] !! Check validity of `modelName` in pipeline
3542
+ * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
3543
+ * TODO: [🏢] !! Check validity of `temperature` in pipeline
3544
+ */
3545
+
3546
+ /**
3547
+ * Prepare pipeline from string (markdown) format to JSON format
3548
+ *
3549
+ * Note: This function does not validate logic of the pipeline
3550
+ * Note: This function acts as part of compilation process
3551
+ */
3552
+ function preparePipeline(pipeline, options) {
3553
+ return __awaiter(this, void 0, void 0, function () {
3554
+ var _a, maxParallelCount,
3555
+ /*
3556
+ <- TODO: [🧠][0] `promptbookVersion` */
3557
+ knowledgeSources /*
3558
+ <- TODO: [🧊] `knowledgePieces` */, personas /*
3559
+ <- TODO: [🧊] `preparations` */, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared;
3560
+ var _this = this;
3561
+ return __generator(this, function (_b) {
3562
+ switch (_b.label) {
3563
+ case 0:
3564
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
3565
+ knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
3566
+ currentPreparation = {
3567
+ id: 1,
3568
+ // TODO: [🍥]> date: $currentDate(),
3569
+ promptbookVersion: PROMPTBOOK_VERSION,
3570
+ modelUsage: ZERO_USAGE,
3571
+ };
3572
+ preparations = [
3573
+ // ...preparations
3574
+ // <- TODO: [🧊]
3575
+ currentPreparation,
3576
+ ];
3577
+ preparedPersonas = new Array(personas.length);
3578
+ return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () {
3579
+ var modelRequirements, preparedPersona;
3580
+ return __generator(this, function (_a) {
3581
+ switch (_a.label) {
3582
+ case 0: return [4 /*yield*/, preparePersona(persona.description, options)];
3583
+ case 1:
3584
+ modelRequirements = _a.sent();
3585
+ preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] });
3586
+ preparedPersonas[index] = preparedPersona;
3587
+ return [2 /*return*/];
3588
+ }
3589
+ });
3590
+ }); })];
3591
+ case 1:
3592
+ _b.sent();
3593
+ knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
3594
+ return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, options)];
3595
+ case 2:
3596
+ partialknowledgePiecesPrepared = _b.sent();
3597
+ knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
3598
+ // ----- /Knowledge preparation -----
3599
+ // TODO: !!!!! Add context to each template (if missing)
3600
+ // TODO: !!!!! Apply samples to each template (if missing)
3601
+ return [2 /*return*/, __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations })];
3602
+ }
3603
+ });
3604
+ });
3605
+ }
3606
+ /**
3607
+ * TODO: !!!!! Index the samples and maybe templates
3608
+ * TODO: [🔼] !!! Export via `@promptbook/core`
3609
+ * TODO: Write tests for `preparePipeline`
3610
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
3611
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
3612
+ * TODO: [🎐] !!!! Use here countTotalUsage
3613
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
3614
+ */
3615
+
3616
+ /**
3617
+ * Tests if given string is valid URL.
3618
+ *
3619
+ * Note: This does not check if the file exists only if the path is valid
3620
+ */
3621
+ function isValidFilePath(filePath) {
3622
+ if (typeof filePath !== 'string') {
3623
+ return false;
3624
+ }
3625
+ var filePathSlashes = filePath.split('\\').join('/');
3626
+ // Absolute Unix path: /hello.txt
3627
+ if (/^(\/)/i.test(filePathSlashes)) {
3628
+ return true;
3629
+ }
3630
+ // Absolute Windows path: /hello.txt
3631
+ if (/^([A-Z]{1,2}:\/?)\//i.test(filePathSlashes)) {
3632
+ return true;
3633
+ }
3634
+ // Relative path: ./hello.txt
3635
+ if (/^(\.\.?\/)+/i.test(filePathSlashes)) {
3636
+ return true;
3637
+ }
3638
+ return false;
3639
+ }
3640
+
3641
+ /**
3642
+ * Parses the knowledge command
3643
+ *
3644
+ * @see ./KNOWLEDGE-README.md for more details
3645
+ * @private within the commands folder
3646
+ */
3647
+ var knowledgeCommandParser = {
3648
+ /**
3649
+ * Name of the command
3650
+ */
3651
+ name: 'KNOWLEDGE',
3652
+ /**
3653
+ * BOILERPLATE command can be used in:
3654
+ */
3655
+ usagePlaces: ['PIPELINE_HEAD'],
3656
+ /**
3657
+ * Description of the KNOWLEDGE command
3658
+ */
3659
+ description: "Tells promptbook which external knowledge to use",
3660
+ /**
3661
+ * Link to discussion
3662
+ */
3663
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/41',
3664
+ /**
3665
+ * Example usages of the KNOWLEDGE command
3666
+ */
3667
+ examples: [
3668
+ 'KNOWLEDGE https://www.pavolhejny.com/',
3669
+ 'KNOWLEDGE ./hejny-cv.txt',
3670
+ 'KNOWLEDGE ./hejny-cv.md',
3671
+ 'KNOWLEDGE ./hejny-cv.pdf',
3672
+ 'KNOWLEDGE ./hejny-cv.docx',
3673
+ ],
3674
+ /**
3675
+ * Parses the KNOWLEDGE command
3676
+ */
3677
+ parse: function (input) {
3678
+ var args = input.args;
3679
+ var source = args[0];
3680
+ if (source === undefined) {
3681
+ throw new ParsingError("Source is not defined");
3682
+ }
3683
+ if (source.startsWith('http://')) {
3684
+ throw new ParsingError("Source is not secure");
3685
+ }
3686
+ if (!(isValidFilePath(source) || isValidUrl(source))) {
3687
+ throw new ParsingError("Source not valid");
3688
+ }
3689
+ if (source.startsWith('../') || source.startsWith('/') || /^[A-Z]:[\\/]+/i.test(source)) {
3690
+ throw new ParsingError("Source cannot be outside of the .ptbk.md folder");
3691
+ }
3692
+ return {
3693
+ type: 'KNOWLEDGE',
3694
+ source: source,
3695
+ };
3696
+ },
3697
+ /**
3698
+ * Note: Prototype of [🍧] (remove this comment after full implementation)
3699
+ */
3700
+ applyToPipelineJson: function (personaCommand, subjects) {
3701
+ var source = personaCommand.source;
3702
+ var pipelineJson = subjects.pipelineJson;
3703
+ var name = titleToName(source);
3704
+ pipelineJson.knowledgeSources.push({
3705
+ name: name,
3706
+ source: source,
3707
+ });
3708
+ },
3709
+ };
3710
+
2748
3711
  /**
2749
- * 11:11
3712
+ * Parses the persona command
3713
+ *
3714
+ * @see ./PERSONA-README.md for more details
3715
+ * @private within the commands folder
2750
3716
  */
3717
+ var personaCommandParser = {
3718
+ /**
3719
+ * Name of the command
3720
+ */
3721
+ name: 'PERSONA',
3722
+ /**
3723
+ * Aliases for the PERSONA command
3724
+ */
3725
+ aliasNames: ['PERSON'],
3726
+ /**
3727
+ * PERSONA command can be used in:
3728
+ */
3729
+ usagePlaces: ['PIPELINE_HEAD', 'PIPELINE_TEMPLATE'],
3730
+ /**
3731
+ * Description of the PERSONA command
3732
+ */
3733
+ description: "Persona command is used to specify who the system is, it will be transformed into system message, top_t,...",
3734
+ /**
3735
+ * Link to discussion
3736
+ */
3737
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/22',
3738
+ /**
3739
+ * Example usages of the PERSONA command
3740
+ */
3741
+ examples: ['PERSONA Jane, skilled copywriter', 'PERSONA Joe, male 28 years old, programmer'],
3742
+ /**
3743
+ * Parses the PERSONA command
3744
+ */
3745
+ parse: function (input) {
3746
+ var rawArgs = input.rawArgs;
3747
+ var _a = __read(rawArgs.split(/[,;:]/, 2), 2), personaNameRaw = _a[0], personaDescriptionRaw = _a[1];
3748
+ var personaName = (personaNameRaw || '').trim();
3749
+ if (personaName === '') {
3750
+ throw new ParsingError("You must set name for the persona");
3751
+ }
3752
+ var personaDescription = (personaDescriptionRaw || '').trim();
3753
+ if (personaDescription === '') {
3754
+ personaDescription = null;
3755
+ }
3756
+ return {
3757
+ type: 'PERSONA',
3758
+ personaName: personaName,
3759
+ personaDescription: personaDescription,
3760
+ };
3761
+ },
3762
+ /**
3763
+ * Note: Prototype of [🍧] (remove this comment after full implementation)
3764
+ */
3765
+ applyToPipelineJson: function (personaCommand, subjects) {
3766
+ var personaName = personaCommand.personaName, personaDescription = personaCommand.personaDescription;
3767
+ var pipelineJson = subjects.pipelineJson, templateJson = subjects.templateJson;
3768
+ if (templateJson !== null) {
3769
+ if (templateJson.blockType !== 'PROMPT_TEMPLATE') {
3770
+ throw new ParsingError("PERSONA command can be used only in PROMPT_TEMPLATE block");
3771
+ }
3772
+ templateJson.personaName = personaName;
3773
+ }
3774
+ var persona = pipelineJson.personas.find(function (persona) { return persona.name === personaName; });
3775
+ if (persona === undefined) {
3776
+ pipelineJson.personas.push({
3777
+ name: personaName,
3778
+ description: personaDescription || '',
3779
+ });
3780
+ return;
3781
+ }
3782
+ if (persona.description === personaDescription) {
3783
+ return;
3784
+ }
3785
+ if (personaDescription === null) {
3786
+ return;
3787
+ }
3788
+ if (persona.description === '') {
3789
+ persona.description = personaDescription;
3790
+ return;
3791
+ }
3792
+ console.warn(spaceTrim__default["default"]("\n\n Persona \"".concat(personaName, "\" is defined multiple times with different description:\n\n First definition:\n ").concat(persona.description, "\n\n Second definition:\n ").concat(personaDescription, "\n\n ")));
3793
+ persona.description += spaceTrim__default["default"]('\n\n' + personaDescription);
3794
+ },
3795
+ };
2751
3796
 
2752
3797
  /**
2753
3798
  * Removes Markdown formatting tags from a string.
@@ -2867,7 +3912,7 @@
2867
3912
  /**
2868
3913
  * Link to discussion
2869
3914
  */
2870
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/64',
3915
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/64',
2871
3916
  /**
2872
3917
  * Example usages of the BLOCK command
2873
3918
  */
@@ -2887,7 +3932,7 @@
2887
3932
  'Knowledge BLOCK',
2888
3933
  // 'Knowledge', // <- Note: [⛱] For execution blocks which are also separate commands shortcut does not work
2889
3934
  //---
2890
- /* TODO: !!!! Not implemented block types will be in examples in future -> */
3935
+ /* Note: Not implemented block types will be in examples in future -> */
2891
3936
  'Instrument BLOCK',
2892
3937
  // 'Instrument', // <- Note: [⛱]
2893
3938
  'Action BLOCK',
@@ -2895,6 +3940,7 @@
2895
3940
  //---
2896
3941
  /* <- TODO: [🧠] Maybe dynamic */
2897
3942
  ],
3943
+ // TODO: [♓️] order: -10 /* <- Note: Putting before other commands */
2898
3944
  /**
2899
3945
  * Parses the BLOCK command
2900
3946
  */
@@ -2903,19 +3949,19 @@
2903
3949
  normalized = normalized.split('EXAMPLE').join('SAMPLE');
2904
3950
  var blockTypes = BlockTypes.filter(function (blockType) { return normalized.includes(blockType); });
2905
3951
  if (blockTypes.length !== 1) {
2906
- // console.log('!!!', { blockType });
2907
3952
  throw new ParsingError(spaceTrim__default["default"](function (block) { return "\n Unknown block type in BLOCK command\n\n Supported block types are:\n ".concat(block(BlockTypes.join(', ')), "\n "); }));
2908
3953
  }
2909
- // TODO: !!!! Not supported yet
3954
+ var blockType = blockTypes[0];
2910
3955
  return {
2911
3956
  type: 'BLOCK',
2912
- blockType: blockTypes[0],
3957
+ blockType: blockType,
2913
3958
  };
2914
3959
  },
2915
3960
  };
2916
3961
 
2917
3962
  /**
2918
3963
  * Units of text measurement
3964
+ * @see https://github.com/webgptorg/promptbook/discussions/30
2919
3965
  */
2920
3966
  var EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'];
2921
3967
  /**
@@ -3011,7 +4057,7 @@
3011
4057
  /**
3012
4058
  * Link to discussion
3013
4059
  */
3014
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/30',
4060
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/30',
3015
4061
  /**
3016
4062
  * Example usages of the EXPECT command
3017
4063
  */
@@ -3129,7 +4175,7 @@
3129
4175
  /**
3130
4176
  * Link to discussion
3131
4177
  */
3132
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/66',
4178
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/66',
3133
4179
  /**
3134
4180
  * Example usages of the JOKER command
3135
4181
  */
@@ -3152,94 +4198,11 @@
3152
4198
  };
3153
4199
 
3154
4200
  /**
3155
- * Tests if given string is valid URL.
3156
- *
3157
- * Note: This does not check if the file exists only if the path is valid
3158
- */
3159
- function isValidFilePath(filePath) {
3160
- if (typeof filePath !== 'string') {
3161
- return false;
3162
- }
3163
- var filePathSlashes = filePath.split('\\').join('/');
3164
- // Absolute Unix path: /hello.txt
3165
- if (/^(\/)/i.test(filePathSlashes)) {
3166
- return true;
3167
- }
3168
- // Absolute Windows path: /hello.txt
3169
- if (/^([A-Z]{1,2}:\/?)\//i.test(filePathSlashes)) {
3170
- return true;
3171
- }
3172
- // Relative path: ./hello.txt
3173
- if (/^(\.\.?\/)+/i.test(filePathSlashes)) {
3174
- return true;
3175
- }
3176
- return false;
3177
- }
3178
-
3179
- /**
3180
- * Parses the knowledge command
4201
+ * @@@
3181
4202
  *
3182
- * @see ./KNOWLEDGE-README.md for more details
3183
- * @private within the commands folder
4203
+ * @private for `ModelVariant` and `modelCommandParser`
3184
4204
  */
3185
- var knowledgeCommandParser = {
3186
- /**
3187
- * Name of the command
3188
- */
3189
- name: 'KNOWLEDGE',
3190
- /**
3191
- * BOILERPLATE command can be used in:
3192
- */
3193
- usagePlaces: ['PIPELINE_HEAD'],
3194
- /**
3195
- * Description of the KNOWLEDGE command
3196
- */
3197
- description: "Tells promptbook which external knowledge to use",
3198
- /**
3199
- * Link to discussion
3200
- */
3201
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/41',
3202
- /**
3203
- * Example usages of the KNOWLEDGE command
3204
- */
3205
- examples: [
3206
- 'KNOWLEDGE https://www.pavolhejny.com/',
3207
- 'KNOWLEDGE ./hejny-cv.txt',
3208
- 'KNOWLEDGE ./hejny-cv.md',
3209
- 'KNOWLEDGE ./hejny-cv.pdf',
3210
- 'KNOWLEDGE ./hejny-cv.docx',
3211
- ],
3212
- /**
3213
- * Parses the KNOWLEDGE command
3214
- */
3215
- parse: function (input) {
3216
- var args = input.args;
3217
- var source = args[0];
3218
- if (source === undefined) {
3219
- throw new ParsingError("Source is not defined");
3220
- }
3221
- if (source.startsWith('http://')) {
3222
- throw new ParsingError("Source is not secure");
3223
- }
3224
- if (!(isValidFilePath(source) || isValidUrl(source))) {
3225
- throw new ParsingError("Source not valid");
3226
- }
3227
- if (source.startsWith('../') || source.startsWith('/') || /^[A-Z]:[\\/]+/i.test(source)) {
3228
- throw new ParsingError("Source cannot be outside of the .ptbk.md folder");
3229
- }
3230
- return {
3231
- type: 'KNOWLEDGE',
3232
- source: source,
3233
- };
3234
- },
3235
- };
3236
-
3237
4205
  var MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
3238
- /**
3239
- * TODO: Maybe figure out better word than "variant"
3240
- * TODO: Add here more requirement options like max context size, max tokens, etc.
3241
- * TODO: [👙][🧠] Just selecting gpt3 or gpt4 level of model
3242
- */
3243
4206
 
3244
4207
  /**
3245
4208
  * Parses the model command
@@ -3255,7 +4218,11 @@
3255
4218
  /**
3256
4219
  * BOILERPLATE command can be used in:
3257
4220
  */
3258
- usagePlaces: ['PIPELINE_HEAD', 'PIPELINE_TEMPLATE'],
4221
+ usagePlaces: [
4222
+ 'PIPELINE_HEAD',
4223
+ // <- TODO: [🧠][❔] Should there be possibility to set MODEL for entire pipeline?
4224
+ 'PIPELINE_TEMPLATE',
4225
+ ],
3259
4226
  /**
3260
4227
  * Description of the MODEL command
3261
4228
  */
@@ -3263,7 +4230,7 @@
3263
4230
  /**
3264
4231
  * Link to discussion
3265
4232
  */
3266
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/67',
4233
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/67',
3267
4234
  /**
3268
4235
  * Example usages of the MODEL command
3269
4236
  */
@@ -3343,7 +4310,7 @@
3343
4310
  /**
3344
4311
  * Link to discussion
3345
4312
  */
3346
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/68',
4313
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/68',
3347
4314
  /**
3348
4315
  * Example usages of the PARAMETER command
3349
4316
  */
@@ -3377,59 +4344,6 @@
3377
4344
  },
3378
4345
  };
3379
4346
 
3380
- /**
3381
- * Parses the persona command
3382
- *
3383
- * @see ./PERSONA-README.md for more details
3384
- * @private within the commands folder
3385
- */
3386
- var personaCommandParser = {
3387
- /**
3388
- * Name of the command
3389
- */
3390
- name: 'PERSONA',
3391
- /**
3392
- * Aliases for the PERSONA command
3393
- */
3394
- aliasNames: ['PERSON'],
3395
- /**
3396
- * PERSONA command can be used in:
3397
- */
3398
- usagePlaces: ['PIPELINE_HEAD', 'PIPELINE_TEMPLATE'],
3399
- /**
3400
- * Description of the PERSONA command
3401
- */
3402
- description: "Persona command is used to specify who the system is, it will be transformed into system message, top_t,...",
3403
- /**
3404
- * Link to discussion
3405
- */
3406
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/22',
3407
- /**
3408
- * Example usages of the PERSONA command
3409
- */
3410
- examples: ['PERSONA Jane, skilled copywriter', 'PERSONA Joe, male 28 years old, programmer'],
3411
- /**
3412
- * Parses the PERSONA command
3413
- */
3414
- parse: function (input) {
3415
- var rawArgs = input.rawArgs;
3416
- var _a = __read(rawArgs.split(/[,;:]/, 2), 2), personaNameRaw = _a[0], personaDescriptionRaw = _a[1];
3417
- var personaName = (personaNameRaw || '').trim();
3418
- if (personaName === '') {
3419
- throw new ParsingError("You must set name for the persona");
3420
- }
3421
- var personaDescription = (personaDescriptionRaw || '').trim();
3422
- if (personaDescription === '') {
3423
- personaDescription = null;
3424
- }
3425
- return {
3426
- type: 'PERSONA',
3427
- personaName: personaName,
3428
- personaDescription: personaDescription,
3429
- };
3430
- },
3431
- };
3432
-
3433
4347
  function isValidJavascriptName(javascriptName) {
3434
4348
  if (typeof javascriptName !== 'string') {
3435
4349
  return false;
@@ -3460,7 +4374,7 @@
3460
4374
  /**
3461
4375
  * Link to discussion
3462
4376
  */
3463
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/31',
4377
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/31',
3464
4378
  /**
3465
4379
  * Example usages of the POSTPROCESS command
3466
4380
  */
@@ -3515,7 +4429,7 @@
3515
4429
  /**
3516
4430
  * Link to discussion
3517
4431
  */
3518
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/69',
4432
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/69',
3519
4433
  /**
3520
4434
  * Example usages of the PROMPTBOOK_VERSION command
3521
4435
  */
@@ -3568,7 +4482,7 @@
3568
4482
  /**
3569
4483
  * Link to discussion
3570
4484
  */
3571
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/70',
4485
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/70',
3572
4486
  /**
3573
4487
  * Example usages of the URL command
3574
4488
  */
@@ -3616,22 +4530,6 @@
3616
4530
  },
3617
4531
  };
3618
4532
 
3619
- /**
3620
- * Returns the same value that is passed as argument.
3621
- * No side effects.
3622
- *
3623
- * Note: It can be usefull for leveling indentation
3624
- *
3625
- * @param value any values
3626
- * @returns the same values
3627
- */
3628
- function just(value) {
3629
- if (value === undefined) {
3630
- return undefined;
3631
- }
3632
- return value;
3633
- }
3634
-
3635
4533
  /**
3636
4534
  * Parses the action command
3637
4535
  *
@@ -3654,7 +4552,7 @@
3654
4552
  /**
3655
4553
  * Link to discussion
3656
4554
  */
3657
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/72',
4555
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/72',
3658
4556
  /**
3659
4557
  * Example usages of the ACTION command
3660
4558
  */
@@ -3663,7 +4561,8 @@
3663
4561
  * Parses the ACTION command
3664
4562
  */
3665
4563
  parse: function (input) {
3666
- input.args;
4564
+ var args = input.args;
4565
+ TODO_USE(args);
3667
4566
  return {
3668
4567
  type: 'ACTION',
3669
4568
  };
@@ -3692,7 +4591,7 @@
3692
4591
  /**
3693
4592
  * Link to discussion
3694
4593
  */
3695
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/71',
4594
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/71',
3696
4595
  /**
3697
4596
  * Example usages of the INSTRUMENT command
3698
4597
  */
@@ -3701,7 +4600,8 @@
3701
4600
  * Parses the INSTRUMENT command
3702
4601
  */
3703
4602
  parse: function (input) {
3704
- input.args;
4603
+ var args = input.args;
4604
+ TODO_USE(args);
3705
4605
  return {
3706
4606
  type: 'INSTRUMENT',
3707
4607
  };
@@ -3734,7 +4634,7 @@
3734
4634
  /**
3735
4635
  * Link to discussion
3736
4636
  */
3737
- discussionUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
4637
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
3738
4638
  /**
3739
4639
  * Example usages of the BOILERPLATE command
3740
4640
  */
@@ -3759,6 +4659,7 @@
3759
4659
  };
3760
4660
  /**
3761
4661
  * TODO: [💐] Implement BOILERPLATE command into `pipelineStringToJsonSync` function
4662
+ * Note: [⚪] This should never be in any released package
3762
4663
  */
3763
4664
 
3764
4665
  /**
@@ -3777,7 +4678,7 @@
3777
4678
  actionCommandParser,
3778
4679
  instrumentCommandParser,
3779
4680
  personaCommandParser,
3780
- boilerplateCommandParser, // <- TODO: !!!! Only in development, remove in production
4681
+ boilerplateCommandParser, // <- TODO: !! Only in development, remove in production
3781
4682
  ];
3782
4683
 
3783
4684
  /**
@@ -3863,18 +4764,18 @@
3863
4764
  }));
3864
4765
  }
3865
4766
  /**
3866
- * !!!
4767
+ * @@@
3867
4768
  */
3868
4769
  function getSupportedCommandsMessage() {
3869
4770
  return COMMANDS.flatMap(function (_a) {
3870
- var name = _a.name, aliasNames = _a.aliasNames, description = _a.description, discussionUrl = _a.discussionUrl;
4771
+ var name = _a.name, aliasNames = _a.aliasNames, description = _a.description, documentationUrl = _a.documentationUrl;
3871
4772
  return __spreadArray([
3872
- "- **".concat(name, "** ").concat(description, ", see [discussion](").concat(discussionUrl, ")")
4773
+ "- **".concat(name, "** ").concat(description, ", see [discussion](").concat(documentationUrl, ")")
3873
4774
  ], __read((aliasNames || []).map(function (aliasName) { return " - **".concat(aliasName, "** Alias for **").concat(name, "**"); })), false);
3874
4775
  }).join('\n');
3875
4776
  }
3876
4777
  /**
3877
- * !!!
4778
+ * @@@
3878
4779
  */
3879
4780
  function parseCommandVariant(input) {
3880
4781
  var e_1, _a;
@@ -3883,7 +4784,6 @@
3883
4784
  var _loop_1 = function (commandParser) {
3884
4785
  var name_1 = commandParser.name, aliasNames = commandParser.aliasNames, deprecatedNames = commandParser.deprecatedNames, parse = commandParser.parse;
3885
4786
  var names = __spreadArray(__spreadArray([name_1], __read((aliasNames || [])), false), __read((deprecatedNames || [])), false);
3886
- // console.log('!!!', { commandName, names });
3887
4787
  if (names.includes(commandName)) {
3888
4788
  try {
3889
4789
  return { value: parse({ usagePlace: usagePlace, raw: raw, rawArgs: rawArgs, normalized: normalized, args: args }) };
@@ -3993,25 +4893,42 @@
3993
4893
  var e_1, _a;
3994
4894
  var codeBlocks = [];
3995
4895
  var lines = markdown.split('\n');
4896
+ // Note: [0] Ensure that the last block notated by gt > will be closed
4897
+ lines.push('');
3996
4898
  var currentCodeBlock = null;
3997
4899
  try {
3998
4900
  for (var lines_1 = __values(lines), lines_1_1 = lines_1.next(); !lines_1_1.done; lines_1_1 = lines_1.next()) {
3999
4901
  var line = lines_1_1.value;
4902
+ if (line.startsWith('> ') || line === '>') {
4903
+ if (currentCodeBlock === null) {
4904
+ currentCodeBlock = { blockNotation: '>', language: null, content: '' };
4905
+ } /* not else */
4906
+ if (currentCodeBlock.blockNotation === '>') {
4907
+ if (currentCodeBlock.content !== '') {
4908
+ currentCodeBlock.content += '\n';
4909
+ }
4910
+ currentCodeBlock.content += line.slice(2);
4911
+ }
4912
+ }
4913
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '>' /* <- Note: [0] */) {
4914
+ codeBlocks.push(currentCodeBlock);
4915
+ currentCodeBlock = null;
4916
+ }
4917
+ /* not else */
4000
4918
  if (line.startsWith('```')) {
4001
4919
  var language = line.slice(3).trim() || null;
4002
4920
  if (currentCodeBlock === null) {
4003
- currentCodeBlock = { language: language, content: '' };
4921
+ currentCodeBlock = { blockNotation: '```', language: language, content: '' };
4004
4922
  }
4005
4923
  else {
4006
4924
  if (language !== null) {
4007
- // [🌻]
4008
- throw new Error("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
4925
+ throw new ParsingError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed and already opening new ").concat(language, " code block"));
4009
4926
  }
4010
4927
  codeBlocks.push(currentCodeBlock);
4011
4928
  currentCodeBlock = null;
4012
4929
  }
4013
4930
  }
4014
- else if (currentCodeBlock !== null) {
4931
+ else if (currentCodeBlock !== null && currentCodeBlock.blockNotation === '```') {
4015
4932
  if (currentCodeBlock.content !== '') {
4016
4933
  currentCodeBlock.content += '\n';
4017
4934
  }
@@ -4027,11 +4944,13 @@
4027
4944
  finally { if (e_1) throw e_1.error; }
4028
4945
  }
4029
4946
  if (currentCodeBlock !== null) {
4030
- // [🌻]
4031
- throw new Error("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
4947
+ throw new ParsingError("".concat(capitalize(currentCodeBlock.language || 'the'), " code block was not closed at the end of the markdown"));
4032
4948
  }
4033
4949
  return codeBlocks;
4034
4950
  }
4951
+ /**
4952
+ * TODO: Maybe name for `blockNotation` instead of '```' and '>'
4953
+ */
4035
4954
 
4036
4955
  /**
4037
4956
  * Extracts exactly ONE code block from markdown.
@@ -4049,13 +4968,12 @@
4049
4968
  function extractOneBlockFromMarkdown(markdown) {
4050
4969
  var codeBlocks = extractAllBlocksFromMarkdown(markdown);
4051
4970
  if (codeBlocks.length !== 1) {
4052
- // TODO: Report more specific place where the error happened
4053
- throw new Error(/* <- [🌻] */ 'There should be exactly one code block in the markdown');
4971
+ throw new ParsingError(spaceTrim__default["default"](function (block) { return "\n There should be exactly 1 code block, found ".concat(codeBlocks.length, " code blocks\n\n ").concat(block(codeBlocks.map(function (block, i) { return "Block ".concat(i + 1, ":\n").concat(block.content); }).join('\n\n\n')), "\n "); }));
4054
4972
  }
4055
4973
  return codeBlocks[0];
4056
4974
  }
4057
4975
  /***
4058
- * TODO: [🍓][🌻] !!! Decide of this is internal util, external util OR validator/postprocessor
4976
+ * TODO: [🍓][🌻] Decide of this is internal util, external util OR validator/postprocessor
4059
4977
  */
4060
4978
 
4061
4979
  /**
@@ -4065,13 +4983,13 @@
4065
4983
  var _a, _b;
4066
4984
  var lines = value.split('\n');
4067
4985
  if (!lines[0].startsWith('#')) {
4068
- throw new Error('Markdown section must start with heading');
4986
+ throw new ParsingError('Markdown section must start with heading');
4069
4987
  }
4070
4988
  var title = lines[0].replace(/^#+\s*/, '');
4071
4989
  var level = (_b = (_a = lines[0].match(/^#+/)) === null || _a === void 0 ? void 0 : _a[0].length) !== null && _b !== void 0 ? _b : 0;
4072
4990
  var content = spaceTrim__default["default"](lines.slice(1).join('\n'));
4073
4991
  if (level < 1 || level > 6) {
4074
- throw new Error('Markdown section must have heading level between 1 and 6');
4992
+ throw new ParsingError('Markdown section must have heading level between 1 and 6');
4075
4993
  }
4076
4994
  return { title: title, level: level, content: content };
4077
4995
  }
@@ -4208,203 +5126,14 @@
4208
5126
  }
4209
5127
 
4210
5128
  /**
4211
- * Create difference set of two sets.
4212
- *
4213
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
4214
- */
4215
- function difference(a, b, isEqual) {
4216
- var e_1, _a;
4217
- if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
4218
- var diff = new Set();
4219
- var _loop_1 = function (itemA) {
4220
- if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
4221
- diff.add(itemA);
4222
- }
4223
- };
4224
- try {
4225
- for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
4226
- var itemA = _c.value;
4227
- _loop_1(itemA);
4228
- }
4229
- }
4230
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4231
- finally {
4232
- try {
4233
- if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
4234
- }
4235
- finally { if (e_1) throw e_1.error; }
4236
- }
4237
- return diff;
4238
- }
4239
-
4240
- /**
4241
- * Creates a new set with all elements that are present in either set
4242
- *
4243
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
4244
- */
4245
- function union() {
4246
- var e_1, _a, e_2, _b;
4247
- var sets = [];
4248
- for (var _i = 0; _i < arguments.length; _i++) {
4249
- sets[_i] = arguments[_i];
4250
- }
4251
- var union = new Set();
4252
- try {
4253
- for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
4254
- var set = sets_1_1.value;
4255
- try {
4256
- for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
4257
- var item = _d.value;
4258
- union.add(item);
4259
- }
4260
- }
4261
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
4262
- finally {
4263
- try {
4264
- if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
4265
- }
4266
- finally { if (e_2) throw e_2.error; }
4267
- }
4268
- }
4269
- }
4270
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4271
- finally {
4272
- try {
4273
- if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
4274
- }
4275
- finally { if (e_1) throw e_1.error; }
4276
- }
4277
- return union;
4278
- }
4279
-
4280
- /**
4281
- * Parses the template and returns the list of all parameter names
4282
- *
4283
- * @param template the template with parameters in {curly} braces
4284
- * @returns the list of parameter names
4285
- */
4286
- function extractParameters(template) {
4287
- var e_1, _a;
4288
- var matches = template.matchAll(/{\w+}/g);
4289
- var parameterNames = new Set();
4290
- try {
4291
- for (var matches_1 = __values(matches), matches_1_1 = matches_1.next(); !matches_1_1.done; matches_1_1 = matches_1.next()) {
4292
- var match = matches_1_1.value;
4293
- var parameterName = match[0].slice(1, -1);
4294
- parameterNames.add(parameterName);
4295
- }
4296
- }
4297
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4298
- finally {
4299
- try {
4300
- if (matches_1_1 && !matches_1_1.done && (_a = matches_1.return)) _a.call(matches_1);
4301
- }
4302
- finally { if (e_1) throw e_1.error; }
4303
- }
4304
- return parameterNames;
4305
- }
4306
-
4307
- /**
4308
- * Parses the given script and returns the list of all used variables that are not defined in the script
4309
- *
4310
- * @param script from which to extract the variables
4311
- * @returns the list of variable names
4312
- * @throws {ParsingError} if the script is invalid
4313
- */
4314
- function extractVariables(script) {
4315
- var variables = new Set();
4316
- script = "(()=>{".concat(script, "})()");
4317
- try {
4318
- for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
4319
- try {
4320
- eval(script);
4321
- }
4322
- catch (error) {
4323
- if (!(error instanceof ReferenceError)) {
4324
- throw error;
4325
- }
4326
- var undefinedName = error.message.split(' ')[0];
4327
- /*
4328
- Note: Parsing the error
4329
- [ReferenceError: thing is not defined]
4330
- */
4331
- if (!undefinedName) {
4332
- throw error;
4333
- }
4334
- if (script.includes(undefinedName + '(')) {
4335
- script = "const ".concat(undefinedName, " = ()=>'';") + script;
4336
- }
4337
- else {
4338
- variables.add(undefinedName);
4339
- script = "const ".concat(undefinedName, " = '';") + script;
4340
- }
4341
- }
4342
- }
4343
- catch (error) {
4344
- if (!(error instanceof Error)) {
4345
- throw error;
4346
- }
4347
- throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Can not extract variables from the script\n\n ".concat(block(error.name), ": ").concat(block(error.message), "\n "); }));
4348
- }
4349
- return variables;
4350
- }
4351
- /**
4352
- * TODO: [🔣] Support for multiple languages - python, java,...
4353
- */
4354
-
4355
- /**
4356
- * Parses the prompt template and returns the set of all used parameters
4357
- *
4358
- * @param promptTemplate the template with used parameters
4359
- * @returns the set of parameter names
4360
- * @throws {ParsingError} if the script is invalid
4361
- */
4362
- function extractParametersFromPromptTemplate(promptTemplate) {
4363
- var e_1, _a, e_2, _b;
4364
- var parameterNames = new Set();
4365
- try {
4366
- for (var _c = __values(__spreadArray(__spreadArray(__spreadArray([], __read(extractParameters(promptTemplate.title)), false), __read(extractParameters(promptTemplate.description || '')), false), __read(extractParameters(promptTemplate.content)), false)), _d = _c.next(); !_d.done; _d = _c.next()) {
4367
- var parameterName = _d.value;
4368
- parameterNames.add(parameterName);
4369
- }
4370
- }
4371
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
4372
- finally {
4373
- try {
4374
- if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
4375
- }
4376
- finally { if (e_1) throw e_1.error; }
4377
- }
4378
- if (promptTemplate.blockType === 'SCRIPT') {
4379
- try {
4380
- for (var _e = __values(extractVariables(promptTemplate.content)), _f = _e.next(); !_f.done; _f = _e.next()) {
4381
- var parameterName = _f.value;
4382
- parameterNames.add(parameterName);
4383
- }
4384
- }
4385
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
4386
- finally {
4387
- try {
4388
- if (_f && !_f.done && (_b = _e.return)) _b.call(_e);
4389
- }
4390
- finally { if (e_2) throw e_2.error; }
4391
- }
4392
- }
4393
- return parameterNames;
4394
- }
4395
- /**
4396
- * TODO: [🔣] If script require contentLanguage
4397
- */
4398
-
4399
- /**
4400
- * Compile promptbook from string (markdown) format to JSON format synchronously
5129
+ * Compile pipeline from string (markdown) format to JSON format synchronously
4401
5130
  *
4402
- * Note: There are two similar functions:
5131
+ * Note: There are 3 similar functions:
4403
5132
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
4404
5133
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
5134
+ * - `preparePipeline` - just one step in the compilation process
4405
5135
  *
4406
5136
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
4407
- * @param options - Options and tools for the compilation
4408
5137
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
4409
5138
  * @throws {ParsingError} if the promptbook string is not valid
4410
5139
  *
@@ -4420,7 +5149,10 @@
4420
5149
  description: undefined /* <- Note: Putting here placeholder to keep `description` on top at final JSON */,
4421
5150
  parameters: [],
4422
5151
  promptTemplates: [],
4423
- knowledge: [],
5152
+ knowledgeSources: [],
5153
+ knowledgePieces: [],
5154
+ personas: [],
5155
+ preparations: [],
4424
5156
  };
4425
5157
  // =============================================================
4426
5158
  // Note: 1️⃣ Parsing of the markdown into object
@@ -4450,7 +5182,7 @@
4450
5182
  existingParameter.description &&
4451
5183
  existingParameter.description !== parameterDescription &&
4452
5184
  parameterDescription) {
4453
- throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is defined multiple times with different description.\n\n First definition:\n ").concat(block(existingParameter.description || '[undefined]'), "\n\n Second definition:\n ").concat(block(parameterDescription || '[undefined]'), "\n "); }));
5185
+ throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Parameter {".concat(parameterName, "} is defined multiple times with different description:\n\n First definition:\n ").concat(block(existingParameter.description || '[undefined]'), "\n\n Second definition:\n ").concat(block(parameterDescription || '[undefined]'), "\n "); }));
4454
5186
  }
4455
5187
  if (existingParameter) {
4456
5188
  if (parameterDescription) {
@@ -4469,11 +5201,12 @@
4469
5201
  // =============================================================
4470
5202
  // Note: 3️⃣ Process pipeline head
4471
5203
  pipelineJson.title = pipelineHead.title;
4472
- // TODO: [1] DRY description
5204
+ // TODO: [🎾][1] DRY description
4473
5205
  var description = pipelineHead.content;
4474
- // Note: Remove codeblocks - TODO: Maybe put this into util (exported from `@promptbool/utils`)
5206
+ // Note: Remove codeblocks - TODO: [🎾] Make util removeAllBlocksFromMarkdown (exported from `@promptbool/utils`)
4475
5207
  description = description.split(/^```.*^```/gms).join('');
4476
- //Note: Remove lists and return statement - TODO: Maybe put this into util (exported from `@promptbool/utils`)
5208
+ description = description.split(/^>.*$/gm).join('');
5209
+ //Note: Remove lists and return statement - TODO: [🎾] Make util (exported from `@promptbool/utils`)
4477
5210
  description = description.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join('');
4478
5211
  description = spaceTrim.spaceTrim(description);
4479
5212
  if (description === '') {
@@ -4501,7 +5234,7 @@
4501
5234
  pipelineJson.pipelineUrl = command.pipelineUrl.href;
4502
5235
  break;
4503
5236
  case 'KNOWLEDGE':
4504
- console.error(new NotYetImplementedError('Knowledge is not implemented yet'));
5237
+ knowledgeCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: null });
4505
5238
  break;
4506
5239
  case 'ACTION':
4507
5240
  console.error(new NotYetImplementedError('Actions are not implemented yet'));
@@ -4510,7 +5243,8 @@
4510
5243
  console.error(new NotYetImplementedError('Instruments are not implemented yet'));
4511
5244
  break;
4512
5245
  case 'PERSONA':
4513
- console.error(new NotYetImplementedError('Personas are not implemented yet'));
5246
+ personaCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: null });
5247
+ // <- Note: Prototype of [🍧] (remove this comment after full implementation)
4514
5248
  break;
4515
5249
  case 'BOILERPLATE':
4516
5250
  throw new ParsingError('BOILERPLATE command is only for testing purposes and should not be used in the .ptbk.md file'); // <- TODO: [🚞]
@@ -4533,29 +5267,83 @@
4533
5267
  // TODO: Parse prompt template description (the content out of the codeblock and lists)
4534
5268
  var templateModelRequirements = __assign({}, defaultModelRequirements);
4535
5269
  var listItems_3 = extractAllListItemsFromMarkdown(section.content);
4536
- var dependentParameterNames = new Set();
4537
- var blockType = 'PROMPT_TEMPLATE';
4538
- var jokers = [];
4539
- var postprocessing = [];
4540
- var expectAmount = {};
4541
- var expectFormat = undefined;
4542
- var isBlockTypeChanged = false;
5270
+ var lastLine = section.content.split('\n').pop();
5271
+ var resultingParameterNameMatch = /^->\s*\{(?<resultingParamName>[a-z0-9_]+)\}/im.exec(lastLine);
5272
+ var resultingParameterName = null;
5273
+ if (resultingParameterNameMatch &&
5274
+ resultingParameterNameMatch.groups !== undefined &&
5275
+ resultingParameterNameMatch.groups.resultingParamName !== undefined) {
5276
+ resultingParameterName = resultingParameterNameMatch.groups.resultingParamName;
5277
+ }
5278
+ var expectResultingParameterName = function () {
5279
+ if (resultingParameterName !== null) {
5280
+ return resultingParameterName;
5281
+ }
5282
+ throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Template section must end with -> {parameterName}\n\n Invalid section:\n ".concat(block(
5283
+ // TODO: Show code of invalid sections each time + DRY
5284
+ section.content
5285
+ .split('\n')
5286
+ .map(function (line) { return " | ".concat(line); } /* <- TODO: [🚞] */)
5287
+ .join('\n')), "\n "); }));
5288
+ };
5289
+ var _e = extractOneBlockFromMarkdown(section.content), language = _e.language, content = _e.content;
5290
+ // TODO: [🎾][1] DRY description
5291
+ var description_1 = section.content;
5292
+ // Note: Remove codeblocks - TODO: [🎾]
5293
+ description_1 = description_1.split(/^```.*^```/gms).join('');
5294
+ description_1 = description_1.split(/^>.*$/gm).join('');
5295
+ //Note: Remove lists and return statement - TODO: [🎾]
5296
+ description_1 = description_1.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join('');
5297
+ description_1 = spaceTrim.spaceTrim(description_1);
5298
+ if (description_1 === '') {
5299
+ description_1 = undefined;
5300
+ }
5301
+ var templateJson = {
5302
+ blockType: 'PROMPT_TEMPLATE',
5303
+ name: titleToName(section.title),
5304
+ title: section.title,
5305
+ description: description_1,
5306
+ modelRequirements: templateModelRequirements,
5307
+ content: content,
5308
+ };
5309
+ /**
5310
+ * This is nessesary because block type can be
5311
+ * - Set zero times, so anticipate 'PROMPT_TEMPLATE'
5312
+ * - Set one time
5313
+ * - Set more times - throw error
5314
+ *
5315
+ * Note: [2]
5316
+ */
5317
+ var isBlockTypeSet = false;
4543
5318
  try {
4544
5319
  for (var listItems_2 = (e_3 = void 0, __values(listItems_3)), listItems_2_1 = listItems_2.next(); !listItems_2_1.done; listItems_2_1 = listItems_2.next()) {
4545
5320
  var listItem = listItems_2_1.value;
4546
5321
  var command = parseCommand(listItem, 'PIPELINE_TEMPLATE');
5322
+ // TODO [🍧][♓️] List commands and before apply order them
4547
5323
  switch (command.type) {
4548
5324
  // TODO: [🍧] Use here applyToPipelineJson and remove switch statement
4549
5325
  case 'BLOCK':
4550
- if (isBlockTypeChanged) {
5326
+ if (isBlockTypeSet) {
4551
5327
  throw new ParsingError('Block type is already defined in the prompt template. It can be defined only once.');
4552
5328
  }
4553
5329
  if (command.blockType === 'SAMPLE') {
4554
- console.error(new NotYetImplementedError('Block type SAMPLE is not implemented yet'));
5330
+ expectResultingParameterName();
5331
+ var parameter = pipelineJson.parameters.find(function (param) { return param.name === resultingParameterName; });
5332
+ if (parameter === undefined) {
5333
+ throw new UnexpectedError("Can not find parameter {".concat(resultingParameterName, "} to assign sample value"));
5334
+ }
5335
+ parameter.sampleValues = parameter.sampleValues || [];
5336
+ parameter.sampleValues.push(content);
4555
5337
  return "continue-templates";
4556
5338
  }
4557
5339
  if (command.blockType === 'KNOWLEDGE') {
4558
- console.error(new NotYetImplementedError('Knowledge is not implemented yet'));
5340
+ knowledgeCommandParser.applyToPipelineJson({
5341
+ type: 'KNOWLEDGE',
5342
+ source: content, // <- TODO: [🐝] !!! Work with KNOWLEDGE which not referring to the source file or website, but its content itself
5343
+ }, {
5344
+ pipelineJson: pipelineJson,
5345
+ templateJson: templateJson,
5346
+ });
4559
5347
  return "continue-templates";
4560
5348
  }
4561
5349
  if (command.blockType === 'ACTION') {
@@ -4566,35 +5354,37 @@
4566
5354
  console.error(new NotYetImplementedError('Instruments are not implemented yet'));
4567
5355
  return "continue-templates";
4568
5356
  }
4569
- blockType = command.blockType;
4570
- isBlockTypeChanged = true;
5357
+ expectResultingParameterName();
5358
+ templateJson.blockType = command.blockType;
5359
+ isBlockTypeSet = true; //<- Note: [2]
4571
5360
  break;
4572
5361
  case 'EXPECT_AMOUNT':
4573
5362
  // eslint-disable-next-line no-case-declarations
4574
5363
  var unit = command.unit.toLowerCase();
4575
- expectAmount[unit] = expectAmount[unit] || {};
5364
+ templateJson.expectations = templateJson.expectations || {};
5365
+ templateJson.expectations[unit] = templateJson.expectations[unit] || {};
4576
5366
  if (command.sign === 'MINIMUM' || command.sign === 'EXACTLY') {
4577
- if (expectAmount[unit].min !== undefined) {
4578
- throw new ParsingError("Already defined minumum ".concat(expectAmount[unit].min, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
5367
+ if (templateJson.expectations[unit].min !== undefined) {
5368
+ throw new ParsingError("Already defined minumum ".concat(templateJson.expectations[unit].min, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
4579
5369
  }
4580
- expectAmount[unit].min = command.amount;
5370
+ templateJson.expectations[unit].min = command.amount;
4581
5371
  } /* not else */
4582
5372
  if (command.sign === 'MAXIMUM' || command.sign === 'EXACTLY') {
4583
- if (expectAmount[unit].max !== undefined) {
4584
- throw new ParsingError("Already defined maximum ".concat(expectAmount[unit].max, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
5373
+ if (templateJson.expectations[unit].max !== undefined) {
5374
+ throw new ParsingError("Already defined maximum ".concat(templateJson.expectations[unit].max, " ").concat(command.unit.toLowerCase(), ", now trying to redefine it to ").concat(command.amount));
4585
5375
  }
4586
- expectAmount[unit].max = command.amount;
5376
+ templateJson.expectations[unit].max = command.amount;
4587
5377
  }
4588
5378
  break;
4589
5379
  case 'EXPECT_FORMAT':
4590
- if (expectFormat !== undefined && command.format !== expectFormat) {
4591
- throw new ParsingError("Expect format is already defined to \"".concat(expectFormat, "\". Now you try to redefine it by \"").concat(command.format, "\"."));
5380
+ if (templateJson.expectFormat !== undefined && command.format !== templateJson.expectFormat) {
5381
+ throw new ParsingError(spaceTrim.spaceTrim("\n Expect format is already defined to \"".concat(templateJson.expectFormat, "\".\n Now you try to redefine it by \"").concat(command.format, "\".\n ")));
4592
5382
  }
4593
- expectFormat = command.format;
5383
+ templateJson.expectFormat = command.format;
4594
5384
  break;
4595
5385
  case 'JOKER':
4596
- jokers.push(command.parameterName);
4597
- dependentParameterNames.add(command.parameterName);
5386
+ templateJson.jokerParameterNames = templateJson.jokerParameterNames || [];
5387
+ templateJson.jokerParameterNames.push(command.parameterName);
4598
5388
  break;
4599
5389
  case 'MODEL':
4600
5390
  templateModelRequirements[command.key] = command.value;
@@ -4604,19 +5394,24 @@
4604
5394
  defineParam(command);
4605
5395
  break;
4606
5396
  case 'POSTPROCESS':
4607
- postprocessing.push(command.functionName);
5397
+ templateJson.postprocessingFunctionNames = templateJson.postprocessingFunctionNames || [];
5398
+ templateJson.postprocessingFunctionNames.push(command.functionName);
4608
5399
  break;
4609
5400
  case 'KNOWLEDGE':
4610
- console.error(new NotYetImplementedError('Knowledge is not implemented yet'));
5401
+ // TODO: [👙] The knowledge is maybe relevant for just this template
5402
+ knowledgeCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: templateJson });
4611
5403
  break;
4612
5404
  case 'ACTION':
5405
+ // TODO: [👙] The action is maybe relevant for just this template
4613
5406
  console.error(new NotYetImplementedError('Actions are not implemented yet'));
4614
5407
  break;
4615
5408
  case 'INSTRUMENT':
5409
+ // TODO: [👙] The instrument is maybe relevant for just this template
4616
5410
  console.error(new NotYetImplementedError('Instruments are not implemented yet'));
4617
5411
  break;
4618
5412
  case 'PERSONA':
4619
- console.error(new NotYetImplementedError('Personas are not implemented yet'));
5413
+ personaCommandParser.applyToPipelineJson(command, { pipelineJson: pipelineJson, templateJson: templateJson });
5414
+ // <- Note: Prototype of [🍧] (remove this comment after full implementation)
4620
5415
  break;
4621
5416
  case 'BOILERPLATE':
4622
5417
  console.error(new ParsingError('BOILERPLATE command is only for testing purposes and should not be used in the .ptbk.md file'));
@@ -4634,70 +5429,29 @@
4634
5429
  }
4635
5430
  finally { if (e_3) throw e_3.error; }
4636
5431
  }
4637
- var _e = extractOneBlockFromMarkdown(section.content), language = _e.language, content = _e.content;
4638
- if (blockType === 'SCRIPT') {
5432
+ // TODO: [🍧] Should be done in BLOCK command
5433
+ if (templateJson.blockType === 'SCRIPT') {
4639
5434
  if (!language) {
4640
5435
  throw new ParsingError('You must specify the language of the script in the prompt template');
4641
5436
  }
4642
- else if (!SUPPORTED_SCRIPT_LANGUAGES.includes(language)) {
5437
+ if (!SUPPORTED_SCRIPT_LANGUAGES.includes(language)) {
4643
5438
  throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Script language ".concat(language, " is not supported.\n\n Supported languages are:\n ").concat(block(SUPPORTED_SCRIPT_LANGUAGES.join(', ')), "\n\n "); }));
4644
5439
  }
5440
+ templateJson.contentLanguage = language;
4645
5441
  }
4646
- var lastLine = section.content.split('\n').pop();
4647
- var match = /^->\s*\{(?<resultingParamName>[a-z0-9_]+)\}/im.exec(lastLine);
4648
- if (!match || match.groups === undefined || match.groups.resultingParamName === undefined) {
4649
- throw new ParsingError(spaceTrim.spaceTrim(function (block) { return "\n Each section must end with -> {parameterName}\n\n Invalid section:\n ".concat(block(
4650
- // TODO: Show code of invalid sections each time + DRY
4651
- section.content
4652
- .split('\n')
4653
- .map(function (line) { return " | ".concat(line); } /* <- TODO: [🚞] */)
4654
- .join('\n')), "\n "); }));
4655
- }
4656
- var resultingParameterName = match.groups.resultingParamName;
4657
- // TODO: [1] DRY description
4658
- var description_1 = section.content;
4659
- // Note: Remove codeblocks
4660
- description_1 = description_1.split(/^```.*^```/gms).join('');
4661
- //Note: Remove lists and return statement
4662
- description_1 = description_1.split(/^(?:(?:-)|(?:\d\))|(?:`?->))\s+.*$/gm).join('');
4663
- description_1 = spaceTrim.spaceTrim(description_1);
4664
- if (description_1 === '') {
4665
- description_1 = undefined;
4666
- }
4667
- if (Object.keys(jokers).length === 0) {
4668
- jokers = undefined;
4669
- }
4670
- if (Object.keys(expectAmount).length === 0) {
4671
- expectAmount = undefined;
4672
- }
4673
- if (Object.keys(postprocessing).length === 0) {
4674
- postprocessing = undefined;
4675
- }
4676
- dependentParameterNames = union(dependentParameterNames, extractParametersFromPromptTemplate(__assign(__assign({}, section), { description: description_1, blockType: blockType, content: content })));
5442
+ // TODO: [🍧][❔] Should be done in BLOCK command
4677
5443
  if (templateModelRequirements.modelVariant === undefined) {
4678
5444
  templateModelRequirements.modelVariant = 'CHAT';
4679
5445
  }
4680
- dependentParameterNames = difference(dependentParameterNames, new Set(RESERVED_PARAMETER_NAMES));
4681
- var template = {
4682
- name: titleToName(section.title),
4683
- title: section.title,
4684
- description: description_1,
4685
- dependentParameterNames: Array.from(dependentParameterNames),
4686
- blockType: blockType,
4687
- jokers: jokers,
4688
- postprocessing: postprocessing,
4689
- expectations: expectAmount,
4690
- expectFormat: expectFormat,
4691
- modelRequirements: templateModelRequirements,
4692
- contentLanguage: blockType === 'SCRIPT' ? language : undefined,
4693
- content: content,
4694
- resultingParameterName: resultingParameterName,
4695
- };
4696
- if (blockType !== 'PROMPT_TEMPLATE') {
4697
- delete template.modelRequirements;
5446
+ templateJson.dependentParameterNames = Array.from(extractParametersFromPromptTemplate(templateJson));
5447
+ // TODO: [🍧][❔] Remove this condition - modelRequirements should be put here via BLOCK command not removed when PROMPT_TEMPLATE
5448
+ if (templateJson.blockType !== 'PROMPT_TEMPLATE') {
5449
+ delete templateJson.modelRequirements;
4698
5450
  }
5451
+ // TODO: [🍧] Make this better - for example each command parser can call and apply this
5452
+ templateJson.resultingParameterName = expectResultingParameterName( /* <- Note: This is once more redundant */);
4699
5453
  // TODO: [🍧] What actually about preparation and pushing the block into `promptTemplates`
4700
- pipelineJson.promptTemplates.push(template /* <- !!! */);
5454
+ pipelineJson.promptTemplates.push(templateJson);
4701
5455
  };
4702
5456
  try {
4703
5457
  // =============================================================
@@ -4764,14 +5518,17 @@
4764
5518
  * TODO: Use spaceTrim more effectively
4765
5519
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
4766
5520
  * TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
5521
+ * TODO: [♈] Probbably move expectations from templates to parameters
5522
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
4767
5523
  */
4768
5524
 
4769
5525
  /**
4770
- * Compile promptbook from string (markdown) format to JSON format
5526
+ * Compile pipeline from string (markdown) format to JSON format
4771
5527
  *
4772
- * Note: There are two similar functions:
5528
+ * Note: There are 3 similar functions:
4773
5529
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
4774
5530
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
5531
+ * - `preparePipeline` - just one step in the compilation process
4775
5532
  *
4776
5533
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
4777
5534
  * @param options - Options and tools for the compilation
@@ -4782,22 +5539,18 @@
4782
5539
  * Note: This function acts as compilation process
4783
5540
  */
4784
5541
  function pipelineStringToJson(pipelineString, options) {
4785
- if (options === void 0) { options = {}; }
5542
+ if (options === void 0) { options = { llmTools: null }; }
4786
5543
  return __awaiter(this, void 0, void 0, function () {
4787
- var llmTools, pipelineJson, knowledge;
5544
+ var llmTools, pipelineJson;
4788
5545
  return __generator(this, function (_a) {
4789
5546
  switch (_a.label) {
4790
5547
  case 0:
4791
5548
  llmTools = options.llmTools;
4792
5549
  pipelineJson = pipelineStringToJsonSync(pipelineString);
4793
- if (!llmTools) return [3 /*break*/, 2];
4794
- return [4 /*yield*/, prepareKnowledgeFromMarkdown({
4795
- content: 'Roses are red, violets are blue, programmers use Promptbook, users too',
4796
- llmTools: llmTools,
4797
- })];
5550
+ if (!(llmTools !== null)) return [3 /*break*/, 2];
5551
+ return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
4798
5552
  case 1:
4799
- knowledge = _a.sent();
4800
- pipelineJson = __assign(__assign({}, pipelineJson), { knowledge: __spreadArray(__spreadArray([], __read((pipelineJson.knowledge || [])), false), __read(knowledge), false) });
5553
+ pipelineJson = _a.sent();
4801
5554
  _a.label = 2;
4802
5555
  case 2: return [2 /*return*/, pipelineJson];
4803
5556
  }
@@ -4806,12 +5559,13 @@
4806
5559
  }
4807
5560
  /**
4808
5561
  * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
5562
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
4809
5563
  */
4810
5564
 
4811
5565
  /**
4812
5566
  * Add or modify an auto-generated section in a markdown file
4813
5567
  *
4814
- * @private within the package
5568
+ * @private within the repository
4815
5569
  */
4816
5570
  function addAutoGeneratedSection(content, options) {
4817
5571
  var sectionName = options.sectionName, sectionContent = options.sectionContent;
@@ -4823,7 +5577,10 @@
4823
5577
  }
4824
5578
  var placeForSection = removeContentComments(content).match(/^##.*$/im);
4825
5579
  if (!placeForSection) {
4826
- throw new Error("No place where to put the section <!--".concat(sectionName, "-->"));
5580
+ throw new ParsingError(
5581
+ // <- [🧠] Maybe something better than `ParsingError`
5582
+ "No place where to put the section <!--".concat(sectionName, "-->"));
5583
+ // <- [🚞]
4827
5584
  }
4828
5585
  var _a = __read(placeForSection, 1), heading = _a[0];
4829
5586
  return content.replace(heading, "<!--".concat(sectionName, "-->\n").concat(warningLine, "\n").concat(sectionContent, "\n<!--/").concat(sectionName, "-->\n\n").concat(heading));
@@ -4980,6 +5737,27 @@
4980
5737
  * TODO: [🕌] When more than 2 functionalities, split into separate functions
4981
5738
  */
4982
5739
 
5740
+ /**
5741
+ * Stringify the PipelineJson with proper formatting
5742
+ *
5743
+ * Note: [0] It can be used for more JSON types like whole collection of pipelines, single knowledge piece, etc.
5744
+ * Note: In contrast to JSON.stringify, this function ensures that **embedding index** is on single line
5745
+ */
5746
+ function stringifyPipelineJson(pipeline) {
5747
+ var pipelineJsonStringified = JSON.stringify(pipeline, null, 4);
5748
+ for (var i = 0; i < LOOP_LIMIT; i++) {
5749
+ pipelineJsonStringified = pipelineJsonStringified.replace(/(-?0\.\d+),[\n\s]+(-?0\.\d+)/gms, "$1".concat(REPLACING_NONCE, "$2"));
5750
+ }
5751
+ pipelineJsonStringified = pipelineJsonStringified.split(REPLACING_NONCE).join(', ');
5752
+ pipelineJsonStringified += '\n';
5753
+ return pipelineJsonStringified;
5754
+ }
5755
+ /**
5756
+ * TODO: !!!! Not Working propperly @see https://promptbook.studio/samples/mixed-knowledge.ptbk.md
5757
+ * TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
5758
+ * TODO: [🧠] Maybe more elegant solution than replacing via regex
5759
+ */
5760
+
4983
5761
  /**
4984
5762
  * This error indicates that the pipeline collection cannot be propperly loaded
4985
5763
  */
@@ -4994,6 +5772,20 @@
4994
5772
  return CollectionError;
4995
5773
  }(Error));
4996
5774
 
5775
+ /**
5776
+ * This error type indicates that the version of the pipeline is not matching the expected version
5777
+ */
5778
+ var VersionMismatchError = /** @class */ (function (_super) {
5779
+ __extends(VersionMismatchError, _super);
5780
+ function VersionMismatchError(message, expectedVersion) {
5781
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n This error indicates that there is error with compatibility\n\n Expected version:\n ").concat(expectedVersion, "\n\n Current version:\n ").concat(PROMPTBOOK_VERSION, "\n\n "); })) || this;
5782
+ _this.name = 'UnexpectedError';
5783
+ Object.setPrototypeOf(_this, VersionMismatchError.prototype);
5784
+ return _this;
5785
+ }
5786
+ return VersionMismatchError;
5787
+ }(Error));
5788
+
4997
5789
  /**
4998
5790
  * Pretty print an embedding vector for logging
4999
5791
  */
@@ -5081,6 +5873,9 @@
5081
5873
  };
5082
5874
  return SimplePromptInterfaceTools;
5083
5875
  }());
5876
+ /**
5877
+ * Note: [🔵] This code should never be published outside of `@promptbook/browser`
5878
+ */
5084
5879
 
5085
5880
  /**
5086
5881
  * Default options for generating an execution report string
@@ -5093,7 +5888,7 @@
5093
5888
  /**
5094
5889
  * Format either small or big number
5095
5890
  *
5096
- * @private within the package
5891
+ * @private within the repository
5097
5892
  */
5098
5893
  function formatNumber(value) {
5099
5894
  if (value === 0) {
@@ -5113,7 +5908,7 @@
5113
5908
  /**
5114
5909
  * Create a markdown table from a 2D array of strings
5115
5910
  *
5116
- * @private within the package
5911
+ * @private within the repository
5117
5912
  */
5118
5913
  function createMarkdownTable(table) {
5119
5914
  var columnWidths = table.reduce(function (widths, row) {
@@ -5141,7 +5936,7 @@
5141
5936
  /**
5142
5937
  * Function createMarkdownChart will draw a chart in markdown from ⬛+🟦 tiles
5143
5938
  *
5144
- * @private within the package
5939
+ * @private within the repository
5145
5940
  */
5146
5941
  function createMarkdownChart(options) {
5147
5942
  var e_1, _a;
@@ -5194,7 +5989,7 @@
5194
5989
  /**
5195
5990
  * Count the duration of working time
5196
5991
  *
5197
- * @private within the package
5992
+ * @private within the repository
5198
5993
  */
5199
5994
  function countWorkingDuration(items) {
5200
5995
  var e_1, _a;
@@ -5406,6 +6201,7 @@
5406
6201
  exports.ReferenceError = ReferenceError$1;
5407
6202
  exports.SimplePromptInterfaceTools = SimplePromptInterfaceTools;
5408
6203
  exports.UnexpectedError = UnexpectedError;
6204
+ exports.VersionMismatchError = VersionMismatchError;
5409
6205
  exports.addUsage = addUsage;
5410
6206
  exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
5411
6207
  exports.checkExpectations = checkExpectations;
@@ -5423,7 +6219,10 @@
5423
6219
  exports.pipelineStringToJson = pipelineStringToJson;
5424
6220
  exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
5425
6221
  exports.prepareKnowledgeFromMarkdown = prepareKnowledgeFromMarkdown;
6222
+ exports.preparePipeline = preparePipeline;
5426
6223
  exports.prettifyPipelineString = prettifyPipelineString;
6224
+ exports.stringifyPipelineJson = stringifyPipelineJson;
6225
+ exports.unpreparePipeline = unpreparePipeline;
5427
6226
  exports.usageToWorktime = usageToWorktime;
5428
6227
  exports.validatePipeline = validatePipeline;
5429
6228