@promptbook/core 0.72.0-8 → 0.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +8 -8
  2. package/esm/index.es.js +1630 -2236
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/browser.index.d.ts +2 -0
  5. package/esm/typings/src/_packages/cli.index.d.ts +20 -0
  6. package/esm/typings/src/_packages/core.index.d.ts +42 -30
  7. package/esm/typings/src/_packages/documents.index.d.ts +8 -0
  8. package/esm/typings/src/_packages/legacy-documents.index.d.ts +8 -0
  9. package/esm/typings/src/_packages/markdown-utils.index.d.ts +6 -0
  10. package/esm/typings/src/_packages/node.index.d.ts +12 -4
  11. package/esm/typings/src/_packages/pdf.index.d.ts +8 -0
  12. package/esm/typings/src/_packages/types.index.d.ts +33 -17
  13. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  14. package/esm/typings/src/_packages/website-crawler.index.d.ts +8 -0
  15. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  16. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  17. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  18. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +5 -3
  19. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  20. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  21. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  22. package/esm/typings/src/config.d.ts +26 -13
  23. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +3 -1
  24. package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
  25. package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -3
  26. package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
  27. package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
  28. package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
  29. package/esm/typings/src/executables/locateApp.d.ts +33 -0
  30. package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
  31. package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
  32. package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
  33. package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts} +1 -1
  34. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  35. package/esm/typings/src/execution/Executables.d.ts +18 -0
  36. package/esm/typings/src/execution/ExecutionTools.d.ts +32 -6
  37. package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
  38. package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
  39. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  40. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
  41. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  42. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +3 -12
  43. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -18
  44. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -19
  45. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  46. package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
  47. package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts +13 -0
  48. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  49. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  50. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  51. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  52. package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts} +4 -1
  53. package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts} +5 -2
  54. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts} +3 -3
  55. package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts} +4 -11
  56. package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts} +4 -3
  57. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromEnv.d.ts → register/$provideLlmToolsFromEnv.d.ts} +6 -5
  58. package/esm/typings/src/llm-providers/_common/{$registeredLlmToolsMessage.d.ts → register/$registeredLlmToolsMessage.d.ts} +5 -2
  59. package/esm/typings/src/llm-providers/_common/{LlmToolsConfiguration.d.ts → register/LlmToolsConfiguration.d.ts} +6 -5
  60. package/esm/typings/src/llm-providers/_common/{LlmToolsMetadata.d.ts → register/LlmToolsMetadata.d.ts} +5 -4
  61. package/esm/typings/src/llm-providers/_common/{LlmToolsOptions.d.ts → register/LlmToolsOptions.d.ts} +4 -1
  62. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfiguration.d.ts → register/createLlmToolsFromConfiguration.d.ts} +12 -4
  63. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  64. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  65. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +5 -5
  66. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  67. package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +4 -3
  68. package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +4 -3
  69. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  70. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -3
  71. package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +4 -3
  72. package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +4 -3
  73. package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
  74. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  75. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +4 -4
  76. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +4 -4
  77. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -4
  78. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  79. package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
  80. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
  81. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +4 -4
  82. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  83. package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +5 -4
  84. package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +5 -4
  85. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
  86. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +16 -7
  87. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
  88. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +7 -13
  89. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
  90. package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +11 -16
  91. package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +24 -9
  92. package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
  93. package/esm/typings/src/personas/preparePersona.d.ts +2 -1
  94. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +10 -28
  95. package/esm/typings/src/prepare/preparePipeline.d.ts +2 -1
  96. package/esm/typings/src/prepare/prepareTemplates.d.ts +3 -2
  97. package/esm/typings/src/scrapers/_common/Converter.d.ts +4 -10
  98. package/esm/typings/src/scrapers/_common/Scraper.d.ts +5 -17
  99. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -1
  100. package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
  101. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +12 -0
  102. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +15 -0
  103. package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +12 -0
  104. package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +13 -0
  105. package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +13 -0
  106. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +41 -0
  107. package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +12 -0
  108. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +3 -2
  109. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -2
  110. package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts} +18 -12
  111. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +20 -0
  112. package/esm/typings/src/scrapers/document/register-constructor.d.ts +13 -0
  113. package/esm/typings/src/scrapers/document/register-metadata.d.ts +24 -0
  114. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +43 -0
  115. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +20 -0
  116. package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +13 -0
  117. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +24 -0
  118. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +29 -0
  119. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +20 -0
  120. package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +13 -0
  121. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +24 -0
  122. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +40 -0
  123. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +20 -0
  124. package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +13 -0
  125. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +24 -0
  126. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +50 -0
  127. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +20 -0
  128. package/esm/typings/src/scrapers/website/register-constructor.d.ts +13 -0
  129. package/esm/typings/src/scrapers/website/register-metadata.d.ts +24 -0
  130. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  131. package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +2 -2
  132. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  133. package/esm/typings/src/scripting/python/PythonExecutionTools.d.ts +3 -3
  134. package/esm/typings/src/scripting/typescript/TypescriptExecutionTools.d.ts +3 -3
  135. package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +5 -3
  136. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  137. package/esm/typings/src/storage/{utils → memory/utils}/PrefixStorage.d.ts +1 -1
  138. package/esm/typings/src/storage/{utils → memory/utils}/makePromptbookStorageFromWebStorage.d.ts +1 -1
  139. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  140. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  141. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  142. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  143. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  144. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  145. package/esm/typings/src/types/Prompt.d.ts +2 -1
  146. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  147. package/esm/typings/src/types/typeAliases.d.ts +18 -8
  148. package/esm/typings/src/utils/$Register.d.ts +20 -7
  149. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  150. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  151. package/esm/typings/src/utils/emojis.d.ts +1 -1
  152. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +5 -3
  153. package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
  154. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +2 -1
  155. package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
  156. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  157. package/esm/typings/src/utils/files/isDirectoryExisting.d.ts +14 -0
  158. package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
  159. package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
  160. package/esm/typings/src/utils/files/isFileExisting.test.d.ts +1 -0
  161. package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
  162. package/esm/typings/src/utils/files/listAllFiles.test.d.ts +1 -0
  163. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  164. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  165. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  166. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  167. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  168. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  169. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  170. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  171. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  172. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  173. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  174. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  175. package/package.json +3 -6
  176. package/umd/index.umd.js +1650 -2245
  177. package/umd/index.umd.js.map +1 -1
  178. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  179. package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts +0 -37
  180. package/esm/typings/src/scrapers/index.d.ts +0 -8
  181. package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +0 -29
  182. package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +0 -35
  183. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  184. package/esm/typings/src/scrapers/website/websiteScraper.d.ts +0 -43
  185. package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
  186. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
  187. package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +0 -15
  188. package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
  189. /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
  190. /package/esm/typings/src/{scrapers/website/websiteScraper.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
  191. /package/esm/typings/src/{utils/files/$isDirectoryExisting.test.d.ts → executables/locateApp.test.d.ts} +0 -0
  192. /package/esm/typings/src/scrapers/document/{documentScraper.test.d.ts → DocumentScraper.test.d.ts} +0 -0
  193. /package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.test.d.ts → LegacyDocumentScraper.test.d.ts} +0 -0
  194. /package/esm/typings/src/scrapers/markdown/{markdownScraper.test.d.ts → MarkdownScraper.test.d.ts} +0 -0
  195. /package/esm/typings/src/{utils/files/$isFileExisting.test.d.ts → scrapers/website/utils/createShowdownConverter.test.d.ts} +0 -0
  196. /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
package/esm/index.es.js CHANGED
@@ -1,17 +1,11 @@
1
1
  import spaceTrim, { spaceTrim as spaceTrim$1 } from 'spacetrim';
2
2
  import { format } from 'prettier';
3
3
  import parserHtml from 'prettier/parser-html';
4
- import { stat, access, constants, mkdir, rm, readFile, rmdir, rename, readdir, writeFile } from 'fs/promises';
5
- import { basename, join, dirname } from 'path';
6
- import { spawn } from 'child_process';
7
- import colors from 'colors';
8
4
  import { forTime } from 'waitasecond';
5
+ import { unparse, parse } from 'papaparse';
6
+ import { join, basename } from 'path';
9
7
  import { SHA256 } from 'crypto-js';
10
8
  import hexEncoder from 'crypto-js/enc-hex';
11
- import { unparse, parse } from 'papaparse';
12
- import { Readability } from '@mozilla/readability';
13
- import { JSDOM } from 'jsdom';
14
- import { Converter } from 'showdown';
15
9
  import { lookup } from 'mime-types';
16
10
  import sha256 from 'crypto-js/sha256';
17
11
  import moment from 'moment';
@@ -20,7 +14,7 @@ import moment from 'moment';
20
14
  /**
21
15
  * The version of the Promptbook library
22
16
  */
23
- var PROMPTBOOK_VERSION = '0.72.0-7';
17
+ var PROMPTBOOK_VERSION = '0.72.0-34';
24
18
  // TODO: [main] !!!! List here all the versions and annotate + put into script
25
19
 
26
20
  /*! *****************************************************************************
@@ -227,7 +221,6 @@ function pipelineJsonToString(pipelineJson) {
227
221
  pipelineString += '\n\n';
228
222
  pipelineString += description;
229
223
  }
230
- // TODO:> const commands: Array<Command>
231
224
  var commands = [];
232
225
  if (pipelineUrl) {
233
226
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -283,7 +276,6 @@ function pipelineJsonToString(pipelineJson) {
283
276
  pipelineString += '\n\n';
284
277
  pipelineString += description_1;
285
278
  }
286
- // TODO:> const commands: Array<Command>
287
279
  var commands_1 = [];
288
280
  var contentLanguage = 'text';
289
281
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -643,7 +635,7 @@ var GENERATOR_WARNING = "\u26A0\uFE0F WARNING: This code has been generated so t
643
635
  *
644
636
  * @public exported from `@promptbook/core`
645
637
  */
646
- var CLAIM = "Supercharge LLM models with Promptbook";
638
+ var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
647
639
  // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
648
640
  /**
649
641
  * The maximum number of iterations for a loops
@@ -658,37 +650,44 @@ var LOOP_LIMIT = 1000;
658
650
  */
659
651
  var IMMEDIATE_TIME = 10;
660
652
  /**
661
- * The maximum number of (LLM) tasks running in parallel
653
+ * The maximum length of the (generated) filename
662
654
  *
663
655
  * @public exported from `@promptbook/core`
664
656
  */
665
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
657
+ var MAX_FILENAME_LENGTH = 30;
666
658
  /**
667
- * The maximum number of attempts to execute LLM task before giving up
659
+ * Strategy for caching the intermediate results for knowledge sources
668
660
  *
669
661
  * @public exported from `@promptbook/core`
670
662
  */
671
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
663
+ var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
664
+ // <- TODO: [😡] Change to 'VISIBLE'
672
665
  /**
673
- * The maximum length of the (generated) filename
666
+ * The maximum number of (LLM) tasks running in parallel
674
667
  *
675
668
  * @public exported from `@promptbook/core`
676
669
  */
677
- var MAX_FILENAME_LENGTH = 30;
670
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
671
+ /**
672
+ * The maximum number of attempts to execute LLM task before giving up
673
+ *
674
+ * @public exported from `@promptbook/core`
675
+ */
676
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
678
677
  /**
679
678
  * @@@
680
679
  * TODO: [🐝][main] !!! Use
681
680
  *
682
681
  * @public exported from `@promptbook/core`
683
682
  */
684
- var MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = 3;
683
+ var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = 3;
685
684
  /**
686
685
  * @@@
687
686
  * TODO: [🐝][main] !!! Use
688
687
  *
689
688
  * @public exported from `@promptbook/core`
690
689
  */
691
- var MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
690
+ var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
692
691
  /**
693
692
  * Where to store the cache of executions for promptbook CLI
694
693
  *
@@ -696,7 +695,7 @@ var MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
696
695
  *
697
696
  * @public exported from `@promptbook/core`
698
697
  */
699
- var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
698
+ var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
700
699
  /**
701
700
  * Where to store the scrape cache
702
701
  *
@@ -704,13 +703,13 @@ var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
704
703
  *
705
704
  * @public exported from `@promptbook/core`
706
705
  */
707
- var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
706
+ var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
708
707
  /**
709
708
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
710
709
  *
711
710
  * @public exported from `@promptbook/core`
712
711
  */
713
- var PIPELINE_COLLECTION_BASE_FILENAME = "index";
712
+ var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
714
713
  /**
715
714
  * Nonce which is used for replacing things in strings
716
715
  *
@@ -783,7 +782,13 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
783
782
  *
784
783
  * @public exported from `@promptbook/core`
785
784
  */
786
- var IS_VERBOSE = false;
785
+ var DEFAULT_IS_VERBOSE = false;
786
+ /**
787
+ * @@@
788
+ *
789
+ * @public exported from `@promptbook/core`
790
+ */
791
+ var DEFAULT_IS_AUTO_INSTALLED = false;
787
792
  /**
788
793
  * @@@
789
794
  *
@@ -1532,7 +1537,7 @@ function createCollectionFromUrl(url, options) {
1532
1537
  return __generator(this, function (_d) {
1533
1538
  switch (_d.label) {
1534
1539
  case 0:
1535
- _a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
1540
+ _a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
1536
1541
  collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
1537
1542
  return __generator(this, function (_a) {
1538
1543
  if (isVerbose) {
@@ -1654,19 +1659,19 @@ var TemplateTypes = [
1654
1659
  ];
1655
1660
 
1656
1661
  /**
1657
- * This error indicates that the promptbook can not retrieve knowledge from external sources
1662
+ * This error type indicates that some tools are missing for pipeline execution or preparation
1658
1663
  *
1659
1664
  * @public exported from `@promptbook/core`
1660
1665
  */
1661
- var KnowledgeScrapeError = /** @class */ (function (_super) {
1662
- __extends(KnowledgeScrapeError, _super);
1663
- function KnowledgeScrapeError(message) {
1664
- var _this = _super.call(this, message) || this;
1665
- _this.name = 'KnowledgeScrapeError';
1666
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
1666
+ var MissingToolsError = /** @class */ (function (_super) {
1667
+ __extends(MissingToolsError, _super);
1668
+ function MissingToolsError(message) {
1669
+ var _this = _super.call(this, spaceTrim$1(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
1670
+ _this.name = 'MissingToolsError';
1671
+ Object.setPrototypeOf(_this, MissingToolsError.prototype);
1667
1672
  return _this;
1668
1673
  }
1669
- return KnowledgeScrapeError;
1674
+ return MissingToolsError;
1670
1675
  }(Error));
1671
1676
 
1672
1677
  /**
@@ -1746,1042 +1751,187 @@ function forEachAsync(array, options, callbackfunction) {
1746
1751
  }
1747
1752
 
1748
1753
  /**
1749
- * This error type indicates that some tools are missing for pipeline execution or preparation
1754
+ * Represents the usage with no resources consumed
1750
1755
  *
1751
1756
  * @public exported from `@promptbook/core`
1752
1757
  */
1753
- var MissingToolsError = /** @class */ (function (_super) {
1754
- __extends(MissingToolsError, _super);
1755
- function MissingToolsError(message) {
1756
- var _this = _super.call(this, spaceTrim$1(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
1757
- _this.name = 'MissingToolsError';
1758
- Object.setPrototypeOf(_this, MissingToolsError.prototype);
1759
- return _this;
1760
- }
1761
- return MissingToolsError;
1762
- }(Error));
1763
-
1758
+ var ZERO_USAGE = $deepFreeze({
1759
+ price: { value: 0 },
1760
+ input: {
1761
+ tokensCount: { value: 0 },
1762
+ charactersCount: { value: 0 },
1763
+ wordsCount: { value: 0 },
1764
+ sentencesCount: { value: 0 },
1765
+ linesCount: { value: 0 },
1766
+ paragraphsCount: { value: 0 },
1767
+ pagesCount: { value: 0 },
1768
+ },
1769
+ output: {
1770
+ tokensCount: { value: 0 },
1771
+ charactersCount: { value: 0 },
1772
+ wordsCount: { value: 0 },
1773
+ sentencesCount: { value: 0 },
1774
+ linesCount: { value: 0 },
1775
+ paragraphsCount: { value: 0 },
1776
+ pagesCount: { value: 0 },
1777
+ },
1778
+ });
1764
1779
  /**
1765
- * Detects if the code is running in a Node.js environment
1766
- *
1767
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
1780
+ * Represents the usage with unknown resources consumed
1768
1781
  *
1769
- * @public exported from `@promptbook/utils`
1782
+ * @public exported from `@promptbook/core`
1770
1783
  */
1771
- var $isRunningInNode = new Function("\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n");
1784
+ var UNCERTAIN_USAGE = $deepFreeze({
1785
+ price: { value: 0, isUncertain: true },
1786
+ input: {
1787
+ tokensCount: { value: 0, isUncertain: true },
1788
+ charactersCount: { value: 0, isUncertain: true },
1789
+ wordsCount: { value: 0, isUncertain: true },
1790
+ sentencesCount: { value: 0, isUncertain: true },
1791
+ linesCount: { value: 0, isUncertain: true },
1792
+ paragraphsCount: { value: 0, isUncertain: true },
1793
+ pagesCount: { value: 0, isUncertain: true },
1794
+ },
1795
+ output: {
1796
+ tokensCount: { value: 0, isUncertain: true },
1797
+ charactersCount: { value: 0, isUncertain: true },
1798
+ wordsCount: { value: 0, isUncertain: true },
1799
+ sentencesCount: { value: 0, isUncertain: true },
1800
+ linesCount: { value: 0, isUncertain: true },
1801
+ paragraphsCount: { value: 0, isUncertain: true },
1802
+ pagesCount: { value: 0, isUncertain: true },
1803
+ },
1804
+ });
1772
1805
 
1773
1806
  /**
1774
- * This error type indicates that you try to use a feature that is not available in the current environment
1807
+ * This error indicates errors during the execution of the pipeline
1775
1808
  *
1776
1809
  * @public exported from `@promptbook/core`
1777
1810
  */
1778
- var EnvironmentMismatchError = /** @class */ (function (_super) {
1779
- __extends(EnvironmentMismatchError, _super);
1780
- function EnvironmentMismatchError(message) {
1811
+ var PipelineExecutionError = /** @class */ (function (_super) {
1812
+ __extends(PipelineExecutionError, _super);
1813
+ function PipelineExecutionError(message) {
1781
1814
  var _this = _super.call(this, message) || this;
1782
- _this.name = 'EnvironmentMismatchError';
1783
- Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
1815
+ _this.name = 'PipelineExecutionError';
1816
+ Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
1784
1817
  return _this;
1785
1818
  }
1786
- return EnvironmentMismatchError;
1819
+ return PipelineExecutionError;
1787
1820
  }(Error));
1788
1821
 
1789
1822
  /**
1790
- * Normalize options for `execCommand` and `execCommands`
1823
+ * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1791
1824
  *
1792
- * @private internal utility of `execCommand` and `execCommands`
1825
+ * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1826
+ * @public exported from `@promptbook/core`
1793
1827
  */
1794
- function execCommandNormalizeOptions(options) {
1795
- var _a;
1796
- var _b, _c, _d;
1797
- var command;
1798
- var cwd;
1799
- var crashOnError;
1800
- var args = [];
1801
- var timeout;
1802
- if (typeof options === 'string') {
1803
- // TODO: [1] DRY default values
1804
- command = options;
1805
- cwd = process.cwd();
1806
- crashOnError = true;
1807
- timeout = Infinity;
1808
- }
1809
- else {
1810
- /*
1811
- TODO:
1812
- if ((options as any).commands !== undefined) {
1813
- commands = (options as any).commands;
1814
- } else {
1815
- commands = [(options as any).command];
1828
+ var MultipleLlmExecutionTools = /** @class */ (function () {
1829
+ /**
1830
+ * Gets array of execution tools in order of priority
1831
+ */
1832
+ function MultipleLlmExecutionTools() {
1833
+ var llmExecutionTools = [];
1834
+ for (var _i = 0; _i < arguments.length; _i++) {
1835
+ llmExecutionTools[_i] = arguments[_i];
1816
1836
  }
1817
- */
1818
- // TODO: [1] DRY default values
1819
- command = options.command;
1820
- cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
1821
- crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
1822
- timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
1823
- }
1824
- // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
1825
- var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
1826
- .map(function (_a) {
1827
- var _b = __read(_a, 1), match = _b[0];
1828
- return match;
1829
- })
1830
- .filter(function (arg) { return arg !== ''; });
1831
- if (_.length > 1) {
1832
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1833
- _a = __read(_), command = _a[0], args = _a.slice(1);
1834
- }
1835
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1836
- if (options.args) {
1837
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1838
- args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
1839
- }
1840
- var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
1841
- if (['ts-node'].includes(humanReadableCommand)) {
1842
- humanReadableCommand += " ".concat(args[1]);
1843
- }
1844
- return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
1845
- }
1846
- // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
1847
-
1848
- /**
1849
- * Run one command in a shell
1850
- *
1851
- * Note: There are 2 similar functions in the codebase:
1852
- * - `$execCommand` which runs a single command
1853
- * - `$execCommands` which runs multiple commands
1854
- *
1855
- * @public exported from `@promptbook/node`
1856
- */
1857
- function $execCommand(options) {
1858
- if (!$isRunningInNode()) {
1859
- throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
1837
+ this.llmExecutionTools = llmExecutionTools;
1860
1838
  }
1861
- return new Promise(
1862
- // <- TODO: [🧱] Implement in a functional (not new Class) way
1863
- function (resolve, reject) {
1864
- // eslint-disable-next-line prefer-const
1865
- var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
1866
- if (timeout !== Infinity) {
1867
- // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
1868
- forTime(timeout).then(function () {
1869
- if (crashOnError) {
1870
- reject(new Error("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms")));
1871
- }
1872
- else {
1873
- console.warn("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms but continues running"));
1874
- resolve('Command exceeded time limit');
1875
- }
1876
- });
1877
- }
1878
- if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
1879
- command = "".concat(command, ".cmd");
1880
- }
1881
- // !!!!!! Verbose mode - to all consoles
1882
- console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
1883
- try {
1884
- var commandProcess = spawn(command, args, { cwd: cwd, shell: true });
1885
- commandProcess.on('message', function (message) {
1886
- console.info({ message: message });
1887
- });
1888
- var output_1 = [];
1889
- commandProcess.stdout.on('data', function (stdout) {
1890
- output_1.push(stdout.toString());
1891
- console.info(stdout.toString());
1892
- });
1893
- commandProcess.stderr.on('data', function (stderr) {
1894
- output_1.push(stderr.toString());
1895
- if (stderr.toString().trim()) {
1896
- console.warn(stderr.toString());
1897
- }
1898
- });
1899
- var finishWithCode = function (code) {
1900
- if (code !== 0) {
1901
- if (crashOnError) {
1902
- reject(new Error(output_1.join('\n').trim() ||
1903
- "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
1904
- }
1905
- else {
1906
- console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
1907
- resolve(spaceTrim$1(output_1.join('\n')));
1908
- }
1909
- }
1910
- else {
1911
- resolve(spaceTrim$1(output_1.join('\n')));
1839
+ Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
1840
+ get: function () {
1841
+ return 'Multiple LLM Providers';
1842
+ },
1843
+ enumerable: false,
1844
+ configurable: true
1845
+ });
1846
+ Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
1847
+ get: function () {
1848
+ return this.llmExecutionTools.map(function (_a, index) {
1849
+ var title = _a.title;
1850
+ return "".concat(index + 1, ") `").concat(title, "`");
1851
+ }).join('\n');
1852
+ },
1853
+ enumerable: false,
1854
+ configurable: true
1855
+ });
1856
+ /**
1857
+ * Check the configuration of all execution tools
1858
+ */
1859
+ MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
1860
+ return __awaiter(this, void 0, void 0, function () {
1861
+ var _a, _b, llmExecutionTools, e_1_1;
1862
+ var e_1, _c;
1863
+ return __generator(this, function (_d) {
1864
+ switch (_d.label) {
1865
+ case 0:
1866
+ _d.trys.push([0, 5, 6, 7]);
1867
+ _a = __values(this.llmExecutionTools), _b = _a.next();
1868
+ _d.label = 1;
1869
+ case 1:
1870
+ if (!!_b.done) return [3 /*break*/, 4];
1871
+ llmExecutionTools = _b.value;
1872
+ return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
1873
+ case 2:
1874
+ _d.sent();
1875
+ _d.label = 3;
1876
+ case 3:
1877
+ _b = _a.next();
1878
+ return [3 /*break*/, 1];
1879
+ case 4: return [3 /*break*/, 7];
1880
+ case 5:
1881
+ e_1_1 = _d.sent();
1882
+ e_1 = { error: e_1_1 };
1883
+ return [3 /*break*/, 7];
1884
+ case 6:
1885
+ try {
1886
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
1887
+ }
1888
+ finally { if (e_1) throw e_1.error; }
1889
+ return [7 /*endfinally*/];
1890
+ case 7: return [2 /*return*/];
1912
1891
  }
1913
- };
1914
- commandProcess.on('close', finishWithCode);
1915
- commandProcess.on('exit', finishWithCode);
1916
- commandProcess.on('disconnect', function () {
1917
- // Note: Unexpected disconnection should always result in rejection
1918
- reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
1919
1892
  });
1920
- commandProcess.on('error', function (error) {
1921
- if (crashOnError) {
1922
- reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
1923
- }
1924
- else {
1925
- console.warn(error);
1926
- resolve(spaceTrim$1(output_1.join('\n')));
1927
- }
1928
- });
1929
- }
1930
- catch (error) {
1931
- // Note: Unexpected error in sync code should always result in rejection
1932
- reject(error);
1933
- }
1934
- });
1935
- }
1936
- /**
1937
- * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
1938
- */
1939
-
1940
- /**
1941
- * Checks if the file exists
1942
- *
1943
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
1944
- *
1945
- * @private within the repository
1946
- */
1947
- function $isFileExisting(filename) {
1948
- return __awaiter(this, void 0, void 0, function () {
1949
- var isReadAccessAllowed, isFile;
1950
- return __generator(this, function (_a) {
1951
- switch (_a.label) {
1952
- case 0:
1953
- if (!$isRunningInNode()) {
1954
- throw new EnvironmentMismatchError('Function `$isFileExisting` works only in Node environment.js');
1955
- }
1956
- return [4 /*yield*/, access(filename, constants.R_OK)
1957
- .then(function () { return true; })
1958
- .catch(function () { return false; })];
1959
- case 1:
1960
- isReadAccessAllowed = _a.sent();
1961
- if (!isReadAccessAllowed) {
1962
- return [2 /*return*/, false];
1963
- }
1964
- return [4 /*yield*/, stat(filename)
1965
- .then(function (fileStat) { return fileStat.isFile(); })
1966
- .catch(function () { return false; })];
1967
- case 2:
1968
- isFile = _a.sent();
1969
- return [2 /*return*/, isFile];
1970
- }
1971
- });
1972
- });
1973
- }
1974
- /**
1975
- * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
1976
- * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
1977
- * TODO: [🖇] What about symlinks?
1978
- */
1979
-
1980
- /**
1981
- * Get the file extension from a file name
1982
- *
1983
- * @private within the repository
1984
- */
1985
- function getFileExtension(value) {
1986
- var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
1987
- return match ? match[1].toLowerCase() : null;
1988
- }
1989
-
1990
- var defaultDiacriticsRemovalMap = [
1991
- {
1992
- base: 'A',
1993
- letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
1994
- },
1995
- { base: 'AA', letters: '\uA732' },
1996
- { base: 'AE', letters: '\u00C6\u01FC\u01E2' },
1997
- { base: 'AO', letters: '\uA734' },
1998
- { base: 'AU', letters: '\uA736' },
1999
- { base: 'AV', letters: '\uA738\uA73A' },
2000
- { base: 'AY', letters: '\uA73C' },
2001
- {
2002
- base: 'B',
2003
- letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
2004
- },
2005
- {
2006
- base: 'C',
2007
- letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
2008
- },
2009
- {
2010
- base: 'D',
2011
- letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
2012
- },
2013
- { base: 'DZ', letters: '\u01F1\u01C4' },
2014
- { base: 'Dz', letters: '\u01F2\u01C5' },
2015
- {
2016
- base: 'E',
2017
- letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
2018
- },
2019
- { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
2020
- {
2021
- base: 'G',
2022
- letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
2023
- },
2024
- {
2025
- base: 'H',
2026
- letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
2027
- },
2028
- {
2029
- base: 'I',
2030
- letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
2031
- },
2032
- { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
2033
- {
2034
- base: 'K',
2035
- letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
2036
- },
2037
- {
2038
- base: 'L',
2039
- letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
2040
- },
2041
- { base: 'LJ', letters: '\u01C7' },
2042
- { base: 'Lj', letters: '\u01C8' },
2043
- { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
2044
- {
2045
- base: 'N',
2046
- letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
2047
- },
2048
- { base: 'NJ', letters: '\u01CA' },
2049
- { base: 'Nj', letters: '\u01CB' },
2050
- {
2051
- base: 'O',
2052
- letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
2053
- },
2054
- { base: 'OI', letters: '\u01A2' },
2055
- { base: 'OO', letters: '\uA74E' },
2056
- { base: 'OU', letters: '\u0222' },
2057
- { base: 'OE', letters: '\u008C\u0152' },
2058
- { base: 'oe', letters: '\u009C\u0153' },
2059
- {
2060
- base: 'P',
2061
- letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
2062
- },
2063
- { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
2064
- {
2065
- base: 'R',
2066
- letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
2067
- },
2068
- {
2069
- base: 'S',
2070
- letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
2071
- },
2072
- {
2073
- base: 'T',
2074
- letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
2075
- },
2076
- { base: 'TZ', letters: '\uA728' },
2077
- {
2078
- base: 'U',
2079
- letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
2080
- },
2081
- { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
2082
- { base: 'VY', letters: '\uA760' },
2083
- {
2084
- base: 'W',
2085
- letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
2086
- },
2087
- { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
2088
- {
2089
- base: 'Y',
2090
- letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
2091
- },
2092
- {
2093
- base: 'Z',
2094
- letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
2095
- },
2096
- {
2097
- base: 'a',
2098
- letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
2099
- },
2100
- { base: 'aa', letters: '\uA733' },
2101
- { base: 'ae', letters: '\u00E6\u01FD\u01E3' },
2102
- { base: 'ao', letters: '\uA735' },
2103
- { base: 'au', letters: '\uA737' },
2104
- { base: 'av', letters: '\uA739\uA73B' },
2105
- { base: 'ay', letters: '\uA73D' },
2106
- {
2107
- base: 'b',
2108
- letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
2109
- },
2110
- {
2111
- base: 'c',
2112
- letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
2113
- },
2114
- {
2115
- base: 'd',
2116
- letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
2117
- },
2118
- { base: 'dz', letters: '\u01F3\u01C6' },
2119
- {
2120
- base: 'e',
2121
- letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
2122
- },
2123
- { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
2124
- {
2125
- base: 'g',
2126
- letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
2127
- },
2128
- {
2129
- base: 'h',
2130
- letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
2131
- },
2132
- { base: 'hv', letters: '\u0195' },
2133
- {
2134
- base: 'i',
2135
- letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
2136
- },
2137
- { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
2138
- {
2139
- base: 'k',
2140
- letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
2141
- },
2142
- {
2143
- base: 'l',
2144
- letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
2145
- },
2146
- { base: 'lj', letters: '\u01C9' },
2147
- { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
2148
- {
2149
- base: 'n',
2150
- letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
2151
- },
2152
- { base: 'nj', letters: '\u01CC' },
2153
- {
2154
- base: 'o',
2155
- letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
2156
- },
2157
- { base: 'oi', letters: '\u01A3' },
2158
- { base: 'ou', letters: '\u0223' },
2159
- { base: 'oo', letters: '\uA74F' },
2160
- {
2161
- base: 'p',
2162
- letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
2163
- },
2164
- { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
2165
- {
2166
- base: 'r',
2167
- letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
2168
- },
2169
- {
2170
- base: 's',
2171
- letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
2172
- },
2173
- {
2174
- base: 't',
2175
- letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
2176
- },
2177
- { base: 'tz', letters: '\uA729' },
2178
- {
2179
- base: 'u',
2180
- letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
2181
- },
2182
- { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
2183
- { base: 'vy', letters: '\uA761' },
2184
- {
2185
- base: 'w',
2186
- letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
2187
- },
2188
- { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
2189
- {
2190
- base: 'y',
2191
- letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
2192
- },
2193
- {
2194
- base: 'z',
2195
- letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
2196
- },
2197
- ];
2198
- /**
2199
- * Map of letters from diacritic variant to diacritless variant
2200
- * Contains lowercase and uppercase separatelly
2201
- *
2202
- * > "á" => "a"
2203
- * > "ě" => "e"
2204
- * > "Ă" => "A"
2205
- * > ...
2206
- *
2207
- * @public exported from `@promptbook/utils`
2208
- */
2209
- var DIACRITIC_VARIANTS_LETTERS = {};
2210
- // tslint:disable-next-line: prefer-for-of
2211
- for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
2212
- var letters = defaultDiacriticsRemovalMap[i].letters;
2213
- // tslint:disable-next-line: prefer-for-of
2214
- for (var j = 0; j < letters.length; j++) {
2215
- DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
2216
- }
2217
- }
2218
- // <- TODO: [🍓] Put to maker function to save execution time if not needed
2219
- /*
2220
- @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
2221
- Licensed under the Apache License, Version 2.0 (the "License");
2222
- you may not use this file except in compliance with the License.
2223
- You may obtain a copy of the License at
2224
-
2225
- http://www.apache.org/licenses/LICENSE-2.0
2226
-
2227
- Unless required by applicable law or agreed to in writing, software
2228
- distributed under the License is distributed on an "AS IS" BASIS,
2229
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2230
- See the License for the specific language governing permissions and
2231
- limitations under the License.
2232
- */
2233
-
2234
- /**
2235
- * @@@
2236
- *
2237
- * @param input @@@
2238
- * @returns @@@
2239
- * @public exported from `@promptbook/utils`
2240
- */
2241
- function removeDiacritics(input) {
2242
- /*eslint no-control-regex: "off"*/
2243
- return input.replace(/[^\u0000-\u007E]/g, function (a) {
2244
- return DIACRITIC_VARIANTS_LETTERS[a] || a;
2245
- });
2246
- }
2247
- /**
2248
- * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
2249
- */
2250
-
2251
- /**
2252
- * @@@
2253
- *
2254
- * @param text @@@
2255
- * @returns @@@
2256
- * @example 'hello-world'
2257
- * @example 'i-love-promptbook'
2258
- * @public exported from `@promptbook/utils`
2259
- */
2260
- function normalizeToKebabCase(text) {
2261
- var e_1, _a;
2262
- text = removeDiacritics(text);
2263
- var charType;
2264
- var lastCharType = 'OTHER';
2265
- var normalizedName = '';
2266
- try {
2267
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
2268
- var char = text_1_1.value;
2269
- var normalizedChar = void 0;
2270
- if (/^[a-z]$/.test(char)) {
2271
- charType = 'LOWERCASE';
2272
- normalizedChar = char;
2273
- }
2274
- else if (/^[A-Z]$/.test(char)) {
2275
- charType = 'UPPERCASE';
2276
- normalizedChar = char.toLowerCase();
2277
- }
2278
- else if (/^[0-9]$/.test(char)) {
2279
- charType = 'NUMBER';
2280
- normalizedChar = char;
2281
- }
2282
- else {
2283
- charType = 'OTHER';
2284
- normalizedChar = '-';
2285
- }
2286
- if (charType !== lastCharType &&
2287
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
2288
- !(lastCharType === 'NUMBER') &&
2289
- !(charType === 'NUMBER')) {
2290
- normalizedName += '-';
2291
- }
2292
- normalizedName += normalizedChar;
2293
- lastCharType = charType;
2294
- }
2295
- }
2296
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2297
- finally {
2298
- try {
2299
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
2300
- }
2301
- finally { if (e_1) throw e_1.error; }
2302
- }
2303
- normalizedName = normalizedName.split(/-+/g).join('-');
2304
- normalizedName = normalizedName.split(/-?\/-?/g).join('/');
2305
- normalizedName = normalizedName.replace(/^-/, '');
2306
- normalizedName = normalizedName.replace(/-$/, '');
2307
- return normalizedName;
2308
- }
2309
-
2310
- /**
2311
- * Removes emojis from a string and fix whitespaces
2312
- *
2313
- * @param text with emojis
2314
- * @returns text without emojis
2315
- * @public exported from `@promptbook/utils`
2316
- */
2317
- function removeEmojis(text) {
2318
- // Replace emojis (and also ZWJ sequence) with hyphens
2319
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
2320
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
2321
- text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
2322
- text = text.replace(/\p{Extended_Pictographic}/gu, '');
2323
- return text;
2324
- }
2325
-
2326
- /**
2327
- * Tests if given string is valid URL.
2328
- *
2329
- * Note: This does not check if the file exists only if the path is valid
2330
- * @public exported from `@promptbook/utils`
2331
- */
2332
- function isValidFilePath(filename) {
2333
- if (typeof filename !== 'string') {
2334
- return false;
2335
- }
2336
- var filenameSlashes = filename.split('\\').join('/');
2337
- // Absolute Unix path: /hello.txt
2338
- if (/^(\/)/i.test(filenameSlashes)) {
2339
- return true;
2340
- }
2341
- // Absolute Windows path: /hello.txt
2342
- if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
2343
- return true;
2344
- }
2345
- // Relative path: ./hello.txt
2346
- if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
2347
- return true;
2348
- }
2349
- return false;
2350
- }
2351
-
2352
- /**
2353
- * @@@
2354
- *
2355
- * @param value @@@
2356
- * @returns @@@
2357
- * @example @@@
2358
- * @public exported from `@promptbook/utils`
2359
- */
2360
- function titleToName(value) {
2361
- if (isValidUrl(value)) {
2362
- value = value.replace(/^https?:\/\//, '');
2363
- value = value.replace(/\.html$/, '');
2364
- }
2365
- else if (isValidFilePath(value)) {
2366
- value = basename(value);
2367
- // Note: Keeping extension in the name
2368
- }
2369
- value = value.split('/').join('-');
2370
- value = removeEmojis(value);
2371
- value = normalizeToKebabCase(value);
2372
- // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
2373
- return value;
2374
- }
2375
-
2376
- /**
2377
- * @@@
2378
- *
2379
- * @private for `FileCacheStorage`
2380
- */
2381
- function nameToSubfolderPath(name) {
2382
- return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
2383
- }
2384
-
2385
- /**
2386
- * Just marks a place of place where should be something implemented
2387
- * No side effects.
2388
- *
2389
- * Note: It can be usefull suppressing eslint errors of unused variables
2390
- *
2391
- * @param value any values
2392
- * @returns void
2393
- * @private within the repository
2394
- */
2395
- function TODO_USE() {
2396
- var value = [];
2397
- for (var _i = 0; _i < arguments.length; _i++) {
2398
- value[_i] = arguments[_i];
2399
- }
2400
- }
2401
-
2402
- /**
2403
- * Create a filename for intermediate cache for scrapers
2404
- *
2405
- * Note: It also checks if directory exists and creates it if not
2406
- *
2407
- * @private as internal utility for scrapers
2408
- */
2409
- function getScraperIntermediateSource(source, options) {
2410
- return __awaiter(this, void 0, void 0, function () {
2411
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
2412
- return __generator(this, function (_a) {
2413
- switch (_a.label) {
2414
- case 0:
2415
- sourceFilename = source.filename, url = source.url;
2416
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
2417
- hash = SHA256(
2418
- // <- TODO: [🥬] Encapsulate sha256 to some private utility function
2419
- hexEncoder.parse(sourceFilename || url || 'untitled'))
2420
- .toString( /* hex */)
2421
- .substring(0, 20);
2422
- semanticName = normalizeToKebabCase(titleToName((sourceFilename || url || '').split('intermediate').join(''))).substring(0, 20);
2423
- pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
2424
- name = pieces.join('-').split('--').join('-');
2425
- // <- TODO: Use MAX_FILENAME_LENGTH
2426
- TODO_USE(rootDirname); // <- TODO: !!!!!!
2427
- cacheFilename = join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
2428
- cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
2429
- .join('/') +
2430
- '.' +
2431
- extension;
2432
- return [4 /*yield*/, mkdir(dirname(cacheFilename), { recursive: true })];
2433
- case 1:
2434
- _a.sent();
2435
- isDestroyed = true;
2436
- fileHandler = {
2437
- filename: cacheFilename,
2438
- get isDestroyed() {
2439
- return isDestroyed;
2440
- },
2441
- destroy: function () {
2442
- return __awaiter(this, void 0, void 0, function () {
2443
- return __generator(this, function (_a) {
2444
- switch (_a.label) {
2445
- case 0:
2446
- if (!isCacheCleaned) return [3 /*break*/, 2];
2447
- if (isVerbose) {
2448
- console.info('legacyDocumentScraper: Clening cache');
2449
- }
2450
- return [4 /*yield*/, rm(cacheFilename)];
2451
- case 1:
2452
- _a.sent();
2453
- _a.label = 2;
2454
- case 2:
2455
- isDestroyed = true;
2456
- return [2 /*return*/];
2457
- }
2458
- });
2459
- });
2460
- },
2461
- };
2462
- return [2 /*return*/, fileHandler];
2463
- }
2464
- });
2465
- });
2466
- }
2467
- /**
2468
- * Note: Not using `FileCacheStorage` for two reasons:
2469
- * 1) Need to store more than serialized JSONs
2470
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
2471
- * TODO: [🐱‍🐉][🧠] Make some smart crop
2472
- */
2473
-
2474
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
2475
-
2476
- /**
2477
- * This error indicates errors during the execution of the pipeline
2478
- *
2479
- * @public exported from `@promptbook/core`
2480
- */
2481
- var PipelineExecutionError = /** @class */ (function (_super) {
2482
- __extends(PipelineExecutionError, _super);
2483
- function PipelineExecutionError(message) {
2484
- var _this = _super.call(this, message) || this;
2485
- _this.name = 'PipelineExecutionError';
2486
- Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
2487
- return _this;
2488
- }
2489
- return PipelineExecutionError;
2490
- }(Error));
2491
-
2492
- /**
2493
- * This error indicates that the pipeline collection cannot be propperly loaded
2494
- *
2495
- * @public exported from `@promptbook/core`
2496
- */
2497
- var CollectionError = /** @class */ (function (_super) {
2498
- __extends(CollectionError, _super);
2499
- function CollectionError(message) {
2500
- var _this = _super.call(this, message) || this;
2501
- _this.name = 'CollectionError';
2502
- Object.setPrototypeOf(_this, CollectionError.prototype);
2503
- return _this;
2504
- }
2505
- return CollectionError;
2506
- }(Error));
2507
-
2508
- /**
2509
- * This error occurs when some expectation is not met in the execution of the pipeline
2510
- *
2511
- * @public exported from `@promptbook/core`
2512
- * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
2513
- * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
2514
- * Note: This is a kindof subtype of PipelineExecutionError
2515
- */
2516
- var ExpectError = /** @class */ (function (_super) {
2517
- __extends(ExpectError, _super);
2518
- function ExpectError(message) {
2519
- var _this = _super.call(this, message) || this;
2520
- _this.name = 'ExpectError';
2521
- Object.setPrototypeOf(_this, ExpectError.prototype);
2522
- return _this;
2523
- }
2524
- return ExpectError;
2525
- }(Error));
2526
-
2527
- /**
2528
- * This error type indicates that some limit was reached
2529
- *
2530
- * @public exported from `@promptbook/core`
2531
- */
2532
- var LimitReachedError = /** @class */ (function (_super) {
2533
- __extends(LimitReachedError, _super);
2534
- function LimitReachedError(message) {
2535
- var _this = _super.call(this, message) || this;
2536
- _this.name = 'LimitReachedError';
2537
- Object.setPrototypeOf(_this, LimitReachedError.prototype);
2538
- return _this;
2539
- }
2540
- return LimitReachedError;
2541
- }(Error));
2542
-
2543
- /**
2544
- * This error type indicates that some part of the code is not implemented yet
2545
- *
2546
- * @public exported from `@promptbook/core`
2547
- */
2548
- var NotYetImplementedError = /** @class */ (function (_super) {
2549
- __extends(NotYetImplementedError, _super);
2550
- function NotYetImplementedError(message) {
2551
- var _this = _super.call(this, spaceTrim$1(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2552
- _this.name = 'NotYetImplementedError';
2553
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2554
- return _this;
2555
- }
2556
- return NotYetImplementedError;
2557
- }(Error));
2558
-
2559
- /**
2560
- * Index of all custom errors
2561
- *
2562
- * @public exported from `@promptbook/core`
2563
- */
2564
- var ERRORS = {
2565
- ExpectError: ExpectError,
2566
- CollectionError: CollectionError,
2567
- EnvironmentMismatchError: EnvironmentMismatchError,
2568
- LimitReachedError: LimitReachedError,
2569
- NotFoundError: NotFoundError,
2570
- NotYetImplementedError: NotYetImplementedError,
2571
- ParseError: ParseError,
2572
- PipelineExecutionError: PipelineExecutionError,
2573
- PipelineLogicError: PipelineLogicError,
2574
- PipelineUrlError: PipelineUrlError,
2575
- UnexpectedError: UnexpectedError,
2576
- // TODO: [🪑]> VersionMismatchError,
2577
- };
2578
-
2579
- /**
2580
- * Deserializes the error object
2581
- *
2582
- * @public exported from `@promptbook/utils`
2583
- */
2584
- function deserializeError(error) {
2585
- if (error.name === 'Error') {
2586
- return new Error(error.message);
2587
- }
2588
- var CustomError = ERRORS[error.name];
2589
- return new CustomError(error.message);
2590
- }
2591
-
2592
- /**
2593
- * Asserts that the execution of a Promptbook is successful
2594
- *
2595
- * @param executionResult - The partial result of the Promptbook execution
2596
- * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2597
- * @public exported from `@promptbook/core`
2598
- */
2599
- function assertsExecutionSuccessful(executionResult) {
2600
- var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2601
- if (isSuccessful === true) {
2602
- return;
2603
- }
2604
- if (errors.length === 0) {
2605
- throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
2606
- }
2607
- else if (errors.length === 1) {
2608
- throw deserializeError(errors[0]);
2609
- }
2610
- else {
2611
- throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
2612
- .map(function (_a, index) {
2613
- var name = _a.name, stack = _a.stack, message = _a.message;
2614
- return spaceTrim$1(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
2615
- })
2616
- .join('\n')), "\n "); }));
2617
- }
2618
- }
2619
- /**
2620
- * TODO: [🧠] Can this return type be better typed than void
2621
- */
2622
-
2623
- /**
2624
- * Determine if the pipeline is fully prepared
2625
- *
2626
- * @public exported from `@promptbook/core`
2627
- */
2628
- function isPipelinePrepared(pipeline) {
2629
- // Note: Ignoring `pipeline.preparations` @@@
2630
- // Note: Ignoring `pipeline.knowledgePieces` @@@
2631
- if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2632
- return false;
2633
- }
2634
- if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2635
- return false;
2636
- }
2637
- /*
2638
- TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2639
- > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2640
- > return false;
2641
- > }
2642
- */
2643
- return true;
2644
- }
2645
- /**
2646
- * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2647
- * TODO: [🐠] Maybe base this on `makeValidator`
2648
- * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2649
- * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2650
- * - [🏍] ? Is context in each template
2651
- * - [♨] Are samples prepared
2652
- * - [♨] Are templates prepared
2653
- */
2654
-
2655
- /**
2656
- * Serializes an error into a [🚉] JSON-serializable object
2657
- *
2658
- * @public exported from `@promptbook/utils`
2659
- */
2660
- function serializeError(error) {
2661
- var name = error.name, message = error.message, stack = error.stack;
2662
- if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2663
- throw new UnexpectedError(spaceTrim(function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2664
- }
2665
- return {
2666
- name: name,
2667
- message: message,
2668
- stack: stack,
2669
- };
2670
- }
2671
-
2672
- /**
2673
- * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2674
- *
2675
- * Note: Internal utility of `joinLlmExecutionTools` but exposed type
2676
- * @public exported from `@promptbook/types`
2677
- */
2678
- var MultipleLlmExecutionTools = /** @class */ (function () {
2679
- /**
2680
- * Gets array of execution tools in order of priority
2681
- */
2682
- function MultipleLlmExecutionTools() {
2683
- var llmExecutionTools = [];
2684
- for (var _i = 0; _i < arguments.length; _i++) {
2685
- llmExecutionTools[_i] = arguments[_i];
2686
- }
2687
- this.llmExecutionTools = llmExecutionTools;
2688
- }
2689
- Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
2690
- get: function () {
2691
- return 'Multiple LLM Providers';
2692
- },
2693
- enumerable: false,
2694
- configurable: true
2695
- });
2696
- Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2697
- get: function () {
2698
- return this.llmExecutionTools.map(function (_a, index) {
2699
- var title = _a.title;
2700
- return "".concat(index + 1, ") `").concat(title, "`");
2701
- }).join('\n');
2702
- },
2703
- enumerable: false,
2704
- configurable: true
2705
- });
2706
- /**
2707
- * Check the configuration of all execution tools
2708
- */
2709
- MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
2710
- return __awaiter(this, void 0, void 0, function () {
2711
- var _a, _b, llmExecutionTools, e_1_1;
2712
- var e_1, _c;
2713
- return __generator(this, function (_d) {
2714
- switch (_d.label) {
2715
- case 0:
2716
- _d.trys.push([0, 5, 6, 7]);
2717
- _a = __values(this.llmExecutionTools), _b = _a.next();
2718
- _d.label = 1;
2719
- case 1:
2720
- if (!!_b.done) return [3 /*break*/, 4];
2721
- llmExecutionTools = _b.value;
2722
- return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
2723
- case 2:
2724
- _d.sent();
2725
- _d.label = 3;
2726
- case 3:
2727
- _b = _a.next();
2728
- return [3 /*break*/, 1];
2729
- case 4: return [3 /*break*/, 7];
2730
- case 5:
2731
- e_1_1 = _d.sent();
2732
- e_1 = { error: e_1_1 };
2733
- return [3 /*break*/, 7];
2734
- case 6:
2735
- try {
2736
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2737
- }
2738
- finally { if (e_1) throw e_1.error; }
2739
- return [7 /*endfinally*/];
2740
- case 7: return [2 /*return*/];
2741
- }
2742
- });
2743
- });
2744
- };
2745
- /**
2746
- * List all available models that can be used
2747
- * This lists is a combination of all available models from all execution tools
2748
- */
2749
- MultipleLlmExecutionTools.prototype.listModels = function () {
2750
- return __awaiter(this, void 0, void 0, function () {
2751
- var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
2752
- var e_2, _c;
2753
- return __generator(this, function (_d) {
2754
- switch (_d.label) {
2755
- case 0:
2756
- availableModels = [];
2757
- _d.label = 1;
2758
- case 1:
2759
- _d.trys.push([1, 6, 7, 8]);
2760
- _a = __values(this.llmExecutionTools), _b = _a.next();
2761
- _d.label = 2;
2762
- case 2:
2763
- if (!!_b.done) return [3 /*break*/, 5];
2764
- llmExecutionTools = _b.value;
2765
- return [4 /*yield*/, llmExecutionTools.listModels()];
2766
- case 3:
2767
- models = _d.sent();
2768
- availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
2769
- _d.label = 4;
2770
- case 4:
2771
- _b = _a.next();
2772
- return [3 /*break*/, 2];
2773
- case 5: return [3 /*break*/, 8];
2774
- case 6:
2775
- e_2_1 = _d.sent();
2776
- e_2 = { error: e_2_1 };
2777
- return [3 /*break*/, 8];
2778
- case 7:
2779
- try {
2780
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2781
- }
2782
- finally { if (e_2) throw e_2.error; }
2783
- return [7 /*endfinally*/];
2784
- case 8: return [2 /*return*/, availableModels];
1893
+ });
1894
+ };
1895
+ /**
1896
+ * List all available models that can be used
1897
+ * This lists is a combination of all available models from all execution tools
1898
+ */
1899
+ MultipleLlmExecutionTools.prototype.listModels = function () {
1900
+ return __awaiter(this, void 0, void 0, function () {
1901
+ var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
1902
+ var e_2, _c;
1903
+ return __generator(this, function (_d) {
1904
+ switch (_d.label) {
1905
+ case 0:
1906
+ availableModels = [];
1907
+ _d.label = 1;
1908
+ case 1:
1909
+ _d.trys.push([1, 6, 7, 8]);
1910
+ _a = __values(this.llmExecutionTools), _b = _a.next();
1911
+ _d.label = 2;
1912
+ case 2:
1913
+ if (!!_b.done) return [3 /*break*/, 5];
1914
+ llmExecutionTools = _b.value;
1915
+ return [4 /*yield*/, llmExecutionTools.listModels()];
1916
+ case 3:
1917
+ models = _d.sent();
1918
+ availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
1919
+ _d.label = 4;
1920
+ case 4:
1921
+ _b = _a.next();
1922
+ return [3 /*break*/, 2];
1923
+ case 5: return [3 /*break*/, 8];
1924
+ case 6:
1925
+ e_2_1 = _d.sent();
1926
+ e_2 = { error: e_2_1 };
1927
+ return [3 /*break*/, 8];
1928
+ case 7:
1929
+ try {
1930
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
1931
+ }
1932
+ finally { if (e_2) throw e_2.error; }
1933
+ return [7 /*endfinally*/];
1934
+ case 8: return [2 /*return*/, availableModels];
2785
1935
  }
2786
1936
  });
2787
1937
  });
@@ -2961,25 +2111,6 @@ function joinLlmExecutionTools() {
2961
2111
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2962
2112
  */
2963
2113
 
2964
- /**
2965
- * Takes an item or an array of items and returns an array of items
2966
- *
2967
- * 1) Any item except array and undefined returns array with that one item (also null)
2968
- * 2) Undefined returns empty array
2969
- * 3) Array returns itself
2970
- *
2971
- * @private internal utility
2972
- */
2973
- function arrayableToArray(input) {
2974
- if (input === undefined) {
2975
- return [];
2976
- }
2977
- if (input instanceof Array) {
2978
- return input;
2979
- }
2980
- return [input];
2981
- }
2982
-
2983
2114
  /**
2984
2115
  * @@@
2985
2116
  *
@@ -2995,66 +2126,13 @@ function deepClone(objectValue) {
2995
2126
  > if (value && typeof value === 'object') {
2996
2127
  > deepClone(value);
2997
2128
  > }
2998
- > }
2999
- > return Object.assign({}, objectValue);
3000
- */
3001
- }
3002
- /**
3003
- * TODO: [🧠] Is there a way how to meaningfully test this utility
3004
- */
3005
-
3006
- /**
3007
- * Represents the usage with no resources consumed
3008
- *
3009
- * @public exported from `@promptbook/core`
3010
- */
3011
- var ZERO_USAGE = $deepFreeze({
3012
- price: { value: 0 },
3013
- input: {
3014
- tokensCount: { value: 0 },
3015
- charactersCount: { value: 0 },
3016
- wordsCount: { value: 0 },
3017
- sentencesCount: { value: 0 },
3018
- linesCount: { value: 0 },
3019
- paragraphsCount: { value: 0 },
3020
- pagesCount: { value: 0 },
3021
- },
3022
- output: {
3023
- tokensCount: { value: 0 },
3024
- charactersCount: { value: 0 },
3025
- wordsCount: { value: 0 },
3026
- sentencesCount: { value: 0 },
3027
- linesCount: { value: 0 },
3028
- paragraphsCount: { value: 0 },
3029
- pagesCount: { value: 0 },
3030
- },
3031
- });
2129
+ > }
2130
+ > return Object.assign({}, objectValue);
2131
+ */
2132
+ }
3032
2133
  /**
3033
- * Represents the usage with unknown resources consumed
3034
- *
3035
- * @public exported from `@promptbook/core`
2134
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
3036
2135
  */
3037
- var UNCERTAIN_USAGE = $deepFreeze({
3038
- price: { value: 0, isUncertain: true },
3039
- input: {
3040
- tokensCount: { value: 0, isUncertain: true },
3041
- charactersCount: { value: 0, isUncertain: true },
3042
- wordsCount: { value: 0, isUncertain: true },
3043
- sentencesCount: { value: 0, isUncertain: true },
3044
- linesCount: { value: 0, isUncertain: true },
3045
- paragraphsCount: { value: 0, isUncertain: true },
3046
- pagesCount: { value: 0, isUncertain: true },
3047
- },
3048
- output: {
3049
- tokensCount: { value: 0, isUncertain: true },
3050
- charactersCount: { value: 0, isUncertain: true },
3051
- wordsCount: { value: 0, isUncertain: true },
3052
- sentencesCount: { value: 0, isUncertain: true },
3053
- linesCount: { value: 0, isUncertain: true },
3054
- paragraphsCount: { value: 0, isUncertain: true },
3055
- pagesCount: { value: 0, isUncertain: true },
3056
- },
3057
- });
3058
2136
 
3059
2137
  /**
3060
2138
  * Function `addUsage` will add multiple usages into one
@@ -3128,6 +2206,292 @@ function addUsage() {
3128
2206
  }, deepClone(ZERO_USAGE));
3129
2207
  }
3130
2208
 
2209
+ /**
2210
+ * Intercepts LLM tools and counts total usage of the tools
2211
+ *
2212
+ * @param llmTools LLM tools to be intercepted with usage counting
2213
+ * @returns LLM tools with same functionality with added total cost counting
2214
+ * @public exported from `@promptbook/core`
2215
+ */
2216
+ function countTotalUsage(llmTools) {
2217
+ var _this = this;
2218
+ var totalUsage = ZERO_USAGE;
2219
+ var proxyTools = {
2220
+ get title() {
2221
+ // TODO: [🧠] Maybe put here some suffix
2222
+ return llmTools.title;
2223
+ },
2224
+ get description() {
2225
+ // TODO: [🧠] Maybe put here some suffix
2226
+ return llmTools.description;
2227
+ },
2228
+ checkConfiguration: function () {
2229
+ return __awaiter(this, void 0, void 0, function () {
2230
+ return __generator(this, function (_a) {
2231
+ return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
2232
+ });
2233
+ });
2234
+ },
2235
+ listModels: function () {
2236
+ return /* not await */ llmTools.listModels();
2237
+ },
2238
+ getTotalUsage: function () {
2239
+ // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
2240
+ return totalUsage;
2241
+ },
2242
+ };
2243
+ if (llmTools.callChatModel !== undefined) {
2244
+ proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2245
+ var promptResult;
2246
+ return __generator(this, function (_a) {
2247
+ switch (_a.label) {
2248
+ case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
2249
+ case 1:
2250
+ promptResult = _a.sent();
2251
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2252
+ return [2 /*return*/, promptResult];
2253
+ }
2254
+ });
2255
+ }); };
2256
+ }
2257
+ if (llmTools.callCompletionModel !== undefined) {
2258
+ proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2259
+ var promptResult;
2260
+ return __generator(this, function (_a) {
2261
+ switch (_a.label) {
2262
+ case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
2263
+ case 1:
2264
+ promptResult = _a.sent();
2265
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2266
+ return [2 /*return*/, promptResult];
2267
+ }
2268
+ });
2269
+ }); };
2270
+ }
2271
+ if (llmTools.callEmbeddingModel !== undefined) {
2272
+ proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2273
+ var promptResult;
2274
+ return __generator(this, function (_a) {
2275
+ switch (_a.label) {
2276
+ case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
2277
+ case 1:
2278
+ promptResult = _a.sent();
2279
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2280
+ return [2 /*return*/, promptResult];
2281
+ }
2282
+ });
2283
+ }); };
2284
+ }
2285
+ // <- Note: [🤖]
2286
+ return proxyTools;
2287
+ }
2288
+ /**
2289
+ * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
2290
+ * TODO: [🧠] Is there some meaningfull way how to test this util
2291
+ * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
2292
+ * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
2293
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2294
+ */
2295
+
2296
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
2297
+
2298
+ /**
2299
+ * This error indicates that the pipeline collection cannot be propperly loaded
2300
+ *
2301
+ * @public exported from `@promptbook/core`
2302
+ */
2303
+ var CollectionError = /** @class */ (function (_super) {
2304
+ __extends(CollectionError, _super);
2305
+ function CollectionError(message) {
2306
+ var _this = _super.call(this, message) || this;
2307
+ _this.name = 'CollectionError';
2308
+ Object.setPrototypeOf(_this, CollectionError.prototype);
2309
+ return _this;
2310
+ }
2311
+ return CollectionError;
2312
+ }(Error));
2313
+
2314
+ /**
2315
+ * This error type indicates that you try to use a feature that is not available in the current environment
2316
+ *
2317
+ * @public exported from `@promptbook/core`
2318
+ */
2319
+ var EnvironmentMismatchError = /** @class */ (function (_super) {
2320
+ __extends(EnvironmentMismatchError, _super);
2321
+ function EnvironmentMismatchError(message) {
2322
+ var _this = _super.call(this, message) || this;
2323
+ _this.name = 'EnvironmentMismatchError';
2324
+ Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
2325
+ return _this;
2326
+ }
2327
+ return EnvironmentMismatchError;
2328
+ }(Error));
2329
+
2330
+ /**
2331
+ * This error occurs when some expectation is not met in the execution of the pipeline
2332
+ *
2333
+ * @public exported from `@promptbook/core`
2334
+ * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
2335
+ * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
2336
+ * Note: This is a kindof subtype of PipelineExecutionError
2337
+ */
2338
+ var ExpectError = /** @class */ (function (_super) {
2339
+ __extends(ExpectError, _super);
2340
+ function ExpectError(message) {
2341
+ var _this = _super.call(this, message) || this;
2342
+ _this.name = 'ExpectError';
2343
+ Object.setPrototypeOf(_this, ExpectError.prototype);
2344
+ return _this;
2345
+ }
2346
+ return ExpectError;
2347
+ }(Error));
2348
+
2349
+ /**
2350
+ * This error type indicates that some limit was reached
2351
+ *
2352
+ * @public exported from `@promptbook/core`
2353
+ */
2354
+ var LimitReachedError = /** @class */ (function (_super) {
2355
+ __extends(LimitReachedError, _super);
2356
+ function LimitReachedError(message) {
2357
+ var _this = _super.call(this, message) || this;
2358
+ _this.name = 'LimitReachedError';
2359
+ Object.setPrototypeOf(_this, LimitReachedError.prototype);
2360
+ return _this;
2361
+ }
2362
+ return LimitReachedError;
2363
+ }(Error));
2364
+
2365
+ /**
2366
+ * This error type indicates that some part of the code is not implemented yet
2367
+ *
2368
+ * @public exported from `@promptbook/core`
2369
+ */
2370
+ var NotYetImplementedError = /** @class */ (function (_super) {
2371
+ __extends(NotYetImplementedError, _super);
2372
+ function NotYetImplementedError(message) {
2373
+ var _this = _super.call(this, spaceTrim$1(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2374
+ _this.name = 'NotYetImplementedError';
2375
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2376
+ return _this;
2377
+ }
2378
+ return NotYetImplementedError;
2379
+ }(Error));
2380
+
2381
+ /**
2382
+ * Index of all custom errors
2383
+ *
2384
+ * @public exported from `@promptbook/core`
2385
+ */
2386
+ var ERRORS = {
2387
+ ExpectError: ExpectError,
2388
+ CollectionError: CollectionError,
2389
+ EnvironmentMismatchError: EnvironmentMismatchError,
2390
+ LimitReachedError: LimitReachedError,
2391
+ NotFoundError: NotFoundError,
2392
+ NotYetImplementedError: NotYetImplementedError,
2393
+ ParseError: ParseError,
2394
+ PipelineExecutionError: PipelineExecutionError,
2395
+ PipelineLogicError: PipelineLogicError,
2396
+ PipelineUrlError: PipelineUrlError,
2397
+ UnexpectedError: UnexpectedError,
2398
+ // TODO: [🪑]> VersionMismatchError,
2399
+ };
2400
+
2401
+ /**
2402
+ * Deserializes the error object
2403
+ *
2404
+ * @public exported from `@promptbook/utils`
2405
+ */
2406
+ function deserializeError(error) {
2407
+ if (error.name === 'Error') {
2408
+ return new Error(error.message);
2409
+ }
2410
+ var CustomError = ERRORS[error.name];
2411
+ return new CustomError(error.message);
2412
+ }
2413
+
2414
+ /**
2415
+ * Asserts that the execution of a Promptbook is successful
2416
+ *
2417
+ * @param executionResult - The partial result of the Promptbook execution
2418
+ * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2419
+ * @public exported from `@promptbook/core`
2420
+ */
2421
+ function assertsExecutionSuccessful(executionResult) {
2422
+ var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2423
+ if (isSuccessful === true) {
2424
+ return;
2425
+ }
2426
+ if (errors.length === 0) {
2427
+ throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
2428
+ }
2429
+ else if (errors.length === 1) {
2430
+ throw deserializeError(errors[0]);
2431
+ }
2432
+ else {
2433
+ throw new PipelineExecutionError(spaceTrim$1(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
2434
+ .map(function (_a, index) {
2435
+ var name = _a.name, stack = _a.stack, message = _a.message;
2436
+ return spaceTrim$1(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
2437
+ })
2438
+ .join('\n')), "\n "); }));
2439
+ }
2440
+ }
2441
+ /**
2442
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2443
+ * TODO: [🧠] Can this return type be better typed than void
2444
+ */
2445
+
2446
+ /**
2447
+ * Determine if the pipeline is fully prepared
2448
+ *
2449
+ * @public exported from `@promptbook/core`
2450
+ */
2451
+ function isPipelinePrepared(pipeline) {
2452
+ // Note: Ignoring `pipeline.preparations` @@@
2453
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2454
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2455
+ return false;
2456
+ }
2457
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2458
+ return false;
2459
+ }
2460
+ /*
2461
+ TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2462
+ > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2463
+ > return false;
2464
+ > }
2465
+ */
2466
+ return true;
2467
+ }
2468
+ /**
2469
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2470
+ * TODO: [🐠] Maybe base this on `makeValidator`
2471
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2472
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2473
+ * - [🏍] ? Is context in each template
2474
+ * - [♨] Are samples prepared
2475
+ * - [♨] Are templates prepared
2476
+ */
2477
+
2478
+ /**
2479
+ * Serializes an error into a [🚉] JSON-serializable object
2480
+ *
2481
+ * @public exported from `@promptbook/utils`
2482
+ */
2483
+ function serializeError(error) {
2484
+ var name = error.name, message = error.message, stack = error.stack;
2485
+ if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2486
+ throw new UnexpectedError(spaceTrim(function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2487
+ }
2488
+ return {
2489
+ name: name,
2490
+ message: message,
2491
+ stack: stack,
2492
+ };
2493
+ }
2494
+
3131
2495
  /**
3132
2496
  * Parses the given script and returns the list of all used variables that are not defined in the script
3133
2497
  *
@@ -3333,6 +2697,23 @@ function union() {
3333
2697
  return union;
3334
2698
  }
3335
2699
 
2700
+ /**
2701
+ * Just marks a place of place where should be something implemented
2702
+ * No side effects.
2703
+ *
2704
+ * Note: It can be usefull suppressing eslint errors of unused variables
2705
+ *
2706
+ * @param value any values
2707
+ * @returns void
2708
+ * @private within the repository
2709
+ */
2710
+ function TODO_USE() {
2711
+ var value = [];
2712
+ for (var _i = 0; _i < arguments.length; _i++) {
2713
+ value[_i] = arguments[_i];
2714
+ }
2715
+ }
2716
+
3336
2717
  /**
3337
2718
  * This error indicates problems parsing the format value
3338
2719
  *
@@ -3846,6 +3227,25 @@ function extractJsonBlock(markdown) {
3846
3227
  * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
3847
3228
  */
3848
3229
 
3230
+ /**
3231
+ * Takes an item or an array of items and returns an array of items
3232
+ *
3233
+ * 1) Any item except array and undefined returns array with that one item (also null)
3234
+ * 2) Undefined returns empty array
3235
+ * 3) Array returns itself
3236
+ *
3237
+ * @private internal utility
3238
+ */
3239
+ function arrayableToArray(input) {
3240
+ if (input === undefined) {
3241
+ return [];
3242
+ }
3243
+ if (input instanceof Array) {
3244
+ return input;
3245
+ }
3246
+ return [input];
3247
+ }
3248
+
3849
3249
  /**
3850
3250
  * Just says that the variable is not used but should be kept
3851
3251
  * No side effects.
@@ -3947,69 +3347,330 @@ function replaceParameters(template, parameters) {
3947
3347
  }
3948
3348
 
3949
3349
  /**
3950
- * Counts number of characters in the text
3350
+ * Counts number of characters in the text
3351
+ *
3352
+ * @public exported from `@promptbook/utils`
3353
+ */
3354
+ function countCharacters(text) {
3355
+ // Remove null characters
3356
+ text = text.replace(/\0/g, '');
3357
+ // Replace emojis (and also ZWJ sequence) with hyphens
3358
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
3359
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
3360
+ text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
3361
+ return text.length;
3362
+ }
3363
+
3364
+ /**
3365
+ * Counts number of lines in the text
3366
+ *
3367
+ * @public exported from `@promptbook/utils`
3368
+ */
3369
+ function countLines(text) {
3370
+ if (text === '') {
3371
+ return 0;
3372
+ }
3373
+ return text.split('\n').length;
3374
+ }
3375
+
3376
+ /**
3377
+ * Counts number of pages in the text
3378
+ *
3379
+ * @public exported from `@promptbook/utils`
3380
+ */
3381
+ function countPages(text) {
3382
+ var sentencesPerPage = 5; // Assuming each page has 5 sentences
3383
+ var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3384
+ var pageCount = Math.ceil(sentences.length / sentencesPerPage);
3385
+ return pageCount;
3386
+ }
3387
+
3388
+ /**
3389
+ * Counts number of paragraphs in the text
3390
+ *
3391
+ * @public exported from `@promptbook/utils`
3392
+ */
3393
+ function countParagraphs(text) {
3394
+ return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
3395
+ }
3396
+
3397
+ /**
3398
+ * Split text into sentences
3399
+ *
3400
+ * @public exported from `@promptbook/utils`
3401
+ */
3402
+ function splitIntoSentences(text) {
3403
+ return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3404
+ }
3405
+ /**
3406
+ * Counts number of sentences in the text
3407
+ *
3408
+ * @public exported from `@promptbook/utils`
3409
+ */
3410
+ function countSentences(text) {
3411
+ return splitIntoSentences(text).length;
3412
+ }
3413
+
3414
+ var defaultDiacriticsRemovalMap = [
3415
+ {
3416
+ base: 'A',
3417
+ letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
3418
+ },
3419
+ { base: 'AA', letters: '\uA732' },
3420
+ { base: 'AE', letters: '\u00C6\u01FC\u01E2' },
3421
+ { base: 'AO', letters: '\uA734' },
3422
+ { base: 'AU', letters: '\uA736' },
3423
+ { base: 'AV', letters: '\uA738\uA73A' },
3424
+ { base: 'AY', letters: '\uA73C' },
3425
+ {
3426
+ base: 'B',
3427
+ letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
3428
+ },
3429
+ {
3430
+ base: 'C',
3431
+ letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
3432
+ },
3433
+ {
3434
+ base: 'D',
3435
+ letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
3436
+ },
3437
+ { base: 'DZ', letters: '\u01F1\u01C4' },
3438
+ { base: 'Dz', letters: '\u01F2\u01C5' },
3439
+ {
3440
+ base: 'E',
3441
+ letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
3442
+ },
3443
+ { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
3444
+ {
3445
+ base: 'G',
3446
+ letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
3447
+ },
3448
+ {
3449
+ base: 'H',
3450
+ letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
3451
+ },
3452
+ {
3453
+ base: 'I',
3454
+ letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
3455
+ },
3456
+ { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
3457
+ {
3458
+ base: 'K',
3459
+ letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
3460
+ },
3461
+ {
3462
+ base: 'L',
3463
+ letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
3464
+ },
3465
+ { base: 'LJ', letters: '\u01C7' },
3466
+ { base: 'Lj', letters: '\u01C8' },
3467
+ { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
3468
+ {
3469
+ base: 'N',
3470
+ letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
3471
+ },
3472
+ { base: 'NJ', letters: '\u01CA' },
3473
+ { base: 'Nj', letters: '\u01CB' },
3474
+ {
3475
+ base: 'O',
3476
+ letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
3477
+ },
3478
+ { base: 'OI', letters: '\u01A2' },
3479
+ { base: 'OO', letters: '\uA74E' },
3480
+ { base: 'OU', letters: '\u0222' },
3481
+ { base: 'OE', letters: '\u008C\u0152' },
3482
+ { base: 'oe', letters: '\u009C\u0153' },
3483
+ {
3484
+ base: 'P',
3485
+ letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
3486
+ },
3487
+ { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
3488
+ {
3489
+ base: 'R',
3490
+ letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
3491
+ },
3492
+ {
3493
+ base: 'S',
3494
+ letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
3495
+ },
3496
+ {
3497
+ base: 'T',
3498
+ letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
3499
+ },
3500
+ { base: 'TZ', letters: '\uA728' },
3501
+ {
3502
+ base: 'U',
3503
+ letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
3504
+ },
3505
+ { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
3506
+ { base: 'VY', letters: '\uA760' },
3507
+ {
3508
+ base: 'W',
3509
+ letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
3510
+ },
3511
+ { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
3512
+ {
3513
+ base: 'Y',
3514
+ letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
3515
+ },
3516
+ {
3517
+ base: 'Z',
3518
+ letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
3519
+ },
3520
+ {
3521
+ base: 'a',
3522
+ letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
3523
+ },
3524
+ { base: 'aa', letters: '\uA733' },
3525
+ { base: 'ae', letters: '\u00E6\u01FD\u01E3' },
3526
+ { base: 'ao', letters: '\uA735' },
3527
+ { base: 'au', letters: '\uA737' },
3528
+ { base: 'av', letters: '\uA739\uA73B' },
3529
+ { base: 'ay', letters: '\uA73D' },
3530
+ {
3531
+ base: 'b',
3532
+ letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
3533
+ },
3534
+ {
3535
+ base: 'c',
3536
+ letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
3537
+ },
3538
+ {
3539
+ base: 'd',
3540
+ letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
3541
+ },
3542
+ { base: 'dz', letters: '\u01F3\u01C6' },
3543
+ {
3544
+ base: 'e',
3545
+ letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
3546
+ },
3547
+ { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
3548
+ {
3549
+ base: 'g',
3550
+ letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
3551
+ },
3552
+ {
3553
+ base: 'h',
3554
+ letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
3555
+ },
3556
+ { base: 'hv', letters: '\u0195' },
3557
+ {
3558
+ base: 'i',
3559
+ letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
3560
+ },
3561
+ { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
3562
+ {
3563
+ base: 'k',
3564
+ letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
3565
+ },
3566
+ {
3567
+ base: 'l',
3568
+ letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
3569
+ },
3570
+ { base: 'lj', letters: '\u01C9' },
3571
+ { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
3572
+ {
3573
+ base: 'n',
3574
+ letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
3575
+ },
3576
+ { base: 'nj', letters: '\u01CC' },
3577
+ {
3578
+ base: 'o',
3579
+ letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
3580
+ },
3581
+ { base: 'oi', letters: '\u01A3' },
3582
+ { base: 'ou', letters: '\u0223' },
3583
+ { base: 'oo', letters: '\uA74F' },
3584
+ {
3585
+ base: 'p',
3586
+ letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
3587
+ },
3588
+ { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
3589
+ {
3590
+ base: 'r',
3591
+ letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
3592
+ },
3593
+ {
3594
+ base: 's',
3595
+ letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
3596
+ },
3597
+ {
3598
+ base: 't',
3599
+ letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
3600
+ },
3601
+ { base: 'tz', letters: '\uA729' },
3602
+ {
3603
+ base: 'u',
3604
+ letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
3605
+ },
3606
+ { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
3607
+ { base: 'vy', letters: '\uA761' },
3608
+ {
3609
+ base: 'w',
3610
+ letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
3611
+ },
3612
+ { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
3613
+ {
3614
+ base: 'y',
3615
+ letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
3616
+ },
3617
+ {
3618
+ base: 'z',
3619
+ letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
3620
+ },
3621
+ ];
3622
+ /**
3623
+ * Map of letters from diacritic variant to diacritless variant
3624
+ * Contains lowercase and uppercase separatelly
3951
3625
  *
3952
- * @public exported from `@promptbook/utils`
3953
- */
3954
- function countCharacters(text) {
3955
- // Remove null characters
3956
- text = text.replace(/\0/g, '');
3957
- // Replace emojis (and also ZWJ sequence) with hyphens
3958
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
3959
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
3960
- text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
3961
- return text.length;
3962
- }
3963
-
3964
- /**
3965
- * Counts number of lines in the text
3626
+ * > "á" => "a"
3627
+ * > "ě" => "e"
3628
+ * > "Ă" => "A"
3629
+ * > ...
3966
3630
  *
3967
3631
  * @public exported from `@promptbook/utils`
3968
3632
  */
3969
- function countLines(text) {
3970
- if (text === '') {
3971
- return 0;
3633
+ var DIACRITIC_VARIANTS_LETTERS = {};
3634
+ // tslint:disable-next-line: prefer-for-of
3635
+ for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
3636
+ var letters = defaultDiacriticsRemovalMap[i].letters;
3637
+ // tslint:disable-next-line: prefer-for-of
3638
+ for (var j = 0; j < letters.length; j++) {
3639
+ DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
3972
3640
  }
3973
- return text.split('\n').length;
3974
3641
  }
3642
+ // <- TODO: [🍓] Put to maker function to save execution time if not needed
3643
+ /*
3644
+ @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
3645
+ Licensed under the Apache License, Version 2.0 (the "License");
3646
+ you may not use this file except in compliance with the License.
3647
+ You may obtain a copy of the License at
3975
3648
 
3976
- /**
3977
- * Counts number of pages in the text
3978
- *
3979
- * @public exported from `@promptbook/utils`
3980
- */
3981
- function countPages(text) {
3982
- var sentencesPerPage = 5; // Assuming each page has 5 sentences
3983
- var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3984
- var pageCount = Math.ceil(sentences.length / sentencesPerPage);
3985
- return pageCount;
3986
- }
3649
+ http://www.apache.org/licenses/LICENSE-2.0
3987
3650
 
3988
- /**
3989
- * Counts number of paragraphs in the text
3990
- *
3991
- * @public exported from `@promptbook/utils`
3992
- */
3993
- function countParagraphs(text) {
3994
- return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
3995
- }
3651
+ Unless required by applicable law or agreed to in writing, software
3652
+ distributed under the License is distributed on an "AS IS" BASIS,
3653
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3654
+ See the License for the specific language governing permissions and
3655
+ limitations under the License.
3656
+ */
3996
3657
 
3997
3658
  /**
3998
- * Split text into sentences
3659
+ * @@@
3999
3660
  *
3661
+ * @param input @@@
3662
+ * @returns @@@
4000
3663
  * @public exported from `@promptbook/utils`
4001
3664
  */
4002
- function splitIntoSentences(text) {
4003
- return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3665
+ function removeDiacritics(input) {
3666
+ /*eslint no-control-regex: "off"*/
3667
+ return input.replace(/[^\u0000-\u007E]/g, function (a) {
3668
+ return DIACRITIC_VARIANTS_LETTERS[a] || a;
3669
+ });
4004
3670
  }
4005
3671
  /**
4006
- * Counts number of sentences in the text
4007
- *
4008
- * @public exported from `@promptbook/utils`
3672
+ * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
4009
3673
  */
4010
- function countSentences(text) {
4011
- return splitIntoSentences(text).length;
4012
- }
4013
3674
 
4014
3675
  /**
4015
3676
  * Counts number of words in the text
@@ -4107,18 +3768,19 @@ function isPassingExpectations(expectations, value) {
4107
3768
  */
4108
3769
  function executeAttempts(options) {
4109
3770
  return __awaiter(this, void 0, void 0, function () {
4110
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
3771
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4111
3772
  return __generator(this, function (_a) {
4112
3773
  switch (_a.label) {
4113
3774
  case 0:
4114
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4115
- maxExecutionAttempts = settings.maxExecutionAttempts;
3775
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
4116
3776
  $ongoingTemplateResult = {
4117
3777
  $result: null,
4118
3778
  $resultString: null,
4119
3779
  $expectError: null,
4120
3780
  $scriptPipelineExecutionErrors: [],
4121
3781
  };
3782
+ _llms = arrayableToArray(tools.llm);
3783
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
4122
3784
  _loop_1 = function (attempt) {
4123
3785
  var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
4124
3786
  var e_1, _q, e_3, _r, e_2, _s;
@@ -4187,7 +3849,9 @@ function executeAttempts(options) {
4187
3849
  return [3 /*break*/, 9];
4188
3850
  case 4:
4189
3851
  _d = $ongoingTemplateResult;
4190
- return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3852
+ return [4 /*yield*/, llmTools.callChatModel(
3853
+ // <- TODO: [🧁] Check that `callChatModel` is defined
3854
+ $deepFreeze($ongoingTemplateResult.$prompt))];
4191
3855
  case 5:
4192
3856
  _d.$chatResult = _t.sent();
4193
3857
  // TODO: [🍬] Destroy chatThread
@@ -4196,7 +3860,9 @@ function executeAttempts(options) {
4196
3860
  return [3 /*break*/, 10];
4197
3861
  case 6:
4198
3862
  _e = $ongoingTemplateResult;
4199
- return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3863
+ return [4 /*yield*/, llmTools.callCompletionModel(
3864
+ // <- TODO: [🧁] Check that `callCompletionModel` is defined
3865
+ $deepFreeze($ongoingTemplateResult.$prompt))];
4200
3866
  case 7:
4201
3867
  _e.$completionResult = _t.sent();
4202
3868
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
@@ -4472,12 +4138,12 @@ function executeAttempts(options) {
4472
4138
  */
4473
4139
  function executeFormatSubvalues(options) {
4474
4140
  return __awaiter(this, void 0, void 0, function () {
4475
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4141
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4476
4142
  var _this = this;
4477
4143
  return __generator(this, function (_a) {
4478
4144
  switch (_a.label) {
4479
4145
  case 0:
4480
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4146
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4481
4147
  if (template.foreach === undefined) {
4482
4148
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4483
4149
  }
@@ -4507,7 +4173,7 @@ function executeFormatSubvalues(options) {
4507
4173
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4508
4174
  }
4509
4175
  if (formatDefinition.formatName === 'CSV') {
4510
- formatSettings = settings.csvSettings;
4176
+ formatSettings = csvSettings;
4511
4177
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4512
4178
  }
4513
4179
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4660,13 +4326,12 @@ function getReservedParametersForTemplate(options) {
4660
4326
  */
4661
4327
  function executeTemplate(options) {
4662
4328
  return __awaiter(this, void 0, void 0, function () {
4663
- var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4664
- var e_1, _f, _g;
4665
- return __generator(this, function (_h) {
4666
- switch (_h.label) {
4329
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4330
+ var e_1, _g, _h;
4331
+ return __generator(this, function (_j) {
4332
+ switch (_j.label) {
4667
4333
  case 0:
4668
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, llmTools = options.llmTools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4669
- maxExecutionAttempts = settings.maxExecutionAttempts;
4334
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4670
4335
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4671
4336
  title = currentTemplate.title;
4672
4337
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -4681,7 +4346,7 @@ function executeTemplate(options) {
4681
4346
  // <- [🍸]
4682
4347
  })];
4683
4348
  case 1:
4684
- _h.sent();
4349
+ _j.sent();
4685
4350
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
4686
4351
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
4687
4352
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4692,15 +4357,15 @@ function executeTemplate(options) {
4692
4357
  .map(function (name) { return "{".concat(name, "}"); })
4693
4358
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4694
4359
  }
4695
- _b = (_a = Object).freeze;
4696
- _c = [{}];
4360
+ _c = (_b = Object).freeze;
4361
+ _d = [{}];
4697
4362
  return [4 /*yield*/, getReservedParametersForTemplate({
4698
4363
  preparedPipeline: preparedPipeline,
4699
4364
  template: currentTemplate,
4700
4365
  pipelineIdentification: pipelineIdentification,
4701
4366
  })];
4702
4367
  case 2:
4703
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4368
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
4704
4369
  definedParameterNames = new Set(Object.keys(definedParameters));
4705
4370
  parameters = {};
4706
4371
  _loop_1 = function (parameterName) {
@@ -4720,15 +4385,15 @@ function executeTemplate(options) {
4720
4385
  try {
4721
4386
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4722
4387
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4723
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4724
- parameterName = _e.value;
4388
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
4389
+ parameterName = _f.value;
4725
4390
  _loop_1(parameterName);
4726
4391
  }
4727
4392
  }
4728
4393
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4729
4394
  finally {
4730
4395
  try {
4731
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4396
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
4732
4397
  }
4733
4398
  finally { if (e_1) throw e_1.error; }
4734
4399
  }
@@ -4748,13 +4413,11 @@ function executeTemplate(options) {
4748
4413
  template: currentTemplate,
4749
4414
  preparedPipeline: preparedPipeline,
4750
4415
  tools: tools,
4751
- llmTools: llmTools,
4752
- settings: settings,
4753
4416
  $executionReport: $executionReport,
4754
4417
  pipelineIdentification: pipelineIdentification,
4755
4418
  })];
4756
4419
  case 3:
4757
- resultString = _h.sent();
4420
+ resultString = _j.sent();
4758
4421
  return [4 /*yield*/, onProgress({
4759
4422
  name: name,
4760
4423
  title: title,
@@ -4766,12 +4429,12 @@ function executeTemplate(options) {
4766
4429
  // <- [🍸]
4767
4430
  })];
4768
4431
  case 4:
4769
- _h.sent();
4770
- return [2 /*return*/, Object.freeze((_g = {},
4771
- _g[currentTemplate.resultingParameterName] =
4432
+ _j.sent();
4433
+ return [2 /*return*/, Object.freeze((_h = {},
4434
+ _h[currentTemplate.resultingParameterName] =
4772
4435
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4773
4436
  resultString,
4774
- _g))];
4437
+ _h))];
4775
4438
  }
4776
4439
  });
4777
4440
  });
@@ -4830,18 +4493,15 @@ function filterJustOutputParameters(options) {
4830
4493
  */
4831
4494
  function executePipeline(options) {
4832
4495
  return __awaiter(this, void 0, void 0, function () {
4833
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, llmTools, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4496
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4834
4497
  var e_1, _f, e_2, _g;
4835
4498
  return __generator(this, function (_h) {
4836
4499
  switch (_h.label) {
4837
4500
  case 0:
4838
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4839
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4501
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4840
4502
  preparedPipeline = options.preparedPipeline;
4841
- llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
4842
4503
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4843
- return [4 /*yield*/, preparePipeline(pipeline, {
4844
- llmTools: llmTools,
4504
+ return [4 /*yield*/, preparePipeline(pipeline, tools, {
4845
4505
  rootDirname: rootDirname,
4846
4506
  isVerbose: isVerbose,
4847
4507
  maxParallelCount: maxParallelCount,
@@ -5023,13 +4683,7 @@ function executePipeline(options) {
5023
4683
  return [3 /*break*/, 4];
5024
4684
  case 3:
5025
4685
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
5026
- work_1 = executeTemplate({
5027
- currentTemplate: currentTemplate,
5028
- preparedPipeline: preparedPipeline,
5029
- parametersToPass: parametersToPass,
5030
- tools: tools,
5031
- llmTools: llmTools,
5032
- onProgress: function (progress) {
4686
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
5033
4687
  if (isReturned) {
5034
4688
  throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
5035
4689
  .split('\n')
@@ -5039,11 +4693,7 @@ function executePipeline(options) {
5039
4693
  if (onProgress) {
5040
4694
  onProgress(progress);
5041
4695
  }
5042
- },
5043
- settings: settings,
5044
- $executionReport: executionReport,
5045
- pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
5046
- })
4696
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
5047
4697
  .then(function (newParametersToPass) {
5048
4698
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
5049
4699
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -5147,8 +4797,7 @@ function executePipeline(options) {
5147
4797
  */
5148
4798
  function createPipelineExecutor(options) {
5149
4799
  var _this = this;
5150
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
5151
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
4800
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
5152
4801
  validatePipeline(pipeline);
5153
4802
  var pipelineIdentification = (function () {
5154
4803
  // Note: This is a 😐 implementation of [🚞]
@@ -5182,14 +4831,12 @@ function createPipelineExecutor(options) {
5182
4831
  tools: tools,
5183
4832
  onProgress: onProgress,
5184
4833
  pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
5185
- settings: {
5186
- maxExecutionAttempts: maxExecutionAttempts,
5187
- maxParallelCount: maxParallelCount,
5188
- csvSettings: csvSettings,
5189
- isVerbose: isVerbose,
5190
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5191
- rootDirname: rootDirname,
5192
- },
4834
+ maxExecutionAttempts: maxExecutionAttempts,
4835
+ maxParallelCount: maxParallelCount,
4836
+ csvSettings: csvSettings,
4837
+ isVerbose: isVerbose,
4838
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4839
+ rootDirname: rootDirname,
5193
4840
  })];
5194
4841
  });
5195
4842
  }); };
@@ -5200,678 +4847,531 @@ function createPipelineExecutor(options) {
5200
4847
  */
5201
4848
 
5202
4849
  /**
5203
- * Scraper for markdown files
4850
+ * Prepares the persona for the pipeline
4851
+ *
4852
+ * @see https://github.com/webgptorg/promptbook/discussions/22
4853
+ * @public exported from `@promptbook/core`
4854
+ */
4855
+ function preparePersona(personaDescription, tools, options) {
4856
+ return __awaiter(this, void 0, void 0, function () {
4857
+ var _a, isVerbose, collection, preparePersonaExecutor, _b, _llms, llmTools, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
4858
+ var _c;
4859
+ return __generator(this, function (_d) {
4860
+ switch (_d.label) {
4861
+ case 0:
4862
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4863
+ if (tools === undefined || tools.llm === undefined) {
4864
+ throw new MissingToolsError('LLM tools are required for preparing persona');
4865
+ }
4866
+ collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
4867
+ _b = createPipelineExecutor;
4868
+ _c = {};
4869
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
4870
+ case 1:
4871
+ preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
4872
+ _c.tools = tools,
4873
+ _c)]);
4874
+ _llms = arrayableToArray(tools.llm);
4875
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
4876
+ return [4 /*yield*/, llmTools.listModels()];
4877
+ case 2:
4878
+ availableModels = _d.sent();
4879
+ availableModelNames = availableModels
4880
+ .filter(function (_a) {
4881
+ var modelVariant = _a.modelVariant;
4882
+ return modelVariant === 'CHAT';
4883
+ })
4884
+ .map(function (_a) {
4885
+ var modelName = _a.modelName;
4886
+ return modelName;
4887
+ })
4888
+ .join(',');
4889
+ return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
4890
+ case 3:
4891
+ result = _d.sent();
4892
+ assertsExecutionSuccessful(result);
4893
+ outputParameters = result.outputParameters;
4894
+ modelRequirementsRaw = outputParameters.modelRequirements;
4895
+ modelRequirements = JSON.parse(modelRequirementsRaw);
4896
+ if (isVerbose) {
4897
+ console.info("PERSONA ".concat(personaDescription), modelRequirements);
4898
+ }
4899
+ modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
4900
+ return [2 /*return*/, {
4901
+ modelVariant: 'CHAT',
4902
+ modelName: modelName,
4903
+ systemMessage: systemMessage,
4904
+ temperature: temperature,
4905
+ }];
4906
+ }
4907
+ });
4908
+ });
4909
+ }
4910
+ /**
4911
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4912
+ * TODO: [🏢] !! Check validity of `modelName` in pipeline
4913
+ * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4914
+ * TODO: [🏢] !! Check validity of `temperature` in pipeline
4915
+ */
4916
+
4917
+ /**
4918
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
5204
4919
  *
5205
- * @see `documentationUrl` for more details
5206
4920
  * @public exported from `@promptbook/core`
5207
4921
  */
5208
- var markdownScraper = {
5209
- /**
5210
- * Mime types that this scraper can handle
5211
- */
5212
- mimeTypes: ['text/markdown', 'text/plain'],
5213
- /**
5214
- * Link to documentation
5215
- */
5216
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5217
- /**
5218
- * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
5219
- */
5220
- scrape: function (source, options) {
5221
- return __awaiter(this, void 0, void 0, function () {
5222
- var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, knowledgeContent, result, outputParameters, knowledgePiecesRaw, knowledgeTextPieces, knowledge;
5223
- var _f, _g, _h;
5224
- var _this = this;
5225
- return __generator(this, function (_j) {
5226
- switch (_j.label) {
5227
- case 0:
5228
- llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5229
- if (llmTools === undefined) {
5230
- throw new MissingToolsError('LLM tools are required for scraping external files');
5231
- // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
5232
- }
5233
- TODO_USE(maxParallelCount); // <- [🪂]
5234
- collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
5235
- _c = createPipelineExecutor;
5236
- _f = {};
5237
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
5238
- case 1:
5239
- prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
5240
- _f.tools = {
5241
- llm: llmTools,
5242
- },
5243
- _f)]);
5244
- _d = createPipelineExecutor;
5245
- _g = {};
5246
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
5247
- case 2:
5248
- prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
5249
- _g.tools = {
5250
- llm: llmTools,
5251
- },
5252
- _g)]);
5253
- _e = createPipelineExecutor;
5254
- _h = {};
5255
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
5256
- case 3:
5257
- prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
5258
- _h.tools = {
5259
- llm: llmTools,
5260
- },
5261
- _h)]);
5262
- return [4 /*yield*/, source.asText()];
5263
- case 4:
5264
- knowledgeContent = _j.sent();
5265
- return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
5266
- case 5:
5267
- result = _j.sent();
5268
- assertsExecutionSuccessful(result);
5269
- outputParameters = result.outputParameters;
5270
- knowledgePiecesRaw = outputParameters.knowledgePieces;
5271
- knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
5272
- // <- TODO: [main] !!!!! Smarter split and filter out empty pieces
5273
- if (isVerbose) {
5274
- console.info('knowledgeTextPieces:', knowledgeTextPieces);
5275
- }
5276
- return [4 /*yield*/, Promise.all(
5277
- // TODO: [🪂] !! Do not send all at once but in chunks
5278
- knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
5279
- var name, title, knowledgePieceContent, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
5280
- return __generator(this, function (_c) {
5281
- switch (_c.label) {
5282
- case 0:
5283
- name = "piece-".concat(i);
5284
- title = spaceTrim(knowledgeTextPiece.substring(0, 100));
5285
- knowledgePieceContent = spaceTrim(knowledgeTextPiece);
5286
- keywords = [];
5287
- index = [];
5288
- _c.label = 1;
5289
- case 1:
5290
- _c.trys.push([1, 7, , 8]);
5291
- return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
5292
- case 2:
5293
- titleResult = _c.sent();
5294
- _a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
5295
- title = spaceTrim(titleRaw) /* <- TODO: Maybe do in pipeline */;
5296
- name = titleToName(title);
5297
- return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
5298
- case 3:
5299
- keywordsResult = _c.sent();
5300
- _b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
5301
- keywords = (keywordsRaw || '')
5302
- .split(',')
5303
- .map(function (keyword) { return keyword.trim(); })
5304
- .filter(function (keyword) { return keyword !== ''; });
5305
- if (isVerbose) {
5306
- console.info("Keywords for \"".concat(title, "\":"), keywords);
5307
- }
5308
- if (!!llmTools.callEmbeddingModel) return [3 /*break*/, 4];
5309
- // TODO: [🟥] Detect browser / node and make it colorfull
5310
- console.error('No callEmbeddingModel function provided');
5311
- return [3 /*break*/, 6];
5312
- case 4: return [4 /*yield*/, llmTools.callEmbeddingModel({
5313
- title: "Embedding for ".concat(title) /* <- Note: No impact on embedding result itself, just for logging */,
5314
- parameters: {},
5315
- content: knowledgePieceContent,
5316
- modelRequirements: {
5317
- modelVariant: 'EMBEDDING',
5318
- },
5319
- })];
5320
- case 5:
5321
- embeddingResult = _c.sent();
5322
- index.push({
5323
- modelName: embeddingResult.modelName,
5324
- position: embeddingResult.content,
5325
- });
5326
- _c.label = 6;
5327
- case 6: return [3 /*break*/, 8];
5328
- case 7:
5329
- error_1 = _c.sent();
5330
- // Note: Here is expected error:
5331
- // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
5332
- if (!(error_1 instanceof PipelineExecutionError)) {
5333
- throw error_1;
5334
- }
5335
- // TODO: [🟥] Detect browser / node and make it colorfull
5336
- console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
5337
- return [3 /*break*/, 8];
5338
- case 8: return [2 /*return*/, {
5339
- name: name,
5340
- title: title,
5341
- content: knowledgePieceContent,
5342
- keywords: keywords,
5343
- index: index,
5344
- // <- TODO: [☀] sources,
5345
- }];
5346
- }
5347
- });
5348
- }); }))];
5349
- case 6:
5350
- knowledge = _j.sent();
5351
- return [2 /*return*/, knowledge];
5352
- }
5353
- });
5354
- });
5355
- },
5356
- } /* TODO: [🦷] as const */;
4922
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4923
+ __extends(KnowledgeScrapeError, _super);
4924
+ function KnowledgeScrapeError(message) {
4925
+ var _this = _super.call(this, message) || this;
4926
+ _this.name = 'KnowledgeScrapeError';
4927
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4928
+ return _this;
4929
+ }
4930
+ return KnowledgeScrapeError;
4931
+ }(Error));
4932
+
5357
4933
  /**
5358
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5359
- * TODO: [🪂] Do it in parallel 11:11
5360
- * TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
5361
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4934
+ * @@@
4935
+ *
4936
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4937
+ *
4938
+ * @private internal function of `$Register`
5362
4939
  */
4940
+ function $getGlobalScope() {
4941
+ return Function('return this')();
4942
+ }
5363
4943
 
5364
4944
  /**
5365
- * Scraper of .docx and .odt files
4945
+ * @@@
5366
4946
  *
5367
- * @see `documentationUrl` for more details
5368
- * @public exported from `@promptbook/core`
4947
+ * @param text @@@
4948
+ * @returns @@@
4949
+ * @example 'HELLO_WORLD'
4950
+ * @example 'I_LOVE_PROMPTBOOK'
4951
+ * @public exported from `@promptbook/utils`
5369
4952
  */
5370
- var documentScraper = {
5371
- /**
5372
- * Mime types that this scraper can handle
5373
- */
5374
- mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
5375
- /**
5376
- * Link to documentation
5377
- */
5378
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5379
- /**
5380
- * Convert the `.docx` or `.odt` to `.md` file and returns intermediate source
5381
- *
5382
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5383
- */
5384
- $convert: function (source, options) {
5385
- return __awaiter(this, void 0, void 0, function () {
5386
- var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, command_1;
5387
- return __generator(this, function (_e) {
5388
- switch (_e.label) {
5389
- case 0:
5390
- _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5391
- if (!$isRunningInNode()) {
5392
- throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
5393
- }
5394
- if (externalProgramsPaths.pandocPath === undefined) {
5395
- throw new MissingToolsError('Pandoc is required for scraping .docx files');
5396
- }
5397
- if (source.filename === null) {
5398
- // TODO: [🧠] Maybe save file as temporary
5399
- throw new KnowledgeScrapeError('When parsing .docx file, it must be real file in the file system');
5400
- }
5401
- extension = getFileExtension(source.filename);
5402
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5403
- rootDirname: rootDirname,
5404
- cacheDirname: cacheDirname,
5405
- isCacheCleaned: isCacheCleaned,
5406
- extension: 'md',
5407
- isVerbose: isVerbose,
5408
- })];
5409
- case 1:
5410
- cacheFilehandler = _e.sent();
5411
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5412
- case 2:
5413
- if (!!(_e.sent())) return [3 /*break*/, 5];
5414
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
5415
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
5416
- return [4 /*yield*/, $execCommand(command_1)];
5417
- case 3:
5418
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
5419
- _e.sent();
5420
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5421
- case 4:
5422
- // Note: [0]
5423
- if (!(_e.sent())) {
5424
- throw new UnexpectedError(spaceTrim(function (block) { return "\n File that was supposed to be created by Pandoc does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
5425
- }
5426
- _e.label = 5;
5427
- case 5: return [2 /*return*/, cacheFilehandler];
5428
- }
5429
- });
5430
- });
5431
- },
5432
- /**
5433
- * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
5434
- */
5435
- scrape: function (source, options) {
5436
- return __awaiter(this, void 0, void 0, function () {
5437
- var cacheFilehandler, markdownSource, knowledge;
5438
- return __generator(this, function (_a) {
5439
- switch (_a.label) {
5440
- case 0: return [4 /*yield*/, documentScraper.$convert(source, options)];
5441
- case 1:
5442
- cacheFilehandler = _a.sent();
5443
- markdownSource = {
5444
- source: source.source,
5445
- filename: cacheFilehandler.filename,
5446
- url: null,
5447
- mimeType: 'text/markdown',
5448
- asText: function () {
5449
- return __awaiter(this, void 0, void 0, function () {
5450
- return __generator(this, function (_a) {
5451
- switch (_a.label) {
5452
- case 0: return [4 /*yield*/, readFile(cacheFilehandler.filename, 'utf-8')];
5453
- case 1:
5454
- // Note: [0] In $convert we check that the file exists
5455
- return [2 /*return*/, _a.sent()];
5456
- }
5457
- });
5458
- });
5459
- },
5460
- asJson: function () {
5461
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5462
- },
5463
- asBlob: function () {
5464
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5465
- },
5466
- };
5467
- knowledge = markdownScraper.scrape(markdownSource, options);
5468
- return [4 /*yield*/, cacheFilehandler.destroy()];
5469
- case 2:
5470
- _a.sent();
5471
- return [2 /*return*/, knowledge];
5472
- }
5473
- });
5474
- });
5475
- },
5476
- } /* TODO: [🦷] as const */;
4953
+ function normalizeTo_SCREAMING_CASE(text) {
4954
+ var e_1, _a;
4955
+ var charType;
4956
+ var lastCharType = 'OTHER';
4957
+ var normalizedName = '';
4958
+ try {
4959
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4960
+ var char = text_1_1.value;
4961
+ var normalizedChar = void 0;
4962
+ if (/^[a-z]$/.test(char)) {
4963
+ charType = 'LOWERCASE';
4964
+ normalizedChar = char.toUpperCase();
4965
+ }
4966
+ else if (/^[A-Z]$/.test(char)) {
4967
+ charType = 'UPPERCASE';
4968
+ normalizedChar = char;
4969
+ }
4970
+ else if (/^[0-9]$/.test(char)) {
4971
+ charType = 'NUMBER';
4972
+ normalizedChar = char;
4973
+ }
4974
+ else {
4975
+ charType = 'OTHER';
4976
+ normalizedChar = '_';
4977
+ }
4978
+ if (charType !== lastCharType &&
4979
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4980
+ !(lastCharType === 'NUMBER') &&
4981
+ !(charType === 'NUMBER')) {
4982
+ normalizedName += '_';
4983
+ }
4984
+ normalizedName += normalizedChar;
4985
+ lastCharType = charType;
4986
+ }
4987
+ }
4988
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4989
+ finally {
4990
+ try {
4991
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
4992
+ }
4993
+ finally { if (e_1) throw e_1.error; }
4994
+ }
4995
+ normalizedName = normalizedName.replace(/_+/g, '_');
4996
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
4997
+ normalizedName = normalizedName.replace(/^_/, '');
4998
+ normalizedName = normalizedName.replace(/_$/, '');
4999
+ return normalizedName;
5000
+ }
5477
5001
  /**
5478
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5479
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5480
- * TODO: [🪂] Do it in parallel 11:11
5481
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5482
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5002
+ * TODO: Tests
5003
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
5004
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
5005
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
5006
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
5007
+ * TODO: [🌺] Use some intermediate util splitWords
5483
5008
  */
5484
5009
 
5485
5010
  /**
5486
- * Scraper for .docx files
5011
+ * @@@
5487
5012
  *
5488
- * @see `documentationUrl` for more details
5489
- * @public exported from `@promptbook/core`
5013
+ * @param text @@@
5014
+ * @returns @@@
5015
+ * @example 'hello_world'
5016
+ * @example 'i_love_promptbook'
5017
+ * @public exported from `@promptbook/utils`
5490
5018
  */
5491
- var legacyDocumentScraper = {
5492
- /**
5493
- * Mime types that this scraper can handle
5494
- */
5495
- mimeTypes: ['application/msword', 'text/rtf'],
5496
- /**
5497
- * Link to documentation
5498
- */
5499
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5500
- /**
5501
- * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
5502
- *
5503
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5504
- */
5505
- $convert: function (source, options) {
5506
- return __awaiter(this, void 0, void 0, function () {
5507
- var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
5508
- return __generator(this, function (_e) {
5509
- switch (_e.label) {
5510
- case 0:
5511
- _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5512
- if (!$isRunningInNode()) {
5513
- throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
5514
- }
5515
- if (externalProgramsPaths.libreOfficePath === undefined) {
5516
- throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
5517
- }
5518
- if (source.filename === null) {
5519
- // TODO: [🧠] Maybe save file as temporary
5520
- throw new KnowledgeScrapeError('When parsing .doc or .rtf file, it must be real file in the file system');
5521
- }
5522
- extension = getFileExtension(source.filename);
5523
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5524
- rootDirname: rootDirname,
5525
- cacheDirname: cacheDirname,
5526
- isCacheCleaned: isCacheCleaned,
5527
- extension: 'docx',
5528
- isVerbose: isVerbose,
5529
- })];
5530
- case 1:
5531
- cacheFilehandler = _e.sent();
5532
- if (isVerbose) {
5533
- console.info("documentScraper: Converting .".concat(extension, " -> .docx"));
5534
- }
5535
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5536
- case 2:
5537
- if (!!(_e.sent())) return [3 /*break*/, 8];
5538
- documentSourceOutdirPathForLibreOffice_1 = join(dirname(cacheFilehandler.filename), 'libreoffice')
5539
- .split('\\')
5540
- .join('/');
5541
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
5542
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
5543
- return [4 /*yield*/, $execCommand(command_1)];
5544
- case 3:
5545
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
5546
- _e.sent();
5547
- return [4 /*yield*/, readdir(documentSourceOutdirPathForLibreOffice_1)];
5548
- case 4:
5549
- files_1 = _e.sent();
5550
- if (files_1.length !== 1) {
5551
- throw new UnexpectedError(spaceTrim(function (block) { return "\n Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n "); }));
5552
- }
5553
- file = files_1[0];
5554
- return [4 /*yield*/, rename(join(documentSourceOutdirPathForLibreOffice_1, file), cacheFilehandler.filename)];
5555
- case 5:
5556
- _e.sent();
5557
- return [4 /*yield*/, rmdir(documentSourceOutdirPathForLibreOffice_1)];
5558
- case 6:
5559
- _e.sent();
5560
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5561
- case 7:
5562
- if (!(_e.sent())) {
5563
- throw new UnexpectedError(spaceTrim(function (block) { return "\n File that was supposed to be created by LibreOffice does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
5564
- }
5565
- _e.label = 8;
5566
- case 8: return [2 /*return*/, cacheFilehandler];
5567
- }
5568
- });
5569
- });
5570
- },
5571
- /**
5572
- * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
5573
- */
5574
- scrape: function (source, options) {
5575
- return __awaiter(this, void 0, void 0, function () {
5576
- var cacheFilehandler, markdownSource, knowledge;
5577
- return __generator(this, function (_a) {
5578
- switch (_a.label) {
5579
- case 0: return [4 /*yield*/, legacyDocumentScraper.$convert(source, options)];
5580
- case 1:
5581
- cacheFilehandler = _a.sent();
5582
- markdownSource = {
5583
- source: source.source,
5584
- filename: cacheFilehandler.filename,
5585
- url: null,
5586
- mimeType: 'text/markdown',
5587
- asText: function () {
5588
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asText`');
5589
- },
5590
- asJson: function () {
5591
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
5592
- },
5593
- asBlob: function () {
5594
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
5595
- },
5596
- };
5597
- knowledge = documentScraper.scrape(markdownSource, options);
5598
- return [4 /*yield*/, cacheFilehandler.destroy()];
5599
- case 2:
5600
- _a.sent();
5601
- return [2 /*return*/, knowledge];
5602
- }
5603
- });
5604
- });
5605
- },
5606
- } /* TODO: [🦷] as const */;
5019
+ function normalizeTo_snake_case(text) {
5020
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
5021
+ }
5022
+
5607
5023
  /**
5608
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5609
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5610
- * TODO: [🪂] Do it in parallel 11:11
5611
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5612
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5024
+ * Register is @@@
5025
+ *
5026
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
5027
+ *
5028
+ * @private internal utility, exported are only signleton instances of this class
5613
5029
  */
5030
+ var $Register = /** @class */ (function () {
5031
+ function $Register(registerName) {
5032
+ this.registerName = registerName;
5033
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
5034
+ var globalScope = $getGlobalScope();
5035
+ if (globalScope[storageName] === undefined) {
5036
+ globalScope[storageName] = [];
5037
+ }
5038
+ else if (!Array.isArray(globalScope[storageName])) {
5039
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
5040
+ }
5041
+ this.storage = globalScope[storageName];
5042
+ }
5043
+ $Register.prototype.list = function () {
5044
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
5045
+ return this.storage;
5046
+ };
5047
+ $Register.prototype.register = function (registered) {
5048
+ var packageName = registered.packageName, className = registered.className;
5049
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
5050
+ var existingRegistration = this.storage[existingRegistrationIndex];
5051
+ if (!existingRegistration) {
5052
+ this.storage.push(registered);
5053
+ }
5054
+ else {
5055
+ this.storage[existingRegistrationIndex] = registered;
5056
+ }
5057
+ return {
5058
+ registerName: this.registerName,
5059
+ packageName: packageName,
5060
+ className: className,
5061
+ get isDestroyed() {
5062
+ return false;
5063
+ },
5064
+ destroy: function () {
5065
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
5066
+ },
5067
+ };
5068
+ };
5069
+ return $Register;
5070
+ }());
5614
5071
 
5615
5072
  /**
5616
- * Scraper for .docx files
5073
+ * @@@
5617
5074
  *
5618
- * @see `documentationUrl` for more details
5075
+ * Note: `$` is used to indicate that this interacts with the global scope
5076
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5619
5077
  * @public exported from `@promptbook/core`
5620
5078
  */
5621
- var pdfScraper = {
5622
- /**
5623
- * Mime types that this scraper can handle
5624
- */
5625
- mimeTypes: ['application/pdf'],
5626
- /**
5627
- * Link to documentation
5628
- */
5629
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5630
- /**
5631
- * Converts the `.pdf` file to `.md` file and returns intermediate source
5632
- */
5633
- $convert: function (source, options) {
5634
- return __awaiter(this, void 0, void 0, function () {
5635
- return __generator(this, function (_a) {
5636
- TODO_USE(source);
5637
- TODO_USE(options);
5638
- throw new NotYetImplementedError('PDF conversion not yet implemented');
5639
- });
5640
- });
5641
- },
5642
- /**
5643
- * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
5644
- */
5645
- scrape: function (source, options) {
5646
- return __awaiter(this, void 0, void 0, function () {
5647
- return __generator(this, function (_a) {
5648
- TODO_USE(source);
5649
- TODO_USE(options);
5650
- /*
5651
- const {
5652
- externalProgramsPaths = {},
5653
- cacheDirname = SCRAPE_CACHE_DIRNAME,
5654
- isCacheCleaned = false,
5655
- isVerbose = IS_VERBOSE,
5656
- } = options;
5657
- */
5658
- throw new NotYetImplementedError('PDF scraping not yet implemented');
5659
- });
5660
- });
5661
- },
5662
- } /* TODO: [🦷] as const */;
5079
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
5663
5080
  /**
5664
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5665
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5666
- * TODO: [🪂] Do it in parallel 11:11
5667
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5668
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5081
+ * TODO: [®] DRY Register logic
5669
5082
  */
5670
5083
 
5671
5084
  /**
5672
- * A converter instance that uses showdown and highlight extensions
5085
+ * @@@
5673
5086
  *
5674
- * @type {Converter}
5675
- * @private for markdown and html knowledge scrapers
5087
+ * Note: `$` is used to indicate that this interacts with the global scope
5088
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5089
+ * @public exported from `@promptbook/core`
5676
5090
  */
5677
- var markdownConverter = new Converter({
5678
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
5679
- /*
5680
- > import showdownHighlight from 'showdown-highlight';
5681
- > extensions: [
5682
- > showdownHighlight({
5683
- > // Whether to add the classes to the <pre> tag, default is false
5684
- > pre: true,
5685
- > // Whether to use hljs' auto language detection, default is true
5686
- > auto_detection: true,
5687
- > }),
5688
- > ],
5689
- */
5690
- });
5091
+ var $scrapersRegister = new $Register('scraper_constructors');
5691
5092
  /**
5692
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
5693
- * TODO: !!!!!! Lazy-make converter
5093
+ * TODO: [®] DRY Register logic
5694
5094
  */
5695
5095
 
5696
5096
  /**
5697
- * Scraper for .docx files
5097
+ * Creates a message with all registered scrapers
5698
5098
  *
5699
- * @see `documentationUrl` for more details
5700
- * @public exported from `@promptbook/core`
5099
+ * Note: This function is used to create a (error) message when there is no scraper for particular mime type
5100
+ *
5101
+ * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
5701
5102
  */
5702
- var websiteScraper = {
5703
- /**
5704
- * Mime types that this scraper can handle
5705
- */
5706
- mimeTypes: ['text/html'],
5707
- /**
5708
- * Link to documentation
5709
- */
5710
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5103
+ function $registeredScrapersMessage() {
5104
+ var e_1, _a, e_2, _b;
5711
5105
  /**
5712
- * Convert the website to `.md` file and returns intermediate source
5713
- *
5714
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5106
+ * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
5715
5107
  */
5716
- $convert: function (source, options) {
5717
- return __awaiter(this, void 0, void 0, function () {
5718
- var
5719
- // TODO: [🧠] Maybe in node use headless browser not just JSDOM
5720
- // externalProgramsPaths = {},
5721
- rootDirname, _a, cacheDirname, _b, isCacheCleaned, _c, isVerbose, jsdom, _d, reader, article, html, i, cacheFilehandler, markdown;
5722
- return __generator(this, function (_e) {
5723
- switch (_e.label) {
5724
- case 0:
5725
- rootDirname = options.rootDirname, _a = options.cacheDirname, cacheDirname = _a === void 0 ? SCRAPE_CACHE_DIRNAME : _a, _b = options.isCacheCleaned, isCacheCleaned = _b === void 0 ? false : _b, _c = options.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
5726
- // TODO: !!!!!! Does this work in browser? Make it work.
5727
- if (source.url === null) {
5728
- throw new KnowledgeScrapeError('Website scraper requires URL');
5729
- }
5730
- _d = JSDOM.bind;
5731
- return [4 /*yield*/, source.asText()];
5732
- case 1:
5733
- jsdom = new (_d.apply(JSDOM, [void 0, _e.sent(), {
5734
- url: source.url,
5735
- }]))();
5736
- reader = new Readability(jsdom.window.document);
5737
- article = reader.parse();
5738
- console.log(article);
5739
- return [4 /*yield*/, forTime(10000)];
5740
- case 2:
5741
- _e.sent();
5742
- html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
5743
- // Note: Unwrap html such as it is convertable by `markdownConverter`
5744
- for (i = 0; i < 2; i++) {
5745
- html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
5746
- }
5747
- if (html.includes('<div')) {
5748
- html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
5749
- }
5750
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5751
- rootDirname: rootDirname,
5752
- cacheDirname: cacheDirname,
5753
- isCacheCleaned: isCacheCleaned,
5754
- extension: 'html',
5755
- isVerbose: isVerbose,
5756
- })];
5757
- case 3:
5758
- cacheFilehandler = _e.sent();
5759
- return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
5760
- case 4:
5761
- _e.sent();
5762
- markdown = markdownConverter.makeMarkdown(html, jsdom.window.document);
5763
- return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
5764
- }
5765
- });
5108
+ var all = [];
5109
+ var _loop_1 = function (packageName, className) {
5110
+ if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
5111
+ return "continue";
5112
+ }
5113
+ all.push({ packageName: packageName, className: className });
5114
+ };
5115
+ try {
5116
+ for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
5117
+ var _e = _d.value, packageName = _e.packageName, className = _e.className;
5118
+ _loop_1(packageName, className);
5119
+ }
5120
+ }
5121
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5122
+ finally {
5123
+ try {
5124
+ if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
5125
+ }
5126
+ finally { if (e_1) throw e_1.error; }
5127
+ }
5128
+ var _loop_2 = function (packageName, className) {
5129
+ if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
5130
+ return "continue";
5131
+ }
5132
+ all.push({ packageName: packageName, className: className });
5133
+ };
5134
+ try {
5135
+ for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
5136
+ var _h = _g.value, packageName = _h.packageName, className = _h.className;
5137
+ _loop_2(packageName, className);
5138
+ }
5139
+ }
5140
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
5141
+ finally {
5142
+ try {
5143
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
5144
+ }
5145
+ finally { if (e_2) throw e_2.error; }
5146
+ }
5147
+ var metadata = all.map(function (metadata) {
5148
+ var isMetadataAviailable = $scrapersMetadataRegister
5149
+ .list()
5150
+ .find(function (_a) {
5151
+ var packageName = _a.packageName, className = _a.className;
5152
+ return metadata.packageName === packageName && metadata.className === className;
5766
5153
  });
5767
- },
5768
- /**
5769
- * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
5770
- */
5771
- scrape: function (source, options) {
5772
- return __awaiter(this, void 0, void 0, function () {
5773
- var cacheFilehandler, markdownSource, knowledge;
5774
- return __generator(this, function (_a) {
5775
- switch (_a.label) {
5776
- case 0: return [4 /*yield*/, websiteScraper.$convert(source, options)];
5777
- case 1:
5778
- cacheFilehandler = _a.sent();
5779
- markdownSource = {
5780
- source: source.source,
5781
- filename: cacheFilehandler.filename,
5782
- url: null,
5783
- mimeType: 'text/markdown',
5784
- asText: function () {
5785
- return cacheFilehandler.markdown;
5786
- },
5787
- asJson: function () {
5788
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5789
- },
5790
- asBlob: function () {
5791
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5792
- },
5793
- };
5794
- knowledge = markdownScraper.scrape(markdownSource, options);
5795
- return [4 /*yield*/, cacheFilehandler.destroy()];
5796
- case 2:
5797
- _a.sent();
5798
- return [2 /*return*/, knowledge];
5799
- }
5800
- });
5154
+ var isInstalled = $scrapersRegister
5155
+ .list()
5156
+ .find(function (_a) {
5157
+ var packageName = _a.packageName, className = _a.className;
5158
+ return metadata.packageName === packageName && metadata.className === className;
5801
5159
  });
5802
- },
5803
- } /* TODO: [🦷] as const */;
5160
+ return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
5161
+ });
5162
+ if (metadata.length === 0) {
5163
+ return "No scrapers are available";
5164
+ }
5165
+ return spaceTrim(function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
5166
+ .map(function (_a, i) {
5167
+ var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
5168
+ var more;
5169
+ if (just(false)) {
5170
+ more = '';
5171
+ }
5172
+ else if (!isMetadataAviailable && !isInstalled) {
5173
+ // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
5174
+ more = "(not installed and no metadata, looks like a unexpected behavior)";
5175
+ }
5176
+ else if (isMetadataAviailable && !isInstalled) {
5177
+ // TODO: [�][�]
5178
+ more = "(not installed)";
5179
+ }
5180
+ else if (!isMetadataAviailable && isInstalled) {
5181
+ more = "(no metadata, looks like a unexpected behavior)";
5182
+ }
5183
+ else if (isMetadataAviailable && isInstalled) {
5184
+ more = "(installed)";
5185
+ }
5186
+ else {
5187
+ more = "(unknown state, looks like a unexpected behavior)";
5188
+ }
5189
+ return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(more);
5190
+ })
5191
+ .join('\n')), "\n "); });
5192
+ }
5804
5193
  /**
5805
- * TODO: !!!!!! Put into separate package
5806
- * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
5807
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5808
- * TODO: [🪂] Do it in parallel 11:11
5809
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5810
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5194
+ * TODO: [®] DRY Register logic
5811
5195
  */
5812
5196
 
5813
- // TODO: [🦖] !!!!!! Pass scrapers as dependency,
5814
5197
  /**
5815
5198
  * @@@
5816
5199
  *
5817
- * @private because this will be replaced by a system of one scraper per package [🦖]
5818
- * TODO: [🦖] System for scrapers NOT public exported from `@promptbook/core`
5200
+ * @param text @@@
5201
+ * @returns @@@
5202
+ * @example 'hello-world'
5203
+ * @example 'i-love-promptbook'
5204
+ * @public exported from `@promptbook/utils`
5819
5205
  */
5820
- var SCRAPERS = [
5821
- markdownScraper,
5822
- documentScraper,
5823
- legacyDocumentScraper,
5824
- pdfScraper,
5825
- websiteScraper,
5826
- // <- Note: [♓️] This is the order of the scrapers for knowledge, BUT consider some better (more explicit) way to do this
5827
- ];
5206
+ function normalizeToKebabCase(text) {
5207
+ var e_1, _a;
5208
+ text = removeDiacritics(text);
5209
+ var charType;
5210
+ var lastCharType = 'OTHER';
5211
+ var normalizedName = '';
5212
+ try {
5213
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5214
+ var char = text_1_1.value;
5215
+ var normalizedChar = void 0;
5216
+ if (/^[a-z]$/.test(char)) {
5217
+ charType = 'LOWERCASE';
5218
+ normalizedChar = char;
5219
+ }
5220
+ else if (/^[A-Z]$/.test(char)) {
5221
+ charType = 'UPPERCASE';
5222
+ normalizedChar = char.toLowerCase();
5223
+ }
5224
+ else if (/^[0-9]$/.test(char)) {
5225
+ charType = 'NUMBER';
5226
+ normalizedChar = char;
5227
+ }
5228
+ else {
5229
+ charType = 'OTHER';
5230
+ normalizedChar = '-';
5231
+ }
5232
+ if (charType !== lastCharType &&
5233
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
5234
+ !(lastCharType === 'NUMBER') &&
5235
+ !(charType === 'NUMBER')) {
5236
+ normalizedName += '-';
5237
+ }
5238
+ normalizedName += normalizedChar;
5239
+ lastCharType = charType;
5240
+ }
5241
+ }
5242
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5243
+ finally {
5244
+ try {
5245
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5246
+ }
5247
+ finally { if (e_1) throw e_1.error; }
5248
+ }
5249
+ normalizedName = normalizedName.split(/-+/g).join('-');
5250
+ normalizedName = normalizedName.split(/-?\/-?/g).join('/');
5251
+ normalizedName = normalizedName.replace(/^-/, '');
5252
+ normalizedName = normalizedName.replace(/-$/, '');
5253
+ return normalizedName;
5254
+ }
5828
5255
 
5829
5256
  /**
5830
5257
  * Creates unique name for the source
5831
5258
  *
5832
5259
  * @private within the repository
5833
5260
  */
5834
- function sourceContentToName(sourceContent) {
5835
- // TODO: !!!!!! Better name for source than gibberish hash
5836
- var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
5837
- // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5838
- .toString( /* hex */)
5839
- .substring(0, 20);
5840
- // <- TODO: [🥬] Make some system for hashes and ids of promptbook
5841
- var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
5842
- var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
5843
- var name = pieces.join('-').split('--').join('-');
5844
- // <- TODO: Use MAX_FILENAME_LENGTH
5845
- return name;
5261
+ function sourceContentToName(sourceContent) {
5262
+ var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
5263
+ // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5264
+ .toString( /* hex */)
5265
+ .substring(0, 20);
5266
+ // <- TODO: [🥬] Make some system for hashes and ids of promptbook
5267
+ var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
5268
+ var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
5269
+ var name = pieces.join('-').split('--').join('-');
5270
+ // <- TODO: Use MAX_FILENAME_LENGTH
5271
+ return name;
5272
+ }
5273
+ /**
5274
+ * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
5275
+ */
5276
+
5277
+ /**
5278
+ * Convert file extension to mime type
5279
+ *
5280
+ * @private within the repository
5281
+ */
5282
+ function extensionToMimeType(value) {
5283
+ return lookup(value) || 'application/octet-stream';
5284
+ }
5285
+
5286
+ /**
5287
+ * Get the file extension from a file name
5288
+ *
5289
+ * @private within the repository
5290
+ */
5291
+ function getFileExtension(value) {
5292
+ var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
5293
+ return match ? match[1].toLowerCase() : null;
5294
+ }
5295
+
5296
+ /**
5297
+ * Checks if the file exists
5298
+ *
5299
+ * @private within the repository
5300
+ */
5301
+ function isFileExisting(filename, fs) {
5302
+ return __awaiter(this, void 0, void 0, function () {
5303
+ var isReadAccessAllowed, isFile;
5304
+ return __generator(this, function (_a) {
5305
+ switch (_a.label) {
5306
+ case 0: return [4 /*yield*/, fs
5307
+ .access(filename, fs.constants.R_OK)
5308
+ .then(function () { return true; })
5309
+ .catch(function () { return false; })];
5310
+ case 1:
5311
+ isReadAccessAllowed = _a.sent();
5312
+ if (!isReadAccessAllowed) {
5313
+ return [2 /*return*/, false];
5314
+ }
5315
+ return [4 /*yield*/, fs
5316
+ .stat(filename)
5317
+ .then(function (fileStat) { return fileStat.isFile(); })
5318
+ .catch(function () { return false; })];
5319
+ case 2:
5320
+ isFile = _a.sent();
5321
+ return [2 /*return*/, isFile];
5322
+ }
5323
+ });
5324
+ });
5846
5325
  }
5847
5326
  /**
5848
- * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
5327
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
5328
+ * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
5329
+ * TODO: [🖇] What about symlinks?
5849
5330
  */
5850
5331
 
5851
5332
  /**
5852
- * Convert file extension to mime type
5333
+ * Tests if given string is valid URL.
5853
5334
  *
5854
- * @private within the repository
5335
+ * Note: This does not check if the file exists only if the path is valid
5336
+ * @public exported from `@promptbook/utils`
5855
5337
  */
5856
- function extensionToMimeType(value) {
5857
- return lookup(value) || 'application/octet-stream';
5338
+ function isValidFilePath(filename) {
5339
+ if (typeof filename !== 'string') {
5340
+ return false;
5341
+ }
5342
+ var filenameSlashes = filename.split('\\').join('/');
5343
+ // Absolute Unix path: /hello.txt
5344
+ if (/^(\/)/i.test(filenameSlashes)) {
5345
+ return true;
5346
+ }
5347
+ // Absolute Windows path: /hello.txt
5348
+ if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
5349
+ return true;
5350
+ }
5351
+ // Relative path: ./hello.txt
5352
+ if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
5353
+ return true;
5354
+ }
5355
+ return false;
5858
5356
  }
5859
5357
 
5860
5358
  /**
5861
5359
  * @@@
5862
5360
  *
5863
- * @private for scraper utilities
5361
+ * @public exported from `@promptbook/core`
5864
5362
  */
5865
- function makeKnowledgeSourceHandler(knowledgeSource, options) {
5363
+ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5866
5364
  var _a;
5867
5365
  return __awaiter(this, void 0, void 0, function () {
5868
- var sourceContent, name, _b, _c, rootDirname, _d, isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
5366
+ var sourceContent, name, _b, _c, rootDirname, _d,
5367
+ // <- TODO: process.cwd() if running in Node.js
5368
+ isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
5869
5369
  return __generator(this, function (_e) {
5870
5370
  switch (_e.label) {
5871
5371
  case 0:
5872
5372
  sourceContent = knowledgeSource.sourceContent;
5873
5373
  name = knowledgeSource.name;
5874
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5374
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
5875
5375
  TODO_USE(isVerbose);
5876
5376
  if (!name) {
5877
5377
  name = sourceContentToName(sourceContent);
@@ -5887,19 +5387,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
5887
5387
  filename: null,
5888
5388
  url: url,
5889
5389
  mimeType: mimeType,
5890
- asBlob: function () {
5891
- return __awaiter(this, void 0, void 0, function () {
5892
- var content;
5893
- return __generator(this, function (_a) {
5894
- switch (_a.label) {
5895
- case 0: return [4 /*yield*/, response_1.blob()];
5896
- case 1:
5897
- content = _a.sent();
5898
- return [2 /*return*/, content];
5899
- }
5900
- });
5901
- });
5902
- },
5390
+ /*
5391
+ TODO: [🥽]
5392
+ > async asBlob() {
5393
+ > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
5394
+ > const content = await response.blob();
5395
+ > return content;
5396
+ > },
5397
+ */
5903
5398
  asJson: function () {
5904
5399
  return __awaiter(this, void 0, void 0, function () {
5905
5400
  var content;
@@ -5929,8 +5424,9 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
5929
5424
  }];
5930
5425
  case 2:
5931
5426
  if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
5932
- if (!$isRunningInNode()) {
5933
- throw new EnvironmentMismatchError('Importing knowledge source file works only in Node.js environment');
5427
+ if (tools.fs === undefined) {
5428
+ throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
5429
+ // <- TODO: [🧠] What is the best error type here`
5934
5430
  }
5935
5431
  if (rootDirname === null) {
5936
5432
  throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
@@ -5938,35 +5434,31 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
5938
5434
  }
5939
5435
  filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
5940
5436
  fileExtension = getFileExtension(filename_1);
5941
- mimeType_1 = extensionToMimeType(fileExtension || '');
5942
- return [4 /*yield*/, $isFileExisting(filename_1)];
5437
+ mimeType = extensionToMimeType(fileExtension || '');
5438
+ return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
5943
5439
  case 3:
5944
5440
  if (!(_e.sent())) {
5945
5441
  throw new NotFoundError(spaceTrim(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
5946
5442
  }
5947
- // TODO: !!!!!! Test security file - file is scoped to the project (maybe do this in `filesystemTools`)
5443
+ // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
5948
5444
  return [2 /*return*/, {
5949
5445
  source: name,
5950
5446
  filename: filename_1,
5951
5447
  url: null,
5952
- mimeType: mimeType_1,
5953
- asBlob: function () {
5954
- return __awaiter(this, void 0, void 0, function () {
5955
- var content;
5956
- return __generator(this, function (_a) {
5957
- switch (_a.label) {
5958
- case 0: return [4 /*yield*/, readFile(filename_1)];
5959
- case 1:
5960
- content = _a.sent();
5961
- // <- Note: Its OK to use sync in tooling for tests
5962
- return [2 /*return*/, new Blob([
5963
- content,
5964
- // <- TODO: !!!!!! Maybe not working
5965
- ], { type: mimeType_1 })];
5966
- }
5967
- });
5968
- });
5969
- },
5448
+ mimeType: mimeType,
5449
+ /*
5450
+ TODO: [🥽]
5451
+ > async asBlob() {
5452
+ > const content = await tools.fs!.readFile(filename);
5453
+ > return new Blob(
5454
+ > [
5455
+ > content,
5456
+ > // <- TODO: [🥽] This is NOT tested, test it
5457
+ > ],
5458
+ > { type: mimeType },
5459
+ > );
5460
+ > },
5461
+ */
5970
5462
  asJson: function () {
5971
5463
  return __awaiter(this, void 0, void 0, function () {
5972
5464
  var _a, _b;
@@ -5974,7 +5466,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
5974
5466
  switch (_c.label) {
5975
5467
  case 0:
5976
5468
  _b = (_a = JSON).parse;
5977
- return [4 /*yield*/, readFile(filename_1, 'utf-8')];
5469
+ return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
5978
5470
  case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
5979
5471
  }
5980
5472
  });
@@ -5984,7 +5476,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
5984
5476
  return __awaiter(this, void 0, void 0, function () {
5985
5477
  return __generator(this, function (_a) {
5986
5478
  switch (_a.label) {
5987
- case 0: return [4 /*yield*/, readFile(filename_1, 'utf-8')];
5479
+ case 0: return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
5988
5480
  case 1: return [2 /*return*/, _a.sent()];
5989
5481
  }
5990
5482
  });
@@ -6002,9 +5494,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
6002
5494
  asJson: function () {
6003
5495
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
6004
5496
  },
6005
- asBlob: function () {
6006
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
6007
- },
5497
+ /*
5498
+ TODO: [🥽]
5499
+ > asBlob() {
5500
+ > throw new UnexpectedError(
5501
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
5502
+ > );
5503
+ > },
5504
+ */
6008
5505
  }];
6009
5506
  }
6010
5507
  });
@@ -6017,262 +5514,106 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
6017
5514
  * @see https://github.com/webgptorg/promptbook/discussions/41
6018
5515
  * @public exported from `@promptbook/core`
6019
5516
  */
6020
- function prepareKnowledgePieces(knowledgeSources, options) {
5517
+ function prepareKnowledgePieces(knowledgeSources, tools, options) {
6021
5518
  return __awaiter(this, void 0, void 0, function () {
6022
5519
  var _a, maxParallelCount, rootDirname, _b, isVerbose, knowledgePreparedUnflatten, knowledgePrepared;
6023
5520
  var _this = this;
6024
5521
  return __generator(this, function (_c) {
6025
5522
  switch (_c.label) {
6026
5523
  case 0:
6027
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5524
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
6028
5525
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
6029
5526
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
6030
- var partialPieces, sourceHandler, SCRAPERS_1, SCRAPERS_1_1, scraper, partialPiecesUnchecked, e_1_1, pieces;
6031
- var e_1, _a;
6032
- return __generator(this, function (_b) {
6033
- switch (_b.label) {
5527
+ var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
5528
+ var e_1, _c;
5529
+ return __generator(this, function (_d) {
5530
+ switch (_d.label) {
6034
5531
  case 0:
6035
5532
  partialPieces = null;
6036
- return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
5533
+ return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
6037
5534
  case 1:
6038
- sourceHandler = _b.sent();
6039
- _b.label = 2;
5535
+ sourceHandler = _d.sent();
5536
+ _d.label = 2;
6040
5537
  case 2:
6041
- _b.trys.push([2, 7, 8, 9]);
6042
- SCRAPERS_1 = __values(SCRAPERS), SCRAPERS_1_1 = SCRAPERS_1.next();
6043
- _b.label = 3;
5538
+ _d.trys.push([2, 7, 8, 9]);
5539
+ _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
5540
+ _d.label = 3;
6044
5541
  case 3:
6045
- if (!!SCRAPERS_1_1.done) return [3 /*break*/, 6];
6046
- scraper = SCRAPERS_1_1.value;
6047
- if (!scraper.mimeTypes.includes(sourceHandler.mimeType)
5542
+ if (!!_b.done) return [3 /*break*/, 6];
5543
+ scraper = _b.value;
5544
+ if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
6048
5545
  // <- TODO: [🦔] Implement mime-type wildcards
6049
5546
  ) {
6050
5547
  return [3 /*break*/, 5];
6051
5548
  }
6052
- return [4 /*yield*/, scraper.scrape(sourceHandler, options)];
5549
+ return [4 /*yield*/, scraper.scrape(sourceHandler)];
6053
5550
  case 4:
6054
- partialPiecesUnchecked = _b.sent();
5551
+ partialPiecesUnchecked = _d.sent();
6055
5552
  if (partialPiecesUnchecked !== null) {
6056
- partialPieces = partialPiecesUnchecked;
5553
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
5554
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
6057
5555
  return [3 /*break*/, 6];
6058
5556
  }
6059
- _b.label = 5;
5557
+ _d.label = 5;
6060
5558
  case 5:
6061
- SCRAPERS_1_1 = SCRAPERS_1.next();
5559
+ _b = _a.next();
6062
5560
  return [3 /*break*/, 3];
6063
5561
  case 6: return [3 /*break*/, 9];
6064
5562
  case 7:
6065
- e_1_1 = _b.sent();
5563
+ e_1_1 = _d.sent();
6066
5564
  e_1 = { error: e_1_1 };
6067
5565
  return [3 /*break*/, 9];
6068
5566
  case 8:
6069
5567
  try {
6070
- if (SCRAPERS_1_1 && !SCRAPERS_1_1.done && (_a = SCRAPERS_1.return)) _a.call(SCRAPERS_1);
5568
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
6071
5569
  }
6072
5570
  finally { if (e_1) throw e_1.error; }
6073
5571
  return [7 /*endfinally*/];
6074
5572
  case 9:
6075
5573
  if (partialPieces === null) {
6076
- throw new KnowledgeScrapeError(spaceTrim(function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n Available scrapers:\n ").concat(block(SCRAPERS.flatMap(function (scraper) { return scraper.mimeTypes; })
6077
- .map(function (mimeType) { return "- ".concat(mimeType); })
6078
- .join('\n')), "\n\n\n "); }));
5574
+ throw new KnowledgeScrapeError(spaceTrim(function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
6079
5575
  }
6080
5576
  pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
6081
5577
  {
6082
5578
  name: knowledgeSource.name,
6083
5579
  // line, column <- TODO: [☀]
6084
5580
  // <- TODO: [❎]
6085
- },
6086
- ] })); });
6087
- knowledgePreparedUnflatten[index] = pieces;
6088
- return [2 /*return*/];
6089
- }
6090
- });
6091
- }); })];
6092
- case 1:
6093
- _c.sent();
6094
- knowledgePrepared = knowledgePreparedUnflatten.flat();
6095
- return [2 /*return*/, knowledgePrepared];
6096
- }
6097
- });
6098
- });
6099
- }
6100
- /*
6101
- TODO: [🧊] This is how it can look in future
6102
- > type PrepareKnowledgeKnowledge = {
6103
- > /**
6104
- > * Unprepared knowledge
6105
- > * /
6106
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
6107
- > };
6108
- >
6109
- > export async function prepareKnowledgePieces(
6110
- > knowledge: PrepareKnowledgeKnowledge,
6111
- > options: PrepareAndScrapeOptions,
6112
- > ):
6113
- */
6114
- /**
6115
- * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
6116
- * Put `knowledgePieces` into `PrepareKnowledgeOptions`
6117
- * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
6118
- * TODO: [🧠][❎] Do here propper M:N mapping
6119
- * [x] One source can make multiple pieces
6120
- * [ ] One piece can have multiple sources
6121
- */
6122
-
6123
- /**
6124
- * Intercepts LLM tools and counts total usage of the tools
6125
- *
6126
- * @param llmTools LLM tools to be intercepted with usage counting
6127
- * @returns LLM tools with same functionality with added total cost counting
6128
- * @public exported from `@promptbook/core`
6129
- */
6130
- function countTotalUsage(llmTools) {
6131
- var _this = this;
6132
- var totalUsage = ZERO_USAGE;
6133
- var proxyTools = {
6134
- get title() {
6135
- // TODO: [🧠] Maybe put here some suffix
6136
- return llmTools.title;
6137
- },
6138
- get description() {
6139
- // TODO: [🧠] Maybe put here some suffix
6140
- return llmTools.description;
6141
- },
6142
- checkConfiguration: function () {
6143
- return __awaiter(this, void 0, void 0, function () {
6144
- return __generator(this, function (_a) {
6145
- return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
6146
- });
6147
- });
6148
- },
6149
- listModels: function () {
6150
- return /* not await */ llmTools.listModels();
6151
- },
6152
- getTotalUsage: function () {
6153
- // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
6154
- return totalUsage;
6155
- },
6156
- };
6157
- if (llmTools.callChatModel !== undefined) {
6158
- proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6159
- var promptResult;
6160
- return __generator(this, function (_a) {
6161
- switch (_a.label) {
6162
- case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
6163
- case 1:
6164
- promptResult = _a.sent();
6165
- totalUsage = addUsage(totalUsage, promptResult.usage);
6166
- return [2 /*return*/, promptResult];
6167
- }
6168
- });
6169
- }); };
6170
- }
6171
- if (llmTools.callCompletionModel !== undefined) {
6172
- proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6173
- var promptResult;
6174
- return __generator(this, function (_a) {
6175
- switch (_a.label) {
6176
- case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
6177
- case 1:
6178
- promptResult = _a.sent();
6179
- totalUsage = addUsage(totalUsage, promptResult.usage);
6180
- return [2 /*return*/, promptResult];
6181
- }
6182
- });
6183
- }); };
6184
- }
6185
- if (llmTools.callEmbeddingModel !== undefined) {
6186
- proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6187
- var promptResult;
6188
- return __generator(this, function (_a) {
6189
- switch (_a.label) {
6190
- case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
6191
- case 1:
6192
- promptResult = _a.sent();
6193
- totalUsage = addUsage(totalUsage, promptResult.usage);
6194
- return [2 /*return*/, promptResult];
6195
- }
6196
- });
6197
- }); };
6198
- }
6199
- // <- Note: [🤖]
6200
- return proxyTools;
6201
- }
6202
- /**
6203
- * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
6204
- * TODO: [🧠] Is there some meaningfull way how to test this util
6205
- * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
6206
- * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
6207
- * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
6208
- */
6209
-
6210
- /**
6211
- * Prepares the persona for the pipeline
6212
- *
6213
- * @see https://github.com/webgptorg/promptbook/discussions/22
6214
- * @public exported from `@promptbook/core`
6215
- */
6216
- function preparePersona(personaDescription, options) {
6217
- return __awaiter(this, void 0, void 0, function () {
6218
- var llmTools, _a, isVerbose, collection, preparePersonaExecutor, _b, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
6219
- var _c;
6220
- return __generator(this, function (_d) {
6221
- switch (_d.label) {
6222
- case 0:
6223
- llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
6224
- if (llmTools === undefined) {
6225
- throw new MissingToolsError('LLM tools are required for preparing persona');
6226
- }
6227
- collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
6228
- _b = createPipelineExecutor;
6229
- _c = {};
6230
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
6231
- case 1:
6232
- preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
6233
- _c.tools = {
6234
- llm: llmTools,
6235
- },
6236
- _c)]);
6237
- return [4 /*yield*/, llmTools.listModels()];
6238
- case 2:
6239
- availableModels = _d.sent();
6240
- availableModelNames = availableModels
6241
- .filter(function (_a) {
6242
- var modelVariant = _a.modelVariant;
6243
- return modelVariant === 'CHAT';
6244
- })
6245
- .map(function (_a) {
6246
- var modelName = _a.modelName;
6247
- return modelName;
6248
- })
6249
- .join(',');
6250
- return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
6251
- case 3:
6252
- result = _d.sent();
6253
- assertsExecutionSuccessful(result);
6254
- outputParameters = result.outputParameters;
6255
- modelRequirementsRaw = outputParameters.modelRequirements;
6256
- modelRequirements = JSON.parse(modelRequirementsRaw);
6257
- if (isVerbose) {
6258
- console.info("PERSONA ".concat(personaDescription), modelRequirements);
6259
- }
6260
- modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
6261
- return [2 /*return*/, {
6262
- modelVariant: 'CHAT',
6263
- modelName: modelName,
6264
- systemMessage: systemMessage,
6265
- temperature: temperature,
6266
- }];
5581
+ },
5582
+ ] })); });
5583
+ knowledgePreparedUnflatten[index] = pieces;
5584
+ return [2 /*return*/];
5585
+ }
5586
+ });
5587
+ }); })];
5588
+ case 1:
5589
+ _c.sent();
5590
+ knowledgePrepared = knowledgePreparedUnflatten.flat();
5591
+ return [2 /*return*/, knowledgePrepared];
6267
5592
  }
6268
5593
  });
6269
5594
  });
6270
5595
  }
5596
+ /*
5597
+ TODO: [🧊] This is how it can look in future
5598
+ > type PrepareKnowledgeKnowledge = {
5599
+ > /**
5600
+ > * Unprepared knowledge
5601
+ > * /
5602
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
5603
+ > };
5604
+ >
5605
+ > export async function prepareKnowledgePieces(
5606
+ > knowledge: PrepareKnowledgeKnowledge,
5607
+ > options: PrepareAndScrapeOptions,
5608
+ > ):
5609
+ */
6271
5610
  /**
6272
- * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
6273
- * TODO: [🏢] !! Check validity of `modelName` in pipeline
6274
- * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
6275
- * TODO: [🏢] !! Check validity of `temperature` in pipeline
5611
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
5612
+ * Put `knowledgePieces` into `PrepareKnowledgeOptions`
5613
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
5614
+ * TODO: [🧠][❎] Do here propper M:N mapping
5615
+ * [x] One source can make multiple pieces
5616
+ * [ ] One piece can have multiple sources
6276
5617
  */
6277
5618
 
6278
5619
  /**
@@ -6309,20 +5650,18 @@ function clonePipeline(pipeline) {
6309
5650
  *
6310
5651
  * @public exported from `@promptbook/core`
6311
5652
  */
6312
- function prepareTemplates(pipeline, options) {
5653
+ function prepareTemplates(pipeline, tools, options) {
6313
5654
  return __awaiter(this, void 0, void 0, function () {
6314
5655
  var _a, maxParallelCount, templates, parameters, knowledgePiecesCount, templatesPrepared;
6315
5656
  var _this = this;
6316
5657
  return __generator(this, function (_b) {
6317
5658
  switch (_b.label) {
6318
5659
  case 0:
6319
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
5660
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
6320
5661
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
6321
5662
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
6322
5663
  TODO_USE(parameters);
6323
- templatesPrepared = new Array(
6324
- // <- TODO: [🧱] Implement in a functional (not new Class) way
6325
- templates.length);
5664
+ templatesPrepared = new Array(templates.length);
6326
5665
  return [4 /*yield*/, forEachAsync(templates, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (template, index) { return __awaiter(_this, void 0, void 0, function () {
6327
5666
  var dependentParameterNames, preparedContent, preparedTemplate;
6328
5667
  return __generator(this, function (_a) {
@@ -6365,14 +5704,14 @@ function prepareTemplates(pipeline, options) {
6365
5704
  * Note: When the pipeline is already prepared, it returns the same pipeline
6366
5705
  * @public exported from `@promptbook/core`
6367
5706
  */
6368
- function preparePipeline(pipeline, options) {
5707
+ function preparePipeline(pipeline, tools, options) {
6369
5708
  return __awaiter(this, void 0, void 0, function () {
6370
- var llmTools, rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
5709
+ var rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
6371
5710
  /*
6372
5711
  <- TODO: [🧠][🪑] `promptbookVersion` */
6373
5712
  knowledgeSources /*
6374
5713
  <- TODO: [🧊] `knowledgePieces` */, personas /*
6375
- <- TODO: [🧊] `preparations` */, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
5714
+ <- TODO: [🧊] `preparations` */, _llms, llmTools, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
6376
5715
  var _this = this;
6377
5716
  return __generator(this, function (_c) {
6378
5717
  switch (_c.label) {
@@ -6380,11 +5719,13 @@ function preparePipeline(pipeline, options) {
6380
5719
  if (isPipelinePrepared(pipeline)) {
6381
5720
  return [2 /*return*/, pipeline];
6382
5721
  }
6383
- llmTools = options.llmTools, rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5722
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
6384
5723
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
6385
- if (llmTools === undefined) {
5724
+ if (tools === undefined || tools.llm === undefined) {
6386
5725
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
6387
5726
  }
5727
+ _llms = arrayableToArray(tools.llm);
5728
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
6388
5729
  llmToolsWithUsage = countTotalUsage(llmTools);
6389
5730
  currentPreparation = {
6390
5731
  id: 1,
@@ -6397,15 +5738,12 @@ function preparePipeline(pipeline, options) {
6397
5738
  // <- TODO: [🧊]
6398
5739
  currentPreparation,
6399
5740
  ];
6400
- preparedPersonas = new Array(
6401
- // <- TODO: [🧱] Implement in a functional (not new Class) way
6402
- personas.length);
5741
+ preparedPersonas = new Array(personas.length);
6403
5742
  return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () {
6404
5743
  var modelRequirements, preparedPersona;
6405
5744
  return __generator(this, function (_a) {
6406
5745
  switch (_a.label) {
6407
- case 0: return [4 /*yield*/, preparePersona(persona.description, {
6408
- llmTools: llmToolsWithUsage,
5746
+ case 0: return [4 /*yield*/, preparePersona(persona.description, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
6409
5747
  rootDirname: rootDirname,
6410
5748
  maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
6411
5749
  isVerbose: isVerbose,
@@ -6421,7 +5759,7 @@ function preparePipeline(pipeline, options) {
6421
5759
  case 1:
6422
5760
  _c.sent();
6423
5761
  knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
6424
- return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, options), { llmTools: llmToolsWithUsage, rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
5762
+ return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), __assign(__assign({}, options), { rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
6425
5763
  case 2:
6426
5764
  partialknowledgePiecesPrepared = _c.sent();
6427
5765
  knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
@@ -6429,8 +5767,7 @@ function preparePipeline(pipeline, options) {
6429
5767
  parameters: parameters,
6430
5768
  templates: templates,
6431
5769
  knowledgePiecesCount: knowledgePiecesPrepared.length,
6432
- }, {
6433
- llmTools: llmToolsWithUsage,
5770
+ }, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
6434
5771
  rootDirname: rootDirname,
6435
5772
  maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
6436
5773
  isVerbose: isVerbose,
@@ -6440,7 +5777,9 @@ function preparePipeline(pipeline, options) {
6440
5777
  // ----- /Templates preparation -----
6441
5778
  // Note: Count total usage
6442
5779
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
6443
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
5780
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
5781
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
5782
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
6444
5783
  }
6445
5784
  });
6446
5785
  });
@@ -6487,6 +5826,7 @@ var knowledgeCommandParser = {
6487
5826
  'KNOWLEDGE ./hejny-cv.md',
6488
5827
  'KNOWLEDGE ./hejny-cv.pdf',
6489
5828
  'KNOWLEDGE ./hejny-cv.docx',
5829
+ // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
6490
5830
  ],
6491
5831
  /**
6492
5832
  * Parses the KNOWLEDGE command
@@ -6959,72 +6299,6 @@ var expectCommandParser = {
6959
6299
  },
6960
6300
  };
6961
6301
 
6962
- /**
6963
- * @@@
6964
- *
6965
- * @param text @@@
6966
- * @returns @@@
6967
- * @example 'HELLO_WORLD'
6968
- * @example 'I_LOVE_PROMPTBOOK'
6969
- * @public exported from `@promptbook/utils`
6970
- */
6971
- function normalizeTo_SCREAMING_CASE(text) {
6972
- var e_1, _a;
6973
- var charType;
6974
- var lastCharType = 'OTHER';
6975
- var normalizedName = '';
6976
- try {
6977
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
6978
- var char = text_1_1.value;
6979
- var normalizedChar = void 0;
6980
- if (/^[a-z]$/.test(char)) {
6981
- charType = 'LOWERCASE';
6982
- normalizedChar = char.toUpperCase();
6983
- }
6984
- else if (/^[A-Z]$/.test(char)) {
6985
- charType = 'UPPERCASE';
6986
- normalizedChar = char;
6987
- }
6988
- else if (/^[0-9]$/.test(char)) {
6989
- charType = 'NUMBER';
6990
- normalizedChar = char;
6991
- }
6992
- else {
6993
- charType = 'OTHER';
6994
- normalizedChar = '_';
6995
- }
6996
- if (charType !== lastCharType &&
6997
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
6998
- !(lastCharType === 'NUMBER') &&
6999
- !(charType === 'NUMBER')) {
7000
- normalizedName += '_';
7001
- }
7002
- normalizedName += normalizedChar;
7003
- lastCharType = charType;
7004
- }
7005
- }
7006
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
7007
- finally {
7008
- try {
7009
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
7010
- }
7011
- finally { if (e_1) throw e_1.error; }
7012
- }
7013
- normalizedName = normalizedName.replace(/_+/g, '_');
7014
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
7015
- normalizedName = normalizedName.replace(/^_/, '');
7016
- normalizedName = normalizedName.replace(/_$/, '');
7017
- return normalizedName;
7018
- }
7019
- /**
7020
- * TODO: Tests
7021
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
7022
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
7023
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
7024
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
7025
- * TODO: [🌺] Use some intermediate util splitWords
7026
- */
7027
-
7028
6302
  /**
7029
6303
  * @@@
7030
6304
  *
@@ -7089,6 +6363,22 @@ function normalizeTo_camelCase(text, _isFirstLetterCapital) {
7089
6363
  * TODO: [🌺] Use some intermediate util splitWords
7090
6364
  */
7091
6365
 
6366
+ /**
6367
+ * Removes emojis from a string and fix whitespaces
6368
+ *
6369
+ * @param text with emojis
6370
+ * @returns text without emojis
6371
+ * @public exported from `@promptbook/utils`
6372
+ */
6373
+ function removeEmojis(text) {
6374
+ // Replace emojis (and also ZWJ sequence) with hyphens
6375
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
6376
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
6377
+ text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
6378
+ text = text.replace(/\p{Extended_Pictographic}/gu, '');
6379
+ return text;
6380
+ }
6381
+
7092
6382
  /**
7093
6383
  * Removes quotes from a string
7094
6384
  *
@@ -8836,6 +8126,30 @@ function removeContentComments(content) {
8836
8126
  return spaceTrim$1(content.replace(/<!--(.*?)-->/gs, ''));
8837
8127
  }
8838
8128
 
8129
+ /**
8130
+ * @@@
8131
+ *
8132
+ * @param value @@@
8133
+ * @returns @@@
8134
+ * @example @@@
8135
+ * @public exported from `@promptbook/utils`
8136
+ */
8137
+ function titleToName(value) {
8138
+ if (isValidUrl(value)) {
8139
+ value = value.replace(/^https?:\/\//, '');
8140
+ value = value.replace(/\.html$/, '');
8141
+ }
8142
+ else if (isValidFilePath(value)) {
8143
+ value = basename(value);
8144
+ // Note: Keeping extension in the name
8145
+ }
8146
+ value = value.split('/').join('-');
8147
+ value = removeEmojis(value);
8148
+ value = normalizeToKebabCase(value);
8149
+ // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
8150
+ return value;
8151
+ }
8152
+
8839
8153
  /**
8840
8154
  * Compile pipeline from string (markdown) format to JSON format synchronously
8841
8155
  *
@@ -9170,21 +8484,21 @@ function pipelineStringToJsonSync(pipelineString) {
9170
8484
  * Note: This function acts as compilation process
9171
8485
  *
9172
8486
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8487
+ * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
9173
8488
  * @param options - Options and tools for the compilation
9174
8489
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
9175
8490
  * @throws {ParseError} if the promptbook string is not valid
9176
8491
  * @public exported from `@promptbook/core`
9177
8492
  */
9178
- function pipelineStringToJson(pipelineString, options) {
8493
+ function pipelineStringToJson(pipelineString, tools, options) {
9179
8494
  return __awaiter(this, void 0, void 0, function () {
9180
- var llmTools, pipelineJson;
8495
+ var pipelineJson;
9181
8496
  return __generator(this, function (_a) {
9182
8497
  switch (_a.label) {
9183
8498
  case 0:
9184
- llmTools = (options || {}).llmTools;
9185
8499
  pipelineJson = pipelineStringToJsonSync(pipelineString);
9186
- if (!(llmTools !== undefined)) return [3 /*break*/, 2];
9187
- return [4 /*yield*/, preparePipeline(pipelineJson, options || {
8500
+ if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8501
+ return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
9188
8502
  rootDirname: null,
9189
8503
  })];
9190
8504
  case 1:
@@ -9501,56 +8815,6 @@ function usageToHuman(usage) {
9501
8815
  * TODO: [🏛] Maybe make some markdown builder
9502
8816
  */
9503
8817
 
9504
- /**
9505
- * @@@
9506
- *
9507
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
9508
- *
9509
- * @private internal function of `$Register`
9510
- */
9511
- function $getGlobalScope() {
9512
- return Function('return this')();
9513
- }
9514
-
9515
- /**
9516
- * Register is @@@
9517
- *
9518
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
9519
- *
9520
- * @private internal utility, exported are only signleton instances of this class
9521
- */
9522
- var $Register = /** @class */ (function () {
9523
- function $Register(storageName) {
9524
- this.storageName = storageName;
9525
- storageName = "_promptbook_".concat(storageName);
9526
- var globalScope = $getGlobalScope();
9527
- if (globalScope[storageName] === undefined) {
9528
- globalScope[storageName] = [];
9529
- }
9530
- else if (!Array.isArray(globalScope[storageName])) {
9531
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
9532
- }
9533
- this.storage = globalScope[storageName];
9534
- }
9535
- $Register.prototype.list = function () {
9536
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
9537
- return this.storage;
9538
- };
9539
- $Register.prototype.register = function (registered) {
9540
- // <- TODO: What to return here
9541
- var packageName = registered.packageName, className = registered.className;
9542
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
9543
- var existingRegistration = this.storage[existingRegistrationIndex];
9544
- if (!existingRegistration) {
9545
- this.storage.push(registered);
9546
- }
9547
- else {
9548
- this.storage[existingRegistrationIndex] = registered;
9549
- }
9550
- };
9551
- return $Register;
9552
- }());
9553
-
9554
8818
  /**
9555
8819
  * @@@
9556
8820
  *
@@ -9559,6 +8823,9 @@ var $Register = /** @class */ (function () {
9559
8823
  * @public exported from `@promptbook/core`
9560
8824
  */
9561
8825
  var $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
8826
+ /**
8827
+ * TODO: [®] DRY Register logic
8828
+ */
9562
8829
 
9563
8830
  /**
9564
8831
  * @@@
@@ -9568,13 +8835,16 @@ var $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
9568
8835
  * @public exported from `@promptbook/core`
9569
8836
  */
9570
8837
  var $llmToolsRegister = new $Register('llm_execution_tools_constructors');
8838
+ /**
8839
+ * TODO: [®] DRY Register logic
8840
+ */
9571
8841
 
9572
8842
  /**
9573
8843
  * Creates a message with all registered LLM tools
9574
8844
  *
9575
8845
  * Note: This function is used to create a (error) message when there is no constructor for some LLM provider
9576
8846
  *
9577
- * @private internal function of `createLlmToolsFromConfiguration` and `createLlmToolsFromEnv`
8847
+ * @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv`
9578
8848
  */
9579
8849
  function $registeredLlmToolsMessage() {
9580
8850
  var e_1, _a, e_2, _b;
@@ -9666,6 +8936,9 @@ function $registeredLlmToolsMessage() {
9666
8936
  })
9667
8937
  .join('\n')), "\n "); });
9668
8938
  }
8939
+ /**
8940
+ * TODO: [®] DRY Register logic
8941
+ */
9669
8942
 
9670
8943
  /**
9671
8944
  * @@@
@@ -9677,7 +8950,7 @@ function $registeredLlmToolsMessage() {
9677
8950
  */
9678
8951
  function createLlmToolsFromConfiguration(configuration, options) {
9679
8952
  if (options === void 0) { options = {}; }
9680
- var _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
8953
+ var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
9681
8954
  var llmTools = configuration.map(function (llmConfiguration) {
9682
8955
  var registeredItem = $llmToolsRegister
9683
8956
  .list()
@@ -9688,17 +8961,18 @@ function createLlmToolsFromConfiguration(configuration, options) {
9688
8961
  if (registeredItem === undefined) {
9689
8962
  throw new Error(spaceTrim(function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
9690
8963
  }
9691
- return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
8964
+ return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
9692
8965
  });
9693
8966
  return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
9694
8967
  }
9695
8968
  /**
9696
- * TODO: [🎌] Togethere with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
8969
+ * TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
9697
8970
  * TODO: [🧠][🎌] Dynamically install required providers
9698
8971
  * TODO: @@@ write discussion about this - wizzard
9699
8972
  * TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
9700
8973
  * TODO: [🧠] Is there some meaningfull way how to test this util
9701
8974
  * TODO: This should be maybe not under `_common` but under `utils`
8975
+ * TODO: [®] DRY Register logic
9702
8976
  */
9703
8977
 
9704
8978
  /**
@@ -9777,11 +9051,7 @@ function $currentDate() {
9777
9051
  function cacheLlmTools(llmTools, options) {
9778
9052
  var _this = this;
9779
9053
  if (options === void 0) { options = {}; }
9780
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a,
9781
- // <- TODO: [🧱] Implement in a functional (not new Class) way
9782
- _b = options.isReloaded,
9783
- // <- TODO: [🧱] Implement in a functional (not new Class) way
9784
- isReloaded = _b === void 0 ? false : _b;
9054
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
9785
9055
  var proxyTools = __assign(__assign({}, llmTools), {
9786
9056
  // <- Note: [🥫]
9787
9057
  get title() {
@@ -9803,7 +9073,7 @@ function cacheLlmTools(llmTools, options) {
9803
9073
  key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
9804
9074
  '-' +
9805
9075
  sha256(hexEncoder.parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
9806
- if (!!isReloaded) return [3 /*break*/, 2];
9076
+ if (!!isCacheReloaded) return [3 /*break*/, 2];
9807
9077
  return [4 /*yield*/, storage.getItem(key)];
9808
9078
  case 1:
9809
9079
  _a = _c.sent();
@@ -9931,9 +9201,9 @@ function limitTotalUsage(llmTools, options) {
9931
9201
  */
9932
9202
 
9933
9203
  /**
9934
- * @@@ registration1 of default configuration for Anthropic Claude
9204
+ * Registration of LLM provider metadata
9935
9205
  *
9936
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9206
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
9937
9207
  *
9938
9208
  * @public exported from `@promptbook/core`
9939
9209
  * @public exported from `@promptbook/cli`
@@ -9972,9 +9242,9 @@ var _AnthropicClaudeMetadataRegistration = $llmToolsMetadataRegister.register({
9972
9242
  });
9973
9243
 
9974
9244
  /**
9975
- * @@@ registration1 of default configuration for Azure Open AI
9245
+ * Registration of LLM provider metadata
9976
9246
  *
9977
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9247
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
9978
9248
  *
9979
9249
  * @public exported from `@promptbook/core`
9980
9250
  * @public exported from `@promptbook/cli`
@@ -10019,9 +9289,9 @@ var _AzureOpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
10019
9289
  });
10020
9290
 
10021
9291
  /**
10022
- * @@@ registration1 of default configuration for Open AI
9292
+ * Registration of LLM provider metadata
10023
9293
  *
10024
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9294
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
10025
9295
  *
10026
9296
  * @public exported from `@promptbook/core`
10027
9297
  * @public exported from `@promptbook/cli`
@@ -10100,6 +9370,130 @@ var _OpenAiAssistantMetadataRegistration = $llmToolsMetadataRegister.register({
10100
9370
  },
10101
9371
  });
10102
9372
 
9373
+ /**
9374
+ * Metadata of the scraper
9375
+ *
9376
+ * @private within the scraper directory
9377
+ */
9378
+ var legacyDocumentScraperMetadata = $deepFreeze({
9379
+ title: 'LegacyDocument scraper',
9380
+ packageName: '@promptbook/legacy-documents',
9381
+ className: 'LegacyDocumentScraper',
9382
+ mimeTypes: ['application/msword', 'text/rtf'],
9383
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9384
+ isAvilableInBrowser: false,
9385
+ requiredExecutables: [
9386
+ 'Pandoc',
9387
+ 'LibreOffice',
9388
+ // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
9389
+ ],
9390
+ }); /* <- TODO: [🤛] */
9391
+ /**
9392
+ * Registration of known scraper metadata
9393
+ *
9394
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9395
+ *
9396
+ * @public exported from `@promptbook/core`
9397
+ * @public exported from `@promptbook/cli`
9398
+ */
9399
+ var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
9400
+
9401
+ /**
9402
+ * Metadata of the scraper
9403
+ *
9404
+ * @private within the scraper directory
9405
+ */
9406
+ var documentScraperMetadata = $deepFreeze({
9407
+ title: 'Document scraper',
9408
+ packageName: '@promptbook/documents',
9409
+ className: 'DocumentScraper',
9410
+ mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
9411
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9412
+ isAvilableInBrowser: false,
9413
+ requiredExecutables: ['Pandoc'],
9414
+ }); /* <- TODO: [🤛] */
9415
+ /**
9416
+ * Registration of known scraper metadata
9417
+ *
9418
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9419
+ *
9420
+ * @public exported from `@promptbook/core`
9421
+ * @public exported from `@promptbook/cli`
9422
+ */
9423
+ var _DocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(documentScraperMetadata);
9424
+
9425
+ /**
9426
+ * Metadata of the scraper
9427
+ *
9428
+ * @private within the scraper directory
9429
+ */
9430
+ var markdownScraperMetadata = $deepFreeze({
9431
+ title: 'Markdown scraper',
9432
+ packageName: '@promptbook/markdown-utils',
9433
+ className: 'MarkdownScraper',
9434
+ mimeTypes: ['text/markdown', 'text/plain'],
9435
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9436
+ isAvilableInBrowser: true,
9437
+ requiredExecutables: [],
9438
+ }); /* <- TODO: [🤛] */
9439
+ /**
9440
+ * Registration of known scraper metadata
9441
+ *
9442
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9443
+ *
9444
+ * @public exported from `@promptbook/core`
9445
+ * @public exported from `@promptbook/cli`
9446
+ */
9447
+ var _MarkdownScraperMetadataRegistration = $scrapersMetadataRegister.register(markdownScraperMetadata);
9448
+
9449
+ /**
9450
+ * Metadata of the scraper
9451
+ *
9452
+ * @private within the scraper directory
9453
+ */
9454
+ var pdfScraperMetadata = $deepFreeze({
9455
+ title: 'Pdf scraper',
9456
+ packageName: '@promptbook/pdf',
9457
+ className: 'PdfScraper',
9458
+ mimeTypes: ['application/pdf'],
9459
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9460
+ isAvilableInBrowser: true,
9461
+ requiredExecutables: [],
9462
+ }); /* <- TODO: [🤛] */
9463
+ /**
9464
+ * Registration of known scraper metadata
9465
+ *
9466
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9467
+ *
9468
+ * @public exported from `@promptbook/core`
9469
+ * @public exported from `@promptbook/cli`
9470
+ */
9471
+ var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
9472
+
9473
+ /**
9474
+ * Metadata of the scraper
9475
+ *
9476
+ * @private within the scraper directory
9477
+ */
9478
+ var websiteScraperMetadata = $deepFreeze({
9479
+ title: 'Website scraper',
9480
+ packageName: '@promptbook/website-crawler',
9481
+ className: 'WebsiteScraper',
9482
+ mimeTypes: ['text/html'],
9483
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9484
+ isAvilableInBrowser: false,
9485
+ requiredExecutables: [],
9486
+ }); /* <- TODO: [🤛] */
9487
+ /**
9488
+ * Registration of known scraper metadata
9489
+ *
9490
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9491
+ *
9492
+ * @public exported from `@promptbook/core`
9493
+ * @public exported from `@promptbook/cli`
9494
+ */
9495
+ var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
9496
+
10103
9497
  /**
10104
9498
  * This class behaves like LocalStorage but separates keys by prefix
10105
9499
  *
@@ -10450,5 +9844,5 @@ function executionReportJsonToString(executionReportJson, options) {
10450
9844
  * TODO: [🧠] Should be in generated file GENERATOR_WARNING
10451
9845
  */
10452
9846
 
10453
- export { $llmToolsMetadataRegister, $llmToolsRegister, AbstractFormatError, CLAIM, CallbackInterfaceTools, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_CSV_SETTINGS, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, ERRORS, EXECUTIONS_CACHE_DIRNAME, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, IS_VERBOSE, KnowledgeScrapeError, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_EXECUTION_ATTEMPTS, MAX_FILENAME_LENGTH, MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, MAX_PARALLEL_COUNT, MODEL_VARIANTS, MemoryStorage, MissingToolsError, NotFoundError, NotYetImplementedError, PIPELINE_COLLECTION_BASE_FILENAME, PROMPTBOOK_VERSION, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SCRAPE_CACHE_DIRNAME, TemplateTypes, TextFormatDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, addUsage, assertsExecutionSuccessful, cacheLlmTools, collectionToJson, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, documentScraper, embeddingVectorToString, executionReportJsonToString, isPassingExpectations, isPipelinePrepared, joinLlmExecutionTools, legacyDocumentScraper, limitTotalUsage, markdownScraper, pdfScraper, pipelineJsonToString, pipelineStringToJson, pipelineStringToJsonSync, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTemplates, prettifyPipelineString, stringifyPipelineJson, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, websiteScraper };
9847
+ export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, AbstractFormatError, CLAIM, CallbackInterfaceTools, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_CSV_SETTINGS, DEFAULT_EXECUTIONS_CACHE_DIRNAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, ERRORS, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, KnowledgeScrapeError, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NotFoundError, NotYetImplementedError, PROMPTBOOK_VERSION, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, TemplateTypes, TextFormatDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _DocumentScraperMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, cacheLlmTools, collectionToJson, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, isPassingExpectations, isPipelinePrepared, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, pipelineJsonToString, pipelineStringToJson, pipelineStringToJsonSync, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTemplates, prettifyPipelineString, stringifyPipelineJson, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline };
10454
9848
  //# sourceMappingURL=index.es.js.map