@promptbook/core 0.72.0-8 → 0.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +8 -8
  2. package/esm/index.es.js +1630 -2236
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/browser.index.d.ts +2 -0
  5. package/esm/typings/src/_packages/cli.index.d.ts +20 -0
  6. package/esm/typings/src/_packages/core.index.d.ts +42 -30
  7. package/esm/typings/src/_packages/documents.index.d.ts +8 -0
  8. package/esm/typings/src/_packages/legacy-documents.index.d.ts +8 -0
  9. package/esm/typings/src/_packages/markdown-utils.index.d.ts +6 -0
  10. package/esm/typings/src/_packages/node.index.d.ts +12 -4
  11. package/esm/typings/src/_packages/pdf.index.d.ts +8 -0
  12. package/esm/typings/src/_packages/types.index.d.ts +33 -17
  13. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  14. package/esm/typings/src/_packages/website-crawler.index.d.ts +8 -0
  15. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  16. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  17. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  18. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +5 -3
  19. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  20. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  21. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  22. package/esm/typings/src/config.d.ts +26 -13
  23. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +3 -1
  24. package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
  25. package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -3
  26. package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
  27. package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
  28. package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
  29. package/esm/typings/src/executables/locateApp.d.ts +33 -0
  30. package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
  31. package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
  32. package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
  33. package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts} +1 -1
  34. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  35. package/esm/typings/src/execution/Executables.d.ts +18 -0
  36. package/esm/typings/src/execution/ExecutionTools.d.ts +32 -6
  37. package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
  38. package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
  39. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  40. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
  41. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  42. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +3 -12
  43. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -18
  44. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -19
  45. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  46. package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
  47. package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts +13 -0
  48. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  49. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  50. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  51. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  52. package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts} +4 -1
  53. package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts} +5 -2
  54. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts} +3 -3
  55. package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts} +4 -11
  56. package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts} +4 -3
  57. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromEnv.d.ts → register/$provideLlmToolsFromEnv.d.ts} +6 -5
  58. package/esm/typings/src/llm-providers/_common/{$registeredLlmToolsMessage.d.ts → register/$registeredLlmToolsMessage.d.ts} +5 -2
  59. package/esm/typings/src/llm-providers/_common/{LlmToolsConfiguration.d.ts → register/LlmToolsConfiguration.d.ts} +6 -5
  60. package/esm/typings/src/llm-providers/_common/{LlmToolsMetadata.d.ts → register/LlmToolsMetadata.d.ts} +5 -4
  61. package/esm/typings/src/llm-providers/_common/{LlmToolsOptions.d.ts → register/LlmToolsOptions.d.ts} +4 -1
  62. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfiguration.d.ts → register/createLlmToolsFromConfiguration.d.ts} +12 -4
  63. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  64. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  65. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +5 -5
  66. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  67. package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +4 -3
  68. package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +4 -3
  69. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  70. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -3
  71. package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +4 -3
  72. package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +4 -3
  73. package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
  74. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  75. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +4 -4
  76. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +4 -4
  77. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -4
  78. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  79. package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
  80. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
  81. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +4 -4
  82. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  83. package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +5 -4
  84. package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +5 -4
  85. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
  86. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +16 -7
  87. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
  88. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +7 -13
  89. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
  90. package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +11 -16
  91. package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +24 -9
  92. package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
  93. package/esm/typings/src/personas/preparePersona.d.ts +2 -1
  94. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +10 -28
  95. package/esm/typings/src/prepare/preparePipeline.d.ts +2 -1
  96. package/esm/typings/src/prepare/prepareTemplates.d.ts +3 -2
  97. package/esm/typings/src/scrapers/_common/Converter.d.ts +4 -10
  98. package/esm/typings/src/scrapers/_common/Scraper.d.ts +5 -17
  99. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -1
  100. package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
  101. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +12 -0
  102. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +15 -0
  103. package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +12 -0
  104. package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +13 -0
  105. package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +13 -0
  106. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +41 -0
  107. package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +12 -0
  108. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +3 -2
  109. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -2
  110. package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts} +18 -12
  111. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +20 -0
  112. package/esm/typings/src/scrapers/document/register-constructor.d.ts +13 -0
  113. package/esm/typings/src/scrapers/document/register-metadata.d.ts +24 -0
  114. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +43 -0
  115. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +20 -0
  116. package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +13 -0
  117. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +24 -0
  118. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +29 -0
  119. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +20 -0
  120. package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +13 -0
  121. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +24 -0
  122. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +40 -0
  123. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +20 -0
  124. package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +13 -0
  125. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +24 -0
  126. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +50 -0
  127. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +20 -0
  128. package/esm/typings/src/scrapers/website/register-constructor.d.ts +13 -0
  129. package/esm/typings/src/scrapers/website/register-metadata.d.ts +24 -0
  130. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  131. package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +2 -2
  132. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  133. package/esm/typings/src/scripting/python/PythonExecutionTools.d.ts +3 -3
  134. package/esm/typings/src/scripting/typescript/TypescriptExecutionTools.d.ts +3 -3
  135. package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +5 -3
  136. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  137. package/esm/typings/src/storage/{utils → memory/utils}/PrefixStorage.d.ts +1 -1
  138. package/esm/typings/src/storage/{utils → memory/utils}/makePromptbookStorageFromWebStorage.d.ts +1 -1
  139. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  140. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  141. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  142. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  143. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  144. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  145. package/esm/typings/src/types/Prompt.d.ts +2 -1
  146. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  147. package/esm/typings/src/types/typeAliases.d.ts +18 -8
  148. package/esm/typings/src/utils/$Register.d.ts +20 -7
  149. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  150. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  151. package/esm/typings/src/utils/emojis.d.ts +1 -1
  152. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +5 -3
  153. package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
  154. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +2 -1
  155. package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
  156. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  157. package/esm/typings/src/utils/files/isDirectoryExisting.d.ts +14 -0
  158. package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
  159. package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
  160. package/esm/typings/src/utils/files/isFileExisting.test.d.ts +1 -0
  161. package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
  162. package/esm/typings/src/utils/files/listAllFiles.test.d.ts +1 -0
  163. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  164. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  165. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  166. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  167. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  168. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  169. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  170. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  171. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  172. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  173. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  174. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  175. package/package.json +3 -6
  176. package/umd/index.umd.js +1650 -2245
  177. package/umd/index.umd.js.map +1 -1
  178. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  179. package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts +0 -37
  180. package/esm/typings/src/scrapers/index.d.ts +0 -8
  181. package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +0 -29
  182. package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +0 -35
  183. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  184. package/esm/typings/src/scrapers/website/websiteScraper.d.ts +0 -43
  185. package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
  186. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
  187. package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +0 -15
  188. package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
  189. /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
  190. /package/esm/typings/src/{scrapers/website/websiteScraper.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
  191. /package/esm/typings/src/{utils/files/$isDirectoryExisting.test.d.ts → executables/locateApp.test.d.ts} +0 -0
  192. /package/esm/typings/src/scrapers/document/{documentScraper.test.d.ts → DocumentScraper.test.d.ts} +0 -0
  193. /package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.test.d.ts → LegacyDocumentScraper.test.d.ts} +0 -0
  194. /package/esm/typings/src/scrapers/markdown/{markdownScraper.test.d.ts → MarkdownScraper.test.d.ts} +0 -0
  195. /package/esm/typings/src/{utils/files/$isFileExisting.test.d.ts → scrapers/website/utils/createShowdownConverter.test.d.ts} +0 -0
  196. /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
package/umd/index.umd.js CHANGED
@@ -1,14 +1,13 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('fs/promises'), require('path'), require('child_process'), require('colors'), require('waitasecond'), require('crypto-js'), require('crypto-js/enc-hex'), require('papaparse'), require('@mozilla/readability'), require('jsdom'), require('showdown'), require('mime-types'), require('crypto-js/sha256'), require('moment')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'fs/promises', 'path', 'child_process', 'colors', 'waitasecond', 'crypto-js', 'crypto-js/enc-hex', 'papaparse', '@mozilla/readability', 'jsdom', 'showdown', 'mime-types', 'crypto-js/sha256', 'moment'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.promises, global.path, global.child_process, global.colors, global.waitasecond, global.cryptoJs, global.hexEncoder, global.papaparse, global.readability, global.jsdom, global.showdown, global.mimeTypes, global.sha256, global.moment));
5
- })(this, (function (exports, spaceTrim, prettier, parserHtml, promises, path, child_process, colors, waitasecond, cryptoJs, hexEncoder, papaparse, readability, jsdom, showdown, mimeTypes, sha256, moment) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('path'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('crypto-js/sha256'), require('moment')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'path', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'crypto-js/sha256', 'moment'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.path, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.sha256, global.moment));
5
+ })(this, (function (exports, spaceTrim, prettier, parserHtml, waitasecond, papaparse, path, cryptoJs, hexEncoder, mimeTypes, sha256, moment) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
9
9
  var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
10
10
  var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
11
- var colors__default = /*#__PURE__*/_interopDefaultLegacy(colors);
12
11
  var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
13
12
  var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
14
13
  var moment__default = /*#__PURE__*/_interopDefaultLegacy(moment);
@@ -17,7 +16,7 @@
17
16
  /**
18
17
  * The version of the Promptbook library
19
18
  */
20
- var PROMPTBOOK_VERSION = '0.72.0-7';
19
+ var PROMPTBOOK_VERSION = '0.72.0-34';
21
20
  // TODO: [main] !!!! List here all the versions and annotate + put into script
22
21
 
23
22
  /*! *****************************************************************************
@@ -224,7 +223,6 @@
224
223
  pipelineString += '\n\n';
225
224
  pipelineString += description;
226
225
  }
227
- // TODO:> const commands: Array<Command>
228
226
  var commands = [];
229
227
  if (pipelineUrl) {
230
228
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -280,7 +278,6 @@
280
278
  pipelineString += '\n\n';
281
279
  pipelineString += description_1;
282
280
  }
283
- // TODO:> const commands: Array<Command>
284
281
  var commands_1 = [];
285
282
  var contentLanguage = 'text';
286
283
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -640,7 +637,7 @@
640
637
  *
641
638
  * @public exported from `@promptbook/core`
642
639
  */
643
- var CLAIM = "Supercharge LLM models with Promptbook";
640
+ var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
644
641
  // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
645
642
  /**
646
643
  * The maximum number of iterations for a loops
@@ -655,37 +652,44 @@
655
652
  */
656
653
  var IMMEDIATE_TIME = 10;
657
654
  /**
658
- * The maximum number of (LLM) tasks running in parallel
655
+ * The maximum length of the (generated) filename
659
656
  *
660
657
  * @public exported from `@promptbook/core`
661
658
  */
662
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
659
+ var MAX_FILENAME_LENGTH = 30;
663
660
  /**
664
- * The maximum number of attempts to execute LLM task before giving up
661
+ * Strategy for caching the intermediate results for knowledge sources
665
662
  *
666
663
  * @public exported from `@promptbook/core`
667
664
  */
668
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
665
+ var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
666
+ // <- TODO: [😡] Change to 'VISIBLE'
669
667
  /**
670
- * The maximum length of the (generated) filename
668
+ * The maximum number of (LLM) tasks running in parallel
671
669
  *
672
670
  * @public exported from `@promptbook/core`
673
671
  */
674
- var MAX_FILENAME_LENGTH = 30;
672
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
673
+ /**
674
+ * The maximum number of attempts to execute LLM task before giving up
675
+ *
676
+ * @public exported from `@promptbook/core`
677
+ */
678
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
675
679
  /**
676
680
  * @@@
677
681
  * TODO: [🐝][main] !!! Use
678
682
  *
679
683
  * @public exported from `@promptbook/core`
680
684
  */
681
- var MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = 3;
685
+ var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = 3;
682
686
  /**
683
687
  * @@@
684
688
  * TODO: [🐝][main] !!! Use
685
689
  *
686
690
  * @public exported from `@promptbook/core`
687
691
  */
688
- var MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
692
+ var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
689
693
  /**
690
694
  * Where to store the cache of executions for promptbook CLI
691
695
  *
@@ -693,7 +697,7 @@
693
697
  *
694
698
  * @public exported from `@promptbook/core`
695
699
  */
696
- var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
700
+ var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
697
701
  /**
698
702
  * Where to store the scrape cache
699
703
  *
@@ -701,13 +705,13 @@
701
705
  *
702
706
  * @public exported from `@promptbook/core`
703
707
  */
704
- var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
708
+ var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
705
709
  /**
706
710
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
707
711
  *
708
712
  * @public exported from `@promptbook/core`
709
713
  */
710
- var PIPELINE_COLLECTION_BASE_FILENAME = "index";
714
+ var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
711
715
  /**
712
716
  * Nonce which is used for replacing things in strings
713
717
  *
@@ -780,7 +784,13 @@
780
784
  *
781
785
  * @public exported from `@promptbook/core`
782
786
  */
783
- var IS_VERBOSE = false;
787
+ var DEFAULT_IS_VERBOSE = false;
788
+ /**
789
+ * @@@
790
+ *
791
+ * @public exported from `@promptbook/core`
792
+ */
793
+ var DEFAULT_IS_AUTO_INSTALLED = false;
784
794
  /**
785
795
  * @@@
786
796
  *
@@ -1529,7 +1539,7 @@
1529
1539
  return __generator(this, function (_d) {
1530
1540
  switch (_d.label) {
1531
1541
  case 0:
1532
- _a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
1542
+ _a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
1533
1543
  collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
1534
1544
  return __generator(this, function (_a) {
1535
1545
  if (isVerbose) {
@@ -1651,19 +1661,19 @@
1651
1661
  ];
1652
1662
 
1653
1663
  /**
1654
- * This error indicates that the promptbook can not retrieve knowledge from external sources
1664
+ * This error type indicates that some tools are missing for pipeline execution or preparation
1655
1665
  *
1656
1666
  * @public exported from `@promptbook/core`
1657
1667
  */
1658
- var KnowledgeScrapeError = /** @class */ (function (_super) {
1659
- __extends(KnowledgeScrapeError, _super);
1660
- function KnowledgeScrapeError(message) {
1661
- var _this = _super.call(this, message) || this;
1662
- _this.name = 'KnowledgeScrapeError';
1663
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
1668
+ var MissingToolsError = /** @class */ (function (_super) {
1669
+ __extends(MissingToolsError, _super);
1670
+ function MissingToolsError(message) {
1671
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
1672
+ _this.name = 'MissingToolsError';
1673
+ Object.setPrototypeOf(_this, MissingToolsError.prototype);
1664
1674
  return _this;
1665
1675
  }
1666
- return KnowledgeScrapeError;
1676
+ return MissingToolsError;
1667
1677
  }(Error));
1668
1678
 
1669
1679
  /**
@@ -1743,1042 +1753,187 @@
1743
1753
  }
1744
1754
 
1745
1755
  /**
1746
- * This error type indicates that some tools are missing for pipeline execution or preparation
1756
+ * Represents the usage with no resources consumed
1747
1757
  *
1748
1758
  * @public exported from `@promptbook/core`
1749
1759
  */
1750
- var MissingToolsError = /** @class */ (function (_super) {
1751
- __extends(MissingToolsError, _super);
1752
- function MissingToolsError(message) {
1753
- var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
1754
- _this.name = 'MissingToolsError';
1755
- Object.setPrototypeOf(_this, MissingToolsError.prototype);
1756
- return _this;
1757
- }
1758
- return MissingToolsError;
1759
- }(Error));
1760
-
1760
+ var ZERO_USAGE = $deepFreeze({
1761
+ price: { value: 0 },
1762
+ input: {
1763
+ tokensCount: { value: 0 },
1764
+ charactersCount: { value: 0 },
1765
+ wordsCount: { value: 0 },
1766
+ sentencesCount: { value: 0 },
1767
+ linesCount: { value: 0 },
1768
+ paragraphsCount: { value: 0 },
1769
+ pagesCount: { value: 0 },
1770
+ },
1771
+ output: {
1772
+ tokensCount: { value: 0 },
1773
+ charactersCount: { value: 0 },
1774
+ wordsCount: { value: 0 },
1775
+ sentencesCount: { value: 0 },
1776
+ linesCount: { value: 0 },
1777
+ paragraphsCount: { value: 0 },
1778
+ pagesCount: { value: 0 },
1779
+ },
1780
+ });
1761
1781
  /**
1762
- * Detects if the code is running in a Node.js environment
1763
- *
1764
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
1782
+ * Represents the usage with unknown resources consumed
1765
1783
  *
1766
- * @public exported from `@promptbook/utils`
1784
+ * @public exported from `@promptbook/core`
1767
1785
  */
1768
- var $isRunningInNode = new Function("\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n");
1786
+ var UNCERTAIN_USAGE = $deepFreeze({
1787
+ price: { value: 0, isUncertain: true },
1788
+ input: {
1789
+ tokensCount: { value: 0, isUncertain: true },
1790
+ charactersCount: { value: 0, isUncertain: true },
1791
+ wordsCount: { value: 0, isUncertain: true },
1792
+ sentencesCount: { value: 0, isUncertain: true },
1793
+ linesCount: { value: 0, isUncertain: true },
1794
+ paragraphsCount: { value: 0, isUncertain: true },
1795
+ pagesCount: { value: 0, isUncertain: true },
1796
+ },
1797
+ output: {
1798
+ tokensCount: { value: 0, isUncertain: true },
1799
+ charactersCount: { value: 0, isUncertain: true },
1800
+ wordsCount: { value: 0, isUncertain: true },
1801
+ sentencesCount: { value: 0, isUncertain: true },
1802
+ linesCount: { value: 0, isUncertain: true },
1803
+ paragraphsCount: { value: 0, isUncertain: true },
1804
+ pagesCount: { value: 0, isUncertain: true },
1805
+ },
1806
+ });
1769
1807
 
1770
1808
  /**
1771
- * This error type indicates that you try to use a feature that is not available in the current environment
1809
+ * This error indicates errors during the execution of the pipeline
1772
1810
  *
1773
1811
  * @public exported from `@promptbook/core`
1774
1812
  */
1775
- var EnvironmentMismatchError = /** @class */ (function (_super) {
1776
- __extends(EnvironmentMismatchError, _super);
1777
- function EnvironmentMismatchError(message) {
1813
+ var PipelineExecutionError = /** @class */ (function (_super) {
1814
+ __extends(PipelineExecutionError, _super);
1815
+ function PipelineExecutionError(message) {
1778
1816
  var _this = _super.call(this, message) || this;
1779
- _this.name = 'EnvironmentMismatchError';
1780
- Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
1817
+ _this.name = 'PipelineExecutionError';
1818
+ Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
1781
1819
  return _this;
1782
1820
  }
1783
- return EnvironmentMismatchError;
1821
+ return PipelineExecutionError;
1784
1822
  }(Error));
1785
1823
 
1786
1824
  /**
1787
- * Normalize options for `execCommand` and `execCommands`
1825
+ * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1788
1826
  *
1789
- * @private internal utility of `execCommand` and `execCommands`
1827
+ * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1828
+ * @public exported from `@promptbook/core`
1790
1829
  */
1791
- function execCommandNormalizeOptions(options) {
1792
- var _a;
1793
- var _b, _c, _d;
1794
- var command;
1795
- var cwd;
1796
- var crashOnError;
1797
- var args = [];
1798
- var timeout;
1799
- if (typeof options === 'string') {
1800
- // TODO: [1] DRY default values
1801
- command = options;
1802
- cwd = process.cwd();
1803
- crashOnError = true;
1804
- timeout = Infinity;
1805
- }
1806
- else {
1807
- /*
1808
- TODO:
1809
- if ((options as any).commands !== undefined) {
1810
- commands = (options as any).commands;
1811
- } else {
1812
- commands = [(options as any).command];
1830
+ var MultipleLlmExecutionTools = /** @class */ (function () {
1831
+ /**
1832
+ * Gets array of execution tools in order of priority
1833
+ */
1834
+ function MultipleLlmExecutionTools() {
1835
+ var llmExecutionTools = [];
1836
+ for (var _i = 0; _i < arguments.length; _i++) {
1837
+ llmExecutionTools[_i] = arguments[_i];
1813
1838
  }
1814
- */
1815
- // TODO: [1] DRY default values
1816
- command = options.command;
1817
- cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
1818
- crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
1819
- timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
1820
- }
1821
- // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
1822
- var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
1823
- .map(function (_a) {
1824
- var _b = __read(_a, 1), match = _b[0];
1825
- return match;
1826
- })
1827
- .filter(function (arg) { return arg !== ''; });
1828
- if (_.length > 1) {
1829
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1830
- _a = __read(_), command = _a[0], args = _a.slice(1);
1831
- }
1832
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1833
- if (options.args) {
1834
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1835
- args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
1836
- }
1837
- var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
1838
- if (['ts-node'].includes(humanReadableCommand)) {
1839
- humanReadableCommand += " ".concat(args[1]);
1840
- }
1841
- return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
1842
- }
1843
- // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
1844
-
1845
- /**
1846
- * Run one command in a shell
1847
- *
1848
- * Note: There are 2 similar functions in the codebase:
1849
- * - `$execCommand` which runs a single command
1850
- * - `$execCommands` which runs multiple commands
1851
- *
1852
- * @public exported from `@promptbook/node`
1853
- */
1854
- function $execCommand(options) {
1855
- if (!$isRunningInNode()) {
1856
- throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
1839
+ this.llmExecutionTools = llmExecutionTools;
1857
1840
  }
1858
- return new Promise(
1859
- // <- TODO: [🧱] Implement in a functional (not new Class) way
1860
- function (resolve, reject) {
1861
- // eslint-disable-next-line prefer-const
1862
- var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
1863
- if (timeout !== Infinity) {
1864
- // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
1865
- waitasecond.forTime(timeout).then(function () {
1866
- if (crashOnError) {
1867
- reject(new Error("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms")));
1868
- }
1869
- else {
1870
- console.warn("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms but continues running"));
1871
- resolve('Command exceeded time limit');
1872
- }
1873
- });
1874
- }
1875
- if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
1876
- command = "".concat(command, ".cmd");
1877
- }
1878
- // !!!!!! Verbose mode - to all consoles
1879
- console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
1880
- try {
1881
- var commandProcess = child_process.spawn(command, args, { cwd: cwd, shell: true });
1882
- commandProcess.on('message', function (message) {
1883
- console.info({ message: message });
1884
- });
1885
- var output_1 = [];
1886
- commandProcess.stdout.on('data', function (stdout) {
1887
- output_1.push(stdout.toString());
1888
- console.info(stdout.toString());
1889
- });
1890
- commandProcess.stderr.on('data', function (stderr) {
1891
- output_1.push(stderr.toString());
1892
- if (stderr.toString().trim()) {
1893
- console.warn(stderr.toString());
1894
- }
1895
- });
1896
- var finishWithCode = function (code) {
1897
- if (code !== 0) {
1898
- if (crashOnError) {
1899
- reject(new Error(output_1.join('\n').trim() ||
1900
- "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
1901
- }
1902
- else {
1903
- console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
1904
- resolve(spaceTrim.spaceTrim(output_1.join('\n')));
1905
- }
1906
- }
1907
- else {
1908
- resolve(spaceTrim.spaceTrim(output_1.join('\n')));
1841
+ Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
1842
+ get: function () {
1843
+ return 'Multiple LLM Providers';
1844
+ },
1845
+ enumerable: false,
1846
+ configurable: true
1847
+ });
1848
+ Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
1849
+ get: function () {
1850
+ return this.llmExecutionTools.map(function (_a, index) {
1851
+ var title = _a.title;
1852
+ return "".concat(index + 1, ") `").concat(title, "`");
1853
+ }).join('\n');
1854
+ },
1855
+ enumerable: false,
1856
+ configurable: true
1857
+ });
1858
+ /**
1859
+ * Check the configuration of all execution tools
1860
+ */
1861
+ MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
1862
+ return __awaiter(this, void 0, void 0, function () {
1863
+ var _a, _b, llmExecutionTools, e_1_1;
1864
+ var e_1, _c;
1865
+ return __generator(this, function (_d) {
1866
+ switch (_d.label) {
1867
+ case 0:
1868
+ _d.trys.push([0, 5, 6, 7]);
1869
+ _a = __values(this.llmExecutionTools), _b = _a.next();
1870
+ _d.label = 1;
1871
+ case 1:
1872
+ if (!!_b.done) return [3 /*break*/, 4];
1873
+ llmExecutionTools = _b.value;
1874
+ return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
1875
+ case 2:
1876
+ _d.sent();
1877
+ _d.label = 3;
1878
+ case 3:
1879
+ _b = _a.next();
1880
+ return [3 /*break*/, 1];
1881
+ case 4: return [3 /*break*/, 7];
1882
+ case 5:
1883
+ e_1_1 = _d.sent();
1884
+ e_1 = { error: e_1_1 };
1885
+ return [3 /*break*/, 7];
1886
+ case 6:
1887
+ try {
1888
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
1889
+ }
1890
+ finally { if (e_1) throw e_1.error; }
1891
+ return [7 /*endfinally*/];
1892
+ case 7: return [2 /*return*/];
1909
1893
  }
1910
- };
1911
- commandProcess.on('close', finishWithCode);
1912
- commandProcess.on('exit', finishWithCode);
1913
- commandProcess.on('disconnect', function () {
1914
- // Note: Unexpected disconnection should always result in rejection
1915
- reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
1916
1894
  });
1917
- commandProcess.on('error', function (error) {
1918
- if (crashOnError) {
1919
- reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
1920
- }
1921
- else {
1922
- console.warn(error);
1923
- resolve(spaceTrim.spaceTrim(output_1.join('\n')));
1924
- }
1925
- });
1926
- }
1927
- catch (error) {
1928
- // Note: Unexpected error in sync code should always result in rejection
1929
- reject(error);
1930
- }
1931
- });
1932
- }
1933
- /**
1934
- * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
1935
- */
1936
-
1937
- /**
1938
- * Checks if the file exists
1939
- *
1940
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
1941
- *
1942
- * @private within the repository
1943
- */
1944
- function $isFileExisting(filename) {
1945
- return __awaiter(this, void 0, void 0, function () {
1946
- var isReadAccessAllowed, isFile;
1947
- return __generator(this, function (_a) {
1948
- switch (_a.label) {
1949
- case 0:
1950
- if (!$isRunningInNode()) {
1951
- throw new EnvironmentMismatchError('Function `$isFileExisting` works only in Node environment.js');
1952
- }
1953
- return [4 /*yield*/, promises.access(filename, promises.constants.R_OK)
1954
- .then(function () { return true; })
1955
- .catch(function () { return false; })];
1956
- case 1:
1957
- isReadAccessAllowed = _a.sent();
1958
- if (!isReadAccessAllowed) {
1959
- return [2 /*return*/, false];
1960
- }
1961
- return [4 /*yield*/, promises.stat(filename)
1962
- .then(function (fileStat) { return fileStat.isFile(); })
1963
- .catch(function () { return false; })];
1964
- case 2:
1965
- isFile = _a.sent();
1966
- return [2 /*return*/, isFile];
1967
- }
1968
- });
1969
- });
1970
- }
1971
- /**
1972
- * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
1973
- * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
1974
- * TODO: [🖇] What about symlinks?
1975
- */
1976
-
1977
- /**
1978
- * Get the file extension from a file name
1979
- *
1980
- * @private within the repository
1981
- */
1982
- function getFileExtension(value) {
1983
- var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
1984
- return match ? match[1].toLowerCase() : null;
1985
- }
1986
-
1987
- var defaultDiacriticsRemovalMap = [
1988
- {
1989
- base: 'A',
1990
- letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
1991
- },
1992
- { base: 'AA', letters: '\uA732' },
1993
- { base: 'AE', letters: '\u00C6\u01FC\u01E2' },
1994
- { base: 'AO', letters: '\uA734' },
1995
- { base: 'AU', letters: '\uA736' },
1996
- { base: 'AV', letters: '\uA738\uA73A' },
1997
- { base: 'AY', letters: '\uA73C' },
1998
- {
1999
- base: 'B',
2000
- letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
2001
- },
2002
- {
2003
- base: 'C',
2004
- letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
2005
- },
2006
- {
2007
- base: 'D',
2008
- letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
2009
- },
2010
- { base: 'DZ', letters: '\u01F1\u01C4' },
2011
- { base: 'Dz', letters: '\u01F2\u01C5' },
2012
- {
2013
- base: 'E',
2014
- letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
2015
- },
2016
- { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
2017
- {
2018
- base: 'G',
2019
- letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
2020
- },
2021
- {
2022
- base: 'H',
2023
- letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
2024
- },
2025
- {
2026
- base: 'I',
2027
- letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
2028
- },
2029
- { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
2030
- {
2031
- base: 'K',
2032
- letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
2033
- },
2034
- {
2035
- base: 'L',
2036
- letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
2037
- },
2038
- { base: 'LJ', letters: '\u01C7' },
2039
- { base: 'Lj', letters: '\u01C8' },
2040
- { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
2041
- {
2042
- base: 'N',
2043
- letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
2044
- },
2045
- { base: 'NJ', letters: '\u01CA' },
2046
- { base: 'Nj', letters: '\u01CB' },
2047
- {
2048
- base: 'O',
2049
- letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
2050
- },
2051
- { base: 'OI', letters: '\u01A2' },
2052
- { base: 'OO', letters: '\uA74E' },
2053
- { base: 'OU', letters: '\u0222' },
2054
- { base: 'OE', letters: '\u008C\u0152' },
2055
- { base: 'oe', letters: '\u009C\u0153' },
2056
- {
2057
- base: 'P',
2058
- letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
2059
- },
2060
- { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
2061
- {
2062
- base: 'R',
2063
- letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
2064
- },
2065
- {
2066
- base: 'S',
2067
- letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
2068
- },
2069
- {
2070
- base: 'T',
2071
- letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
2072
- },
2073
- { base: 'TZ', letters: '\uA728' },
2074
- {
2075
- base: 'U',
2076
- letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
2077
- },
2078
- { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
2079
- { base: 'VY', letters: '\uA760' },
2080
- {
2081
- base: 'W',
2082
- letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
2083
- },
2084
- { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
2085
- {
2086
- base: 'Y',
2087
- letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
2088
- },
2089
- {
2090
- base: 'Z',
2091
- letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
2092
- },
2093
- {
2094
- base: 'a',
2095
- letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
2096
- },
2097
- { base: 'aa', letters: '\uA733' },
2098
- { base: 'ae', letters: '\u00E6\u01FD\u01E3' },
2099
- { base: 'ao', letters: '\uA735' },
2100
- { base: 'au', letters: '\uA737' },
2101
- { base: 'av', letters: '\uA739\uA73B' },
2102
- { base: 'ay', letters: '\uA73D' },
2103
- {
2104
- base: 'b',
2105
- letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
2106
- },
2107
- {
2108
- base: 'c',
2109
- letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
2110
- },
2111
- {
2112
- base: 'd',
2113
- letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
2114
- },
2115
- { base: 'dz', letters: '\u01F3\u01C6' },
2116
- {
2117
- base: 'e',
2118
- letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
2119
- },
2120
- { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
2121
- {
2122
- base: 'g',
2123
- letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
2124
- },
2125
- {
2126
- base: 'h',
2127
- letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
2128
- },
2129
- { base: 'hv', letters: '\u0195' },
2130
- {
2131
- base: 'i',
2132
- letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
2133
- },
2134
- { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
2135
- {
2136
- base: 'k',
2137
- letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
2138
- },
2139
- {
2140
- base: 'l',
2141
- letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
2142
- },
2143
- { base: 'lj', letters: '\u01C9' },
2144
- { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
2145
- {
2146
- base: 'n',
2147
- letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
2148
- },
2149
- { base: 'nj', letters: '\u01CC' },
2150
- {
2151
- base: 'o',
2152
- letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
2153
- },
2154
- { base: 'oi', letters: '\u01A3' },
2155
- { base: 'ou', letters: '\u0223' },
2156
- { base: 'oo', letters: '\uA74F' },
2157
- {
2158
- base: 'p',
2159
- letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
2160
- },
2161
- { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
2162
- {
2163
- base: 'r',
2164
- letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
2165
- },
2166
- {
2167
- base: 's',
2168
- letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
2169
- },
2170
- {
2171
- base: 't',
2172
- letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
2173
- },
2174
- { base: 'tz', letters: '\uA729' },
2175
- {
2176
- base: 'u',
2177
- letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
2178
- },
2179
- { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
2180
- { base: 'vy', letters: '\uA761' },
2181
- {
2182
- base: 'w',
2183
- letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
2184
- },
2185
- { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
2186
- {
2187
- base: 'y',
2188
- letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
2189
- },
2190
- {
2191
- base: 'z',
2192
- letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
2193
- },
2194
- ];
2195
- /**
2196
- * Map of letters from diacritic variant to diacritless variant
2197
- * Contains lowercase and uppercase separatelly
2198
- *
2199
- * > "á" => "a"
2200
- * > "ě" => "e"
2201
- * > "Ă" => "A"
2202
- * > ...
2203
- *
2204
- * @public exported from `@promptbook/utils`
2205
- */
2206
- var DIACRITIC_VARIANTS_LETTERS = {};
2207
- // tslint:disable-next-line: prefer-for-of
2208
- for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
2209
- var letters = defaultDiacriticsRemovalMap[i].letters;
2210
- // tslint:disable-next-line: prefer-for-of
2211
- for (var j = 0; j < letters.length; j++) {
2212
- DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
2213
- }
2214
- }
2215
- // <- TODO: [🍓] Put to maker function to save execution time if not needed
2216
- /*
2217
- @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
2218
- Licensed under the Apache License, Version 2.0 (the "License");
2219
- you may not use this file except in compliance with the License.
2220
- You may obtain a copy of the License at
2221
-
2222
- http://www.apache.org/licenses/LICENSE-2.0
2223
-
2224
- Unless required by applicable law or agreed to in writing, software
2225
- distributed under the License is distributed on an "AS IS" BASIS,
2226
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2227
- See the License for the specific language governing permissions and
2228
- limitations under the License.
2229
- */
2230
-
2231
- /**
2232
- * @@@
2233
- *
2234
- * @param input @@@
2235
- * @returns @@@
2236
- * @public exported from `@promptbook/utils`
2237
- */
2238
- function removeDiacritics(input) {
2239
- /*eslint no-control-regex: "off"*/
2240
- return input.replace(/[^\u0000-\u007E]/g, function (a) {
2241
- return DIACRITIC_VARIANTS_LETTERS[a] || a;
2242
- });
2243
- }
2244
- /**
2245
- * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
2246
- */
2247
-
2248
- /**
2249
- * @@@
2250
- *
2251
- * @param text @@@
2252
- * @returns @@@
2253
- * @example 'hello-world'
2254
- * @example 'i-love-promptbook'
2255
- * @public exported from `@promptbook/utils`
2256
- */
2257
- function normalizeToKebabCase(text) {
2258
- var e_1, _a;
2259
- text = removeDiacritics(text);
2260
- var charType;
2261
- var lastCharType = 'OTHER';
2262
- var normalizedName = '';
2263
- try {
2264
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
2265
- var char = text_1_1.value;
2266
- var normalizedChar = void 0;
2267
- if (/^[a-z]$/.test(char)) {
2268
- charType = 'LOWERCASE';
2269
- normalizedChar = char;
2270
- }
2271
- else if (/^[A-Z]$/.test(char)) {
2272
- charType = 'UPPERCASE';
2273
- normalizedChar = char.toLowerCase();
2274
- }
2275
- else if (/^[0-9]$/.test(char)) {
2276
- charType = 'NUMBER';
2277
- normalizedChar = char;
2278
- }
2279
- else {
2280
- charType = 'OTHER';
2281
- normalizedChar = '-';
2282
- }
2283
- if (charType !== lastCharType &&
2284
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
2285
- !(lastCharType === 'NUMBER') &&
2286
- !(charType === 'NUMBER')) {
2287
- normalizedName += '-';
2288
- }
2289
- normalizedName += normalizedChar;
2290
- lastCharType = charType;
2291
- }
2292
- }
2293
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2294
- finally {
2295
- try {
2296
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
2297
- }
2298
- finally { if (e_1) throw e_1.error; }
2299
- }
2300
- normalizedName = normalizedName.split(/-+/g).join('-');
2301
- normalizedName = normalizedName.split(/-?\/-?/g).join('/');
2302
- normalizedName = normalizedName.replace(/^-/, '');
2303
- normalizedName = normalizedName.replace(/-$/, '');
2304
- return normalizedName;
2305
- }
2306
-
2307
- /**
2308
- * Removes emojis from a string and fix whitespaces
2309
- *
2310
- * @param text with emojis
2311
- * @returns text without emojis
2312
- * @public exported from `@promptbook/utils`
2313
- */
2314
- function removeEmojis(text) {
2315
- // Replace emojis (and also ZWJ sequence) with hyphens
2316
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
2317
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
2318
- text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
2319
- text = text.replace(/\p{Extended_Pictographic}/gu, '');
2320
- return text;
2321
- }
2322
-
2323
- /**
2324
- * Tests if given string is valid URL.
2325
- *
2326
- * Note: This does not check if the file exists only if the path is valid
2327
- * @public exported from `@promptbook/utils`
2328
- */
2329
- function isValidFilePath(filename) {
2330
- if (typeof filename !== 'string') {
2331
- return false;
2332
- }
2333
- var filenameSlashes = filename.split('\\').join('/');
2334
- // Absolute Unix path: /hello.txt
2335
- if (/^(\/)/i.test(filenameSlashes)) {
2336
- return true;
2337
- }
2338
- // Absolute Windows path: /hello.txt
2339
- if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
2340
- return true;
2341
- }
2342
- // Relative path: ./hello.txt
2343
- if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
2344
- return true;
2345
- }
2346
- return false;
2347
- }
2348
-
2349
- /**
2350
- * @@@
2351
- *
2352
- * @param value @@@
2353
- * @returns @@@
2354
- * @example @@@
2355
- * @public exported from `@promptbook/utils`
2356
- */
2357
- function titleToName(value) {
2358
- if (isValidUrl(value)) {
2359
- value = value.replace(/^https?:\/\//, '');
2360
- value = value.replace(/\.html$/, '');
2361
- }
2362
- else if (isValidFilePath(value)) {
2363
- value = path.basename(value);
2364
- // Note: Keeping extension in the name
2365
- }
2366
- value = value.split('/').join('-');
2367
- value = removeEmojis(value);
2368
- value = normalizeToKebabCase(value);
2369
- // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
2370
- return value;
2371
- }
2372
-
2373
- /**
2374
- * @@@
2375
- *
2376
- * @private for `FileCacheStorage`
2377
- */
2378
- function nameToSubfolderPath(name) {
2379
- return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
2380
- }
2381
-
2382
- /**
2383
- * Just marks a place of place where should be something implemented
2384
- * No side effects.
2385
- *
2386
- * Note: It can be usefull suppressing eslint errors of unused variables
2387
- *
2388
- * @param value any values
2389
- * @returns void
2390
- * @private within the repository
2391
- */
2392
- function TODO_USE() {
2393
- var value = [];
2394
- for (var _i = 0; _i < arguments.length; _i++) {
2395
- value[_i] = arguments[_i];
2396
- }
2397
- }
2398
-
2399
- /**
2400
- * Create a filename for intermediate cache for scrapers
2401
- *
2402
- * Note: It also checks if directory exists and creates it if not
2403
- *
2404
- * @private as internal utility for scrapers
2405
- */
2406
- function getScraperIntermediateSource(source, options) {
2407
- return __awaiter(this, void 0, void 0, function () {
2408
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
2409
- return __generator(this, function (_a) {
2410
- switch (_a.label) {
2411
- case 0:
2412
- sourceFilename = source.filename, url = source.url;
2413
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
2414
- hash = cryptoJs.SHA256(
2415
- // <- TODO: [🥬] Encapsulate sha256 to some private utility function
2416
- hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
2417
- .toString( /* hex */)
2418
- .substring(0, 20);
2419
- semanticName = normalizeToKebabCase(titleToName((sourceFilename || url || '').split('intermediate').join(''))).substring(0, 20);
2420
- pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
2421
- name = pieces.join('-').split('--').join('-');
2422
- // <- TODO: Use MAX_FILENAME_LENGTH
2423
- TODO_USE(rootDirname); // <- TODO: !!!!!!
2424
- cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
2425
- cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
2426
- .join('/') +
2427
- '.' +
2428
- extension;
2429
- return [4 /*yield*/, promises.mkdir(path.dirname(cacheFilename), { recursive: true })];
2430
- case 1:
2431
- _a.sent();
2432
- isDestroyed = true;
2433
- fileHandler = {
2434
- filename: cacheFilename,
2435
- get isDestroyed() {
2436
- return isDestroyed;
2437
- },
2438
- destroy: function () {
2439
- return __awaiter(this, void 0, void 0, function () {
2440
- return __generator(this, function (_a) {
2441
- switch (_a.label) {
2442
- case 0:
2443
- if (!isCacheCleaned) return [3 /*break*/, 2];
2444
- if (isVerbose) {
2445
- console.info('legacyDocumentScraper: Clening cache');
2446
- }
2447
- return [4 /*yield*/, promises.rm(cacheFilename)];
2448
- case 1:
2449
- _a.sent();
2450
- _a.label = 2;
2451
- case 2:
2452
- isDestroyed = true;
2453
- return [2 /*return*/];
2454
- }
2455
- });
2456
- });
2457
- },
2458
- };
2459
- return [2 /*return*/, fileHandler];
2460
- }
2461
- });
2462
- });
2463
- }
2464
- /**
2465
- * Note: Not using `FileCacheStorage` for two reasons:
2466
- * 1) Need to store more than serialized JSONs
2467
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
2468
- * TODO: [🐱‍🐉][🧠] Make some smart crop
2469
- */
2470
-
2471
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
2472
-
2473
- /**
2474
- * This error indicates errors during the execution of the pipeline
2475
- *
2476
- * @public exported from `@promptbook/core`
2477
- */
2478
- var PipelineExecutionError = /** @class */ (function (_super) {
2479
- __extends(PipelineExecutionError, _super);
2480
- function PipelineExecutionError(message) {
2481
- var _this = _super.call(this, message) || this;
2482
- _this.name = 'PipelineExecutionError';
2483
- Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
2484
- return _this;
2485
- }
2486
- return PipelineExecutionError;
2487
- }(Error));
2488
-
2489
- /**
2490
- * This error indicates that the pipeline collection cannot be propperly loaded
2491
- *
2492
- * @public exported from `@promptbook/core`
2493
- */
2494
- var CollectionError = /** @class */ (function (_super) {
2495
- __extends(CollectionError, _super);
2496
- function CollectionError(message) {
2497
- var _this = _super.call(this, message) || this;
2498
- _this.name = 'CollectionError';
2499
- Object.setPrototypeOf(_this, CollectionError.prototype);
2500
- return _this;
2501
- }
2502
- return CollectionError;
2503
- }(Error));
2504
-
2505
- /**
2506
- * This error occurs when some expectation is not met in the execution of the pipeline
2507
- *
2508
- * @public exported from `@promptbook/core`
2509
- * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
2510
- * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
2511
- * Note: This is a kindof subtype of PipelineExecutionError
2512
- */
2513
- var ExpectError = /** @class */ (function (_super) {
2514
- __extends(ExpectError, _super);
2515
- function ExpectError(message) {
2516
- var _this = _super.call(this, message) || this;
2517
- _this.name = 'ExpectError';
2518
- Object.setPrototypeOf(_this, ExpectError.prototype);
2519
- return _this;
2520
- }
2521
- return ExpectError;
2522
- }(Error));
2523
-
2524
- /**
2525
- * This error type indicates that some limit was reached
2526
- *
2527
- * @public exported from `@promptbook/core`
2528
- */
2529
- var LimitReachedError = /** @class */ (function (_super) {
2530
- __extends(LimitReachedError, _super);
2531
- function LimitReachedError(message) {
2532
- var _this = _super.call(this, message) || this;
2533
- _this.name = 'LimitReachedError';
2534
- Object.setPrototypeOf(_this, LimitReachedError.prototype);
2535
- return _this;
2536
- }
2537
- return LimitReachedError;
2538
- }(Error));
2539
-
2540
- /**
2541
- * This error type indicates that some part of the code is not implemented yet
2542
- *
2543
- * @public exported from `@promptbook/core`
2544
- */
2545
- var NotYetImplementedError = /** @class */ (function (_super) {
2546
- __extends(NotYetImplementedError, _super);
2547
- function NotYetImplementedError(message) {
2548
- var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2549
- _this.name = 'NotYetImplementedError';
2550
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2551
- return _this;
2552
- }
2553
- return NotYetImplementedError;
2554
- }(Error));
2555
-
2556
- /**
2557
- * Index of all custom errors
2558
- *
2559
- * @public exported from `@promptbook/core`
2560
- */
2561
- var ERRORS = {
2562
- ExpectError: ExpectError,
2563
- CollectionError: CollectionError,
2564
- EnvironmentMismatchError: EnvironmentMismatchError,
2565
- LimitReachedError: LimitReachedError,
2566
- NotFoundError: NotFoundError,
2567
- NotYetImplementedError: NotYetImplementedError,
2568
- ParseError: ParseError,
2569
- PipelineExecutionError: PipelineExecutionError,
2570
- PipelineLogicError: PipelineLogicError,
2571
- PipelineUrlError: PipelineUrlError,
2572
- UnexpectedError: UnexpectedError,
2573
- // TODO: [🪑]> VersionMismatchError,
2574
- };
2575
-
2576
- /**
2577
- * Deserializes the error object
2578
- *
2579
- * @public exported from `@promptbook/utils`
2580
- */
2581
- function deserializeError(error) {
2582
- if (error.name === 'Error') {
2583
- return new Error(error.message);
2584
- }
2585
- var CustomError = ERRORS[error.name];
2586
- return new CustomError(error.message);
2587
- }
2588
-
2589
- /**
2590
- * Asserts that the execution of a Promptbook is successful
2591
- *
2592
- * @param executionResult - The partial result of the Promptbook execution
2593
- * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2594
- * @public exported from `@promptbook/core`
2595
- */
2596
- function assertsExecutionSuccessful(executionResult) {
2597
- var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2598
- if (isSuccessful === true) {
2599
- return;
2600
- }
2601
- if (errors.length === 0) {
2602
- throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
2603
- }
2604
- else if (errors.length === 1) {
2605
- throw deserializeError(errors[0]);
2606
- }
2607
- else {
2608
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
2609
- .map(function (_a, index) {
2610
- var name = _a.name, stack = _a.stack, message = _a.message;
2611
- return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
2612
- })
2613
- .join('\n')), "\n "); }));
2614
- }
2615
- }
2616
- /**
2617
- * TODO: [🧠] Can this return type be better typed than void
2618
- */
2619
-
2620
- /**
2621
- * Determine if the pipeline is fully prepared
2622
- *
2623
- * @public exported from `@promptbook/core`
2624
- */
2625
- function isPipelinePrepared(pipeline) {
2626
- // Note: Ignoring `pipeline.preparations` @@@
2627
- // Note: Ignoring `pipeline.knowledgePieces` @@@
2628
- if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2629
- return false;
2630
- }
2631
- if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2632
- return false;
2633
- }
2634
- /*
2635
- TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2636
- > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2637
- > return false;
2638
- > }
2639
- */
2640
- return true;
2641
- }
2642
- /**
2643
- * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2644
- * TODO: [🐠] Maybe base this on `makeValidator`
2645
- * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2646
- * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2647
- * - [🏍] ? Is context in each template
2648
- * - [♨] Are samples prepared
2649
- * - [♨] Are templates prepared
2650
- */
2651
-
2652
- /**
2653
- * Serializes an error into a [🚉] JSON-serializable object
2654
- *
2655
- * @public exported from `@promptbook/utils`
2656
- */
2657
- function serializeError(error) {
2658
- var name = error.name, message = error.message, stack = error.stack;
2659
- if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2660
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2661
- }
2662
- return {
2663
- name: name,
2664
- message: message,
2665
- stack: stack,
2666
- };
2667
- }
2668
-
2669
- /**
2670
- * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2671
- *
2672
- * Note: Internal utility of `joinLlmExecutionTools` but exposed type
2673
- * @public exported from `@promptbook/types`
2674
- */
2675
- var MultipleLlmExecutionTools = /** @class */ (function () {
2676
- /**
2677
- * Gets array of execution tools in order of priority
2678
- */
2679
- function MultipleLlmExecutionTools() {
2680
- var llmExecutionTools = [];
2681
- for (var _i = 0; _i < arguments.length; _i++) {
2682
- llmExecutionTools[_i] = arguments[_i];
2683
- }
2684
- this.llmExecutionTools = llmExecutionTools;
2685
- }
2686
- Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
2687
- get: function () {
2688
- return 'Multiple LLM Providers';
2689
- },
2690
- enumerable: false,
2691
- configurable: true
2692
- });
2693
- Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2694
- get: function () {
2695
- return this.llmExecutionTools.map(function (_a, index) {
2696
- var title = _a.title;
2697
- return "".concat(index + 1, ") `").concat(title, "`");
2698
- }).join('\n');
2699
- },
2700
- enumerable: false,
2701
- configurable: true
2702
- });
2703
- /**
2704
- * Check the configuration of all execution tools
2705
- */
2706
- MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
2707
- return __awaiter(this, void 0, void 0, function () {
2708
- var _a, _b, llmExecutionTools, e_1_1;
2709
- var e_1, _c;
2710
- return __generator(this, function (_d) {
2711
- switch (_d.label) {
2712
- case 0:
2713
- _d.trys.push([0, 5, 6, 7]);
2714
- _a = __values(this.llmExecutionTools), _b = _a.next();
2715
- _d.label = 1;
2716
- case 1:
2717
- if (!!_b.done) return [3 /*break*/, 4];
2718
- llmExecutionTools = _b.value;
2719
- return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
2720
- case 2:
2721
- _d.sent();
2722
- _d.label = 3;
2723
- case 3:
2724
- _b = _a.next();
2725
- return [3 /*break*/, 1];
2726
- case 4: return [3 /*break*/, 7];
2727
- case 5:
2728
- e_1_1 = _d.sent();
2729
- e_1 = { error: e_1_1 };
2730
- return [3 /*break*/, 7];
2731
- case 6:
2732
- try {
2733
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2734
- }
2735
- finally { if (e_1) throw e_1.error; }
2736
- return [7 /*endfinally*/];
2737
- case 7: return [2 /*return*/];
2738
- }
2739
- });
2740
- });
2741
- };
2742
- /**
2743
- * List all available models that can be used
2744
- * This lists is a combination of all available models from all execution tools
2745
- */
2746
- MultipleLlmExecutionTools.prototype.listModels = function () {
2747
- return __awaiter(this, void 0, void 0, function () {
2748
- var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
2749
- var e_2, _c;
2750
- return __generator(this, function (_d) {
2751
- switch (_d.label) {
2752
- case 0:
2753
- availableModels = [];
2754
- _d.label = 1;
2755
- case 1:
2756
- _d.trys.push([1, 6, 7, 8]);
2757
- _a = __values(this.llmExecutionTools), _b = _a.next();
2758
- _d.label = 2;
2759
- case 2:
2760
- if (!!_b.done) return [3 /*break*/, 5];
2761
- llmExecutionTools = _b.value;
2762
- return [4 /*yield*/, llmExecutionTools.listModels()];
2763
- case 3:
2764
- models = _d.sent();
2765
- availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
2766
- _d.label = 4;
2767
- case 4:
2768
- _b = _a.next();
2769
- return [3 /*break*/, 2];
2770
- case 5: return [3 /*break*/, 8];
2771
- case 6:
2772
- e_2_1 = _d.sent();
2773
- e_2 = { error: e_2_1 };
2774
- return [3 /*break*/, 8];
2775
- case 7:
2776
- try {
2777
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2778
- }
2779
- finally { if (e_2) throw e_2.error; }
2780
- return [7 /*endfinally*/];
2781
- case 8: return [2 /*return*/, availableModels];
1895
+ });
1896
+ };
1897
+ /**
1898
+ * List all available models that can be used
1899
+ * This lists is a combination of all available models from all execution tools
1900
+ */
1901
+ MultipleLlmExecutionTools.prototype.listModels = function () {
1902
+ return __awaiter(this, void 0, void 0, function () {
1903
+ var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
1904
+ var e_2, _c;
1905
+ return __generator(this, function (_d) {
1906
+ switch (_d.label) {
1907
+ case 0:
1908
+ availableModels = [];
1909
+ _d.label = 1;
1910
+ case 1:
1911
+ _d.trys.push([1, 6, 7, 8]);
1912
+ _a = __values(this.llmExecutionTools), _b = _a.next();
1913
+ _d.label = 2;
1914
+ case 2:
1915
+ if (!!_b.done) return [3 /*break*/, 5];
1916
+ llmExecutionTools = _b.value;
1917
+ return [4 /*yield*/, llmExecutionTools.listModels()];
1918
+ case 3:
1919
+ models = _d.sent();
1920
+ availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
1921
+ _d.label = 4;
1922
+ case 4:
1923
+ _b = _a.next();
1924
+ return [3 /*break*/, 2];
1925
+ case 5: return [3 /*break*/, 8];
1926
+ case 6:
1927
+ e_2_1 = _d.sent();
1928
+ e_2 = { error: e_2_1 };
1929
+ return [3 /*break*/, 8];
1930
+ case 7:
1931
+ try {
1932
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
1933
+ }
1934
+ finally { if (e_2) throw e_2.error; }
1935
+ return [7 /*endfinally*/];
1936
+ case 8: return [2 /*return*/, availableModels];
2782
1937
  }
2783
1938
  });
2784
1939
  });
@@ -2958,25 +2113,6 @@
2958
2113
  * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2959
2114
  */
2960
2115
 
2961
- /**
2962
- * Takes an item or an array of items and returns an array of items
2963
- *
2964
- * 1) Any item except array and undefined returns array with that one item (also null)
2965
- * 2) Undefined returns empty array
2966
- * 3) Array returns itself
2967
- *
2968
- * @private internal utility
2969
- */
2970
- function arrayableToArray(input) {
2971
- if (input === undefined) {
2972
- return [];
2973
- }
2974
- if (input instanceof Array) {
2975
- return input;
2976
- }
2977
- return [input];
2978
- }
2979
-
2980
2116
  /**
2981
2117
  * @@@
2982
2118
  *
@@ -2992,66 +2128,13 @@
2992
2128
  > if (value && typeof value === 'object') {
2993
2129
  > deepClone(value);
2994
2130
  > }
2995
- > }
2996
- > return Object.assign({}, objectValue);
2997
- */
2998
- }
2999
- /**
3000
- * TODO: [🧠] Is there a way how to meaningfully test this utility
3001
- */
3002
-
3003
- /**
3004
- * Represents the usage with no resources consumed
3005
- *
3006
- * @public exported from `@promptbook/core`
3007
- */
3008
- var ZERO_USAGE = $deepFreeze({
3009
- price: { value: 0 },
3010
- input: {
3011
- tokensCount: { value: 0 },
3012
- charactersCount: { value: 0 },
3013
- wordsCount: { value: 0 },
3014
- sentencesCount: { value: 0 },
3015
- linesCount: { value: 0 },
3016
- paragraphsCount: { value: 0 },
3017
- pagesCount: { value: 0 },
3018
- },
3019
- output: {
3020
- tokensCount: { value: 0 },
3021
- charactersCount: { value: 0 },
3022
- wordsCount: { value: 0 },
3023
- sentencesCount: { value: 0 },
3024
- linesCount: { value: 0 },
3025
- paragraphsCount: { value: 0 },
3026
- pagesCount: { value: 0 },
3027
- },
3028
- });
2131
+ > }
2132
+ > return Object.assign({}, objectValue);
2133
+ */
2134
+ }
3029
2135
  /**
3030
- * Represents the usage with unknown resources consumed
3031
- *
3032
- * @public exported from `@promptbook/core`
2136
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
3033
2137
  */
3034
- var UNCERTAIN_USAGE = $deepFreeze({
3035
- price: { value: 0, isUncertain: true },
3036
- input: {
3037
- tokensCount: { value: 0, isUncertain: true },
3038
- charactersCount: { value: 0, isUncertain: true },
3039
- wordsCount: { value: 0, isUncertain: true },
3040
- sentencesCount: { value: 0, isUncertain: true },
3041
- linesCount: { value: 0, isUncertain: true },
3042
- paragraphsCount: { value: 0, isUncertain: true },
3043
- pagesCount: { value: 0, isUncertain: true },
3044
- },
3045
- output: {
3046
- tokensCount: { value: 0, isUncertain: true },
3047
- charactersCount: { value: 0, isUncertain: true },
3048
- wordsCount: { value: 0, isUncertain: true },
3049
- sentencesCount: { value: 0, isUncertain: true },
3050
- linesCount: { value: 0, isUncertain: true },
3051
- paragraphsCount: { value: 0, isUncertain: true },
3052
- pagesCount: { value: 0, isUncertain: true },
3053
- },
3054
- });
3055
2138
 
3056
2139
  /**
3057
2140
  * Function `addUsage` will add multiple usages into one
@@ -3125,6 +2208,292 @@
3125
2208
  }, deepClone(ZERO_USAGE));
3126
2209
  }
3127
2210
 
2211
+ /**
2212
+ * Intercepts LLM tools and counts total usage of the tools
2213
+ *
2214
+ * @param llmTools LLM tools to be intercepted with usage counting
2215
+ * @returns LLM tools with same functionality with added total cost counting
2216
+ * @public exported from `@promptbook/core`
2217
+ */
2218
+ function countTotalUsage(llmTools) {
2219
+ var _this = this;
2220
+ var totalUsage = ZERO_USAGE;
2221
+ var proxyTools = {
2222
+ get title() {
2223
+ // TODO: [🧠] Maybe put here some suffix
2224
+ return llmTools.title;
2225
+ },
2226
+ get description() {
2227
+ // TODO: [🧠] Maybe put here some suffix
2228
+ return llmTools.description;
2229
+ },
2230
+ checkConfiguration: function () {
2231
+ return __awaiter(this, void 0, void 0, function () {
2232
+ return __generator(this, function (_a) {
2233
+ return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
2234
+ });
2235
+ });
2236
+ },
2237
+ listModels: function () {
2238
+ return /* not await */ llmTools.listModels();
2239
+ },
2240
+ getTotalUsage: function () {
2241
+ // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
2242
+ return totalUsage;
2243
+ },
2244
+ };
2245
+ if (llmTools.callChatModel !== undefined) {
2246
+ proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2247
+ var promptResult;
2248
+ return __generator(this, function (_a) {
2249
+ switch (_a.label) {
2250
+ case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
2251
+ case 1:
2252
+ promptResult = _a.sent();
2253
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2254
+ return [2 /*return*/, promptResult];
2255
+ }
2256
+ });
2257
+ }); };
2258
+ }
2259
+ if (llmTools.callCompletionModel !== undefined) {
2260
+ proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2261
+ var promptResult;
2262
+ return __generator(this, function (_a) {
2263
+ switch (_a.label) {
2264
+ case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
2265
+ case 1:
2266
+ promptResult = _a.sent();
2267
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2268
+ return [2 /*return*/, promptResult];
2269
+ }
2270
+ });
2271
+ }); };
2272
+ }
2273
+ if (llmTools.callEmbeddingModel !== undefined) {
2274
+ proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2275
+ var promptResult;
2276
+ return __generator(this, function (_a) {
2277
+ switch (_a.label) {
2278
+ case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
2279
+ case 1:
2280
+ promptResult = _a.sent();
2281
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2282
+ return [2 /*return*/, promptResult];
2283
+ }
2284
+ });
2285
+ }); };
2286
+ }
2287
+ // <- Note: [🤖]
2288
+ return proxyTools;
2289
+ }
2290
+ /**
2291
+ * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
2292
+ * TODO: [🧠] Is there some meaningfull way how to test this util
2293
+ * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
2294
+ * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
2295
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2296
+ */
2297
+
2298
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
2299
+
2300
+ /**
2301
+ * This error indicates that the pipeline collection cannot be propperly loaded
2302
+ *
2303
+ * @public exported from `@promptbook/core`
2304
+ */
2305
+ var CollectionError = /** @class */ (function (_super) {
2306
+ __extends(CollectionError, _super);
2307
+ function CollectionError(message) {
2308
+ var _this = _super.call(this, message) || this;
2309
+ _this.name = 'CollectionError';
2310
+ Object.setPrototypeOf(_this, CollectionError.prototype);
2311
+ return _this;
2312
+ }
2313
+ return CollectionError;
2314
+ }(Error));
2315
+
2316
+ /**
2317
+ * This error type indicates that you try to use a feature that is not available in the current environment
2318
+ *
2319
+ * @public exported from `@promptbook/core`
2320
+ */
2321
+ var EnvironmentMismatchError = /** @class */ (function (_super) {
2322
+ __extends(EnvironmentMismatchError, _super);
2323
+ function EnvironmentMismatchError(message) {
2324
+ var _this = _super.call(this, message) || this;
2325
+ _this.name = 'EnvironmentMismatchError';
2326
+ Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
2327
+ return _this;
2328
+ }
2329
+ return EnvironmentMismatchError;
2330
+ }(Error));
2331
+
2332
+ /**
2333
+ * This error occurs when some expectation is not met in the execution of the pipeline
2334
+ *
2335
+ * @public exported from `@promptbook/core`
2336
+ * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
2337
+ * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
2338
+ * Note: This is a kindof subtype of PipelineExecutionError
2339
+ */
2340
+ var ExpectError = /** @class */ (function (_super) {
2341
+ __extends(ExpectError, _super);
2342
+ function ExpectError(message) {
2343
+ var _this = _super.call(this, message) || this;
2344
+ _this.name = 'ExpectError';
2345
+ Object.setPrototypeOf(_this, ExpectError.prototype);
2346
+ return _this;
2347
+ }
2348
+ return ExpectError;
2349
+ }(Error));
2350
+
2351
+ /**
2352
+ * This error type indicates that some limit was reached
2353
+ *
2354
+ * @public exported from `@promptbook/core`
2355
+ */
2356
+ var LimitReachedError = /** @class */ (function (_super) {
2357
+ __extends(LimitReachedError, _super);
2358
+ function LimitReachedError(message) {
2359
+ var _this = _super.call(this, message) || this;
2360
+ _this.name = 'LimitReachedError';
2361
+ Object.setPrototypeOf(_this, LimitReachedError.prototype);
2362
+ return _this;
2363
+ }
2364
+ return LimitReachedError;
2365
+ }(Error));
2366
+
2367
+ /**
2368
+ * This error type indicates that some part of the code is not implemented yet
2369
+ *
2370
+ * @public exported from `@promptbook/core`
2371
+ */
2372
+ var NotYetImplementedError = /** @class */ (function (_super) {
2373
+ __extends(NotYetImplementedError, _super);
2374
+ function NotYetImplementedError(message) {
2375
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2376
+ _this.name = 'NotYetImplementedError';
2377
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2378
+ return _this;
2379
+ }
2380
+ return NotYetImplementedError;
2381
+ }(Error));
2382
+
2383
+ /**
2384
+ * Index of all custom errors
2385
+ *
2386
+ * @public exported from `@promptbook/core`
2387
+ */
2388
+ var ERRORS = {
2389
+ ExpectError: ExpectError,
2390
+ CollectionError: CollectionError,
2391
+ EnvironmentMismatchError: EnvironmentMismatchError,
2392
+ LimitReachedError: LimitReachedError,
2393
+ NotFoundError: NotFoundError,
2394
+ NotYetImplementedError: NotYetImplementedError,
2395
+ ParseError: ParseError,
2396
+ PipelineExecutionError: PipelineExecutionError,
2397
+ PipelineLogicError: PipelineLogicError,
2398
+ PipelineUrlError: PipelineUrlError,
2399
+ UnexpectedError: UnexpectedError,
2400
+ // TODO: [🪑]> VersionMismatchError,
2401
+ };
2402
+
2403
+ /**
2404
+ * Deserializes the error object
2405
+ *
2406
+ * @public exported from `@promptbook/utils`
2407
+ */
2408
+ function deserializeError(error) {
2409
+ if (error.name === 'Error') {
2410
+ return new Error(error.message);
2411
+ }
2412
+ var CustomError = ERRORS[error.name];
2413
+ return new CustomError(error.message);
2414
+ }
2415
+
2416
+ /**
2417
+ * Asserts that the execution of a Promptbook is successful
2418
+ *
2419
+ * @param executionResult - The partial result of the Promptbook execution
2420
+ * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2421
+ * @public exported from `@promptbook/core`
2422
+ */
2423
+ function assertsExecutionSuccessful(executionResult) {
2424
+ var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2425
+ if (isSuccessful === true) {
2426
+ return;
2427
+ }
2428
+ if (errors.length === 0) {
2429
+ throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
2430
+ }
2431
+ else if (errors.length === 1) {
2432
+ throw deserializeError(errors[0]);
2433
+ }
2434
+ else {
2435
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
2436
+ .map(function (_a, index) {
2437
+ var name = _a.name, stack = _a.stack, message = _a.message;
2438
+ return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
2439
+ })
2440
+ .join('\n')), "\n "); }));
2441
+ }
2442
+ }
2443
+ /**
2444
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2445
+ * TODO: [🧠] Can this return type be better typed than void
2446
+ */
2447
+
2448
+ /**
2449
+ * Determine if the pipeline is fully prepared
2450
+ *
2451
+ * @public exported from `@promptbook/core`
2452
+ */
2453
+ function isPipelinePrepared(pipeline) {
2454
+ // Note: Ignoring `pipeline.preparations` @@@
2455
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2456
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2457
+ return false;
2458
+ }
2459
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2460
+ return false;
2461
+ }
2462
+ /*
2463
+ TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2464
+ > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2465
+ > return false;
2466
+ > }
2467
+ */
2468
+ return true;
2469
+ }
2470
+ /**
2471
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2472
+ * TODO: [🐠] Maybe base this on `makeValidator`
2473
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2474
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2475
+ * - [🏍] ? Is context in each template
2476
+ * - [♨] Are samples prepared
2477
+ * - [♨] Are templates prepared
2478
+ */
2479
+
2480
+ /**
2481
+ * Serializes an error into a [🚉] JSON-serializable object
2482
+ *
2483
+ * @public exported from `@promptbook/utils`
2484
+ */
2485
+ function serializeError(error) {
2486
+ var name = error.name, message = error.message, stack = error.stack;
2487
+ if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2488
+ throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2489
+ }
2490
+ return {
2491
+ name: name,
2492
+ message: message,
2493
+ stack: stack,
2494
+ };
2495
+ }
2496
+
3128
2497
  /**
3129
2498
  * Parses the given script and returns the list of all used variables that are not defined in the script
3130
2499
  *
@@ -3330,6 +2699,23 @@
3330
2699
  return union;
3331
2700
  }
3332
2701
 
2702
+ /**
2703
+ * Just marks a place of place where should be something implemented
2704
+ * No side effects.
2705
+ *
2706
+ * Note: It can be usefull suppressing eslint errors of unused variables
2707
+ *
2708
+ * @param value any values
2709
+ * @returns void
2710
+ * @private within the repository
2711
+ */
2712
+ function TODO_USE() {
2713
+ var value = [];
2714
+ for (var _i = 0; _i < arguments.length; _i++) {
2715
+ value[_i] = arguments[_i];
2716
+ }
2717
+ }
2718
+
3333
2719
  /**
3334
2720
  * This error indicates problems parsing the format value
3335
2721
  *
@@ -3843,6 +3229,25 @@
3843
3229
  * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
3844
3230
  */
3845
3231
 
3232
+ /**
3233
+ * Takes an item or an array of items and returns an array of items
3234
+ *
3235
+ * 1) Any item except array and undefined returns array with that one item (also null)
3236
+ * 2) Undefined returns empty array
3237
+ * 3) Array returns itself
3238
+ *
3239
+ * @private internal utility
3240
+ */
3241
+ function arrayableToArray(input) {
3242
+ if (input === undefined) {
3243
+ return [];
3244
+ }
3245
+ if (input instanceof Array) {
3246
+ return input;
3247
+ }
3248
+ return [input];
3249
+ }
3250
+
3846
3251
  /**
3847
3252
  * Just says that the variable is not used but should be kept
3848
3253
  * No side effects.
@@ -3944,69 +3349,330 @@
3944
3349
  }
3945
3350
 
3946
3351
  /**
3947
- * Counts number of characters in the text
3352
+ * Counts number of characters in the text
3353
+ *
3354
+ * @public exported from `@promptbook/utils`
3355
+ */
3356
+ function countCharacters(text) {
3357
+ // Remove null characters
3358
+ text = text.replace(/\0/g, '');
3359
+ // Replace emojis (and also ZWJ sequence) with hyphens
3360
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
3361
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
3362
+ text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
3363
+ return text.length;
3364
+ }
3365
+
3366
+ /**
3367
+ * Counts number of lines in the text
3368
+ *
3369
+ * @public exported from `@promptbook/utils`
3370
+ */
3371
+ function countLines(text) {
3372
+ if (text === '') {
3373
+ return 0;
3374
+ }
3375
+ return text.split('\n').length;
3376
+ }
3377
+
3378
+ /**
3379
+ * Counts number of pages in the text
3380
+ *
3381
+ * @public exported from `@promptbook/utils`
3382
+ */
3383
+ function countPages(text) {
3384
+ var sentencesPerPage = 5; // Assuming each page has 5 sentences
3385
+ var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3386
+ var pageCount = Math.ceil(sentences.length / sentencesPerPage);
3387
+ return pageCount;
3388
+ }
3389
+
3390
+ /**
3391
+ * Counts number of paragraphs in the text
3392
+ *
3393
+ * @public exported from `@promptbook/utils`
3394
+ */
3395
+ function countParagraphs(text) {
3396
+ return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
3397
+ }
3398
+
3399
+ /**
3400
+ * Split text into sentences
3401
+ *
3402
+ * @public exported from `@promptbook/utils`
3403
+ */
3404
+ function splitIntoSentences(text) {
3405
+ return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3406
+ }
3407
+ /**
3408
+ * Counts number of sentences in the text
3409
+ *
3410
+ * @public exported from `@promptbook/utils`
3411
+ */
3412
+ function countSentences(text) {
3413
+ return splitIntoSentences(text).length;
3414
+ }
3415
+
3416
+ var defaultDiacriticsRemovalMap = [
3417
+ {
3418
+ base: 'A',
3419
+ letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
3420
+ },
3421
+ { base: 'AA', letters: '\uA732' },
3422
+ { base: 'AE', letters: '\u00C6\u01FC\u01E2' },
3423
+ { base: 'AO', letters: '\uA734' },
3424
+ { base: 'AU', letters: '\uA736' },
3425
+ { base: 'AV', letters: '\uA738\uA73A' },
3426
+ { base: 'AY', letters: '\uA73C' },
3427
+ {
3428
+ base: 'B',
3429
+ letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
3430
+ },
3431
+ {
3432
+ base: 'C',
3433
+ letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
3434
+ },
3435
+ {
3436
+ base: 'D',
3437
+ letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
3438
+ },
3439
+ { base: 'DZ', letters: '\u01F1\u01C4' },
3440
+ { base: 'Dz', letters: '\u01F2\u01C5' },
3441
+ {
3442
+ base: 'E',
3443
+ letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
3444
+ },
3445
+ { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
3446
+ {
3447
+ base: 'G',
3448
+ letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
3449
+ },
3450
+ {
3451
+ base: 'H',
3452
+ letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
3453
+ },
3454
+ {
3455
+ base: 'I',
3456
+ letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
3457
+ },
3458
+ { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
3459
+ {
3460
+ base: 'K',
3461
+ letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
3462
+ },
3463
+ {
3464
+ base: 'L',
3465
+ letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
3466
+ },
3467
+ { base: 'LJ', letters: '\u01C7' },
3468
+ { base: 'Lj', letters: '\u01C8' },
3469
+ { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
3470
+ {
3471
+ base: 'N',
3472
+ letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
3473
+ },
3474
+ { base: 'NJ', letters: '\u01CA' },
3475
+ { base: 'Nj', letters: '\u01CB' },
3476
+ {
3477
+ base: 'O',
3478
+ letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
3479
+ },
3480
+ { base: 'OI', letters: '\u01A2' },
3481
+ { base: 'OO', letters: '\uA74E' },
3482
+ { base: 'OU', letters: '\u0222' },
3483
+ { base: 'OE', letters: '\u008C\u0152' },
3484
+ { base: 'oe', letters: '\u009C\u0153' },
3485
+ {
3486
+ base: 'P',
3487
+ letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
3488
+ },
3489
+ { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
3490
+ {
3491
+ base: 'R',
3492
+ letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
3493
+ },
3494
+ {
3495
+ base: 'S',
3496
+ letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
3497
+ },
3498
+ {
3499
+ base: 'T',
3500
+ letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
3501
+ },
3502
+ { base: 'TZ', letters: '\uA728' },
3503
+ {
3504
+ base: 'U',
3505
+ letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
3506
+ },
3507
+ { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
3508
+ { base: 'VY', letters: '\uA760' },
3509
+ {
3510
+ base: 'W',
3511
+ letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
3512
+ },
3513
+ { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
3514
+ {
3515
+ base: 'Y',
3516
+ letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
3517
+ },
3518
+ {
3519
+ base: 'Z',
3520
+ letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
3521
+ },
3522
+ {
3523
+ base: 'a',
3524
+ letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
3525
+ },
3526
+ { base: 'aa', letters: '\uA733' },
3527
+ { base: 'ae', letters: '\u00E6\u01FD\u01E3' },
3528
+ { base: 'ao', letters: '\uA735' },
3529
+ { base: 'au', letters: '\uA737' },
3530
+ { base: 'av', letters: '\uA739\uA73B' },
3531
+ { base: 'ay', letters: '\uA73D' },
3532
+ {
3533
+ base: 'b',
3534
+ letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
3535
+ },
3536
+ {
3537
+ base: 'c',
3538
+ letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
3539
+ },
3540
+ {
3541
+ base: 'd',
3542
+ letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
3543
+ },
3544
+ { base: 'dz', letters: '\u01F3\u01C6' },
3545
+ {
3546
+ base: 'e',
3547
+ letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
3548
+ },
3549
+ { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
3550
+ {
3551
+ base: 'g',
3552
+ letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
3553
+ },
3554
+ {
3555
+ base: 'h',
3556
+ letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
3557
+ },
3558
+ { base: 'hv', letters: '\u0195' },
3559
+ {
3560
+ base: 'i',
3561
+ letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
3562
+ },
3563
+ { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
3564
+ {
3565
+ base: 'k',
3566
+ letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
3567
+ },
3568
+ {
3569
+ base: 'l',
3570
+ letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
3571
+ },
3572
+ { base: 'lj', letters: '\u01C9' },
3573
+ { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
3574
+ {
3575
+ base: 'n',
3576
+ letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
3577
+ },
3578
+ { base: 'nj', letters: '\u01CC' },
3579
+ {
3580
+ base: 'o',
3581
+ letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
3582
+ },
3583
+ { base: 'oi', letters: '\u01A3' },
3584
+ { base: 'ou', letters: '\u0223' },
3585
+ { base: 'oo', letters: '\uA74F' },
3586
+ {
3587
+ base: 'p',
3588
+ letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
3589
+ },
3590
+ { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
3591
+ {
3592
+ base: 'r',
3593
+ letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
3594
+ },
3595
+ {
3596
+ base: 's',
3597
+ letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
3598
+ },
3599
+ {
3600
+ base: 't',
3601
+ letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
3602
+ },
3603
+ { base: 'tz', letters: '\uA729' },
3604
+ {
3605
+ base: 'u',
3606
+ letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
3607
+ },
3608
+ { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
3609
+ { base: 'vy', letters: '\uA761' },
3610
+ {
3611
+ base: 'w',
3612
+ letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
3613
+ },
3614
+ { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
3615
+ {
3616
+ base: 'y',
3617
+ letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
3618
+ },
3619
+ {
3620
+ base: 'z',
3621
+ letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
3622
+ },
3623
+ ];
3624
+ /**
3625
+ * Map of letters from diacritic variant to diacritless variant
3626
+ * Contains lowercase and uppercase separatelly
3948
3627
  *
3949
- * @public exported from `@promptbook/utils`
3950
- */
3951
- function countCharacters(text) {
3952
- // Remove null characters
3953
- text = text.replace(/\0/g, '');
3954
- // Replace emojis (and also ZWJ sequence) with hyphens
3955
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
3956
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
3957
- text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
3958
- return text.length;
3959
- }
3960
-
3961
- /**
3962
- * Counts number of lines in the text
3628
+ * > "á" => "a"
3629
+ * > "ě" => "e"
3630
+ * > "Ă" => "A"
3631
+ * > ...
3963
3632
  *
3964
3633
  * @public exported from `@promptbook/utils`
3965
3634
  */
3966
- function countLines(text) {
3967
- if (text === '') {
3968
- return 0;
3635
+ var DIACRITIC_VARIANTS_LETTERS = {};
3636
+ // tslint:disable-next-line: prefer-for-of
3637
+ for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
3638
+ var letters = defaultDiacriticsRemovalMap[i].letters;
3639
+ // tslint:disable-next-line: prefer-for-of
3640
+ for (var j = 0; j < letters.length; j++) {
3641
+ DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
3969
3642
  }
3970
- return text.split('\n').length;
3971
3643
  }
3644
+ // <- TODO: [🍓] Put to maker function to save execution time if not needed
3645
+ /*
3646
+ @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
3647
+ Licensed under the Apache License, Version 2.0 (the "License");
3648
+ you may not use this file except in compliance with the License.
3649
+ You may obtain a copy of the License at
3972
3650
 
3973
- /**
3974
- * Counts number of pages in the text
3975
- *
3976
- * @public exported from `@promptbook/utils`
3977
- */
3978
- function countPages(text) {
3979
- var sentencesPerPage = 5; // Assuming each page has 5 sentences
3980
- var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3981
- var pageCount = Math.ceil(sentences.length / sentencesPerPage);
3982
- return pageCount;
3983
- }
3651
+ http://www.apache.org/licenses/LICENSE-2.0
3984
3652
 
3985
- /**
3986
- * Counts number of paragraphs in the text
3987
- *
3988
- * @public exported from `@promptbook/utils`
3989
- */
3990
- function countParagraphs(text) {
3991
- return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
3992
- }
3653
+ Unless required by applicable law or agreed to in writing, software
3654
+ distributed under the License is distributed on an "AS IS" BASIS,
3655
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3656
+ See the License for the specific language governing permissions and
3657
+ limitations under the License.
3658
+ */
3993
3659
 
3994
3660
  /**
3995
- * Split text into sentences
3661
+ * @@@
3996
3662
  *
3663
+ * @param input @@@
3664
+ * @returns @@@
3997
3665
  * @public exported from `@promptbook/utils`
3998
3666
  */
3999
- function splitIntoSentences(text) {
4000
- return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3667
+ function removeDiacritics(input) {
3668
+ /*eslint no-control-regex: "off"*/
3669
+ return input.replace(/[^\u0000-\u007E]/g, function (a) {
3670
+ return DIACRITIC_VARIANTS_LETTERS[a] || a;
3671
+ });
4001
3672
  }
4002
3673
  /**
4003
- * Counts number of sentences in the text
4004
- *
4005
- * @public exported from `@promptbook/utils`
3674
+ * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
4006
3675
  */
4007
- function countSentences(text) {
4008
- return splitIntoSentences(text).length;
4009
- }
4010
3676
 
4011
3677
  /**
4012
3678
  * Counts number of words in the text
@@ -4104,18 +3770,19 @@
4104
3770
  */
4105
3771
  function executeAttempts(options) {
4106
3772
  return __awaiter(this, void 0, void 0, function () {
4107
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
3773
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4108
3774
  return __generator(this, function (_a) {
4109
3775
  switch (_a.label) {
4110
3776
  case 0:
4111
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4112
- maxExecutionAttempts = settings.maxExecutionAttempts;
3777
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
4113
3778
  $ongoingTemplateResult = {
4114
3779
  $result: null,
4115
3780
  $resultString: null,
4116
3781
  $expectError: null,
4117
3782
  $scriptPipelineExecutionErrors: [],
4118
3783
  };
3784
+ _llms = arrayableToArray(tools.llm);
3785
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
4119
3786
  _loop_1 = function (attempt) {
4120
3787
  var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
4121
3788
  var e_1, _q, e_3, _r, e_2, _s;
@@ -4184,7 +3851,9 @@
4184
3851
  return [3 /*break*/, 9];
4185
3852
  case 4:
4186
3853
  _d = $ongoingTemplateResult;
4187
- return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3854
+ return [4 /*yield*/, llmTools.callChatModel(
3855
+ // <- TODO: [🧁] Check that `callChatModel` is defined
3856
+ $deepFreeze($ongoingTemplateResult.$prompt))];
4188
3857
  case 5:
4189
3858
  _d.$chatResult = _t.sent();
4190
3859
  // TODO: [🍬] Destroy chatThread
@@ -4193,7 +3862,9 @@
4193
3862
  return [3 /*break*/, 10];
4194
3863
  case 6:
4195
3864
  _e = $ongoingTemplateResult;
4196
- return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3865
+ return [4 /*yield*/, llmTools.callCompletionModel(
3866
+ // <- TODO: [🧁] Check that `callCompletionModel` is defined
3867
+ $deepFreeze($ongoingTemplateResult.$prompt))];
4197
3868
  case 7:
4198
3869
  _e.$completionResult = _t.sent();
4199
3870
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
@@ -4469,12 +4140,12 @@
4469
4140
  */
4470
4141
  function executeFormatSubvalues(options) {
4471
4142
  return __awaiter(this, void 0, void 0, function () {
4472
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4143
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4473
4144
  var _this = this;
4474
4145
  return __generator(this, function (_a) {
4475
4146
  switch (_a.label) {
4476
4147
  case 0:
4477
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4148
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4478
4149
  if (template.foreach === undefined) {
4479
4150
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4480
4151
  }
@@ -4504,7 +4175,7 @@
4504
4175
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4505
4176
  }
4506
4177
  if (formatDefinition.formatName === 'CSV') {
4507
- formatSettings = settings.csvSettings;
4178
+ formatSettings = csvSettings;
4508
4179
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4509
4180
  }
4510
4181
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4657,13 +4328,12 @@
4657
4328
  */
4658
4329
  function executeTemplate(options) {
4659
4330
  return __awaiter(this, void 0, void 0, function () {
4660
- var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4661
- var e_1, _f, _g;
4662
- return __generator(this, function (_h) {
4663
- switch (_h.label) {
4331
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4332
+ var e_1, _g, _h;
4333
+ return __generator(this, function (_j) {
4334
+ switch (_j.label) {
4664
4335
  case 0:
4665
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, llmTools = options.llmTools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4666
- maxExecutionAttempts = settings.maxExecutionAttempts;
4336
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4667
4337
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4668
4338
  title = currentTemplate.title;
4669
4339
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -4678,7 +4348,7 @@
4678
4348
  // <- [🍸]
4679
4349
  })];
4680
4350
  case 1:
4681
- _h.sent();
4351
+ _j.sent();
4682
4352
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
4683
4353
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
4684
4354
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4689,15 +4359,15 @@
4689
4359
  .map(function (name) { return "{".concat(name, "}"); })
4690
4360
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4691
4361
  }
4692
- _b = (_a = Object).freeze;
4693
- _c = [{}];
4362
+ _c = (_b = Object).freeze;
4363
+ _d = [{}];
4694
4364
  return [4 /*yield*/, getReservedParametersForTemplate({
4695
4365
  preparedPipeline: preparedPipeline,
4696
4366
  template: currentTemplate,
4697
4367
  pipelineIdentification: pipelineIdentification,
4698
4368
  })];
4699
4369
  case 2:
4700
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4370
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
4701
4371
  definedParameterNames = new Set(Object.keys(definedParameters));
4702
4372
  parameters = {};
4703
4373
  _loop_1 = function (parameterName) {
@@ -4717,15 +4387,15 @@
4717
4387
  try {
4718
4388
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4719
4389
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4720
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4721
- parameterName = _e.value;
4390
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
4391
+ parameterName = _f.value;
4722
4392
  _loop_1(parameterName);
4723
4393
  }
4724
4394
  }
4725
4395
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4726
4396
  finally {
4727
4397
  try {
4728
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4398
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
4729
4399
  }
4730
4400
  finally { if (e_1) throw e_1.error; }
4731
4401
  }
@@ -4745,13 +4415,11 @@
4745
4415
  template: currentTemplate,
4746
4416
  preparedPipeline: preparedPipeline,
4747
4417
  tools: tools,
4748
- llmTools: llmTools,
4749
- settings: settings,
4750
4418
  $executionReport: $executionReport,
4751
4419
  pipelineIdentification: pipelineIdentification,
4752
4420
  })];
4753
4421
  case 3:
4754
- resultString = _h.sent();
4422
+ resultString = _j.sent();
4755
4423
  return [4 /*yield*/, onProgress({
4756
4424
  name: name,
4757
4425
  title: title,
@@ -4763,12 +4431,12 @@
4763
4431
  // <- [🍸]
4764
4432
  })];
4765
4433
  case 4:
4766
- _h.sent();
4767
- return [2 /*return*/, Object.freeze((_g = {},
4768
- _g[currentTemplate.resultingParameterName] =
4434
+ _j.sent();
4435
+ return [2 /*return*/, Object.freeze((_h = {},
4436
+ _h[currentTemplate.resultingParameterName] =
4769
4437
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4770
4438
  resultString,
4771
- _g))];
4439
+ _h))];
4772
4440
  }
4773
4441
  });
4774
4442
  });
@@ -4827,18 +4495,15 @@
4827
4495
  */
4828
4496
  function executePipeline(options) {
4829
4497
  return __awaiter(this, void 0, void 0, function () {
4830
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, llmTools, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4498
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4831
4499
  var e_1, _f, e_2, _g;
4832
4500
  return __generator(this, function (_h) {
4833
4501
  switch (_h.label) {
4834
4502
  case 0:
4835
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4836
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4503
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4837
4504
  preparedPipeline = options.preparedPipeline;
4838
- llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
4839
4505
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4840
- return [4 /*yield*/, preparePipeline(pipeline, {
4841
- llmTools: llmTools,
4506
+ return [4 /*yield*/, preparePipeline(pipeline, tools, {
4842
4507
  rootDirname: rootDirname,
4843
4508
  isVerbose: isVerbose,
4844
4509
  maxParallelCount: maxParallelCount,
@@ -5020,13 +4685,7 @@
5020
4685
  return [3 /*break*/, 4];
5021
4686
  case 3:
5022
4687
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
5023
- work_1 = executeTemplate({
5024
- currentTemplate: currentTemplate,
5025
- preparedPipeline: preparedPipeline,
5026
- parametersToPass: parametersToPass,
5027
- tools: tools,
5028
- llmTools: llmTools,
5029
- onProgress: function (progress) {
4688
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
5030
4689
  if (isReturned) {
5031
4690
  throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
5032
4691
  .split('\n')
@@ -5036,11 +4695,7 @@
5036
4695
  if (onProgress) {
5037
4696
  onProgress(progress);
5038
4697
  }
5039
- },
5040
- settings: settings,
5041
- $executionReport: executionReport,
5042
- pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
5043
- })
4698
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
5044
4699
  .then(function (newParametersToPass) {
5045
4700
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
5046
4701
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -5144,8 +4799,7 @@
5144
4799
  */
5145
4800
  function createPipelineExecutor(options) {
5146
4801
  var _this = this;
5147
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
5148
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
4802
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
5149
4803
  validatePipeline(pipeline);
5150
4804
  var pipelineIdentification = (function () {
5151
4805
  // Note: This is a 😐 implementation of [🚞]
@@ -5179,14 +4833,12 @@
5179
4833
  tools: tools,
5180
4834
  onProgress: onProgress,
5181
4835
  pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
5182
- settings: {
5183
- maxExecutionAttempts: maxExecutionAttempts,
5184
- maxParallelCount: maxParallelCount,
5185
- csvSettings: csvSettings,
5186
- isVerbose: isVerbose,
5187
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5188
- rootDirname: rootDirname,
5189
- },
4836
+ maxExecutionAttempts: maxExecutionAttempts,
4837
+ maxParallelCount: maxParallelCount,
4838
+ csvSettings: csvSettings,
4839
+ isVerbose: isVerbose,
4840
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4841
+ rootDirname: rootDirname,
5190
4842
  })];
5191
4843
  });
5192
4844
  }); };
@@ -5197,678 +4849,531 @@
5197
4849
  */
5198
4850
 
5199
4851
  /**
5200
- * Scraper for markdown files
4852
+ * Prepares the persona for the pipeline
4853
+ *
4854
+ * @see https://github.com/webgptorg/promptbook/discussions/22
4855
+ * @public exported from `@promptbook/core`
4856
+ */
4857
+ function preparePersona(personaDescription, tools, options) {
4858
+ return __awaiter(this, void 0, void 0, function () {
4859
+ var _a, isVerbose, collection, preparePersonaExecutor, _b, _llms, llmTools, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
4860
+ var _c;
4861
+ return __generator(this, function (_d) {
4862
+ switch (_d.label) {
4863
+ case 0:
4864
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4865
+ if (tools === undefined || tools.llm === undefined) {
4866
+ throw new MissingToolsError('LLM tools are required for preparing persona');
4867
+ }
4868
+ collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
4869
+ _b = createPipelineExecutor;
4870
+ _c = {};
4871
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
4872
+ case 1:
4873
+ preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
4874
+ _c.tools = tools,
4875
+ _c)]);
4876
+ _llms = arrayableToArray(tools.llm);
4877
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
4878
+ return [4 /*yield*/, llmTools.listModels()];
4879
+ case 2:
4880
+ availableModels = _d.sent();
4881
+ availableModelNames = availableModels
4882
+ .filter(function (_a) {
4883
+ var modelVariant = _a.modelVariant;
4884
+ return modelVariant === 'CHAT';
4885
+ })
4886
+ .map(function (_a) {
4887
+ var modelName = _a.modelName;
4888
+ return modelName;
4889
+ })
4890
+ .join(',');
4891
+ return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
4892
+ case 3:
4893
+ result = _d.sent();
4894
+ assertsExecutionSuccessful(result);
4895
+ outputParameters = result.outputParameters;
4896
+ modelRequirementsRaw = outputParameters.modelRequirements;
4897
+ modelRequirements = JSON.parse(modelRequirementsRaw);
4898
+ if (isVerbose) {
4899
+ console.info("PERSONA ".concat(personaDescription), modelRequirements);
4900
+ }
4901
+ modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
4902
+ return [2 /*return*/, {
4903
+ modelVariant: 'CHAT',
4904
+ modelName: modelName,
4905
+ systemMessage: systemMessage,
4906
+ temperature: temperature,
4907
+ }];
4908
+ }
4909
+ });
4910
+ });
4911
+ }
4912
+ /**
4913
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4914
+ * TODO: [🏢] !! Check validity of `modelName` in pipeline
4915
+ * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4916
+ * TODO: [🏢] !! Check validity of `temperature` in pipeline
4917
+ */
4918
+
4919
+ /**
4920
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
5201
4921
  *
5202
- * @see `documentationUrl` for more details
5203
4922
  * @public exported from `@promptbook/core`
5204
4923
  */
5205
- var markdownScraper = {
5206
- /**
5207
- * Mime types that this scraper can handle
5208
- */
5209
- mimeTypes: ['text/markdown', 'text/plain'],
5210
- /**
5211
- * Link to documentation
5212
- */
5213
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5214
- /**
5215
- * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
5216
- */
5217
- scrape: function (source, options) {
5218
- return __awaiter(this, void 0, void 0, function () {
5219
- var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, knowledgeContent, result, outputParameters, knowledgePiecesRaw, knowledgeTextPieces, knowledge;
5220
- var _f, _g, _h;
5221
- var _this = this;
5222
- return __generator(this, function (_j) {
5223
- switch (_j.label) {
5224
- case 0:
5225
- llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5226
- if (llmTools === undefined) {
5227
- throw new MissingToolsError('LLM tools are required for scraping external files');
5228
- // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
5229
- }
5230
- TODO_USE(maxParallelCount); // <- [🪂]
5231
- collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
5232
- _c = createPipelineExecutor;
5233
- _f = {};
5234
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
5235
- case 1:
5236
- prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
5237
- _f.tools = {
5238
- llm: llmTools,
5239
- },
5240
- _f)]);
5241
- _d = createPipelineExecutor;
5242
- _g = {};
5243
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
5244
- case 2:
5245
- prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
5246
- _g.tools = {
5247
- llm: llmTools,
5248
- },
5249
- _g)]);
5250
- _e = createPipelineExecutor;
5251
- _h = {};
5252
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
5253
- case 3:
5254
- prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
5255
- _h.tools = {
5256
- llm: llmTools,
5257
- },
5258
- _h)]);
5259
- return [4 /*yield*/, source.asText()];
5260
- case 4:
5261
- knowledgeContent = _j.sent();
5262
- return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
5263
- case 5:
5264
- result = _j.sent();
5265
- assertsExecutionSuccessful(result);
5266
- outputParameters = result.outputParameters;
5267
- knowledgePiecesRaw = outputParameters.knowledgePieces;
5268
- knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
5269
- // <- TODO: [main] !!!!! Smarter split and filter out empty pieces
5270
- if (isVerbose) {
5271
- console.info('knowledgeTextPieces:', knowledgeTextPieces);
5272
- }
5273
- return [4 /*yield*/, Promise.all(
5274
- // TODO: [🪂] !! Do not send all at once but in chunks
5275
- knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
5276
- var name, title, knowledgePieceContent, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
5277
- return __generator(this, function (_c) {
5278
- switch (_c.label) {
5279
- case 0:
5280
- name = "piece-".concat(i);
5281
- title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
5282
- knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
5283
- keywords = [];
5284
- index = [];
5285
- _c.label = 1;
5286
- case 1:
5287
- _c.trys.push([1, 7, , 8]);
5288
- return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
5289
- case 2:
5290
- titleResult = _c.sent();
5291
- _a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
5292
- title = spaceTrim__default["default"](titleRaw) /* <- TODO: Maybe do in pipeline */;
5293
- name = titleToName(title);
5294
- return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
5295
- case 3:
5296
- keywordsResult = _c.sent();
5297
- _b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
5298
- keywords = (keywordsRaw || '')
5299
- .split(',')
5300
- .map(function (keyword) { return keyword.trim(); })
5301
- .filter(function (keyword) { return keyword !== ''; });
5302
- if (isVerbose) {
5303
- console.info("Keywords for \"".concat(title, "\":"), keywords);
5304
- }
5305
- if (!!llmTools.callEmbeddingModel) return [3 /*break*/, 4];
5306
- // TODO: [🟥] Detect browser / node and make it colorfull
5307
- console.error('No callEmbeddingModel function provided');
5308
- return [3 /*break*/, 6];
5309
- case 4: return [4 /*yield*/, llmTools.callEmbeddingModel({
5310
- title: "Embedding for ".concat(title) /* <- Note: No impact on embedding result itself, just for logging */,
5311
- parameters: {},
5312
- content: knowledgePieceContent,
5313
- modelRequirements: {
5314
- modelVariant: 'EMBEDDING',
5315
- },
5316
- })];
5317
- case 5:
5318
- embeddingResult = _c.sent();
5319
- index.push({
5320
- modelName: embeddingResult.modelName,
5321
- position: embeddingResult.content,
5322
- });
5323
- _c.label = 6;
5324
- case 6: return [3 /*break*/, 8];
5325
- case 7:
5326
- error_1 = _c.sent();
5327
- // Note: Here is expected error:
5328
- // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
5329
- if (!(error_1 instanceof PipelineExecutionError)) {
5330
- throw error_1;
5331
- }
5332
- // TODO: [🟥] Detect browser / node and make it colorfull
5333
- console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
5334
- return [3 /*break*/, 8];
5335
- case 8: return [2 /*return*/, {
5336
- name: name,
5337
- title: title,
5338
- content: knowledgePieceContent,
5339
- keywords: keywords,
5340
- index: index,
5341
- // <- TODO: [☀] sources,
5342
- }];
5343
- }
5344
- });
5345
- }); }))];
5346
- case 6:
5347
- knowledge = _j.sent();
5348
- return [2 /*return*/, knowledge];
5349
- }
5350
- });
5351
- });
5352
- },
5353
- } /* TODO: [🦷] as const */;
4924
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4925
+ __extends(KnowledgeScrapeError, _super);
4926
+ function KnowledgeScrapeError(message) {
4927
+ var _this = _super.call(this, message) || this;
4928
+ _this.name = 'KnowledgeScrapeError';
4929
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4930
+ return _this;
4931
+ }
4932
+ return KnowledgeScrapeError;
4933
+ }(Error));
4934
+
5354
4935
  /**
5355
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5356
- * TODO: [🪂] Do it in parallel 11:11
5357
- * TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
5358
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4936
+ * @@@
4937
+ *
4938
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4939
+ *
4940
+ * @private internal function of `$Register`
5359
4941
  */
4942
+ function $getGlobalScope() {
4943
+ return Function('return this')();
4944
+ }
5360
4945
 
5361
4946
  /**
5362
- * Scraper of .docx and .odt files
4947
+ * @@@
5363
4948
  *
5364
- * @see `documentationUrl` for more details
5365
- * @public exported from `@promptbook/core`
4949
+ * @param text @@@
4950
+ * @returns @@@
4951
+ * @example 'HELLO_WORLD'
4952
+ * @example 'I_LOVE_PROMPTBOOK'
4953
+ * @public exported from `@promptbook/utils`
5366
4954
  */
5367
- var documentScraper = {
5368
- /**
5369
- * Mime types that this scraper can handle
5370
- */
5371
- mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
5372
- /**
5373
- * Link to documentation
5374
- */
5375
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5376
- /**
5377
- * Convert the `.docx` or `.odt` to `.md` file and returns intermediate source
5378
- *
5379
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5380
- */
5381
- $convert: function (source, options) {
5382
- return __awaiter(this, void 0, void 0, function () {
5383
- var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, command_1;
5384
- return __generator(this, function (_e) {
5385
- switch (_e.label) {
5386
- case 0:
5387
- _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5388
- if (!$isRunningInNode()) {
5389
- throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
5390
- }
5391
- if (externalProgramsPaths.pandocPath === undefined) {
5392
- throw new MissingToolsError('Pandoc is required for scraping .docx files');
5393
- }
5394
- if (source.filename === null) {
5395
- // TODO: [🧠] Maybe save file as temporary
5396
- throw new KnowledgeScrapeError('When parsing .docx file, it must be real file in the file system');
5397
- }
5398
- extension = getFileExtension(source.filename);
5399
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5400
- rootDirname: rootDirname,
5401
- cacheDirname: cacheDirname,
5402
- isCacheCleaned: isCacheCleaned,
5403
- extension: 'md',
5404
- isVerbose: isVerbose,
5405
- })];
5406
- case 1:
5407
- cacheFilehandler = _e.sent();
5408
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5409
- case 2:
5410
- if (!!(_e.sent())) return [3 /*break*/, 5];
5411
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
5412
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
5413
- return [4 /*yield*/, $execCommand(command_1)];
5414
- case 3:
5415
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
5416
- _e.sent();
5417
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5418
- case 4:
5419
- // Note: [0]
5420
- if (!(_e.sent())) {
5421
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n File that was supposed to be created by Pandoc does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
5422
- }
5423
- _e.label = 5;
5424
- case 5: return [2 /*return*/, cacheFilehandler];
5425
- }
5426
- });
5427
- });
5428
- },
5429
- /**
5430
- * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
5431
- */
5432
- scrape: function (source, options) {
5433
- return __awaiter(this, void 0, void 0, function () {
5434
- var cacheFilehandler, markdownSource, knowledge;
5435
- return __generator(this, function (_a) {
5436
- switch (_a.label) {
5437
- case 0: return [4 /*yield*/, documentScraper.$convert(source, options)];
5438
- case 1:
5439
- cacheFilehandler = _a.sent();
5440
- markdownSource = {
5441
- source: source.source,
5442
- filename: cacheFilehandler.filename,
5443
- url: null,
5444
- mimeType: 'text/markdown',
5445
- asText: function () {
5446
- return __awaiter(this, void 0, void 0, function () {
5447
- return __generator(this, function (_a) {
5448
- switch (_a.label) {
5449
- case 0: return [4 /*yield*/, promises.readFile(cacheFilehandler.filename, 'utf-8')];
5450
- case 1:
5451
- // Note: [0] In $convert we check that the file exists
5452
- return [2 /*return*/, _a.sent()];
5453
- }
5454
- });
5455
- });
5456
- },
5457
- asJson: function () {
5458
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5459
- },
5460
- asBlob: function () {
5461
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5462
- },
5463
- };
5464
- knowledge = markdownScraper.scrape(markdownSource, options);
5465
- return [4 /*yield*/, cacheFilehandler.destroy()];
5466
- case 2:
5467
- _a.sent();
5468
- return [2 /*return*/, knowledge];
5469
- }
5470
- });
5471
- });
5472
- },
5473
- } /* TODO: [🦷] as const */;
4955
+ function normalizeTo_SCREAMING_CASE(text) {
4956
+ var e_1, _a;
4957
+ var charType;
4958
+ var lastCharType = 'OTHER';
4959
+ var normalizedName = '';
4960
+ try {
4961
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4962
+ var char = text_1_1.value;
4963
+ var normalizedChar = void 0;
4964
+ if (/^[a-z]$/.test(char)) {
4965
+ charType = 'LOWERCASE';
4966
+ normalizedChar = char.toUpperCase();
4967
+ }
4968
+ else if (/^[A-Z]$/.test(char)) {
4969
+ charType = 'UPPERCASE';
4970
+ normalizedChar = char;
4971
+ }
4972
+ else if (/^[0-9]$/.test(char)) {
4973
+ charType = 'NUMBER';
4974
+ normalizedChar = char;
4975
+ }
4976
+ else {
4977
+ charType = 'OTHER';
4978
+ normalizedChar = '_';
4979
+ }
4980
+ if (charType !== lastCharType &&
4981
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4982
+ !(lastCharType === 'NUMBER') &&
4983
+ !(charType === 'NUMBER')) {
4984
+ normalizedName += '_';
4985
+ }
4986
+ normalizedName += normalizedChar;
4987
+ lastCharType = charType;
4988
+ }
4989
+ }
4990
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4991
+ finally {
4992
+ try {
4993
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
4994
+ }
4995
+ finally { if (e_1) throw e_1.error; }
4996
+ }
4997
+ normalizedName = normalizedName.replace(/_+/g, '_');
4998
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
4999
+ normalizedName = normalizedName.replace(/^_/, '');
5000
+ normalizedName = normalizedName.replace(/_$/, '');
5001
+ return normalizedName;
5002
+ }
5474
5003
  /**
5475
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5476
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5477
- * TODO: [🪂] Do it in parallel 11:11
5478
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5479
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5004
+ * TODO: Tests
5005
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
5006
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
5007
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
5008
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
5009
+ * TODO: [🌺] Use some intermediate util splitWords
5480
5010
  */
5481
5011
 
5482
5012
  /**
5483
- * Scraper for .docx files
5013
+ * @@@
5484
5014
  *
5485
- * @see `documentationUrl` for more details
5486
- * @public exported from `@promptbook/core`
5015
+ * @param text @@@
5016
+ * @returns @@@
5017
+ * @example 'hello_world'
5018
+ * @example 'i_love_promptbook'
5019
+ * @public exported from `@promptbook/utils`
5487
5020
  */
5488
- var legacyDocumentScraper = {
5489
- /**
5490
- * Mime types that this scraper can handle
5491
- */
5492
- mimeTypes: ['application/msword', 'text/rtf'],
5493
- /**
5494
- * Link to documentation
5495
- */
5496
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5497
- /**
5498
- * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
5499
- *
5500
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5501
- */
5502
- $convert: function (source, options) {
5503
- return __awaiter(this, void 0, void 0, function () {
5504
- var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
5505
- return __generator(this, function (_e) {
5506
- switch (_e.label) {
5507
- case 0:
5508
- _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5509
- if (!$isRunningInNode()) {
5510
- throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
5511
- }
5512
- if (externalProgramsPaths.libreOfficePath === undefined) {
5513
- throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
5514
- }
5515
- if (source.filename === null) {
5516
- // TODO: [🧠] Maybe save file as temporary
5517
- throw new KnowledgeScrapeError('When parsing .doc or .rtf file, it must be real file in the file system');
5518
- }
5519
- extension = getFileExtension(source.filename);
5520
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5521
- rootDirname: rootDirname,
5522
- cacheDirname: cacheDirname,
5523
- isCacheCleaned: isCacheCleaned,
5524
- extension: 'docx',
5525
- isVerbose: isVerbose,
5526
- })];
5527
- case 1:
5528
- cacheFilehandler = _e.sent();
5529
- if (isVerbose) {
5530
- console.info("documentScraper: Converting .".concat(extension, " -> .docx"));
5531
- }
5532
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5533
- case 2:
5534
- if (!!(_e.sent())) return [3 /*break*/, 8];
5535
- documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
5536
- .split('\\')
5537
- .join('/');
5538
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
5539
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
5540
- return [4 /*yield*/, $execCommand(command_1)];
5541
- case 3:
5542
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
5543
- _e.sent();
5544
- return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
5545
- case 4:
5546
- files_1 = _e.sent();
5547
- if (files_1.length !== 1) {
5548
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n "); }));
5549
- }
5550
- file = files_1[0];
5551
- return [4 /*yield*/, promises.rename(path.join(documentSourceOutdirPathForLibreOffice_1, file), cacheFilehandler.filename)];
5552
- case 5:
5553
- _e.sent();
5554
- return [4 /*yield*/, promises.rmdir(documentSourceOutdirPathForLibreOffice_1)];
5555
- case 6:
5556
- _e.sent();
5557
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5558
- case 7:
5559
- if (!(_e.sent())) {
5560
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n File that was supposed to be created by LibreOffice does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
5561
- }
5562
- _e.label = 8;
5563
- case 8: return [2 /*return*/, cacheFilehandler];
5564
- }
5565
- });
5566
- });
5567
- },
5568
- /**
5569
- * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
5570
- */
5571
- scrape: function (source, options) {
5572
- return __awaiter(this, void 0, void 0, function () {
5573
- var cacheFilehandler, markdownSource, knowledge;
5574
- return __generator(this, function (_a) {
5575
- switch (_a.label) {
5576
- case 0: return [4 /*yield*/, legacyDocumentScraper.$convert(source, options)];
5577
- case 1:
5578
- cacheFilehandler = _a.sent();
5579
- markdownSource = {
5580
- source: source.source,
5581
- filename: cacheFilehandler.filename,
5582
- url: null,
5583
- mimeType: 'text/markdown',
5584
- asText: function () {
5585
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asText`');
5586
- },
5587
- asJson: function () {
5588
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
5589
- },
5590
- asBlob: function () {
5591
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
5592
- },
5593
- };
5594
- knowledge = documentScraper.scrape(markdownSource, options);
5595
- return [4 /*yield*/, cacheFilehandler.destroy()];
5596
- case 2:
5597
- _a.sent();
5598
- return [2 /*return*/, knowledge];
5599
- }
5600
- });
5601
- });
5602
- },
5603
- } /* TODO: [🦷] as const */;
5021
+ function normalizeTo_snake_case(text) {
5022
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
5023
+ }
5024
+
5604
5025
  /**
5605
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5606
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5607
- * TODO: [🪂] Do it in parallel 11:11
5608
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5609
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5026
+ * Register is @@@
5027
+ *
5028
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
5029
+ *
5030
+ * @private internal utility, exported are only signleton instances of this class
5610
5031
  */
5032
+ var $Register = /** @class */ (function () {
5033
+ function $Register(registerName) {
5034
+ this.registerName = registerName;
5035
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
5036
+ var globalScope = $getGlobalScope();
5037
+ if (globalScope[storageName] === undefined) {
5038
+ globalScope[storageName] = [];
5039
+ }
5040
+ else if (!Array.isArray(globalScope[storageName])) {
5041
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
5042
+ }
5043
+ this.storage = globalScope[storageName];
5044
+ }
5045
+ $Register.prototype.list = function () {
5046
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
5047
+ return this.storage;
5048
+ };
5049
+ $Register.prototype.register = function (registered) {
5050
+ var packageName = registered.packageName, className = registered.className;
5051
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
5052
+ var existingRegistration = this.storage[existingRegistrationIndex];
5053
+ if (!existingRegistration) {
5054
+ this.storage.push(registered);
5055
+ }
5056
+ else {
5057
+ this.storage[existingRegistrationIndex] = registered;
5058
+ }
5059
+ return {
5060
+ registerName: this.registerName,
5061
+ packageName: packageName,
5062
+ className: className,
5063
+ get isDestroyed() {
5064
+ return false;
5065
+ },
5066
+ destroy: function () {
5067
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
5068
+ },
5069
+ };
5070
+ };
5071
+ return $Register;
5072
+ }());
5611
5073
 
5612
5074
  /**
5613
- * Scraper for .docx files
5075
+ * @@@
5614
5076
  *
5615
- * @see `documentationUrl` for more details
5077
+ * Note: `$` is used to indicate that this interacts with the global scope
5078
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5616
5079
  * @public exported from `@promptbook/core`
5617
5080
  */
5618
- var pdfScraper = {
5619
- /**
5620
- * Mime types that this scraper can handle
5621
- */
5622
- mimeTypes: ['application/pdf'],
5623
- /**
5624
- * Link to documentation
5625
- */
5626
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5627
- /**
5628
- * Converts the `.pdf` file to `.md` file and returns intermediate source
5629
- */
5630
- $convert: function (source, options) {
5631
- return __awaiter(this, void 0, void 0, function () {
5632
- return __generator(this, function (_a) {
5633
- TODO_USE(source);
5634
- TODO_USE(options);
5635
- throw new NotYetImplementedError('PDF conversion not yet implemented');
5636
- });
5637
- });
5638
- },
5639
- /**
5640
- * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
5641
- */
5642
- scrape: function (source, options) {
5643
- return __awaiter(this, void 0, void 0, function () {
5644
- return __generator(this, function (_a) {
5645
- TODO_USE(source);
5646
- TODO_USE(options);
5647
- /*
5648
- const {
5649
- externalProgramsPaths = {},
5650
- cacheDirname = SCRAPE_CACHE_DIRNAME,
5651
- isCacheCleaned = false,
5652
- isVerbose = IS_VERBOSE,
5653
- } = options;
5654
- */
5655
- throw new NotYetImplementedError('PDF scraping not yet implemented');
5656
- });
5657
- });
5658
- },
5659
- } /* TODO: [🦷] as const */;
5081
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
5660
5082
  /**
5661
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5662
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5663
- * TODO: [🪂] Do it in parallel 11:11
5664
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5665
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5083
+ * TODO: [®] DRY Register logic
5666
5084
  */
5667
5085
 
5668
5086
  /**
5669
- * A converter instance that uses showdown and highlight extensions
5087
+ * @@@
5670
5088
  *
5671
- * @type {Converter}
5672
- * @private for markdown and html knowledge scrapers
5089
+ * Note: `$` is used to indicate that this interacts with the global scope
5090
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5091
+ * @public exported from `@promptbook/core`
5673
5092
  */
5674
- var markdownConverter = new showdown.Converter({
5675
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
5676
- /*
5677
- > import showdownHighlight from 'showdown-highlight';
5678
- > extensions: [
5679
- > showdownHighlight({
5680
- > // Whether to add the classes to the <pre> tag, default is false
5681
- > pre: true,
5682
- > // Whether to use hljs' auto language detection, default is true
5683
- > auto_detection: true,
5684
- > }),
5685
- > ],
5686
- */
5687
- });
5093
+ var $scrapersRegister = new $Register('scraper_constructors');
5688
5094
  /**
5689
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
5690
- * TODO: !!!!!! Lazy-make converter
5095
+ * TODO: [®] DRY Register logic
5691
5096
  */
5692
5097
 
5693
5098
  /**
5694
- * Scraper for .docx files
5099
+ * Creates a message with all registered scrapers
5695
5100
  *
5696
- * @see `documentationUrl` for more details
5697
- * @public exported from `@promptbook/core`
5101
+ * Note: This function is used to create a (error) message when there is no scraper for particular mime type
5102
+ *
5103
+ * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
5698
5104
  */
5699
- var websiteScraper = {
5700
- /**
5701
- * Mime types that this scraper can handle
5702
- */
5703
- mimeTypes: ['text/html'],
5704
- /**
5705
- * Link to documentation
5706
- */
5707
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5105
+ function $registeredScrapersMessage() {
5106
+ var e_1, _a, e_2, _b;
5708
5107
  /**
5709
- * Convert the website to `.md` file and returns intermediate source
5710
- *
5711
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5108
+ * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
5712
5109
  */
5713
- $convert: function (source, options) {
5714
- return __awaiter(this, void 0, void 0, function () {
5715
- var
5716
- // TODO: [🧠] Maybe in node use headless browser not just JSDOM
5717
- // externalProgramsPaths = {},
5718
- rootDirname, _a, cacheDirname, _b, isCacheCleaned, _c, isVerbose, jsdom$1, _d, reader, article, html, i, cacheFilehandler, markdown;
5719
- return __generator(this, function (_e) {
5720
- switch (_e.label) {
5721
- case 0:
5722
- rootDirname = options.rootDirname, _a = options.cacheDirname, cacheDirname = _a === void 0 ? SCRAPE_CACHE_DIRNAME : _a, _b = options.isCacheCleaned, isCacheCleaned = _b === void 0 ? false : _b, _c = options.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
5723
- // TODO: !!!!!! Does this work in browser? Make it work.
5724
- if (source.url === null) {
5725
- throw new KnowledgeScrapeError('Website scraper requires URL');
5726
- }
5727
- _d = jsdom.JSDOM.bind;
5728
- return [4 /*yield*/, source.asText()];
5729
- case 1:
5730
- jsdom$1 = new (_d.apply(jsdom.JSDOM, [void 0, _e.sent(), {
5731
- url: source.url,
5732
- }]))();
5733
- reader = new readability.Readability(jsdom$1.window.document);
5734
- article = reader.parse();
5735
- console.log(article);
5736
- return [4 /*yield*/, waitasecond.forTime(10000)];
5737
- case 2:
5738
- _e.sent();
5739
- html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
5740
- // Note: Unwrap html such as it is convertable by `markdownConverter`
5741
- for (i = 0; i < 2; i++) {
5742
- html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
5743
- }
5744
- if (html.includes('<div')) {
5745
- html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
5746
- }
5747
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5748
- rootDirname: rootDirname,
5749
- cacheDirname: cacheDirname,
5750
- isCacheCleaned: isCacheCleaned,
5751
- extension: 'html',
5752
- isVerbose: isVerbose,
5753
- })];
5754
- case 3:
5755
- cacheFilehandler = _e.sent();
5756
- return [4 /*yield*/, promises.writeFile(cacheFilehandler.filename, html, 'utf-8')];
5757
- case 4:
5758
- _e.sent();
5759
- markdown = markdownConverter.makeMarkdown(html, jsdom$1.window.document);
5760
- return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
5761
- }
5762
- });
5110
+ var all = [];
5111
+ var _loop_1 = function (packageName, className) {
5112
+ if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
5113
+ return "continue";
5114
+ }
5115
+ all.push({ packageName: packageName, className: className });
5116
+ };
5117
+ try {
5118
+ for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
5119
+ var _e = _d.value, packageName = _e.packageName, className = _e.className;
5120
+ _loop_1(packageName, className);
5121
+ }
5122
+ }
5123
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5124
+ finally {
5125
+ try {
5126
+ if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
5127
+ }
5128
+ finally { if (e_1) throw e_1.error; }
5129
+ }
5130
+ var _loop_2 = function (packageName, className) {
5131
+ if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
5132
+ return "continue";
5133
+ }
5134
+ all.push({ packageName: packageName, className: className });
5135
+ };
5136
+ try {
5137
+ for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
5138
+ var _h = _g.value, packageName = _h.packageName, className = _h.className;
5139
+ _loop_2(packageName, className);
5140
+ }
5141
+ }
5142
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
5143
+ finally {
5144
+ try {
5145
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
5146
+ }
5147
+ finally { if (e_2) throw e_2.error; }
5148
+ }
5149
+ var metadata = all.map(function (metadata) {
5150
+ var isMetadataAviailable = $scrapersMetadataRegister
5151
+ .list()
5152
+ .find(function (_a) {
5153
+ var packageName = _a.packageName, className = _a.className;
5154
+ return metadata.packageName === packageName && metadata.className === className;
5763
5155
  });
5764
- },
5765
- /**
5766
- * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
5767
- */
5768
- scrape: function (source, options) {
5769
- return __awaiter(this, void 0, void 0, function () {
5770
- var cacheFilehandler, markdownSource, knowledge;
5771
- return __generator(this, function (_a) {
5772
- switch (_a.label) {
5773
- case 0: return [4 /*yield*/, websiteScraper.$convert(source, options)];
5774
- case 1:
5775
- cacheFilehandler = _a.sent();
5776
- markdownSource = {
5777
- source: source.source,
5778
- filename: cacheFilehandler.filename,
5779
- url: null,
5780
- mimeType: 'text/markdown',
5781
- asText: function () {
5782
- return cacheFilehandler.markdown;
5783
- },
5784
- asJson: function () {
5785
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5786
- },
5787
- asBlob: function () {
5788
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5789
- },
5790
- };
5791
- knowledge = markdownScraper.scrape(markdownSource, options);
5792
- return [4 /*yield*/, cacheFilehandler.destroy()];
5793
- case 2:
5794
- _a.sent();
5795
- return [2 /*return*/, knowledge];
5796
- }
5797
- });
5156
+ var isInstalled = $scrapersRegister
5157
+ .list()
5158
+ .find(function (_a) {
5159
+ var packageName = _a.packageName, className = _a.className;
5160
+ return metadata.packageName === packageName && metadata.className === className;
5798
5161
  });
5799
- },
5800
- } /* TODO: [🦷] as const */;
5162
+ return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
5163
+ });
5164
+ if (metadata.length === 0) {
5165
+ return "No scrapers are available";
5166
+ }
5167
+ return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
5168
+ .map(function (_a, i) {
5169
+ var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
5170
+ var more;
5171
+ if (just(false)) {
5172
+ more = '';
5173
+ }
5174
+ else if (!isMetadataAviailable && !isInstalled) {
5175
+ // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
5176
+ more = "(not installed and no metadata, looks like a unexpected behavior)";
5177
+ }
5178
+ else if (isMetadataAviailable && !isInstalled) {
5179
+ // TODO: [�][�]
5180
+ more = "(not installed)";
5181
+ }
5182
+ else if (!isMetadataAviailable && isInstalled) {
5183
+ more = "(no metadata, looks like a unexpected behavior)";
5184
+ }
5185
+ else if (isMetadataAviailable && isInstalled) {
5186
+ more = "(installed)";
5187
+ }
5188
+ else {
5189
+ more = "(unknown state, looks like a unexpected behavior)";
5190
+ }
5191
+ return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(more);
5192
+ })
5193
+ .join('\n')), "\n "); });
5194
+ }
5801
5195
  /**
5802
- * TODO: !!!!!! Put into separate package
5803
- * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
5804
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5805
- * TODO: [🪂] Do it in parallel 11:11
5806
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5807
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5196
+ * TODO: [®] DRY Register logic
5808
5197
  */
5809
5198
 
5810
- // TODO: [🦖] !!!!!! Pass scrapers as dependency,
5811
5199
  /**
5812
5200
  * @@@
5813
5201
  *
5814
- * @private because this will be replaced by a system of one scraper per package [🦖]
5815
- * TODO: [🦖] System for scrapers NOT public exported from `@promptbook/core`
5202
+ * @param text @@@
5203
+ * @returns @@@
5204
+ * @example 'hello-world'
5205
+ * @example 'i-love-promptbook'
5206
+ * @public exported from `@promptbook/utils`
5816
5207
  */
5817
- var SCRAPERS = [
5818
- markdownScraper,
5819
- documentScraper,
5820
- legacyDocumentScraper,
5821
- pdfScraper,
5822
- websiteScraper,
5823
- // <- Note: [♓️] This is the order of the scrapers for knowledge, BUT consider some better (more explicit) way to do this
5824
- ];
5208
+ function normalizeToKebabCase(text) {
5209
+ var e_1, _a;
5210
+ text = removeDiacritics(text);
5211
+ var charType;
5212
+ var lastCharType = 'OTHER';
5213
+ var normalizedName = '';
5214
+ try {
5215
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5216
+ var char = text_1_1.value;
5217
+ var normalizedChar = void 0;
5218
+ if (/^[a-z]$/.test(char)) {
5219
+ charType = 'LOWERCASE';
5220
+ normalizedChar = char;
5221
+ }
5222
+ else if (/^[A-Z]$/.test(char)) {
5223
+ charType = 'UPPERCASE';
5224
+ normalizedChar = char.toLowerCase();
5225
+ }
5226
+ else if (/^[0-9]$/.test(char)) {
5227
+ charType = 'NUMBER';
5228
+ normalizedChar = char;
5229
+ }
5230
+ else {
5231
+ charType = 'OTHER';
5232
+ normalizedChar = '-';
5233
+ }
5234
+ if (charType !== lastCharType &&
5235
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
5236
+ !(lastCharType === 'NUMBER') &&
5237
+ !(charType === 'NUMBER')) {
5238
+ normalizedName += '-';
5239
+ }
5240
+ normalizedName += normalizedChar;
5241
+ lastCharType = charType;
5242
+ }
5243
+ }
5244
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5245
+ finally {
5246
+ try {
5247
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5248
+ }
5249
+ finally { if (e_1) throw e_1.error; }
5250
+ }
5251
+ normalizedName = normalizedName.split(/-+/g).join('-');
5252
+ normalizedName = normalizedName.split(/-?\/-?/g).join('/');
5253
+ normalizedName = normalizedName.replace(/^-/, '');
5254
+ normalizedName = normalizedName.replace(/-$/, '');
5255
+ return normalizedName;
5256
+ }
5825
5257
 
5826
5258
  /**
5827
5259
  * Creates unique name for the source
5828
5260
  *
5829
5261
  * @private within the repository
5830
5262
  */
5831
- function sourceContentToName(sourceContent) {
5832
- // TODO: !!!!!! Better name for source than gibberish hash
5833
- var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
5834
- // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5835
- .toString( /* hex */)
5836
- .substring(0, 20);
5837
- // <- TODO: [🥬] Make some system for hashes and ids of promptbook
5838
- var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
5839
- var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
5840
- var name = pieces.join('-').split('--').join('-');
5841
- // <- TODO: Use MAX_FILENAME_LENGTH
5842
- return name;
5263
+ function sourceContentToName(sourceContent) {
5264
+ var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
5265
+ // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5266
+ .toString( /* hex */)
5267
+ .substring(0, 20);
5268
+ // <- TODO: [🥬] Make some system for hashes and ids of promptbook
5269
+ var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
5270
+ var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
5271
+ var name = pieces.join('-').split('--').join('-');
5272
+ // <- TODO: Use MAX_FILENAME_LENGTH
5273
+ return name;
5274
+ }
5275
+ /**
5276
+ * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
5277
+ */
5278
+
5279
+ /**
5280
+ * Convert file extension to mime type
5281
+ *
5282
+ * @private within the repository
5283
+ */
5284
+ function extensionToMimeType(value) {
5285
+ return mimeTypes.lookup(value) || 'application/octet-stream';
5286
+ }
5287
+
5288
+ /**
5289
+ * Get the file extension from a file name
5290
+ *
5291
+ * @private within the repository
5292
+ */
5293
+ function getFileExtension(value) {
5294
+ var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
5295
+ return match ? match[1].toLowerCase() : null;
5296
+ }
5297
+
5298
+ /**
5299
+ * Checks if the file exists
5300
+ *
5301
+ * @private within the repository
5302
+ */
5303
+ function isFileExisting(filename, fs) {
5304
+ return __awaiter(this, void 0, void 0, function () {
5305
+ var isReadAccessAllowed, isFile;
5306
+ return __generator(this, function (_a) {
5307
+ switch (_a.label) {
5308
+ case 0: return [4 /*yield*/, fs
5309
+ .access(filename, fs.constants.R_OK)
5310
+ .then(function () { return true; })
5311
+ .catch(function () { return false; })];
5312
+ case 1:
5313
+ isReadAccessAllowed = _a.sent();
5314
+ if (!isReadAccessAllowed) {
5315
+ return [2 /*return*/, false];
5316
+ }
5317
+ return [4 /*yield*/, fs
5318
+ .stat(filename)
5319
+ .then(function (fileStat) { return fileStat.isFile(); })
5320
+ .catch(function () { return false; })];
5321
+ case 2:
5322
+ isFile = _a.sent();
5323
+ return [2 /*return*/, isFile];
5324
+ }
5325
+ });
5326
+ });
5843
5327
  }
5844
5328
  /**
5845
- * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
5329
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
5330
+ * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
5331
+ * TODO: [🖇] What about symlinks?
5846
5332
  */
5847
5333
 
5848
5334
  /**
5849
- * Convert file extension to mime type
5335
+ * Tests if given string is valid URL.
5850
5336
  *
5851
- * @private within the repository
5337
+ * Note: This does not check if the file exists only if the path is valid
5338
+ * @public exported from `@promptbook/utils`
5852
5339
  */
5853
- function extensionToMimeType(value) {
5854
- return mimeTypes.lookup(value) || 'application/octet-stream';
5340
+ function isValidFilePath(filename) {
5341
+ if (typeof filename !== 'string') {
5342
+ return false;
5343
+ }
5344
+ var filenameSlashes = filename.split('\\').join('/');
5345
+ // Absolute Unix path: /hello.txt
5346
+ if (/^(\/)/i.test(filenameSlashes)) {
5347
+ return true;
5348
+ }
5349
+ // Absolute Windows path: /hello.txt
5350
+ if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
5351
+ return true;
5352
+ }
5353
+ // Relative path: ./hello.txt
5354
+ if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
5355
+ return true;
5356
+ }
5357
+ return false;
5855
5358
  }
5856
5359
 
5857
5360
  /**
5858
5361
  * @@@
5859
5362
  *
5860
- * @private for scraper utilities
5363
+ * @public exported from `@promptbook/core`
5861
5364
  */
5862
- function makeKnowledgeSourceHandler(knowledgeSource, options) {
5365
+ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5863
5366
  var _a;
5864
5367
  return __awaiter(this, void 0, void 0, function () {
5865
- var sourceContent, name, _b, _c, rootDirname, _d, isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
5368
+ var sourceContent, name, _b, _c, rootDirname, _d,
5369
+ // <- TODO: process.cwd() if running in Node.js
5370
+ isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
5866
5371
  return __generator(this, function (_e) {
5867
5372
  switch (_e.label) {
5868
5373
  case 0:
5869
5374
  sourceContent = knowledgeSource.sourceContent;
5870
5375
  name = knowledgeSource.name;
5871
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5376
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
5872
5377
  TODO_USE(isVerbose);
5873
5378
  if (!name) {
5874
5379
  name = sourceContentToName(sourceContent);
@@ -5884,19 +5389,14 @@
5884
5389
  filename: null,
5885
5390
  url: url,
5886
5391
  mimeType: mimeType,
5887
- asBlob: function () {
5888
- return __awaiter(this, void 0, void 0, function () {
5889
- var content;
5890
- return __generator(this, function (_a) {
5891
- switch (_a.label) {
5892
- case 0: return [4 /*yield*/, response_1.blob()];
5893
- case 1:
5894
- content = _a.sent();
5895
- return [2 /*return*/, content];
5896
- }
5897
- });
5898
- });
5899
- },
5392
+ /*
5393
+ TODO: [🥽]
5394
+ > async asBlob() {
5395
+ > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
5396
+ > const content = await response.blob();
5397
+ > return content;
5398
+ > },
5399
+ */
5900
5400
  asJson: function () {
5901
5401
  return __awaiter(this, void 0, void 0, function () {
5902
5402
  var content;
@@ -5926,8 +5426,9 @@
5926
5426
  }];
5927
5427
  case 2:
5928
5428
  if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
5929
- if (!$isRunningInNode()) {
5930
- throw new EnvironmentMismatchError('Importing knowledge source file works only in Node.js environment');
5429
+ if (tools.fs === undefined) {
5430
+ throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
5431
+ // <- TODO: [🧠] What is the best error type here`
5931
5432
  }
5932
5433
  if (rootDirname === null) {
5933
5434
  throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
@@ -5935,35 +5436,31 @@
5935
5436
  }
5936
5437
  filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
5937
5438
  fileExtension = getFileExtension(filename_1);
5938
- mimeType_1 = extensionToMimeType(fileExtension || '');
5939
- return [4 /*yield*/, $isFileExisting(filename_1)];
5439
+ mimeType = extensionToMimeType(fileExtension || '');
5440
+ return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
5940
5441
  case 3:
5941
5442
  if (!(_e.sent())) {
5942
5443
  throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
5943
5444
  }
5944
- // TODO: !!!!!! Test security file - file is scoped to the project (maybe do this in `filesystemTools`)
5445
+ // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
5945
5446
  return [2 /*return*/, {
5946
5447
  source: name,
5947
5448
  filename: filename_1,
5948
5449
  url: null,
5949
- mimeType: mimeType_1,
5950
- asBlob: function () {
5951
- return __awaiter(this, void 0, void 0, function () {
5952
- var content;
5953
- return __generator(this, function (_a) {
5954
- switch (_a.label) {
5955
- case 0: return [4 /*yield*/, promises.readFile(filename_1)];
5956
- case 1:
5957
- content = _a.sent();
5958
- // <- Note: Its OK to use sync in tooling for tests
5959
- return [2 /*return*/, new Blob([
5960
- content,
5961
- // <- TODO: !!!!!! Maybe not working
5962
- ], { type: mimeType_1 })];
5963
- }
5964
- });
5965
- });
5966
- },
5450
+ mimeType: mimeType,
5451
+ /*
5452
+ TODO: [🥽]
5453
+ > async asBlob() {
5454
+ > const content = await tools.fs!.readFile(filename);
5455
+ > return new Blob(
5456
+ > [
5457
+ > content,
5458
+ > // <- TODO: [🥽] This is NOT tested, test it
5459
+ > ],
5460
+ > { type: mimeType },
5461
+ > );
5462
+ > },
5463
+ */
5967
5464
  asJson: function () {
5968
5465
  return __awaiter(this, void 0, void 0, function () {
5969
5466
  var _a, _b;
@@ -5971,7 +5468,7 @@
5971
5468
  switch (_c.label) {
5972
5469
  case 0:
5973
5470
  _b = (_a = JSON).parse;
5974
- return [4 /*yield*/, promises.readFile(filename_1, 'utf-8')];
5471
+ return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
5975
5472
  case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
5976
5473
  }
5977
5474
  });
@@ -5981,7 +5478,7 @@
5981
5478
  return __awaiter(this, void 0, void 0, function () {
5982
5479
  return __generator(this, function (_a) {
5983
5480
  switch (_a.label) {
5984
- case 0: return [4 /*yield*/, promises.readFile(filename_1, 'utf-8')];
5481
+ case 0: return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
5985
5482
  case 1: return [2 /*return*/, _a.sent()];
5986
5483
  }
5987
5484
  });
@@ -5999,9 +5496,14 @@
5999
5496
  asJson: function () {
6000
5497
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
6001
5498
  },
6002
- asBlob: function () {
6003
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
6004
- },
5499
+ /*
5500
+ TODO: [🥽]
5501
+ > asBlob() {
5502
+ > throw new UnexpectedError(
5503
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
5504
+ > );
5505
+ > },
5506
+ */
6005
5507
  }];
6006
5508
  }
6007
5509
  });
@@ -6014,262 +5516,106 @@
6014
5516
  * @see https://github.com/webgptorg/promptbook/discussions/41
6015
5517
  * @public exported from `@promptbook/core`
6016
5518
  */
6017
- function prepareKnowledgePieces(knowledgeSources, options) {
5519
+ function prepareKnowledgePieces(knowledgeSources, tools, options) {
6018
5520
  return __awaiter(this, void 0, void 0, function () {
6019
5521
  var _a, maxParallelCount, rootDirname, _b, isVerbose, knowledgePreparedUnflatten, knowledgePrepared;
6020
5522
  var _this = this;
6021
5523
  return __generator(this, function (_c) {
6022
5524
  switch (_c.label) {
6023
5525
  case 0:
6024
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5526
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
6025
5527
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
6026
5528
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
6027
- var partialPieces, sourceHandler, SCRAPERS_1, SCRAPERS_1_1, scraper, partialPiecesUnchecked, e_1_1, pieces;
6028
- var e_1, _a;
6029
- return __generator(this, function (_b) {
6030
- switch (_b.label) {
5529
+ var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
5530
+ var e_1, _c;
5531
+ return __generator(this, function (_d) {
5532
+ switch (_d.label) {
6031
5533
  case 0:
6032
5534
  partialPieces = null;
6033
- return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
5535
+ return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
6034
5536
  case 1:
6035
- sourceHandler = _b.sent();
6036
- _b.label = 2;
5537
+ sourceHandler = _d.sent();
5538
+ _d.label = 2;
6037
5539
  case 2:
6038
- _b.trys.push([2, 7, 8, 9]);
6039
- SCRAPERS_1 = __values(SCRAPERS), SCRAPERS_1_1 = SCRAPERS_1.next();
6040
- _b.label = 3;
5540
+ _d.trys.push([2, 7, 8, 9]);
5541
+ _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
5542
+ _d.label = 3;
6041
5543
  case 3:
6042
- if (!!SCRAPERS_1_1.done) return [3 /*break*/, 6];
6043
- scraper = SCRAPERS_1_1.value;
6044
- if (!scraper.mimeTypes.includes(sourceHandler.mimeType)
5544
+ if (!!_b.done) return [3 /*break*/, 6];
5545
+ scraper = _b.value;
5546
+ if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
6045
5547
  // <- TODO: [🦔] Implement mime-type wildcards
6046
5548
  ) {
6047
5549
  return [3 /*break*/, 5];
6048
5550
  }
6049
- return [4 /*yield*/, scraper.scrape(sourceHandler, options)];
5551
+ return [4 /*yield*/, scraper.scrape(sourceHandler)];
6050
5552
  case 4:
6051
- partialPiecesUnchecked = _b.sent();
5553
+ partialPiecesUnchecked = _d.sent();
6052
5554
  if (partialPiecesUnchecked !== null) {
6053
- partialPieces = partialPiecesUnchecked;
5555
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
5556
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
6054
5557
  return [3 /*break*/, 6];
6055
5558
  }
6056
- _b.label = 5;
5559
+ _d.label = 5;
6057
5560
  case 5:
6058
- SCRAPERS_1_1 = SCRAPERS_1.next();
5561
+ _b = _a.next();
6059
5562
  return [3 /*break*/, 3];
6060
5563
  case 6: return [3 /*break*/, 9];
6061
5564
  case 7:
6062
- e_1_1 = _b.sent();
5565
+ e_1_1 = _d.sent();
6063
5566
  e_1 = { error: e_1_1 };
6064
5567
  return [3 /*break*/, 9];
6065
5568
  case 8:
6066
5569
  try {
6067
- if (SCRAPERS_1_1 && !SCRAPERS_1_1.done && (_a = SCRAPERS_1.return)) _a.call(SCRAPERS_1);
5570
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
6068
5571
  }
6069
5572
  finally { if (e_1) throw e_1.error; }
6070
5573
  return [7 /*endfinally*/];
6071
5574
  case 9:
6072
5575
  if (partialPieces === null) {
6073
- throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n Available scrapers:\n ").concat(block(SCRAPERS.flatMap(function (scraper) { return scraper.mimeTypes; })
6074
- .map(function (mimeType) { return "- ".concat(mimeType); })
6075
- .join('\n')), "\n\n\n "); }));
5576
+ throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
6076
5577
  }
6077
5578
  pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
6078
5579
  {
6079
5580
  name: knowledgeSource.name,
6080
5581
  // line, column <- TODO: [☀]
6081
5582
  // <- TODO: [❎]
6082
- },
6083
- ] })); });
6084
- knowledgePreparedUnflatten[index] = pieces;
6085
- return [2 /*return*/];
6086
- }
6087
- });
6088
- }); })];
6089
- case 1:
6090
- _c.sent();
6091
- knowledgePrepared = knowledgePreparedUnflatten.flat();
6092
- return [2 /*return*/, knowledgePrepared];
6093
- }
6094
- });
6095
- });
6096
- }
6097
- /*
6098
- TODO: [🧊] This is how it can look in future
6099
- > type PrepareKnowledgeKnowledge = {
6100
- > /**
6101
- > * Unprepared knowledge
6102
- > * /
6103
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
6104
- > };
6105
- >
6106
- > export async function prepareKnowledgePieces(
6107
- > knowledge: PrepareKnowledgeKnowledge,
6108
- > options: PrepareAndScrapeOptions,
6109
- > ):
6110
- */
6111
- /**
6112
- * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
6113
- * Put `knowledgePieces` into `PrepareKnowledgeOptions`
6114
- * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
6115
- * TODO: [🧠][❎] Do here propper M:N mapping
6116
- * [x] One source can make multiple pieces
6117
- * [ ] One piece can have multiple sources
6118
- */
6119
-
6120
- /**
6121
- * Intercepts LLM tools and counts total usage of the tools
6122
- *
6123
- * @param llmTools LLM tools to be intercepted with usage counting
6124
- * @returns LLM tools with same functionality with added total cost counting
6125
- * @public exported from `@promptbook/core`
6126
- */
6127
- function countTotalUsage(llmTools) {
6128
- var _this = this;
6129
- var totalUsage = ZERO_USAGE;
6130
- var proxyTools = {
6131
- get title() {
6132
- // TODO: [🧠] Maybe put here some suffix
6133
- return llmTools.title;
6134
- },
6135
- get description() {
6136
- // TODO: [🧠] Maybe put here some suffix
6137
- return llmTools.description;
6138
- },
6139
- checkConfiguration: function () {
6140
- return __awaiter(this, void 0, void 0, function () {
6141
- return __generator(this, function (_a) {
6142
- return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
6143
- });
6144
- });
6145
- },
6146
- listModels: function () {
6147
- return /* not await */ llmTools.listModels();
6148
- },
6149
- getTotalUsage: function () {
6150
- // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
6151
- return totalUsage;
6152
- },
6153
- };
6154
- if (llmTools.callChatModel !== undefined) {
6155
- proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6156
- var promptResult;
6157
- return __generator(this, function (_a) {
6158
- switch (_a.label) {
6159
- case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
6160
- case 1:
6161
- promptResult = _a.sent();
6162
- totalUsage = addUsage(totalUsage, promptResult.usage);
6163
- return [2 /*return*/, promptResult];
6164
- }
6165
- });
6166
- }); };
6167
- }
6168
- if (llmTools.callCompletionModel !== undefined) {
6169
- proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6170
- var promptResult;
6171
- return __generator(this, function (_a) {
6172
- switch (_a.label) {
6173
- case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
6174
- case 1:
6175
- promptResult = _a.sent();
6176
- totalUsage = addUsage(totalUsage, promptResult.usage);
6177
- return [2 /*return*/, promptResult];
6178
- }
6179
- });
6180
- }); };
6181
- }
6182
- if (llmTools.callEmbeddingModel !== undefined) {
6183
- proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6184
- var promptResult;
6185
- return __generator(this, function (_a) {
6186
- switch (_a.label) {
6187
- case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
6188
- case 1:
6189
- promptResult = _a.sent();
6190
- totalUsage = addUsage(totalUsage, promptResult.usage);
6191
- return [2 /*return*/, promptResult];
6192
- }
6193
- });
6194
- }); };
6195
- }
6196
- // <- Note: [🤖]
6197
- return proxyTools;
6198
- }
6199
- /**
6200
- * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
6201
- * TODO: [🧠] Is there some meaningfull way how to test this util
6202
- * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
6203
- * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
6204
- * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
6205
- */
6206
-
6207
- /**
6208
- * Prepares the persona for the pipeline
6209
- *
6210
- * @see https://github.com/webgptorg/promptbook/discussions/22
6211
- * @public exported from `@promptbook/core`
6212
- */
6213
- function preparePersona(personaDescription, options) {
6214
- return __awaiter(this, void 0, void 0, function () {
6215
- var llmTools, _a, isVerbose, collection, preparePersonaExecutor, _b, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
6216
- var _c;
6217
- return __generator(this, function (_d) {
6218
- switch (_d.label) {
6219
- case 0:
6220
- llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
6221
- if (llmTools === undefined) {
6222
- throw new MissingToolsError('LLM tools are required for preparing persona');
6223
- }
6224
- collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
6225
- _b = createPipelineExecutor;
6226
- _c = {};
6227
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
6228
- case 1:
6229
- preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
6230
- _c.tools = {
6231
- llm: llmTools,
6232
- },
6233
- _c)]);
6234
- return [4 /*yield*/, llmTools.listModels()];
6235
- case 2:
6236
- availableModels = _d.sent();
6237
- availableModelNames = availableModels
6238
- .filter(function (_a) {
6239
- var modelVariant = _a.modelVariant;
6240
- return modelVariant === 'CHAT';
6241
- })
6242
- .map(function (_a) {
6243
- var modelName = _a.modelName;
6244
- return modelName;
6245
- })
6246
- .join(',');
6247
- return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
6248
- case 3:
6249
- result = _d.sent();
6250
- assertsExecutionSuccessful(result);
6251
- outputParameters = result.outputParameters;
6252
- modelRequirementsRaw = outputParameters.modelRequirements;
6253
- modelRequirements = JSON.parse(modelRequirementsRaw);
6254
- if (isVerbose) {
6255
- console.info("PERSONA ".concat(personaDescription), modelRequirements);
6256
- }
6257
- modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
6258
- return [2 /*return*/, {
6259
- modelVariant: 'CHAT',
6260
- modelName: modelName,
6261
- systemMessage: systemMessage,
6262
- temperature: temperature,
6263
- }];
5583
+ },
5584
+ ] })); });
5585
+ knowledgePreparedUnflatten[index] = pieces;
5586
+ return [2 /*return*/];
5587
+ }
5588
+ });
5589
+ }); })];
5590
+ case 1:
5591
+ _c.sent();
5592
+ knowledgePrepared = knowledgePreparedUnflatten.flat();
5593
+ return [2 /*return*/, knowledgePrepared];
6264
5594
  }
6265
5595
  });
6266
5596
  });
6267
5597
  }
5598
+ /*
5599
+ TODO: [🧊] This is how it can look in future
5600
+ > type PrepareKnowledgeKnowledge = {
5601
+ > /**
5602
+ > * Unprepared knowledge
5603
+ > * /
5604
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
5605
+ > };
5606
+ >
5607
+ > export async function prepareKnowledgePieces(
5608
+ > knowledge: PrepareKnowledgeKnowledge,
5609
+ > options: PrepareAndScrapeOptions,
5610
+ > ):
5611
+ */
6268
5612
  /**
6269
- * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
6270
- * TODO: [🏢] !! Check validity of `modelName` in pipeline
6271
- * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
6272
- * TODO: [🏢] !! Check validity of `temperature` in pipeline
5613
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
5614
+ * Put `knowledgePieces` into `PrepareKnowledgeOptions`
5615
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
5616
+ * TODO: [🧠][❎] Do here propper M:N mapping
5617
+ * [x] One source can make multiple pieces
5618
+ * [ ] One piece can have multiple sources
6273
5619
  */
6274
5620
 
6275
5621
  /**
@@ -6306,20 +5652,18 @@
6306
5652
  *
6307
5653
  * @public exported from `@promptbook/core`
6308
5654
  */
6309
- function prepareTemplates(pipeline, options) {
5655
+ function prepareTemplates(pipeline, tools, options) {
6310
5656
  return __awaiter(this, void 0, void 0, function () {
6311
5657
  var _a, maxParallelCount, templates, parameters, knowledgePiecesCount, templatesPrepared;
6312
5658
  var _this = this;
6313
5659
  return __generator(this, function (_b) {
6314
5660
  switch (_b.label) {
6315
5661
  case 0:
6316
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
5662
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
6317
5663
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
6318
5664
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
6319
5665
  TODO_USE(parameters);
6320
- templatesPrepared = new Array(
6321
- // <- TODO: [🧱] Implement in a functional (not new Class) way
6322
- templates.length);
5666
+ templatesPrepared = new Array(templates.length);
6323
5667
  return [4 /*yield*/, forEachAsync(templates, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (template, index) { return __awaiter(_this, void 0, void 0, function () {
6324
5668
  var dependentParameterNames, preparedContent, preparedTemplate;
6325
5669
  return __generator(this, function (_a) {
@@ -6362,14 +5706,14 @@
6362
5706
  * Note: When the pipeline is already prepared, it returns the same pipeline
6363
5707
  * @public exported from `@promptbook/core`
6364
5708
  */
6365
- function preparePipeline(pipeline, options) {
5709
+ function preparePipeline(pipeline, tools, options) {
6366
5710
  return __awaiter(this, void 0, void 0, function () {
6367
- var llmTools, rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
5711
+ var rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
6368
5712
  /*
6369
5713
  <- TODO: [🧠][🪑] `promptbookVersion` */
6370
5714
  knowledgeSources /*
6371
5715
  <- TODO: [🧊] `knowledgePieces` */, personas /*
6372
- <- TODO: [🧊] `preparations` */, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
5716
+ <- TODO: [🧊] `preparations` */, _llms, llmTools, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
6373
5717
  var _this = this;
6374
5718
  return __generator(this, function (_c) {
6375
5719
  switch (_c.label) {
@@ -6377,11 +5721,13 @@
6377
5721
  if (isPipelinePrepared(pipeline)) {
6378
5722
  return [2 /*return*/, pipeline];
6379
5723
  }
6380
- llmTools = options.llmTools, rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5724
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
6381
5725
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
6382
- if (llmTools === undefined) {
5726
+ if (tools === undefined || tools.llm === undefined) {
6383
5727
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
6384
5728
  }
5729
+ _llms = arrayableToArray(tools.llm);
5730
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
6385
5731
  llmToolsWithUsage = countTotalUsage(llmTools);
6386
5732
  currentPreparation = {
6387
5733
  id: 1,
@@ -6394,15 +5740,12 @@
6394
5740
  // <- TODO: [🧊]
6395
5741
  currentPreparation,
6396
5742
  ];
6397
- preparedPersonas = new Array(
6398
- // <- TODO: [🧱] Implement in a functional (not new Class) way
6399
- personas.length);
5743
+ preparedPersonas = new Array(personas.length);
6400
5744
  return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () {
6401
5745
  var modelRequirements, preparedPersona;
6402
5746
  return __generator(this, function (_a) {
6403
5747
  switch (_a.label) {
6404
- case 0: return [4 /*yield*/, preparePersona(persona.description, {
6405
- llmTools: llmToolsWithUsage,
5748
+ case 0: return [4 /*yield*/, preparePersona(persona.description, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
6406
5749
  rootDirname: rootDirname,
6407
5750
  maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
6408
5751
  isVerbose: isVerbose,
@@ -6418,7 +5761,7 @@
6418
5761
  case 1:
6419
5762
  _c.sent();
6420
5763
  knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
6421
- return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, options), { llmTools: llmToolsWithUsage, rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
5764
+ return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), __assign(__assign({}, options), { rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
6422
5765
  case 2:
6423
5766
  partialknowledgePiecesPrepared = _c.sent();
6424
5767
  knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
@@ -6426,8 +5769,7 @@
6426
5769
  parameters: parameters,
6427
5770
  templates: templates,
6428
5771
  knowledgePiecesCount: knowledgePiecesPrepared.length,
6429
- }, {
6430
- llmTools: llmToolsWithUsage,
5772
+ }, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
6431
5773
  rootDirname: rootDirname,
6432
5774
  maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
6433
5775
  isVerbose: isVerbose,
@@ -6437,7 +5779,9 @@
6437
5779
  // ----- /Templates preparation -----
6438
5780
  // Note: Count total usage
6439
5781
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
6440
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
5782
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
5783
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
5784
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
6441
5785
  }
6442
5786
  });
6443
5787
  });
@@ -6484,6 +5828,7 @@
6484
5828
  'KNOWLEDGE ./hejny-cv.md',
6485
5829
  'KNOWLEDGE ./hejny-cv.pdf',
6486
5830
  'KNOWLEDGE ./hejny-cv.docx',
5831
+ // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
6487
5832
  ],
6488
5833
  /**
6489
5834
  * Parses the KNOWLEDGE command
@@ -6956,72 +6301,6 @@
6956
6301
  },
6957
6302
  };
6958
6303
 
6959
- /**
6960
- * @@@
6961
- *
6962
- * @param text @@@
6963
- * @returns @@@
6964
- * @example 'HELLO_WORLD'
6965
- * @example 'I_LOVE_PROMPTBOOK'
6966
- * @public exported from `@promptbook/utils`
6967
- */
6968
- function normalizeTo_SCREAMING_CASE(text) {
6969
- var e_1, _a;
6970
- var charType;
6971
- var lastCharType = 'OTHER';
6972
- var normalizedName = '';
6973
- try {
6974
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
6975
- var char = text_1_1.value;
6976
- var normalizedChar = void 0;
6977
- if (/^[a-z]$/.test(char)) {
6978
- charType = 'LOWERCASE';
6979
- normalizedChar = char.toUpperCase();
6980
- }
6981
- else if (/^[A-Z]$/.test(char)) {
6982
- charType = 'UPPERCASE';
6983
- normalizedChar = char;
6984
- }
6985
- else if (/^[0-9]$/.test(char)) {
6986
- charType = 'NUMBER';
6987
- normalizedChar = char;
6988
- }
6989
- else {
6990
- charType = 'OTHER';
6991
- normalizedChar = '_';
6992
- }
6993
- if (charType !== lastCharType &&
6994
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
6995
- !(lastCharType === 'NUMBER') &&
6996
- !(charType === 'NUMBER')) {
6997
- normalizedName += '_';
6998
- }
6999
- normalizedName += normalizedChar;
7000
- lastCharType = charType;
7001
- }
7002
- }
7003
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
7004
- finally {
7005
- try {
7006
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
7007
- }
7008
- finally { if (e_1) throw e_1.error; }
7009
- }
7010
- normalizedName = normalizedName.replace(/_+/g, '_');
7011
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
7012
- normalizedName = normalizedName.replace(/^_/, '');
7013
- normalizedName = normalizedName.replace(/_$/, '');
7014
- return normalizedName;
7015
- }
7016
- /**
7017
- * TODO: Tests
7018
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
7019
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
7020
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
7021
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
7022
- * TODO: [🌺] Use some intermediate util splitWords
7023
- */
7024
-
7025
6304
  /**
7026
6305
  * @@@
7027
6306
  *
@@ -7086,6 +6365,22 @@
7086
6365
  * TODO: [🌺] Use some intermediate util splitWords
7087
6366
  */
7088
6367
 
6368
+ /**
6369
+ * Removes emojis from a string and fix whitespaces
6370
+ *
6371
+ * @param text with emojis
6372
+ * @returns text without emojis
6373
+ * @public exported from `@promptbook/utils`
6374
+ */
6375
+ function removeEmojis(text) {
6376
+ // Replace emojis (and also ZWJ sequence) with hyphens
6377
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
6378
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
6379
+ text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
6380
+ text = text.replace(/\p{Extended_Pictographic}/gu, '');
6381
+ return text;
6382
+ }
6383
+
7089
6384
  /**
7090
6385
  * Removes quotes from a string
7091
6386
  *
@@ -8833,6 +8128,30 @@
8833
8128
  return spaceTrim.spaceTrim(content.replace(/<!--(.*?)-->/gs, ''));
8834
8129
  }
8835
8130
 
8131
+ /**
8132
+ * @@@
8133
+ *
8134
+ * @param value @@@
8135
+ * @returns @@@
8136
+ * @example @@@
8137
+ * @public exported from `@promptbook/utils`
8138
+ */
8139
+ function titleToName(value) {
8140
+ if (isValidUrl(value)) {
8141
+ value = value.replace(/^https?:\/\//, '');
8142
+ value = value.replace(/\.html$/, '');
8143
+ }
8144
+ else if (isValidFilePath(value)) {
8145
+ value = path.basename(value);
8146
+ // Note: Keeping extension in the name
8147
+ }
8148
+ value = value.split('/').join('-');
8149
+ value = removeEmojis(value);
8150
+ value = normalizeToKebabCase(value);
8151
+ // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
8152
+ return value;
8153
+ }
8154
+
8836
8155
  /**
8837
8156
  * Compile pipeline from string (markdown) format to JSON format synchronously
8838
8157
  *
@@ -9167,21 +8486,21 @@
9167
8486
  * Note: This function acts as compilation process
9168
8487
  *
9169
8488
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8489
+ * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
9170
8490
  * @param options - Options and tools for the compilation
9171
8491
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
9172
8492
  * @throws {ParseError} if the promptbook string is not valid
9173
8493
  * @public exported from `@promptbook/core`
9174
8494
  */
9175
- function pipelineStringToJson(pipelineString, options) {
8495
+ function pipelineStringToJson(pipelineString, tools, options) {
9176
8496
  return __awaiter(this, void 0, void 0, function () {
9177
- var llmTools, pipelineJson;
8497
+ var pipelineJson;
9178
8498
  return __generator(this, function (_a) {
9179
8499
  switch (_a.label) {
9180
8500
  case 0:
9181
- llmTools = (options || {}).llmTools;
9182
8501
  pipelineJson = pipelineStringToJsonSync(pipelineString);
9183
- if (!(llmTools !== undefined)) return [3 /*break*/, 2];
9184
- return [4 /*yield*/, preparePipeline(pipelineJson, options || {
8502
+ if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8503
+ return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
9185
8504
  rootDirname: null,
9186
8505
  })];
9187
8506
  case 1:
@@ -9498,56 +8817,6 @@
9498
8817
  * TODO: [🏛] Maybe make some markdown builder
9499
8818
  */
9500
8819
 
9501
- /**
9502
- * @@@
9503
- *
9504
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
9505
- *
9506
- * @private internal function of `$Register`
9507
- */
9508
- function $getGlobalScope() {
9509
- return Function('return this')();
9510
- }
9511
-
9512
- /**
9513
- * Register is @@@
9514
- *
9515
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
9516
- *
9517
- * @private internal utility, exported are only signleton instances of this class
9518
- */
9519
- var $Register = /** @class */ (function () {
9520
- function $Register(storageName) {
9521
- this.storageName = storageName;
9522
- storageName = "_promptbook_".concat(storageName);
9523
- var globalScope = $getGlobalScope();
9524
- if (globalScope[storageName] === undefined) {
9525
- globalScope[storageName] = [];
9526
- }
9527
- else if (!Array.isArray(globalScope[storageName])) {
9528
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
9529
- }
9530
- this.storage = globalScope[storageName];
9531
- }
9532
- $Register.prototype.list = function () {
9533
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
9534
- return this.storage;
9535
- };
9536
- $Register.prototype.register = function (registered) {
9537
- // <- TODO: What to return here
9538
- var packageName = registered.packageName, className = registered.className;
9539
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
9540
- var existingRegistration = this.storage[existingRegistrationIndex];
9541
- if (!existingRegistration) {
9542
- this.storage.push(registered);
9543
- }
9544
- else {
9545
- this.storage[existingRegistrationIndex] = registered;
9546
- }
9547
- };
9548
- return $Register;
9549
- }());
9550
-
9551
8820
  /**
9552
8821
  * @@@
9553
8822
  *
@@ -9556,6 +8825,9 @@
9556
8825
  * @public exported from `@promptbook/core`
9557
8826
  */
9558
8827
  var $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
8828
+ /**
8829
+ * TODO: [®] DRY Register logic
8830
+ */
9559
8831
 
9560
8832
  /**
9561
8833
  * @@@
@@ -9565,13 +8837,16 @@
9565
8837
  * @public exported from `@promptbook/core`
9566
8838
  */
9567
8839
  var $llmToolsRegister = new $Register('llm_execution_tools_constructors');
8840
+ /**
8841
+ * TODO: [®] DRY Register logic
8842
+ */
9568
8843
 
9569
8844
  /**
9570
8845
  * Creates a message with all registered LLM tools
9571
8846
  *
9572
8847
  * Note: This function is used to create a (error) message when there is no constructor for some LLM provider
9573
8848
  *
9574
- * @private internal function of `createLlmToolsFromConfiguration` and `createLlmToolsFromEnv`
8849
+ * @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv`
9575
8850
  */
9576
8851
  function $registeredLlmToolsMessage() {
9577
8852
  var e_1, _a, e_2, _b;
@@ -9663,6 +8938,9 @@
9663
8938
  })
9664
8939
  .join('\n')), "\n "); });
9665
8940
  }
8941
+ /**
8942
+ * TODO: [®] DRY Register logic
8943
+ */
9666
8944
 
9667
8945
  /**
9668
8946
  * @@@
@@ -9674,7 +8952,7 @@
9674
8952
  */
9675
8953
  function createLlmToolsFromConfiguration(configuration, options) {
9676
8954
  if (options === void 0) { options = {}; }
9677
- var _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
8955
+ var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
9678
8956
  var llmTools = configuration.map(function (llmConfiguration) {
9679
8957
  var registeredItem = $llmToolsRegister
9680
8958
  .list()
@@ -9685,17 +8963,18 @@
9685
8963
  if (registeredItem === undefined) {
9686
8964
  throw new Error(spaceTrim__default["default"](function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
9687
8965
  }
9688
- return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
8966
+ return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
9689
8967
  });
9690
8968
  return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
9691
8969
  }
9692
8970
  /**
9693
- * TODO: [🎌] Togethere with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
8971
+ * TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
9694
8972
  * TODO: [🧠][🎌] Dynamically install required providers
9695
8973
  * TODO: @@@ write discussion about this - wizzard
9696
8974
  * TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
9697
8975
  * TODO: [🧠] Is there some meaningfull way how to test this util
9698
8976
  * TODO: This should be maybe not under `_common` but under `utils`
8977
+ * TODO: [®] DRY Register logic
9699
8978
  */
9700
8979
 
9701
8980
  /**
@@ -9774,11 +9053,7 @@
9774
9053
  function cacheLlmTools(llmTools, options) {
9775
9054
  var _this = this;
9776
9055
  if (options === void 0) { options = {}; }
9777
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a,
9778
- // <- TODO: [🧱] Implement in a functional (not new Class) way
9779
- _b = options.isReloaded,
9780
- // <- TODO: [🧱] Implement in a functional (not new Class) way
9781
- isReloaded = _b === void 0 ? false : _b;
9056
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
9782
9057
  var proxyTools = __assign(__assign({}, llmTools), {
9783
9058
  // <- Note: [🥫]
9784
9059
  get title() {
@@ -9800,7 +9075,7 @@
9800
9075
  key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
9801
9076
  '-' +
9802
9077
  sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
9803
- if (!!isReloaded) return [3 /*break*/, 2];
9078
+ if (!!isCacheReloaded) return [3 /*break*/, 2];
9804
9079
  return [4 /*yield*/, storage.getItem(key)];
9805
9080
  case 1:
9806
9081
  _a = _c.sent();
@@ -9928,9 +9203,9 @@
9928
9203
  */
9929
9204
 
9930
9205
  /**
9931
- * @@@ registration1 of default configuration for Anthropic Claude
9206
+ * Registration of LLM provider metadata
9932
9207
  *
9933
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9208
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
9934
9209
  *
9935
9210
  * @public exported from `@promptbook/core`
9936
9211
  * @public exported from `@promptbook/cli`
@@ -9969,9 +9244,9 @@
9969
9244
  });
9970
9245
 
9971
9246
  /**
9972
- * @@@ registration1 of default configuration for Azure Open AI
9247
+ * Registration of LLM provider metadata
9973
9248
  *
9974
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9249
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
9975
9250
  *
9976
9251
  * @public exported from `@promptbook/core`
9977
9252
  * @public exported from `@promptbook/cli`
@@ -10016,9 +9291,9 @@
10016
9291
  });
10017
9292
 
10018
9293
  /**
10019
- * @@@ registration1 of default configuration for Open AI
9294
+ * Registration of LLM provider metadata
10020
9295
  *
10021
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9296
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
10022
9297
  *
10023
9298
  * @public exported from `@promptbook/core`
10024
9299
  * @public exported from `@promptbook/cli`
@@ -10097,6 +9372,130 @@
10097
9372
  },
10098
9373
  });
10099
9374
 
9375
+ /**
9376
+ * Metadata of the scraper
9377
+ *
9378
+ * @private within the scraper directory
9379
+ */
9380
+ var legacyDocumentScraperMetadata = $deepFreeze({
9381
+ title: 'LegacyDocument scraper',
9382
+ packageName: '@promptbook/legacy-documents',
9383
+ className: 'LegacyDocumentScraper',
9384
+ mimeTypes: ['application/msword', 'text/rtf'],
9385
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9386
+ isAvilableInBrowser: false,
9387
+ requiredExecutables: [
9388
+ 'Pandoc',
9389
+ 'LibreOffice',
9390
+ // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
9391
+ ],
9392
+ }); /* <- TODO: [🤛] */
9393
+ /**
9394
+ * Registration of known scraper metadata
9395
+ *
9396
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9397
+ *
9398
+ * @public exported from `@promptbook/core`
9399
+ * @public exported from `@promptbook/cli`
9400
+ */
9401
+ var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
9402
+
9403
+ /**
9404
+ * Metadata of the scraper
9405
+ *
9406
+ * @private within the scraper directory
9407
+ */
9408
+ var documentScraperMetadata = $deepFreeze({
9409
+ title: 'Document scraper',
9410
+ packageName: '@promptbook/documents',
9411
+ className: 'DocumentScraper',
9412
+ mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
9413
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9414
+ isAvilableInBrowser: false,
9415
+ requiredExecutables: ['Pandoc'],
9416
+ }); /* <- TODO: [🤛] */
9417
+ /**
9418
+ * Registration of known scraper metadata
9419
+ *
9420
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9421
+ *
9422
+ * @public exported from `@promptbook/core`
9423
+ * @public exported from `@promptbook/cli`
9424
+ */
9425
+ var _DocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(documentScraperMetadata);
9426
+
9427
+ /**
9428
+ * Metadata of the scraper
9429
+ *
9430
+ * @private within the scraper directory
9431
+ */
9432
+ var markdownScraperMetadata = $deepFreeze({
9433
+ title: 'Markdown scraper',
9434
+ packageName: '@promptbook/markdown-utils',
9435
+ className: 'MarkdownScraper',
9436
+ mimeTypes: ['text/markdown', 'text/plain'],
9437
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9438
+ isAvilableInBrowser: true,
9439
+ requiredExecutables: [],
9440
+ }); /* <- TODO: [🤛] */
9441
+ /**
9442
+ * Registration of known scraper metadata
9443
+ *
9444
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9445
+ *
9446
+ * @public exported from `@promptbook/core`
9447
+ * @public exported from `@promptbook/cli`
9448
+ */
9449
+ var _MarkdownScraperMetadataRegistration = $scrapersMetadataRegister.register(markdownScraperMetadata);
9450
+
9451
+ /**
9452
+ * Metadata of the scraper
9453
+ *
9454
+ * @private within the scraper directory
9455
+ */
9456
+ var pdfScraperMetadata = $deepFreeze({
9457
+ title: 'Pdf scraper',
9458
+ packageName: '@promptbook/pdf',
9459
+ className: 'PdfScraper',
9460
+ mimeTypes: ['application/pdf'],
9461
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9462
+ isAvilableInBrowser: true,
9463
+ requiredExecutables: [],
9464
+ }); /* <- TODO: [🤛] */
9465
+ /**
9466
+ * Registration of known scraper metadata
9467
+ *
9468
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9469
+ *
9470
+ * @public exported from `@promptbook/core`
9471
+ * @public exported from `@promptbook/cli`
9472
+ */
9473
+ var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
9474
+
9475
+ /**
9476
+ * Metadata of the scraper
9477
+ *
9478
+ * @private within the scraper directory
9479
+ */
9480
+ var websiteScraperMetadata = $deepFreeze({
9481
+ title: 'Website scraper',
9482
+ packageName: '@promptbook/website-crawler',
9483
+ className: 'WebsiteScraper',
9484
+ mimeTypes: ['text/html'],
9485
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9486
+ isAvilableInBrowser: false,
9487
+ requiredExecutables: [],
9488
+ }); /* <- TODO: [🤛] */
9489
+ /**
9490
+ * Registration of known scraper metadata
9491
+ *
9492
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9493
+ *
9494
+ * @public exported from `@promptbook/core`
9495
+ * @public exported from `@promptbook/cli`
9496
+ */
9497
+ var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
9498
+
10100
9499
  /**
10101
9500
  * This class behaves like LocalStorage but separates keys by prefix
10102
9501
  *
@@ -10449,6 +9848,8 @@
10449
9848
 
10450
9849
  exports.$llmToolsMetadataRegister = $llmToolsMetadataRegister;
10451
9850
  exports.$llmToolsRegister = $llmToolsRegister;
9851
+ exports.$scrapersMetadataRegister = $scrapersMetadataRegister;
9852
+ exports.$scrapersRegister = $scrapersRegister;
10452
9853
  exports.AbstractFormatError = AbstractFormatError;
10453
9854
  exports.CLAIM = CLAIM;
10454
9855
  exports.CallbackInterfaceTools = CallbackInterfaceTools;
@@ -10456,29 +9857,33 @@
10456
9857
  exports.CsvFormatDefinition = CsvFormatDefinition;
10457
9858
  exports.CsvFormatError = CsvFormatError;
10458
9859
  exports.DEFAULT_CSV_SETTINGS = DEFAULT_CSV_SETTINGS;
9860
+ exports.DEFAULT_EXECUTIONS_CACHE_DIRNAME = DEFAULT_EXECUTIONS_CACHE_DIRNAME;
9861
+ exports.DEFAULT_INTERMEDIATE_FILES_STRATEGY = DEFAULT_INTERMEDIATE_FILES_STRATEGY;
9862
+ exports.DEFAULT_IS_AUTO_INSTALLED = DEFAULT_IS_AUTO_INSTALLED;
9863
+ exports.DEFAULT_IS_VERBOSE = DEFAULT_IS_VERBOSE;
9864
+ exports.DEFAULT_MAX_EXECUTION_ATTEMPTS = DEFAULT_MAX_EXECUTION_ATTEMPTS;
9865
+ exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
9866
+ exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
9867
+ exports.DEFAULT_MAX_PARALLEL_COUNT = DEFAULT_MAX_PARALLEL_COUNT;
9868
+ exports.DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME;
10459
9869
  exports.DEFAULT_REMOTE_URL = DEFAULT_REMOTE_URL;
10460
9870
  exports.DEFAULT_REMOTE_URL_PATH = DEFAULT_REMOTE_URL_PATH;
9871
+ exports.DEFAULT_SCRAPE_CACHE_DIRNAME = DEFAULT_SCRAPE_CACHE_DIRNAME;
10461
9872
  exports.ERRORS = ERRORS;
10462
- exports.EXECUTIONS_CACHE_DIRNAME = EXECUTIONS_CACHE_DIRNAME;
10463
9873
  exports.EXPECTATION_UNITS = EXPECTATION_UNITS;
10464
9874
  exports.EnvironmentMismatchError = EnvironmentMismatchError;
10465
9875
  exports.ExecutionReportStringOptionsDefaults = ExecutionReportStringOptionsDefaults;
10466
9876
  exports.ExpectError = ExpectError;
10467
- exports.IS_VERBOSE = IS_VERBOSE;
10468
9877
  exports.KnowledgeScrapeError = KnowledgeScrapeError;
10469
9878
  exports.LimitReachedError = LimitReachedError;
10470
9879
  exports.MANDATORY_CSV_SETTINGS = MANDATORY_CSV_SETTINGS;
10471
- exports.MAX_EXECUTION_ATTEMPTS = MAX_EXECUTION_ATTEMPTS;
10472
9880
  exports.MAX_FILENAME_LENGTH = MAX_FILENAME_LENGTH;
10473
- exports.MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
10474
- exports.MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
10475
- exports.MAX_PARALLEL_COUNT = MAX_PARALLEL_COUNT;
10476
9881
  exports.MODEL_VARIANTS = MODEL_VARIANTS;
10477
9882
  exports.MemoryStorage = MemoryStorage;
10478
9883
  exports.MissingToolsError = MissingToolsError;
9884
+ exports.MultipleLlmExecutionTools = MultipleLlmExecutionTools;
10479
9885
  exports.NotFoundError = NotFoundError;
10480
9886
  exports.NotYetImplementedError = NotYetImplementedError;
10481
- exports.PIPELINE_COLLECTION_BASE_FILENAME = PIPELINE_COLLECTION_BASE_FILENAME;
10482
9887
  exports.PROMPTBOOK_VERSION = PROMPTBOOK_VERSION;
10483
9888
  exports.ParseError = ParseError;
10484
9889
  exports.PipelineExecutionError = PipelineExecutionError;
@@ -10486,7 +9891,6 @@
10486
9891
  exports.PipelineUrlError = PipelineUrlError;
10487
9892
  exports.PrefixStorage = PrefixStorage;
10488
9893
  exports.RESERVED_PARAMETER_NAMES = RESERVED_PARAMETER_NAMES;
10489
- exports.SCRAPE_CACHE_DIRNAME = SCRAPE_CACHE_DIRNAME;
10490
9894
  exports.TemplateTypes = TemplateTypes;
10491
9895
  exports.TextFormatDefinition = TextFormatDefinition;
10492
9896
  exports.UNCERTAIN_USAGE = UNCERTAIN_USAGE;
@@ -10494,8 +9898,13 @@
10494
9898
  exports.ZERO_USAGE = ZERO_USAGE;
10495
9899
  exports._AnthropicClaudeMetadataRegistration = _AnthropicClaudeMetadataRegistration;
10496
9900
  exports._AzureOpenAiMetadataRegistration = _AzureOpenAiMetadataRegistration;
9901
+ exports._DocumentScraperMetadataRegistration = _DocumentScraperMetadataRegistration;
9902
+ exports._LegacyDocumentScraperMetadataRegistration = _LegacyDocumentScraperMetadataRegistration;
9903
+ exports._MarkdownScraperMetadataRegistration = _MarkdownScraperMetadataRegistration;
10497
9904
  exports._OpenAiAssistantMetadataRegistration = _OpenAiAssistantMetadataRegistration;
10498
9905
  exports._OpenAiMetadataRegistration = _OpenAiMetadataRegistration;
9906
+ exports._PdfScraperMetadataRegistration = _PdfScraperMetadataRegistration;
9907
+ exports._WebsiteScraperMetadataRegistration = _WebsiteScraperMetadataRegistration;
10499
9908
  exports.addUsage = addUsage;
10500
9909
  exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
10501
9910
  exports.cacheLlmTools = cacheLlmTools;
@@ -10507,16 +9916,13 @@
10507
9916
  exports.createLlmToolsFromConfiguration = createLlmToolsFromConfiguration;
10508
9917
  exports.createPipelineExecutor = createPipelineExecutor;
10509
9918
  exports.createSubcollection = createSubcollection;
10510
- exports.documentScraper = documentScraper;
10511
9919
  exports.embeddingVectorToString = embeddingVectorToString;
10512
9920
  exports.executionReportJsonToString = executionReportJsonToString;
10513
9921
  exports.isPassingExpectations = isPassingExpectations;
10514
9922
  exports.isPipelinePrepared = isPipelinePrepared;
10515
9923
  exports.joinLlmExecutionTools = joinLlmExecutionTools;
10516
- exports.legacyDocumentScraper = legacyDocumentScraper;
10517
9924
  exports.limitTotalUsage = limitTotalUsage;
10518
- exports.markdownScraper = markdownScraper;
10519
- exports.pdfScraper = pdfScraper;
9925
+ exports.makeKnowledgeSourceHandler = makeKnowledgeSourceHandler;
10520
9926
  exports.pipelineJsonToString = pipelineJsonToString;
10521
9927
  exports.pipelineStringToJson = pipelineStringToJson;
10522
9928
  exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
@@ -10530,7 +9936,6 @@
10530
9936
  exports.usageToHuman = usageToHuman;
10531
9937
  exports.usageToWorktime = usageToWorktime;
10532
9938
  exports.validatePipeline = validatePipeline;
10533
- exports.websiteScraper = websiteScraper;
10534
9939
 
10535
9940
  Object.defineProperty(exports, '__esModule', { value: true });
10536
9941