@promptbook/core 0.72.0-8 → 0.72.0-9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +4 -0
  2. package/esm/index.es.js +1624 -2222
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/browser.index.d.ts +2 -0
  5. package/esm/typings/src/_packages/cli.index.d.ts +20 -0
  6. package/esm/typings/src/_packages/core.index.d.ts +20 -14
  7. package/esm/typings/src/_packages/documents.index.d.ts +8 -0
  8. package/esm/typings/src/_packages/legacy-documents.index.d.ts +8 -0
  9. package/esm/typings/src/_packages/markdown-utils.index.d.ts +6 -0
  10. package/esm/typings/src/_packages/node.index.d.ts +10 -4
  11. package/esm/typings/src/_packages/pdf.index.d.ts +8 -0
  12. package/esm/typings/src/_packages/types.index.d.ts +15 -5
  13. package/esm/typings/src/_packages/website-crawler.index.d.ts +8 -0
  14. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +5 -3
  15. package/esm/typings/src/config.d.ts +6 -0
  16. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +3 -1
  17. package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
  18. package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -3
  19. package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts} +1 -1
  20. package/esm/typings/src/execution/ExecutionTools.d.ts +26 -6
  21. package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
  22. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +1 -1
  23. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +1 -6
  24. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +1 -6
  25. package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
  26. package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts +13 -0
  27. package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts} +4 -1
  28. package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts} +5 -2
  29. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts} +3 -3
  30. package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts} +4 -11
  31. package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts} +4 -3
  32. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromEnv.d.ts → register/$provideLlmToolsFromEnv.d.ts} +6 -5
  33. package/esm/typings/src/llm-providers/_common/{$registeredLlmToolsMessage.d.ts → register/$registeredLlmToolsMessage.d.ts} +5 -2
  34. package/esm/typings/src/llm-providers/_common/{LlmToolsConfiguration.d.ts → register/LlmToolsConfiguration.d.ts} +5 -4
  35. package/esm/typings/src/llm-providers/_common/{LlmToolsMetadata.d.ts → register/LlmToolsMetadata.d.ts} +5 -4
  36. package/esm/typings/src/llm-providers/_common/{LlmToolsOptions.d.ts → register/LlmToolsOptions.d.ts} +4 -1
  37. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfiguration.d.ts → register/createLlmToolsFromConfiguration.d.ts} +5 -4
  38. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +3 -3
  39. package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +4 -3
  40. package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +4 -3
  41. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -2
  42. package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +4 -3
  43. package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +4 -3
  44. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +3 -3
  45. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +3 -3
  46. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +1 -0
  47. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
  48. package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +5 -4
  49. package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +5 -4
  50. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +1 -1
  51. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +1 -1
  52. package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +3 -3
  53. package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +2 -2
  54. package/esm/typings/src/personas/preparePersona.d.ts +2 -1
  55. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +8 -7
  56. package/esm/typings/src/prepare/preparePipeline.d.ts +2 -1
  57. package/esm/typings/src/prepare/prepareTemplates.d.ts +2 -1
  58. package/esm/typings/src/scrapers/_common/Converter.d.ts +4 -10
  59. package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -9
  60. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -1
  61. package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
  62. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +12 -0
  63. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +15 -0
  64. package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +12 -0
  65. package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +13 -0
  66. package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +13 -0
  67. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +41 -0
  68. package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +12 -0
  69. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +1 -0
  70. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +2 -1
  71. package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts} +18 -12
  72. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +20 -0
  73. package/esm/typings/src/scrapers/document/register-constructor.d.ts +13 -0
  74. package/esm/typings/src/scrapers/document/register-metadata.d.ts +24 -0
  75. package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.d.ts → LegacyDocumentScraper.d.ts} +18 -12
  76. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +20 -0
  77. package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +13 -0
  78. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +24 -0
  79. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +29 -0
  80. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +20 -0
  81. package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +13 -0
  82. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +24 -0
  83. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +40 -0
  84. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +20 -0
  85. package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +13 -0
  86. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +24 -0
  87. package/esm/typings/src/scrapers/website/{websiteScraper.d.ts → WebsiteScraper.d.ts} +18 -14
  88. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +20 -0
  89. package/esm/typings/src/scrapers/website/register-constructor.d.ts +13 -0
  90. package/esm/typings/src/scrapers/website/register-metadata.d.ts +24 -0
  91. package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +2 -2
  92. package/esm/typings/src/scripting/python/PythonExecutionTools.d.ts +3 -3
  93. package/esm/typings/src/scripting/typescript/TypescriptExecutionTools.d.ts +3 -3
  94. package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +5 -3
  95. package/esm/typings/src/storage/{utils → memory/utils}/PrefixStorage.d.ts +1 -1
  96. package/esm/typings/src/storage/{utils → memory/utils}/makePromptbookStorageFromWebStorage.d.ts +1 -1
  97. package/esm/typings/src/types/typeAliases.d.ts +7 -0
  98. package/esm/typings/src/utils/$Register.d.ts +19 -6
  99. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +1 -1
  100. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -1
  101. package/esm/typings/src/utils/files/isDirectoryExisting.d.ts +14 -0
  102. package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
  103. package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
  104. package/package.json +1 -5
  105. package/umd/index.umd.js +1633 -2223
  106. package/umd/index.umd.js.map +1 -1
  107. package/esm/typings/src/scrapers/index.d.ts +0 -8
  108. package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +0 -29
  109. package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +0 -35
  110. package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +0 -15
  111. package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
  112. /package/esm/typings/src/scrapers/document/{documentScraper.test.d.ts → DocumentScraper.test.d.ts} +0 -0
  113. /package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.test.d.ts → LegacyDocumentScraper.test.d.ts} +0 -0
  114. /package/esm/typings/src/scrapers/markdown/{markdownScraper.test.d.ts → MarkdownScraper.test.d.ts} +0 -0
  115. /package/esm/typings/src/scrapers/website/{websiteScraper.test.d.ts → WebsiteScraper.test.d.ts} +0 -0
  116. /package/esm/typings/src/utils/files/{$isDirectoryExisting.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
  117. /package/esm/typings/src/utils/files/{$isFileExisting.test.d.ts → isFileExisting.test.d.ts} +0 -0
  118. /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → listAllFiles.test.d.ts} +0 -0
package/umd/index.umd.js CHANGED
@@ -1,14 +1,13 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('fs/promises'), require('path'), require('child_process'), require('colors'), require('waitasecond'), require('crypto-js'), require('crypto-js/enc-hex'), require('papaparse'), require('@mozilla/readability'), require('jsdom'), require('showdown'), require('mime-types'), require('crypto-js/sha256'), require('moment')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'fs/promises', 'path', 'child_process', 'colors', 'waitasecond', 'crypto-js', 'crypto-js/enc-hex', 'papaparse', '@mozilla/readability', 'jsdom', 'showdown', 'mime-types', 'crypto-js/sha256', 'moment'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.promises, global.path, global.child_process, global.colors, global.waitasecond, global.cryptoJs, global.hexEncoder, global.papaparse, global.readability, global.jsdom, global.showdown, global.mimeTypes, global.sha256, global.moment));
5
- })(this, (function (exports, spaceTrim, prettier, parserHtml, promises, path, child_process, colors, waitasecond, cryptoJs, hexEncoder, papaparse, readability, jsdom, showdown, mimeTypes, sha256, moment) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('path'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('crypto-js/sha256'), require('moment')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'path', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'crypto-js/sha256', 'moment'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.path, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.sha256, global.moment));
5
+ })(this, (function (exports, spaceTrim, prettier, parserHtml, waitasecond, papaparse, path, cryptoJs, hexEncoder, mimeTypes, sha256, moment) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
9
9
  var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
10
10
  var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
11
- var colors__default = /*#__PURE__*/_interopDefaultLegacy(colors);
12
11
  var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
13
12
  var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
14
13
  var moment__default = /*#__PURE__*/_interopDefaultLegacy(moment);
@@ -17,7 +16,7 @@
17
16
  /**
18
17
  * The version of the Promptbook library
19
18
  */
20
- var PROMPTBOOK_VERSION = '0.72.0-7';
19
+ var PROMPTBOOK_VERSION = '0.72.0-8';
21
20
  // TODO: [main] !!!! List here all the versions and annotate + put into script
22
21
 
23
22
  /*! *****************************************************************************
@@ -781,6 +780,12 @@
781
780
  * @public exported from `@promptbook/core`
782
781
  */
783
782
  var IS_VERBOSE = false;
783
+ /**
784
+ * @@@
785
+ *
786
+ * @public exported from `@promptbook/core`
787
+ */
788
+ var IS_AUTO_INSTALLED = false;
784
789
  /**
785
790
  * @@@
786
791
  *
@@ -1651,19 +1656,19 @@
1651
1656
  ];
1652
1657
 
1653
1658
  /**
1654
- * This error indicates that the promptbook can not retrieve knowledge from external sources
1659
+ * This error type indicates that some tools are missing for pipeline execution or preparation
1655
1660
  *
1656
1661
  * @public exported from `@promptbook/core`
1657
1662
  */
1658
- var KnowledgeScrapeError = /** @class */ (function (_super) {
1659
- __extends(KnowledgeScrapeError, _super);
1660
- function KnowledgeScrapeError(message) {
1661
- var _this = _super.call(this, message) || this;
1662
- _this.name = 'KnowledgeScrapeError';
1663
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
1663
+ var MissingToolsError = /** @class */ (function (_super) {
1664
+ __extends(MissingToolsError, _super);
1665
+ function MissingToolsError(message) {
1666
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
1667
+ _this.name = 'MissingToolsError';
1668
+ Object.setPrototypeOf(_this, MissingToolsError.prototype);
1664
1669
  return _this;
1665
1670
  }
1666
- return KnowledgeScrapeError;
1671
+ return MissingToolsError;
1667
1672
  }(Error));
1668
1673
 
1669
1674
  /**
@@ -1743,1042 +1748,188 @@
1743
1748
  }
1744
1749
 
1745
1750
  /**
1746
- * This error type indicates that some tools are missing for pipeline execution or preparation
1751
+ * Represents the usage with no resources consumed
1747
1752
  *
1748
1753
  * @public exported from `@promptbook/core`
1749
1754
  */
1750
- var MissingToolsError = /** @class */ (function (_super) {
1751
- __extends(MissingToolsError, _super);
1752
- function MissingToolsError(message) {
1753
- var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
1754
- _this.name = 'MissingToolsError';
1755
- Object.setPrototypeOf(_this, MissingToolsError.prototype);
1756
- return _this;
1757
- }
1758
- return MissingToolsError;
1759
- }(Error));
1760
-
1755
+ var ZERO_USAGE = $deepFreeze({
1756
+ price: { value: 0 },
1757
+ input: {
1758
+ tokensCount: { value: 0 },
1759
+ charactersCount: { value: 0 },
1760
+ wordsCount: { value: 0 },
1761
+ sentencesCount: { value: 0 },
1762
+ linesCount: { value: 0 },
1763
+ paragraphsCount: { value: 0 },
1764
+ pagesCount: { value: 0 },
1765
+ },
1766
+ output: {
1767
+ tokensCount: { value: 0 },
1768
+ charactersCount: { value: 0 },
1769
+ wordsCount: { value: 0 },
1770
+ sentencesCount: { value: 0 },
1771
+ linesCount: { value: 0 },
1772
+ paragraphsCount: { value: 0 },
1773
+ pagesCount: { value: 0 },
1774
+ },
1775
+ });
1761
1776
  /**
1762
- * Detects if the code is running in a Node.js environment
1763
- *
1764
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
1777
+ * Represents the usage with unknown resources consumed
1765
1778
  *
1766
- * @public exported from `@promptbook/utils`
1779
+ * @public exported from `@promptbook/core`
1767
1780
  */
1768
- var $isRunningInNode = new Function("\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n");
1781
+ var UNCERTAIN_USAGE = $deepFreeze({
1782
+ price: { value: 0, isUncertain: true },
1783
+ input: {
1784
+ tokensCount: { value: 0, isUncertain: true },
1785
+ charactersCount: { value: 0, isUncertain: true },
1786
+ wordsCount: { value: 0, isUncertain: true },
1787
+ sentencesCount: { value: 0, isUncertain: true },
1788
+ linesCount: { value: 0, isUncertain: true },
1789
+ paragraphsCount: { value: 0, isUncertain: true },
1790
+ pagesCount: { value: 0, isUncertain: true },
1791
+ },
1792
+ output: {
1793
+ tokensCount: { value: 0, isUncertain: true },
1794
+ charactersCount: { value: 0, isUncertain: true },
1795
+ wordsCount: { value: 0, isUncertain: true },
1796
+ sentencesCount: { value: 0, isUncertain: true },
1797
+ linesCount: { value: 0, isUncertain: true },
1798
+ paragraphsCount: { value: 0, isUncertain: true },
1799
+ pagesCount: { value: 0, isUncertain: true },
1800
+ },
1801
+ });
1769
1802
 
1770
1803
  /**
1771
- * This error type indicates that you try to use a feature that is not available in the current environment
1804
+ * This error indicates errors during the execution of the pipeline
1772
1805
  *
1773
1806
  * @public exported from `@promptbook/core`
1774
1807
  */
1775
- var EnvironmentMismatchError = /** @class */ (function (_super) {
1776
- __extends(EnvironmentMismatchError, _super);
1777
- function EnvironmentMismatchError(message) {
1808
+ var PipelineExecutionError = /** @class */ (function (_super) {
1809
+ __extends(PipelineExecutionError, _super);
1810
+ function PipelineExecutionError(message) {
1778
1811
  var _this = _super.call(this, message) || this;
1779
- _this.name = 'EnvironmentMismatchError';
1780
- Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
1812
+ _this.name = 'PipelineExecutionError';
1813
+ Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
1781
1814
  return _this;
1782
1815
  }
1783
- return EnvironmentMismatchError;
1816
+ return PipelineExecutionError;
1784
1817
  }(Error));
1785
1818
 
1786
1819
  /**
1787
- * Normalize options for `execCommand` and `execCommands`
1820
+ * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1788
1821
  *
1789
- * @private internal utility of `execCommand` and `execCommands`
1822
+ * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1823
+ * @public exported from `@promptbook/types`
1824
+ * TODO: !!!!!! Export as runtime class not just type
1790
1825
  */
1791
- function execCommandNormalizeOptions(options) {
1792
- var _a;
1793
- var _b, _c, _d;
1794
- var command;
1795
- var cwd;
1796
- var crashOnError;
1797
- var args = [];
1798
- var timeout;
1799
- if (typeof options === 'string') {
1800
- // TODO: [1] DRY default values
1801
- command = options;
1802
- cwd = process.cwd();
1803
- crashOnError = true;
1804
- timeout = Infinity;
1805
- }
1806
- else {
1807
- /*
1808
- TODO:
1809
- if ((options as any).commands !== undefined) {
1810
- commands = (options as any).commands;
1811
- } else {
1812
- commands = [(options as any).command];
1826
+ var MultipleLlmExecutionTools = /** @class */ (function () {
1827
+ /**
1828
+ * Gets array of execution tools in order of priority
1829
+ */
1830
+ function MultipleLlmExecutionTools() {
1831
+ var llmExecutionTools = [];
1832
+ for (var _i = 0; _i < arguments.length; _i++) {
1833
+ llmExecutionTools[_i] = arguments[_i];
1813
1834
  }
1814
- */
1815
- // TODO: [1] DRY default values
1816
- command = options.command;
1817
- cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
1818
- crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
1819
- timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
1820
- }
1821
- // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
1822
- var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
1823
- .map(function (_a) {
1824
- var _b = __read(_a, 1), match = _b[0];
1825
- return match;
1826
- })
1827
- .filter(function (arg) { return arg !== ''; });
1828
- if (_.length > 1) {
1829
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1830
- _a = __read(_), command = _a[0], args = _a.slice(1);
1831
- }
1832
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1833
- if (options.args) {
1834
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
1835
- args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
1836
- }
1837
- var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
1838
- if (['ts-node'].includes(humanReadableCommand)) {
1839
- humanReadableCommand += " ".concat(args[1]);
1840
- }
1841
- return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
1842
- }
1843
- // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
1844
-
1845
- /**
1846
- * Run one command in a shell
1847
- *
1848
- * Note: There are 2 similar functions in the codebase:
1849
- * - `$execCommand` which runs a single command
1850
- * - `$execCommands` which runs multiple commands
1851
- *
1852
- * @public exported from `@promptbook/node`
1853
- */
1854
- function $execCommand(options) {
1855
- if (!$isRunningInNode()) {
1856
- throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
1835
+ this.llmExecutionTools = llmExecutionTools;
1857
1836
  }
1858
- return new Promise(
1859
- // <- TODO: [🧱] Implement in a functional (not new Class) way
1860
- function (resolve, reject) {
1861
- // eslint-disable-next-line prefer-const
1862
- var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
1863
- if (timeout !== Infinity) {
1864
- // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
1865
- waitasecond.forTime(timeout).then(function () {
1866
- if (crashOnError) {
1867
- reject(new Error("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms")));
1868
- }
1869
- else {
1870
- console.warn("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms but continues running"));
1871
- resolve('Command exceeded time limit');
1872
- }
1873
- });
1874
- }
1875
- if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
1876
- command = "".concat(command, ".cmd");
1877
- }
1878
- // !!!!!! Verbose mode - to all consoles
1879
- console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
1880
- try {
1881
- var commandProcess = child_process.spawn(command, args, { cwd: cwd, shell: true });
1882
- commandProcess.on('message', function (message) {
1883
- console.info({ message: message });
1884
- });
1885
- var output_1 = [];
1886
- commandProcess.stdout.on('data', function (stdout) {
1887
- output_1.push(stdout.toString());
1888
- console.info(stdout.toString());
1889
- });
1890
- commandProcess.stderr.on('data', function (stderr) {
1891
- output_1.push(stderr.toString());
1892
- if (stderr.toString().trim()) {
1893
- console.warn(stderr.toString());
1894
- }
1895
- });
1896
- var finishWithCode = function (code) {
1897
- if (code !== 0) {
1898
- if (crashOnError) {
1899
- reject(new Error(output_1.join('\n').trim() ||
1900
- "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
1901
- }
1902
- else {
1903
- console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
1904
- resolve(spaceTrim.spaceTrim(output_1.join('\n')));
1905
- }
1906
- }
1907
- else {
1908
- resolve(spaceTrim.spaceTrim(output_1.join('\n')));
1837
+ Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
1838
+ get: function () {
1839
+ return 'Multiple LLM Providers';
1840
+ },
1841
+ enumerable: false,
1842
+ configurable: true
1843
+ });
1844
+ Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
1845
+ get: function () {
1846
+ return this.llmExecutionTools.map(function (_a, index) {
1847
+ var title = _a.title;
1848
+ return "".concat(index + 1, ") `").concat(title, "`");
1849
+ }).join('\n');
1850
+ },
1851
+ enumerable: false,
1852
+ configurable: true
1853
+ });
1854
+ /**
1855
+ * Check the configuration of all execution tools
1856
+ */
1857
+ MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
1858
+ return __awaiter(this, void 0, void 0, function () {
1859
+ var _a, _b, llmExecutionTools, e_1_1;
1860
+ var e_1, _c;
1861
+ return __generator(this, function (_d) {
1862
+ switch (_d.label) {
1863
+ case 0:
1864
+ _d.trys.push([0, 5, 6, 7]);
1865
+ _a = __values(this.llmExecutionTools), _b = _a.next();
1866
+ _d.label = 1;
1867
+ case 1:
1868
+ if (!!_b.done) return [3 /*break*/, 4];
1869
+ llmExecutionTools = _b.value;
1870
+ return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
1871
+ case 2:
1872
+ _d.sent();
1873
+ _d.label = 3;
1874
+ case 3:
1875
+ _b = _a.next();
1876
+ return [3 /*break*/, 1];
1877
+ case 4: return [3 /*break*/, 7];
1878
+ case 5:
1879
+ e_1_1 = _d.sent();
1880
+ e_1 = { error: e_1_1 };
1881
+ return [3 /*break*/, 7];
1882
+ case 6:
1883
+ try {
1884
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
1885
+ }
1886
+ finally { if (e_1) throw e_1.error; }
1887
+ return [7 /*endfinally*/];
1888
+ case 7: return [2 /*return*/];
1909
1889
  }
1910
- };
1911
- commandProcess.on('close', finishWithCode);
1912
- commandProcess.on('exit', finishWithCode);
1913
- commandProcess.on('disconnect', function () {
1914
- // Note: Unexpected disconnection should always result in rejection
1915
- reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
1916
1890
  });
1917
- commandProcess.on('error', function (error) {
1918
- if (crashOnError) {
1919
- reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
1920
- }
1921
- else {
1922
- console.warn(error);
1923
- resolve(spaceTrim.spaceTrim(output_1.join('\n')));
1924
- }
1925
- });
1926
- }
1927
- catch (error) {
1928
- // Note: Unexpected error in sync code should always result in rejection
1929
- reject(error);
1930
- }
1931
- });
1932
- }
1933
- /**
1934
- * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
1935
- */
1936
-
1937
- /**
1938
- * Checks if the file exists
1939
- *
1940
- * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
1941
- *
1942
- * @private within the repository
1943
- */
1944
- function $isFileExisting(filename) {
1945
- return __awaiter(this, void 0, void 0, function () {
1946
- var isReadAccessAllowed, isFile;
1947
- return __generator(this, function (_a) {
1948
- switch (_a.label) {
1949
- case 0:
1950
- if (!$isRunningInNode()) {
1951
- throw new EnvironmentMismatchError('Function `$isFileExisting` works only in Node environment.js');
1952
- }
1953
- return [4 /*yield*/, promises.access(filename, promises.constants.R_OK)
1954
- .then(function () { return true; })
1955
- .catch(function () { return false; })];
1956
- case 1:
1957
- isReadAccessAllowed = _a.sent();
1958
- if (!isReadAccessAllowed) {
1959
- return [2 /*return*/, false];
1960
- }
1961
- return [4 /*yield*/, promises.stat(filename)
1962
- .then(function (fileStat) { return fileStat.isFile(); })
1963
- .catch(function () { return false; })];
1964
- case 2:
1965
- isFile = _a.sent();
1966
- return [2 /*return*/, isFile];
1967
- }
1968
- });
1969
- });
1970
- }
1971
- /**
1972
- * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
1973
- * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
1974
- * TODO: [🖇] What about symlinks?
1975
- */
1976
-
1977
- /**
1978
- * Get the file extension from a file name
1979
- *
1980
- * @private within the repository
1981
- */
1982
- function getFileExtension(value) {
1983
- var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
1984
- return match ? match[1].toLowerCase() : null;
1985
- }
1986
-
1987
- var defaultDiacriticsRemovalMap = [
1988
- {
1989
- base: 'A',
1990
- letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
1991
- },
1992
- { base: 'AA', letters: '\uA732' },
1993
- { base: 'AE', letters: '\u00C6\u01FC\u01E2' },
1994
- { base: 'AO', letters: '\uA734' },
1995
- { base: 'AU', letters: '\uA736' },
1996
- { base: 'AV', letters: '\uA738\uA73A' },
1997
- { base: 'AY', letters: '\uA73C' },
1998
- {
1999
- base: 'B',
2000
- letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
2001
- },
2002
- {
2003
- base: 'C',
2004
- letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
2005
- },
2006
- {
2007
- base: 'D',
2008
- letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
2009
- },
2010
- { base: 'DZ', letters: '\u01F1\u01C4' },
2011
- { base: 'Dz', letters: '\u01F2\u01C5' },
2012
- {
2013
- base: 'E',
2014
- letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
2015
- },
2016
- { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
2017
- {
2018
- base: 'G',
2019
- letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
2020
- },
2021
- {
2022
- base: 'H',
2023
- letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
2024
- },
2025
- {
2026
- base: 'I',
2027
- letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
2028
- },
2029
- { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
2030
- {
2031
- base: 'K',
2032
- letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
2033
- },
2034
- {
2035
- base: 'L',
2036
- letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
2037
- },
2038
- { base: 'LJ', letters: '\u01C7' },
2039
- { base: 'Lj', letters: '\u01C8' },
2040
- { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
2041
- {
2042
- base: 'N',
2043
- letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
2044
- },
2045
- { base: 'NJ', letters: '\u01CA' },
2046
- { base: 'Nj', letters: '\u01CB' },
2047
- {
2048
- base: 'O',
2049
- letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
2050
- },
2051
- { base: 'OI', letters: '\u01A2' },
2052
- { base: 'OO', letters: '\uA74E' },
2053
- { base: 'OU', letters: '\u0222' },
2054
- { base: 'OE', letters: '\u008C\u0152' },
2055
- { base: 'oe', letters: '\u009C\u0153' },
2056
- {
2057
- base: 'P',
2058
- letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
2059
- },
2060
- { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
2061
- {
2062
- base: 'R',
2063
- letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
2064
- },
2065
- {
2066
- base: 'S',
2067
- letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
2068
- },
2069
- {
2070
- base: 'T',
2071
- letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
2072
- },
2073
- { base: 'TZ', letters: '\uA728' },
2074
- {
2075
- base: 'U',
2076
- letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
2077
- },
2078
- { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
2079
- { base: 'VY', letters: '\uA760' },
2080
- {
2081
- base: 'W',
2082
- letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
2083
- },
2084
- { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
2085
- {
2086
- base: 'Y',
2087
- letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
2088
- },
2089
- {
2090
- base: 'Z',
2091
- letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
2092
- },
2093
- {
2094
- base: 'a',
2095
- letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
2096
- },
2097
- { base: 'aa', letters: '\uA733' },
2098
- { base: 'ae', letters: '\u00E6\u01FD\u01E3' },
2099
- { base: 'ao', letters: '\uA735' },
2100
- { base: 'au', letters: '\uA737' },
2101
- { base: 'av', letters: '\uA739\uA73B' },
2102
- { base: 'ay', letters: '\uA73D' },
2103
- {
2104
- base: 'b',
2105
- letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
2106
- },
2107
- {
2108
- base: 'c',
2109
- letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
2110
- },
2111
- {
2112
- base: 'd',
2113
- letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
2114
- },
2115
- { base: 'dz', letters: '\u01F3\u01C6' },
2116
- {
2117
- base: 'e',
2118
- letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
2119
- },
2120
- { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
2121
- {
2122
- base: 'g',
2123
- letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
2124
- },
2125
- {
2126
- base: 'h',
2127
- letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
2128
- },
2129
- { base: 'hv', letters: '\u0195' },
2130
- {
2131
- base: 'i',
2132
- letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
2133
- },
2134
- { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
2135
- {
2136
- base: 'k',
2137
- letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
2138
- },
2139
- {
2140
- base: 'l',
2141
- letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
2142
- },
2143
- { base: 'lj', letters: '\u01C9' },
2144
- { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
2145
- {
2146
- base: 'n',
2147
- letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
2148
- },
2149
- { base: 'nj', letters: '\u01CC' },
2150
- {
2151
- base: 'o',
2152
- letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
2153
- },
2154
- { base: 'oi', letters: '\u01A3' },
2155
- { base: 'ou', letters: '\u0223' },
2156
- { base: 'oo', letters: '\uA74F' },
2157
- {
2158
- base: 'p',
2159
- letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
2160
- },
2161
- { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
2162
- {
2163
- base: 'r',
2164
- letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
2165
- },
2166
- {
2167
- base: 's',
2168
- letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
2169
- },
2170
- {
2171
- base: 't',
2172
- letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
2173
- },
2174
- { base: 'tz', letters: '\uA729' },
2175
- {
2176
- base: 'u',
2177
- letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
2178
- },
2179
- { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
2180
- { base: 'vy', letters: '\uA761' },
2181
- {
2182
- base: 'w',
2183
- letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
2184
- },
2185
- { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
2186
- {
2187
- base: 'y',
2188
- letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
2189
- },
2190
- {
2191
- base: 'z',
2192
- letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
2193
- },
2194
- ];
2195
- /**
2196
- * Map of letters from diacritic variant to diacritless variant
2197
- * Contains lowercase and uppercase separatelly
2198
- *
2199
- * > "á" => "a"
2200
- * > "ě" => "e"
2201
- * > "Ă" => "A"
2202
- * > ...
2203
- *
2204
- * @public exported from `@promptbook/utils`
2205
- */
2206
- var DIACRITIC_VARIANTS_LETTERS = {};
2207
- // tslint:disable-next-line: prefer-for-of
2208
- for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
2209
- var letters = defaultDiacriticsRemovalMap[i].letters;
2210
- // tslint:disable-next-line: prefer-for-of
2211
- for (var j = 0; j < letters.length; j++) {
2212
- DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
2213
- }
2214
- }
2215
- // <- TODO: [🍓] Put to maker function to save execution time if not needed
2216
- /*
2217
- @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
2218
- Licensed under the Apache License, Version 2.0 (the "License");
2219
- you may not use this file except in compliance with the License.
2220
- You may obtain a copy of the License at
2221
-
2222
- http://www.apache.org/licenses/LICENSE-2.0
2223
-
2224
- Unless required by applicable law or agreed to in writing, software
2225
- distributed under the License is distributed on an "AS IS" BASIS,
2226
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2227
- See the License for the specific language governing permissions and
2228
- limitations under the License.
2229
- */
2230
-
2231
- /**
2232
- * @@@
2233
- *
2234
- * @param input @@@
2235
- * @returns @@@
2236
- * @public exported from `@promptbook/utils`
2237
- */
2238
- function removeDiacritics(input) {
2239
- /*eslint no-control-regex: "off"*/
2240
- return input.replace(/[^\u0000-\u007E]/g, function (a) {
2241
- return DIACRITIC_VARIANTS_LETTERS[a] || a;
2242
- });
2243
- }
2244
- /**
2245
- * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
2246
- */
2247
-
2248
- /**
2249
- * @@@
2250
- *
2251
- * @param text @@@
2252
- * @returns @@@
2253
- * @example 'hello-world'
2254
- * @example 'i-love-promptbook'
2255
- * @public exported from `@promptbook/utils`
2256
- */
2257
- function normalizeToKebabCase(text) {
2258
- var e_1, _a;
2259
- text = removeDiacritics(text);
2260
- var charType;
2261
- var lastCharType = 'OTHER';
2262
- var normalizedName = '';
2263
- try {
2264
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
2265
- var char = text_1_1.value;
2266
- var normalizedChar = void 0;
2267
- if (/^[a-z]$/.test(char)) {
2268
- charType = 'LOWERCASE';
2269
- normalizedChar = char;
2270
- }
2271
- else if (/^[A-Z]$/.test(char)) {
2272
- charType = 'UPPERCASE';
2273
- normalizedChar = char.toLowerCase();
2274
- }
2275
- else if (/^[0-9]$/.test(char)) {
2276
- charType = 'NUMBER';
2277
- normalizedChar = char;
2278
- }
2279
- else {
2280
- charType = 'OTHER';
2281
- normalizedChar = '-';
2282
- }
2283
- if (charType !== lastCharType &&
2284
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
2285
- !(lastCharType === 'NUMBER') &&
2286
- !(charType === 'NUMBER')) {
2287
- normalizedName += '-';
2288
- }
2289
- normalizedName += normalizedChar;
2290
- lastCharType = charType;
2291
- }
2292
- }
2293
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
2294
- finally {
2295
- try {
2296
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
2297
- }
2298
- finally { if (e_1) throw e_1.error; }
2299
- }
2300
- normalizedName = normalizedName.split(/-+/g).join('-');
2301
- normalizedName = normalizedName.split(/-?\/-?/g).join('/');
2302
- normalizedName = normalizedName.replace(/^-/, '');
2303
- normalizedName = normalizedName.replace(/-$/, '');
2304
- return normalizedName;
2305
- }
2306
-
2307
- /**
2308
- * Removes emojis from a string and fix whitespaces
2309
- *
2310
- * @param text with emojis
2311
- * @returns text without emojis
2312
- * @public exported from `@promptbook/utils`
2313
- */
2314
- function removeEmojis(text) {
2315
- // Replace emojis (and also ZWJ sequence) with hyphens
2316
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
2317
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
2318
- text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
2319
- text = text.replace(/\p{Extended_Pictographic}/gu, '');
2320
- return text;
2321
- }
2322
-
2323
- /**
2324
- * Tests if given string is valid URL.
2325
- *
2326
- * Note: This does not check if the file exists only if the path is valid
2327
- * @public exported from `@promptbook/utils`
2328
- */
2329
- function isValidFilePath(filename) {
2330
- if (typeof filename !== 'string') {
2331
- return false;
2332
- }
2333
- var filenameSlashes = filename.split('\\').join('/');
2334
- // Absolute Unix path: /hello.txt
2335
- if (/^(\/)/i.test(filenameSlashes)) {
2336
- return true;
2337
- }
2338
- // Absolute Windows path: /hello.txt
2339
- if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
2340
- return true;
2341
- }
2342
- // Relative path: ./hello.txt
2343
- if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
2344
- return true;
2345
- }
2346
- return false;
2347
- }
2348
-
2349
- /**
2350
- * @@@
2351
- *
2352
- * @param value @@@
2353
- * @returns @@@
2354
- * @example @@@
2355
- * @public exported from `@promptbook/utils`
2356
- */
2357
- function titleToName(value) {
2358
- if (isValidUrl(value)) {
2359
- value = value.replace(/^https?:\/\//, '');
2360
- value = value.replace(/\.html$/, '');
2361
- }
2362
- else if (isValidFilePath(value)) {
2363
- value = path.basename(value);
2364
- // Note: Keeping extension in the name
2365
- }
2366
- value = value.split('/').join('-');
2367
- value = removeEmojis(value);
2368
- value = normalizeToKebabCase(value);
2369
- // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
2370
- return value;
2371
- }
2372
-
2373
- /**
2374
- * @@@
2375
- *
2376
- * @private for `FileCacheStorage`
2377
- */
2378
- function nameToSubfolderPath(name) {
2379
- return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
2380
- }
2381
-
2382
- /**
2383
- * Just marks a place of place where should be something implemented
2384
- * No side effects.
2385
- *
2386
- * Note: It can be usefull suppressing eslint errors of unused variables
2387
- *
2388
- * @param value any values
2389
- * @returns void
2390
- * @private within the repository
2391
- */
2392
- function TODO_USE() {
2393
- var value = [];
2394
- for (var _i = 0; _i < arguments.length; _i++) {
2395
- value[_i] = arguments[_i];
2396
- }
2397
- }
2398
-
2399
- /**
2400
- * Create a filename for intermediate cache for scrapers
2401
- *
2402
- * Note: It also checks if directory exists and creates it if not
2403
- *
2404
- * @private as internal utility for scrapers
2405
- */
2406
- function getScraperIntermediateSource(source, options) {
2407
- return __awaiter(this, void 0, void 0, function () {
2408
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
2409
- return __generator(this, function (_a) {
2410
- switch (_a.label) {
2411
- case 0:
2412
- sourceFilename = source.filename, url = source.url;
2413
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
2414
- hash = cryptoJs.SHA256(
2415
- // <- TODO: [🥬] Encapsulate sha256 to some private utility function
2416
- hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
2417
- .toString( /* hex */)
2418
- .substring(0, 20);
2419
- semanticName = normalizeToKebabCase(titleToName((sourceFilename || url || '').split('intermediate').join(''))).substring(0, 20);
2420
- pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
2421
- name = pieces.join('-').split('--').join('-');
2422
- // <- TODO: Use MAX_FILENAME_LENGTH
2423
- TODO_USE(rootDirname); // <- TODO: !!!!!!
2424
- cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
2425
- cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
2426
- .join('/') +
2427
- '.' +
2428
- extension;
2429
- return [4 /*yield*/, promises.mkdir(path.dirname(cacheFilename), { recursive: true })];
2430
- case 1:
2431
- _a.sent();
2432
- isDestroyed = true;
2433
- fileHandler = {
2434
- filename: cacheFilename,
2435
- get isDestroyed() {
2436
- return isDestroyed;
2437
- },
2438
- destroy: function () {
2439
- return __awaiter(this, void 0, void 0, function () {
2440
- return __generator(this, function (_a) {
2441
- switch (_a.label) {
2442
- case 0:
2443
- if (!isCacheCleaned) return [3 /*break*/, 2];
2444
- if (isVerbose) {
2445
- console.info('legacyDocumentScraper: Clening cache');
2446
- }
2447
- return [4 /*yield*/, promises.rm(cacheFilename)];
2448
- case 1:
2449
- _a.sent();
2450
- _a.label = 2;
2451
- case 2:
2452
- isDestroyed = true;
2453
- return [2 /*return*/];
2454
- }
2455
- });
2456
- });
2457
- },
2458
- };
2459
- return [2 /*return*/, fileHandler];
2460
- }
2461
- });
2462
- });
2463
- }
2464
- /**
2465
- * Note: Not using `FileCacheStorage` for two reasons:
2466
- * 1) Need to store more than serialized JSONs
2467
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
2468
- * TODO: [🐱‍🐉][🧠] Make some smart crop
2469
- */
2470
-
2471
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
2472
-
2473
- /**
2474
- * This error indicates errors during the execution of the pipeline
2475
- *
2476
- * @public exported from `@promptbook/core`
2477
- */
2478
- var PipelineExecutionError = /** @class */ (function (_super) {
2479
- __extends(PipelineExecutionError, _super);
2480
- function PipelineExecutionError(message) {
2481
- var _this = _super.call(this, message) || this;
2482
- _this.name = 'PipelineExecutionError';
2483
- Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
2484
- return _this;
2485
- }
2486
- return PipelineExecutionError;
2487
- }(Error));
2488
-
2489
- /**
2490
- * This error indicates that the pipeline collection cannot be propperly loaded
2491
- *
2492
- * @public exported from `@promptbook/core`
2493
- */
2494
- var CollectionError = /** @class */ (function (_super) {
2495
- __extends(CollectionError, _super);
2496
- function CollectionError(message) {
2497
- var _this = _super.call(this, message) || this;
2498
- _this.name = 'CollectionError';
2499
- Object.setPrototypeOf(_this, CollectionError.prototype);
2500
- return _this;
2501
- }
2502
- return CollectionError;
2503
- }(Error));
2504
-
2505
- /**
2506
- * This error occurs when some expectation is not met in the execution of the pipeline
2507
- *
2508
- * @public exported from `@promptbook/core`
2509
- * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
2510
- * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
2511
- * Note: This is a kindof subtype of PipelineExecutionError
2512
- */
2513
- var ExpectError = /** @class */ (function (_super) {
2514
- __extends(ExpectError, _super);
2515
- function ExpectError(message) {
2516
- var _this = _super.call(this, message) || this;
2517
- _this.name = 'ExpectError';
2518
- Object.setPrototypeOf(_this, ExpectError.prototype);
2519
- return _this;
2520
- }
2521
- return ExpectError;
2522
- }(Error));
2523
-
2524
- /**
2525
- * This error type indicates that some limit was reached
2526
- *
2527
- * @public exported from `@promptbook/core`
2528
- */
2529
- var LimitReachedError = /** @class */ (function (_super) {
2530
- __extends(LimitReachedError, _super);
2531
- function LimitReachedError(message) {
2532
- var _this = _super.call(this, message) || this;
2533
- _this.name = 'LimitReachedError';
2534
- Object.setPrototypeOf(_this, LimitReachedError.prototype);
2535
- return _this;
2536
- }
2537
- return LimitReachedError;
2538
- }(Error));
2539
-
2540
- /**
2541
- * This error type indicates that some part of the code is not implemented yet
2542
- *
2543
- * @public exported from `@promptbook/core`
2544
- */
2545
- var NotYetImplementedError = /** @class */ (function (_super) {
2546
- __extends(NotYetImplementedError, _super);
2547
- function NotYetImplementedError(message) {
2548
- var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2549
- _this.name = 'NotYetImplementedError';
2550
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2551
- return _this;
2552
- }
2553
- return NotYetImplementedError;
2554
- }(Error));
2555
-
2556
- /**
2557
- * Index of all custom errors
2558
- *
2559
- * @public exported from `@promptbook/core`
2560
- */
2561
- var ERRORS = {
2562
- ExpectError: ExpectError,
2563
- CollectionError: CollectionError,
2564
- EnvironmentMismatchError: EnvironmentMismatchError,
2565
- LimitReachedError: LimitReachedError,
2566
- NotFoundError: NotFoundError,
2567
- NotYetImplementedError: NotYetImplementedError,
2568
- ParseError: ParseError,
2569
- PipelineExecutionError: PipelineExecutionError,
2570
- PipelineLogicError: PipelineLogicError,
2571
- PipelineUrlError: PipelineUrlError,
2572
- UnexpectedError: UnexpectedError,
2573
- // TODO: [🪑]> VersionMismatchError,
2574
- };
2575
-
2576
- /**
2577
- * Deserializes the error object
2578
- *
2579
- * @public exported from `@promptbook/utils`
2580
- */
2581
- function deserializeError(error) {
2582
- if (error.name === 'Error') {
2583
- return new Error(error.message);
2584
- }
2585
- var CustomError = ERRORS[error.name];
2586
- return new CustomError(error.message);
2587
- }
2588
-
2589
- /**
2590
- * Asserts that the execution of a Promptbook is successful
2591
- *
2592
- * @param executionResult - The partial result of the Promptbook execution
2593
- * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2594
- * @public exported from `@promptbook/core`
2595
- */
2596
- function assertsExecutionSuccessful(executionResult) {
2597
- var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2598
- if (isSuccessful === true) {
2599
- return;
2600
- }
2601
- if (errors.length === 0) {
2602
- throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
2603
- }
2604
- else if (errors.length === 1) {
2605
- throw deserializeError(errors[0]);
2606
- }
2607
- else {
2608
- throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
2609
- .map(function (_a, index) {
2610
- var name = _a.name, stack = _a.stack, message = _a.message;
2611
- return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
2612
- })
2613
- .join('\n')), "\n "); }));
2614
- }
2615
- }
2616
- /**
2617
- * TODO: [🧠] Can this return type be better typed than void
2618
- */
2619
-
2620
- /**
2621
- * Determine if the pipeline is fully prepared
2622
- *
2623
- * @public exported from `@promptbook/core`
2624
- */
2625
- function isPipelinePrepared(pipeline) {
2626
- // Note: Ignoring `pipeline.preparations` @@@
2627
- // Note: Ignoring `pipeline.knowledgePieces` @@@
2628
- if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2629
- return false;
2630
- }
2631
- if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2632
- return false;
2633
- }
2634
- /*
2635
- TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2636
- > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2637
- > return false;
2638
- > }
2639
- */
2640
- return true;
2641
- }
2642
- /**
2643
- * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2644
- * TODO: [🐠] Maybe base this on `makeValidator`
2645
- * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2646
- * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2647
- * - [🏍] ? Is context in each template
2648
- * - [♨] Are samples prepared
2649
- * - [♨] Are templates prepared
2650
- */
2651
-
2652
- /**
2653
- * Serializes an error into a [🚉] JSON-serializable object
2654
- *
2655
- * @public exported from `@promptbook/utils`
2656
- */
2657
- function serializeError(error) {
2658
- var name = error.name, message = error.message, stack = error.stack;
2659
- if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2660
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2661
- }
2662
- return {
2663
- name: name,
2664
- message: message,
2665
- stack: stack,
2666
- };
2667
- }
2668
-
2669
- /**
2670
- * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2671
- *
2672
- * Note: Internal utility of `joinLlmExecutionTools` but exposed type
2673
- * @public exported from `@promptbook/types`
2674
- */
2675
- var MultipleLlmExecutionTools = /** @class */ (function () {
2676
- /**
2677
- * Gets array of execution tools in order of priority
2678
- */
2679
- function MultipleLlmExecutionTools() {
2680
- var llmExecutionTools = [];
2681
- for (var _i = 0; _i < arguments.length; _i++) {
2682
- llmExecutionTools[_i] = arguments[_i];
2683
- }
2684
- this.llmExecutionTools = llmExecutionTools;
2685
- }
2686
- Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
2687
- get: function () {
2688
- return 'Multiple LLM Providers';
2689
- },
2690
- enumerable: false,
2691
- configurable: true
2692
- });
2693
- Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
2694
- get: function () {
2695
- return this.llmExecutionTools.map(function (_a, index) {
2696
- var title = _a.title;
2697
- return "".concat(index + 1, ") `").concat(title, "`");
2698
- }).join('\n');
2699
- },
2700
- enumerable: false,
2701
- configurable: true
2702
- });
2703
- /**
2704
- * Check the configuration of all execution tools
2705
- */
2706
- MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
2707
- return __awaiter(this, void 0, void 0, function () {
2708
- var _a, _b, llmExecutionTools, e_1_1;
2709
- var e_1, _c;
2710
- return __generator(this, function (_d) {
2711
- switch (_d.label) {
2712
- case 0:
2713
- _d.trys.push([0, 5, 6, 7]);
2714
- _a = __values(this.llmExecutionTools), _b = _a.next();
2715
- _d.label = 1;
2716
- case 1:
2717
- if (!!_b.done) return [3 /*break*/, 4];
2718
- llmExecutionTools = _b.value;
2719
- return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
2720
- case 2:
2721
- _d.sent();
2722
- _d.label = 3;
2723
- case 3:
2724
- _b = _a.next();
2725
- return [3 /*break*/, 1];
2726
- case 4: return [3 /*break*/, 7];
2727
- case 5:
2728
- e_1_1 = _d.sent();
2729
- e_1 = { error: e_1_1 };
2730
- return [3 /*break*/, 7];
2731
- case 6:
2732
- try {
2733
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2734
- }
2735
- finally { if (e_1) throw e_1.error; }
2736
- return [7 /*endfinally*/];
2737
- case 7: return [2 /*return*/];
2738
- }
2739
- });
2740
- });
2741
- };
2742
- /**
2743
- * List all available models that can be used
2744
- * This lists is a combination of all available models from all execution tools
2745
- */
2746
- MultipleLlmExecutionTools.prototype.listModels = function () {
2747
- return __awaiter(this, void 0, void 0, function () {
2748
- var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
2749
- var e_2, _c;
2750
- return __generator(this, function (_d) {
2751
- switch (_d.label) {
2752
- case 0:
2753
- availableModels = [];
2754
- _d.label = 1;
2755
- case 1:
2756
- _d.trys.push([1, 6, 7, 8]);
2757
- _a = __values(this.llmExecutionTools), _b = _a.next();
2758
- _d.label = 2;
2759
- case 2:
2760
- if (!!_b.done) return [3 /*break*/, 5];
2761
- llmExecutionTools = _b.value;
2762
- return [4 /*yield*/, llmExecutionTools.listModels()];
2763
- case 3:
2764
- models = _d.sent();
2765
- availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
2766
- _d.label = 4;
2767
- case 4:
2768
- _b = _a.next();
2769
- return [3 /*break*/, 2];
2770
- case 5: return [3 /*break*/, 8];
2771
- case 6:
2772
- e_2_1 = _d.sent();
2773
- e_2 = { error: e_2_1 };
2774
- return [3 /*break*/, 8];
2775
- case 7:
2776
- try {
2777
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
2778
- }
2779
- finally { if (e_2) throw e_2.error; }
2780
- return [7 /*endfinally*/];
2781
- case 8: return [2 /*return*/, availableModels];
1891
+ });
1892
+ };
1893
+ /**
1894
+ * List all available models that can be used
1895
+ * This lists is a combination of all available models from all execution tools
1896
+ */
1897
+ MultipleLlmExecutionTools.prototype.listModels = function () {
1898
+ return __awaiter(this, void 0, void 0, function () {
1899
+ var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
1900
+ var e_2, _c;
1901
+ return __generator(this, function (_d) {
1902
+ switch (_d.label) {
1903
+ case 0:
1904
+ availableModels = [];
1905
+ _d.label = 1;
1906
+ case 1:
1907
+ _d.trys.push([1, 6, 7, 8]);
1908
+ _a = __values(this.llmExecutionTools), _b = _a.next();
1909
+ _d.label = 2;
1910
+ case 2:
1911
+ if (!!_b.done) return [3 /*break*/, 5];
1912
+ llmExecutionTools = _b.value;
1913
+ return [4 /*yield*/, llmExecutionTools.listModels()];
1914
+ case 3:
1915
+ models = _d.sent();
1916
+ availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
1917
+ _d.label = 4;
1918
+ case 4:
1919
+ _b = _a.next();
1920
+ return [3 /*break*/, 2];
1921
+ case 5: return [3 /*break*/, 8];
1922
+ case 6:
1923
+ e_2_1 = _d.sent();
1924
+ e_2 = { error: e_2_1 };
1925
+ return [3 /*break*/, 8];
1926
+ case 7:
1927
+ try {
1928
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
1929
+ }
1930
+ finally { if (e_2) throw e_2.error; }
1931
+ return [7 /*endfinally*/];
1932
+ case 8: return [2 /*return*/, availableModels];
2782
1933
  }
2783
1934
  });
2784
1935
  });
@@ -2897,232 +2048,445 @@
2897
2048
  }
2898
2049
  }
2899
2050
  });
2900
- });
2901
- };
2902
- return MultipleLlmExecutionTools;
2903
- }());
2051
+ });
2052
+ };
2053
+ return MultipleLlmExecutionTools;
2054
+ }());
2055
+ /**
2056
+ * TODO: [🧠][🎛] Aggregating multiple models - have result not only from one first aviable model BUT all of them
2057
+ * TODO: [🏖] If no llmTools have for example not defined `callCompletionModel` this will still return object with defined `callCompletionModel` which just throws `PipelineExecutionError`, make it undefined instead
2058
+ * Look how `countTotalUsage` (and `cacheLlmTools`) implements it
2059
+ */
2060
+
2061
+ /**
2062
+ * Joins multiple LLM Execution Tools into one
2063
+ *
2064
+ * @returns {LlmExecutionTools} Single wrapper for multiple LlmExecutionTools
2065
+ *
2066
+ * 0) If there is no LlmExecutionTools, it warns and returns valid but empty LlmExecutionTools
2067
+ * 1) If there is only one LlmExecutionTools, it returns it wrapped in a proxy object
2068
+ * 2) If there are multiple LlmExecutionTools, first will be used first, second will be used if the first hasn`t defined model variant or fails, etc.
2069
+ * 3) When all LlmExecutionTools fail, it throws an error with a list of all errors merged into one
2070
+ *
2071
+ *
2072
+ * Tip: You don't have to use this function directly, just pass an array of LlmExecutionTools to the `ExecutionTools`
2073
+ *
2074
+ * @public exported from `@promptbook/core`
2075
+ */
2076
+ function joinLlmExecutionTools() {
2077
+ var llmExecutionTools = [];
2078
+ for (var _i = 0; _i < arguments.length; _i++) {
2079
+ llmExecutionTools[_i] = arguments[_i];
2080
+ }
2081
+ if (llmExecutionTools.length === 0) {
2082
+ var warningMessage = spaceTrim__default["default"]("\n You have not provided any `LlmExecutionTools`\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ");
2083
+ // TODO: [🟥] Detect browser / node and make it colorfull
2084
+ console.warn(warningMessage);
2085
+ /*
2086
+ return {
2087
+ async listModels() {
2088
+ // TODO: [🟥] Detect browser / node and make it colorfull
2089
+ console.warn(
2090
+ spaceTrim(
2091
+ (block) => `
2092
+
2093
+ You can't list models because you have no LLM Execution Tools defined:
2094
+
2095
+ tl;dr
2096
+
2097
+ ${block(warningMessage)}
2098
+ `,
2099
+ ),
2100
+ );
2101
+ return [];
2102
+ },
2103
+ };
2104
+ */
2105
+ }
2106
+ return new (MultipleLlmExecutionTools.bind.apply(MultipleLlmExecutionTools, __spreadArray([void 0], __read(llmExecutionTools), false)))();
2107
+ }
2108
+ /**
2109
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2110
+ */
2111
+
2112
+ /**
2113
+ * @@@
2114
+ *
2115
+ * @public exported from `@promptbook/utils`
2116
+ */
2117
+ function deepClone(objectValue) {
2118
+ return JSON.parse(JSON.stringify(objectValue));
2119
+ /*
2120
+ TODO: [🧠] Is there a better implementation?
2121
+ > const propertyNames = Object.getOwnPropertyNames(objectValue);
2122
+ > for (const propertyName of propertyNames) {
2123
+ > const value = (objectValue as really_any)[propertyName];
2124
+ > if (value && typeof value === 'object') {
2125
+ > deepClone(value);
2126
+ > }
2127
+ > }
2128
+ > return Object.assign({}, objectValue);
2129
+ */
2130
+ }
2131
+ /**
2132
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
2133
+ */
2134
+
2135
+ /**
2136
+ * Function `addUsage` will add multiple usages into one
2137
+ *
2138
+ * Note: If you provide 0 values, it returns ZERO_USAGE
2139
+ *
2140
+ * @public exported from `@promptbook/core`
2141
+ */
2142
+ function addUsage() {
2143
+ var usageItems = [];
2144
+ for (var _i = 0; _i < arguments.length; _i++) {
2145
+ usageItems[_i] = arguments[_i];
2146
+ }
2147
+ return usageItems.reduce(function (acc, item) {
2148
+ var e_1, _a, e_2, _b;
2149
+ var _c;
2150
+ acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
2151
+ try {
2152
+ for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
2153
+ var key = _e.value;
2154
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2155
+ //@ts-ignore
2156
+ if (item.input[key]) {
2157
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2158
+ //@ts-ignore
2159
+ acc.input[key].value += item.input[key].value || 0;
2160
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2161
+ //@ts-ignore
2162
+ if (item.input[key].isUncertain) {
2163
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2164
+ //@ts-ignore
2165
+ acc.input[key].isUncertain = true;
2166
+ }
2167
+ }
2168
+ }
2169
+ }
2170
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2171
+ finally {
2172
+ try {
2173
+ if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
2174
+ }
2175
+ finally { if (e_1) throw e_1.error; }
2176
+ }
2177
+ try {
2178
+ for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
2179
+ var key = _g.value;
2180
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2181
+ //@ts-ignore
2182
+ if (item.output[key]) {
2183
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2184
+ //@ts-ignore
2185
+ acc.output[key].value += item.output[key].value || 0;
2186
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2187
+ //@ts-ignore
2188
+ if (item.output[key].isUncertain) {
2189
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
2190
+ //@ts-ignore
2191
+ acc.output[key].isUncertain = true;
2192
+ }
2193
+ }
2194
+ }
2195
+ }
2196
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
2197
+ finally {
2198
+ try {
2199
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
2200
+ }
2201
+ finally { if (e_2) throw e_2.error; }
2202
+ }
2203
+ return acc;
2204
+ }, deepClone(ZERO_USAGE));
2205
+ }
2206
+
2207
+ /**
2208
+ * Intercepts LLM tools and counts total usage of the tools
2209
+ *
2210
+ * @param llmTools LLM tools to be intercepted with usage counting
2211
+ * @returns LLM tools with same functionality with added total cost counting
2212
+ * @public exported from `@promptbook/core`
2213
+ */
2214
+ function countTotalUsage(llmTools) {
2215
+ var _this = this;
2216
+ var totalUsage = ZERO_USAGE;
2217
+ var proxyTools = {
2218
+ get title() {
2219
+ // TODO: [🧠] Maybe put here some suffix
2220
+ return llmTools.title;
2221
+ },
2222
+ get description() {
2223
+ // TODO: [🧠] Maybe put here some suffix
2224
+ return llmTools.description;
2225
+ },
2226
+ checkConfiguration: function () {
2227
+ return __awaiter(this, void 0, void 0, function () {
2228
+ return __generator(this, function (_a) {
2229
+ return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
2230
+ });
2231
+ });
2232
+ },
2233
+ listModels: function () {
2234
+ return /* not await */ llmTools.listModels();
2235
+ },
2236
+ getTotalUsage: function () {
2237
+ // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
2238
+ return totalUsage;
2239
+ },
2240
+ };
2241
+ if (llmTools.callChatModel !== undefined) {
2242
+ proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2243
+ var promptResult;
2244
+ return __generator(this, function (_a) {
2245
+ switch (_a.label) {
2246
+ case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
2247
+ case 1:
2248
+ promptResult = _a.sent();
2249
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2250
+ return [2 /*return*/, promptResult];
2251
+ }
2252
+ });
2253
+ }); };
2254
+ }
2255
+ if (llmTools.callCompletionModel !== undefined) {
2256
+ proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2257
+ var promptResult;
2258
+ return __generator(this, function (_a) {
2259
+ switch (_a.label) {
2260
+ case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
2261
+ case 1:
2262
+ promptResult = _a.sent();
2263
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2264
+ return [2 /*return*/, promptResult];
2265
+ }
2266
+ });
2267
+ }); };
2268
+ }
2269
+ if (llmTools.callEmbeddingModel !== undefined) {
2270
+ proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
2271
+ var promptResult;
2272
+ return __generator(this, function (_a) {
2273
+ switch (_a.label) {
2274
+ case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
2275
+ case 1:
2276
+ promptResult = _a.sent();
2277
+ totalUsage = addUsage(totalUsage, promptResult.usage);
2278
+ return [2 /*return*/, promptResult];
2279
+ }
2280
+ });
2281
+ }); };
2282
+ }
2283
+ // <- Note: [🤖]
2284
+ return proxyTools;
2285
+ }
2904
2286
  /**
2905
- * TODO: [🧠][🎛] Aggregating multiple models - have result not only from one first aviable model BUT all of them
2906
- * TODO: [🏖] If no llmTools have for example not defined `callCompletionModel` this will still return object with defined `callCompletionModel` which just throws `PipelineExecutionError`, make it undefined instead
2907
- * Look how `countTotalUsage` (and `cacheLlmTools`) implements it
2287
+ * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
2288
+ * TODO: [🧠] Is there some meaningfull way how to test this util
2289
+ * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
2290
+ * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
2291
+ * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2908
2292
  */
2909
2293
 
2294
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
2295
+
2910
2296
  /**
2911
- * Joins multiple LLM Execution Tools into one
2912
- *
2913
- * @returns {LlmExecutionTools} Single wrapper for multiple LlmExecutionTools
2914
- *
2915
- * 0) If there is no LlmExecutionTools, it warns and returns valid but empty LlmExecutionTools
2916
- * 1) If there is only one LlmExecutionTools, it returns it wrapped in a proxy object
2917
- * 2) If there are multiple LlmExecutionTools, first will be used first, second will be used if the first hasn`t defined model variant or fails, etc.
2918
- * 3) When all LlmExecutionTools fail, it throws an error with a list of all errors merged into one
2919
- *
2920
- *
2921
- * Tip: You don't have to use this function directly, just pass an array of LlmExecutionTools to the `ExecutionTools`
2297
+ * This error indicates that the pipeline collection cannot be propperly loaded
2922
2298
  *
2923
2299
  * @public exported from `@promptbook/core`
2924
2300
  */
2925
- function joinLlmExecutionTools() {
2926
- var llmExecutionTools = [];
2927
- for (var _i = 0; _i < arguments.length; _i++) {
2928
- llmExecutionTools[_i] = arguments[_i];
2301
+ var CollectionError = /** @class */ (function (_super) {
2302
+ __extends(CollectionError, _super);
2303
+ function CollectionError(message) {
2304
+ var _this = _super.call(this, message) || this;
2305
+ _this.name = 'CollectionError';
2306
+ Object.setPrototypeOf(_this, CollectionError.prototype);
2307
+ return _this;
2929
2308
  }
2930
- if (llmExecutionTools.length === 0) {
2931
- var warningMessage = spaceTrim__default["default"]("\n You have not provided any `LlmExecutionTools`\n This means that you won't be able to execute any prompts that require large language models like GPT-4 or Anthropic's Claude.\n\n Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.\n ");
2932
- // TODO: [🟥] Detect browser / node and make it colorfull
2933
- console.warn(warningMessage);
2934
- /*
2935
- return {
2936
- async listModels() {
2937
- // TODO: [🟥] Detect browser / node and make it colorfull
2938
- console.warn(
2939
- spaceTrim(
2940
- (block) => `
2309
+ return CollectionError;
2310
+ }(Error));
2941
2311
 
2942
- You can't list models because you have no LLM Execution Tools defined:
2312
+ /**
2313
+ * This error type indicates that you try to use a feature that is not available in the current environment
2314
+ *
2315
+ * @public exported from `@promptbook/core`
2316
+ */
2317
+ var EnvironmentMismatchError = /** @class */ (function (_super) {
2318
+ __extends(EnvironmentMismatchError, _super);
2319
+ function EnvironmentMismatchError(message) {
2320
+ var _this = _super.call(this, message) || this;
2321
+ _this.name = 'EnvironmentMismatchError';
2322
+ Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
2323
+ return _this;
2324
+ }
2325
+ return EnvironmentMismatchError;
2326
+ }(Error));
2943
2327
 
2944
- tl;dr
2328
+ /**
2329
+ * This error occurs when some expectation is not met in the execution of the pipeline
2330
+ *
2331
+ * @public exported from `@promptbook/core`
2332
+ * Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
2333
+ * Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
2334
+ * Note: This is a kindof subtype of PipelineExecutionError
2335
+ */
2336
+ var ExpectError = /** @class */ (function (_super) {
2337
+ __extends(ExpectError, _super);
2338
+ function ExpectError(message) {
2339
+ var _this = _super.call(this, message) || this;
2340
+ _this.name = 'ExpectError';
2341
+ Object.setPrototypeOf(_this, ExpectError.prototype);
2342
+ return _this;
2343
+ }
2344
+ return ExpectError;
2345
+ }(Error));
2945
2346
 
2946
- ${block(warningMessage)}
2947
- `,
2948
- ),
2949
- );
2950
- return [];
2951
- },
2952
- };
2953
- */
2347
+ /**
2348
+ * This error type indicates that some limit was reached
2349
+ *
2350
+ * @public exported from `@promptbook/core`
2351
+ */
2352
+ var LimitReachedError = /** @class */ (function (_super) {
2353
+ __extends(LimitReachedError, _super);
2354
+ function LimitReachedError(message) {
2355
+ var _this = _super.call(this, message) || this;
2356
+ _this.name = 'LimitReachedError';
2357
+ Object.setPrototypeOf(_this, LimitReachedError.prototype);
2358
+ return _this;
2954
2359
  }
2955
- return new (MultipleLlmExecutionTools.bind.apply(MultipleLlmExecutionTools, __spreadArray([void 0], __read(llmExecutionTools), false)))();
2956
- }
2360
+ return LimitReachedError;
2361
+ }(Error));
2362
+
2957
2363
  /**
2958
- * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
2364
+ * This error type indicates that some part of the code is not implemented yet
2365
+ *
2366
+ * @public exported from `@promptbook/core`
2959
2367
  */
2368
+ var NotYetImplementedError = /** @class */ (function (_super) {
2369
+ __extends(NotYetImplementedError, _super);
2370
+ function NotYetImplementedError(message) {
2371
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2372
+ _this.name = 'NotYetImplementedError';
2373
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2374
+ return _this;
2375
+ }
2376
+ return NotYetImplementedError;
2377
+ }(Error));
2960
2378
 
2961
2379
  /**
2962
- * Takes an item or an array of items and returns an array of items
2380
+ * Index of all custom errors
2963
2381
  *
2964
- * 1) Any item except array and undefined returns array with that one item (also null)
2965
- * 2) Undefined returns empty array
2966
- * 3) Array returns itself
2382
+ * @public exported from `@promptbook/core`
2383
+ */
2384
+ var ERRORS = {
2385
+ ExpectError: ExpectError,
2386
+ CollectionError: CollectionError,
2387
+ EnvironmentMismatchError: EnvironmentMismatchError,
2388
+ LimitReachedError: LimitReachedError,
2389
+ NotFoundError: NotFoundError,
2390
+ NotYetImplementedError: NotYetImplementedError,
2391
+ ParseError: ParseError,
2392
+ PipelineExecutionError: PipelineExecutionError,
2393
+ PipelineLogicError: PipelineLogicError,
2394
+ PipelineUrlError: PipelineUrlError,
2395
+ UnexpectedError: UnexpectedError,
2396
+ // TODO: [🪑]> VersionMismatchError,
2397
+ };
2398
+
2399
+ /**
2400
+ * Deserializes the error object
2967
2401
  *
2968
- * @private internal utility
2402
+ * @public exported from `@promptbook/utils`
2969
2403
  */
2970
- function arrayableToArray(input) {
2971
- if (input === undefined) {
2972
- return [];
2973
- }
2974
- if (input instanceof Array) {
2975
- return input;
2404
+ function deserializeError(error) {
2405
+ if (error.name === 'Error') {
2406
+ return new Error(error.message);
2976
2407
  }
2977
- return [input];
2408
+ var CustomError = ERRORS[error.name];
2409
+ return new CustomError(error.message);
2978
2410
  }
2979
2411
 
2980
2412
  /**
2981
- * @@@
2413
+ * Asserts that the execution of a Promptbook is successful
2982
2414
  *
2983
- * @public exported from `@promptbook/utils`
2415
+ * @param executionResult - The partial result of the Promptbook execution
2416
+ * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2417
+ * @public exported from `@promptbook/core`
2984
2418
  */
2985
- function deepClone(objectValue) {
2986
- return JSON.parse(JSON.stringify(objectValue));
2987
- /*
2988
- TODO: [🧠] Is there a better implementation?
2989
- > const propertyNames = Object.getOwnPropertyNames(objectValue);
2990
- > for (const propertyName of propertyNames) {
2991
- > const value = (objectValue as really_any)[propertyName];
2992
- > if (value && typeof value === 'object') {
2993
- > deepClone(value);
2994
- > }
2995
- > }
2996
- > return Object.assign({}, objectValue);
2997
- */
2419
+ function assertsExecutionSuccessful(executionResult) {
2420
+ var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2421
+ if (isSuccessful === true) {
2422
+ return;
2423
+ }
2424
+ if (errors.length === 0) {
2425
+ throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
2426
+ }
2427
+ else if (errors.length === 1) {
2428
+ throw deserializeError(errors[0]);
2429
+ }
2430
+ else {
2431
+ throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
2432
+ .map(function (_a, index) {
2433
+ var name = _a.name, stack = _a.stack, message = _a.message;
2434
+ return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
2435
+ })
2436
+ .join('\n')), "\n "); }));
2437
+ }
2998
2438
  }
2999
2439
  /**
3000
- * TODO: [🧠] Is there a way how to meaningfully test this utility
2440
+ * TODO: [🧠] Can this return type be better typed than void
3001
2441
  */
3002
2442
 
3003
2443
  /**
3004
- * Represents the usage with no resources consumed
2444
+ * Determine if the pipeline is fully prepared
3005
2445
  *
3006
2446
  * @public exported from `@promptbook/core`
3007
2447
  */
3008
- var ZERO_USAGE = $deepFreeze({
3009
- price: { value: 0 },
3010
- input: {
3011
- tokensCount: { value: 0 },
3012
- charactersCount: { value: 0 },
3013
- wordsCount: { value: 0 },
3014
- sentencesCount: { value: 0 },
3015
- linesCount: { value: 0 },
3016
- paragraphsCount: { value: 0 },
3017
- pagesCount: { value: 0 },
3018
- },
3019
- output: {
3020
- tokensCount: { value: 0 },
3021
- charactersCount: { value: 0 },
3022
- wordsCount: { value: 0 },
3023
- sentencesCount: { value: 0 },
3024
- linesCount: { value: 0 },
3025
- paragraphsCount: { value: 0 },
3026
- pagesCount: { value: 0 },
3027
- },
3028
- });
2448
+ function isPipelinePrepared(pipeline) {
2449
+ // Note: Ignoring `pipeline.preparations` @@@
2450
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2451
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2452
+ return false;
2453
+ }
2454
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2455
+ return false;
2456
+ }
2457
+ /*
2458
+ TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
2459
+ > if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
2460
+ > return false;
2461
+ > }
2462
+ */
2463
+ return true;
2464
+ }
3029
2465
  /**
3030
- * Represents the usage with unknown resources consumed
3031
- *
3032
- * @public exported from `@promptbook/core`
2466
+ * TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
2467
+ * TODO: [🐠] Maybe base this on `makeValidator`
2468
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2469
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2470
+ * - [🏍] ? Is context in each template
2471
+ * - [♨] Are samples prepared
2472
+ * - [♨] Are templates prepared
3033
2473
  */
3034
- var UNCERTAIN_USAGE = $deepFreeze({
3035
- price: { value: 0, isUncertain: true },
3036
- input: {
3037
- tokensCount: { value: 0, isUncertain: true },
3038
- charactersCount: { value: 0, isUncertain: true },
3039
- wordsCount: { value: 0, isUncertain: true },
3040
- sentencesCount: { value: 0, isUncertain: true },
3041
- linesCount: { value: 0, isUncertain: true },
3042
- paragraphsCount: { value: 0, isUncertain: true },
3043
- pagesCount: { value: 0, isUncertain: true },
3044
- },
3045
- output: {
3046
- tokensCount: { value: 0, isUncertain: true },
3047
- charactersCount: { value: 0, isUncertain: true },
3048
- wordsCount: { value: 0, isUncertain: true },
3049
- sentencesCount: { value: 0, isUncertain: true },
3050
- linesCount: { value: 0, isUncertain: true },
3051
- paragraphsCount: { value: 0, isUncertain: true },
3052
- pagesCount: { value: 0, isUncertain: true },
3053
- },
3054
- });
3055
2474
 
3056
2475
  /**
3057
- * Function `addUsage` will add multiple usages into one
3058
- *
3059
- * Note: If you provide 0 values, it returns ZERO_USAGE
2476
+ * Serializes an error into a [🚉] JSON-serializable object
3060
2477
  *
3061
- * @public exported from `@promptbook/core`
2478
+ * @public exported from `@promptbook/utils`
3062
2479
  */
3063
- function addUsage() {
3064
- var usageItems = [];
3065
- for (var _i = 0; _i < arguments.length; _i++) {
3066
- usageItems[_i] = arguments[_i];
2480
+ function serializeError(error) {
2481
+ var name = error.name, message = error.message, stack = error.stack;
2482
+ if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
2483
+ throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
3067
2484
  }
3068
- return usageItems.reduce(function (acc, item) {
3069
- var e_1, _a, e_2, _b;
3070
- var _c;
3071
- acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
3072
- try {
3073
- for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
3074
- var key = _e.value;
3075
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3076
- //@ts-ignore
3077
- if (item.input[key]) {
3078
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3079
- //@ts-ignore
3080
- acc.input[key].value += item.input[key].value || 0;
3081
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3082
- //@ts-ignore
3083
- if (item.input[key].isUncertain) {
3084
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3085
- //@ts-ignore
3086
- acc.input[key].isUncertain = true;
3087
- }
3088
- }
3089
- }
3090
- }
3091
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
3092
- finally {
3093
- try {
3094
- if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
3095
- }
3096
- finally { if (e_1) throw e_1.error; }
3097
- }
3098
- try {
3099
- for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
3100
- var key = _g.value;
3101
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3102
- //@ts-ignore
3103
- if (item.output[key]) {
3104
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3105
- //@ts-ignore
3106
- acc.output[key].value += item.output[key].value || 0;
3107
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3108
- //@ts-ignore
3109
- if (item.output[key].isUncertain) {
3110
- // eslint-disable-next-line @typescript-eslint/ban-ts-comment
3111
- //@ts-ignore
3112
- acc.output[key].isUncertain = true;
3113
- }
3114
- }
3115
- }
3116
- }
3117
- catch (e_2_1) { e_2 = { error: e_2_1 }; }
3118
- finally {
3119
- try {
3120
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
3121
- }
3122
- finally { if (e_2) throw e_2.error; }
3123
- }
3124
- return acc;
3125
- }, deepClone(ZERO_USAGE));
2485
+ return {
2486
+ name: name,
2487
+ message: message,
2488
+ stack: stack,
2489
+ };
3126
2490
  }
3127
2491
 
3128
2492
  /**
@@ -3330,6 +2694,23 @@
3330
2694
  return union;
3331
2695
  }
3332
2696
 
2697
+ /**
2698
+ * Just marks a place of place where should be something implemented
2699
+ * No side effects.
2700
+ *
2701
+ * Note: It can be usefull suppressing eslint errors of unused variables
2702
+ *
2703
+ * @param value any values
2704
+ * @returns void
2705
+ * @private within the repository
2706
+ */
2707
+ function TODO_USE() {
2708
+ var value = [];
2709
+ for (var _i = 0; _i < arguments.length; _i++) {
2710
+ value[_i] = arguments[_i];
2711
+ }
2712
+ }
2713
+
3333
2714
  /**
3334
2715
  * This error indicates problems parsing the format value
3335
2716
  *
@@ -3843,6 +3224,25 @@
3843
3224
  * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
3844
3225
  */
3845
3226
 
3227
+ /**
3228
+ * Takes an item or an array of items and returns an array of items
3229
+ *
3230
+ * 1) Any item except array and undefined returns array with that one item (also null)
3231
+ * 2) Undefined returns empty array
3232
+ * 3) Array returns itself
3233
+ *
3234
+ * @private internal utility
3235
+ */
3236
+ function arrayableToArray(input) {
3237
+ if (input === undefined) {
3238
+ return [];
3239
+ }
3240
+ if (input instanceof Array) {
3241
+ return input;
3242
+ }
3243
+ return [input];
3244
+ }
3245
+
3846
3246
  /**
3847
3247
  * Just says that the variable is not used but should be kept
3848
3248
  * No side effects.
@@ -3944,69 +3344,330 @@
3944
3344
  }
3945
3345
 
3946
3346
  /**
3947
- * Counts number of characters in the text
3347
+ * Counts number of characters in the text
3348
+ *
3349
+ * @public exported from `@promptbook/utils`
3350
+ */
3351
+ function countCharacters(text) {
3352
+ // Remove null characters
3353
+ text = text.replace(/\0/g, '');
3354
+ // Replace emojis (and also ZWJ sequence) with hyphens
3355
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
3356
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
3357
+ text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
3358
+ return text.length;
3359
+ }
3360
+
3361
+ /**
3362
+ * Counts number of lines in the text
3363
+ *
3364
+ * @public exported from `@promptbook/utils`
3365
+ */
3366
+ function countLines(text) {
3367
+ if (text === '') {
3368
+ return 0;
3369
+ }
3370
+ return text.split('\n').length;
3371
+ }
3372
+
3373
+ /**
3374
+ * Counts number of pages in the text
3375
+ *
3376
+ * @public exported from `@promptbook/utils`
3377
+ */
3378
+ function countPages(text) {
3379
+ var sentencesPerPage = 5; // Assuming each page has 5 sentences
3380
+ var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3381
+ var pageCount = Math.ceil(sentences.length / sentencesPerPage);
3382
+ return pageCount;
3383
+ }
3384
+
3385
+ /**
3386
+ * Counts number of paragraphs in the text
3387
+ *
3388
+ * @public exported from `@promptbook/utils`
3389
+ */
3390
+ function countParagraphs(text) {
3391
+ return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
3392
+ }
3393
+
3394
+ /**
3395
+ * Split text into sentences
3396
+ *
3397
+ * @public exported from `@promptbook/utils`
3398
+ */
3399
+ function splitIntoSentences(text) {
3400
+ return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3401
+ }
3402
+ /**
3403
+ * Counts number of sentences in the text
3404
+ *
3405
+ * @public exported from `@promptbook/utils`
3406
+ */
3407
+ function countSentences(text) {
3408
+ return splitIntoSentences(text).length;
3409
+ }
3410
+
3411
+ var defaultDiacriticsRemovalMap = [
3412
+ {
3413
+ base: 'A',
3414
+ letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
3415
+ },
3416
+ { base: 'AA', letters: '\uA732' },
3417
+ { base: 'AE', letters: '\u00C6\u01FC\u01E2' },
3418
+ { base: 'AO', letters: '\uA734' },
3419
+ { base: 'AU', letters: '\uA736' },
3420
+ { base: 'AV', letters: '\uA738\uA73A' },
3421
+ { base: 'AY', letters: '\uA73C' },
3422
+ {
3423
+ base: 'B',
3424
+ letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
3425
+ },
3426
+ {
3427
+ base: 'C',
3428
+ letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
3429
+ },
3430
+ {
3431
+ base: 'D',
3432
+ letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
3433
+ },
3434
+ { base: 'DZ', letters: '\u01F1\u01C4' },
3435
+ { base: 'Dz', letters: '\u01F2\u01C5' },
3436
+ {
3437
+ base: 'E',
3438
+ letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
3439
+ },
3440
+ { base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
3441
+ {
3442
+ base: 'G',
3443
+ letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
3444
+ },
3445
+ {
3446
+ base: 'H',
3447
+ letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
3448
+ },
3449
+ {
3450
+ base: 'I',
3451
+ letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
3452
+ },
3453
+ { base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
3454
+ {
3455
+ base: 'K',
3456
+ letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
3457
+ },
3458
+ {
3459
+ base: 'L',
3460
+ letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
3461
+ },
3462
+ { base: 'LJ', letters: '\u01C7' },
3463
+ { base: 'Lj', letters: '\u01C8' },
3464
+ { base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
3465
+ {
3466
+ base: 'N',
3467
+ letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
3468
+ },
3469
+ { base: 'NJ', letters: '\u01CA' },
3470
+ { base: 'Nj', letters: '\u01CB' },
3471
+ {
3472
+ base: 'O',
3473
+ letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
3474
+ },
3475
+ { base: 'OI', letters: '\u01A2' },
3476
+ { base: 'OO', letters: '\uA74E' },
3477
+ { base: 'OU', letters: '\u0222' },
3478
+ { base: 'OE', letters: '\u008C\u0152' },
3479
+ { base: 'oe', letters: '\u009C\u0153' },
3480
+ {
3481
+ base: 'P',
3482
+ letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
3483
+ },
3484
+ { base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
3485
+ {
3486
+ base: 'R',
3487
+ letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
3488
+ },
3489
+ {
3490
+ base: 'S',
3491
+ letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
3492
+ },
3493
+ {
3494
+ base: 'T',
3495
+ letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
3496
+ },
3497
+ { base: 'TZ', letters: '\uA728' },
3498
+ {
3499
+ base: 'U',
3500
+ letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
3501
+ },
3502
+ { base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
3503
+ { base: 'VY', letters: '\uA760' },
3504
+ {
3505
+ base: 'W',
3506
+ letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
3507
+ },
3508
+ { base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
3509
+ {
3510
+ base: 'Y',
3511
+ letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
3512
+ },
3513
+ {
3514
+ base: 'Z',
3515
+ letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
3516
+ },
3517
+ {
3518
+ base: 'a',
3519
+ letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
3520
+ },
3521
+ { base: 'aa', letters: '\uA733' },
3522
+ { base: 'ae', letters: '\u00E6\u01FD\u01E3' },
3523
+ { base: 'ao', letters: '\uA735' },
3524
+ { base: 'au', letters: '\uA737' },
3525
+ { base: 'av', letters: '\uA739\uA73B' },
3526
+ { base: 'ay', letters: '\uA73D' },
3527
+ {
3528
+ base: 'b',
3529
+ letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
3530
+ },
3531
+ {
3532
+ base: 'c',
3533
+ letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
3534
+ },
3535
+ {
3536
+ base: 'd',
3537
+ letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
3538
+ },
3539
+ { base: 'dz', letters: '\u01F3\u01C6' },
3540
+ {
3541
+ base: 'e',
3542
+ letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
3543
+ },
3544
+ { base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
3545
+ {
3546
+ base: 'g',
3547
+ letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
3548
+ },
3549
+ {
3550
+ base: 'h',
3551
+ letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
3552
+ },
3553
+ { base: 'hv', letters: '\u0195' },
3554
+ {
3555
+ base: 'i',
3556
+ letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
3557
+ },
3558
+ { base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
3559
+ {
3560
+ base: 'k',
3561
+ letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
3562
+ },
3563
+ {
3564
+ base: 'l',
3565
+ letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
3566
+ },
3567
+ { base: 'lj', letters: '\u01C9' },
3568
+ { base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
3569
+ {
3570
+ base: 'n',
3571
+ letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
3572
+ },
3573
+ { base: 'nj', letters: '\u01CC' },
3574
+ {
3575
+ base: 'o',
3576
+ letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
3577
+ },
3578
+ { base: 'oi', letters: '\u01A3' },
3579
+ { base: 'ou', letters: '\u0223' },
3580
+ { base: 'oo', letters: '\uA74F' },
3581
+ {
3582
+ base: 'p',
3583
+ letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
3584
+ },
3585
+ { base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
3586
+ {
3587
+ base: 'r',
3588
+ letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
3589
+ },
3590
+ {
3591
+ base: 's',
3592
+ letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
3593
+ },
3594
+ {
3595
+ base: 't',
3596
+ letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
3597
+ },
3598
+ { base: 'tz', letters: '\uA729' },
3599
+ {
3600
+ base: 'u',
3601
+ letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
3602
+ },
3603
+ { base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
3604
+ { base: 'vy', letters: '\uA761' },
3605
+ {
3606
+ base: 'w',
3607
+ letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
3608
+ },
3609
+ { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
3610
+ {
3611
+ base: 'y',
3612
+ letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
3613
+ },
3614
+ {
3615
+ base: 'z',
3616
+ letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
3617
+ },
3618
+ ];
3619
+ /**
3620
+ * Map of letters from diacritic variant to diacritless variant
3621
+ * Contains lowercase and uppercase separatelly
3948
3622
  *
3949
- * @public exported from `@promptbook/utils`
3950
- */
3951
- function countCharacters(text) {
3952
- // Remove null characters
3953
- text = text.replace(/\0/g, '');
3954
- // Replace emojis (and also ZWJ sequence) with hyphens
3955
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
3956
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
3957
- text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
3958
- return text.length;
3959
- }
3960
-
3961
- /**
3962
- * Counts number of lines in the text
3623
+ * > "á" => "a"
3624
+ * > "ě" => "e"
3625
+ * > "Ă" => "A"
3626
+ * > ...
3963
3627
  *
3964
3628
  * @public exported from `@promptbook/utils`
3965
3629
  */
3966
- function countLines(text) {
3967
- if (text === '') {
3968
- return 0;
3630
+ var DIACRITIC_VARIANTS_LETTERS = {};
3631
+ // tslint:disable-next-line: prefer-for-of
3632
+ for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
3633
+ var letters = defaultDiacriticsRemovalMap[i].letters;
3634
+ // tslint:disable-next-line: prefer-for-of
3635
+ for (var j = 0; j < letters.length; j++) {
3636
+ DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
3969
3637
  }
3970
- return text.split('\n').length;
3971
3638
  }
3639
+ // <- TODO: [🍓] Put to maker function to save execution time if not needed
3640
+ /*
3641
+ @see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
3642
+ Licensed under the Apache License, Version 2.0 (the "License");
3643
+ you may not use this file except in compliance with the License.
3644
+ You may obtain a copy of the License at
3972
3645
 
3973
- /**
3974
- * Counts number of pages in the text
3975
- *
3976
- * @public exported from `@promptbook/utils`
3977
- */
3978
- function countPages(text) {
3979
- var sentencesPerPage = 5; // Assuming each page has 5 sentences
3980
- var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3981
- var pageCount = Math.ceil(sentences.length / sentencesPerPage);
3982
- return pageCount;
3983
- }
3646
+ http://www.apache.org/licenses/LICENSE-2.0
3984
3647
 
3985
- /**
3986
- * Counts number of paragraphs in the text
3987
- *
3988
- * @public exported from `@promptbook/utils`
3989
- */
3990
- function countParagraphs(text) {
3991
- return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
3992
- }
3648
+ Unless required by applicable law or agreed to in writing, software
3649
+ distributed under the License is distributed on an "AS IS" BASIS,
3650
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3651
+ See the License for the specific language governing permissions and
3652
+ limitations under the License.
3653
+ */
3993
3654
 
3994
3655
  /**
3995
- * Split text into sentences
3656
+ * @@@
3996
3657
  *
3658
+ * @param input @@@
3659
+ * @returns @@@
3997
3660
  * @public exported from `@promptbook/utils`
3998
3661
  */
3999
- function splitIntoSentences(text) {
4000
- return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
3662
+ function removeDiacritics(input) {
3663
+ /*eslint no-control-regex: "off"*/
3664
+ return input.replace(/[^\u0000-\u007E]/g, function (a) {
3665
+ return DIACRITIC_VARIANTS_LETTERS[a] || a;
3666
+ });
4001
3667
  }
4002
3668
  /**
4003
- * Counts number of sentences in the text
4004
- *
4005
- * @public exported from `@promptbook/utils`
3669
+ * TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
4006
3670
  */
4007
- function countSentences(text) {
4008
- return splitIntoSentences(text).length;
4009
- }
4010
3671
 
4011
3672
  /**
4012
3673
  * Counts number of words in the text
@@ -4104,11 +3765,11 @@
4104
3765
  */
4105
3766
  function executeAttempts(options) {
4106
3767
  return __awaiter(this, void 0, void 0, function () {
4107
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, llmTools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _loop_1, attempt, state_1;
3768
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4108
3769
  return __generator(this, function (_a) {
4109
3770
  switch (_a.label) {
4110
3771
  case 0:
4111
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, llmTools = options.llmTools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3772
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4112
3773
  maxExecutionAttempts = settings.maxExecutionAttempts;
4113
3774
  $ongoingTemplateResult = {
4114
3775
  $result: null,
@@ -4116,6 +3777,8 @@
4116
3777
  $expectError: null,
4117
3778
  $scriptPipelineExecutionErrors: [],
4118
3779
  };
3780
+ _llms = arrayableToArray(tools.llm);
3781
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
4119
3782
  _loop_1 = function (attempt) {
4120
3783
  var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
4121
3784
  var e_1, _q, e_3, _r, e_2, _s;
@@ -4184,7 +3847,9 @@
4184
3847
  return [3 /*break*/, 9];
4185
3848
  case 4:
4186
3849
  _d = $ongoingTemplateResult;
4187
- return [4 /*yield*/, llmTools.callChatModel($deepFreeze($ongoingTemplateResult.$prompt))];
3850
+ return [4 /*yield*/, llmTools.callChatModel(
3851
+ // <- TODO: [🧁] Check that `callChatModel` is defined
3852
+ $deepFreeze($ongoingTemplateResult.$prompt))];
4188
3853
  case 5:
4189
3854
  _d.$chatResult = _t.sent();
4190
3855
  // TODO: [🍬] Destroy chatThread
@@ -4193,7 +3858,9 @@
4193
3858
  return [3 /*break*/, 10];
4194
3859
  case 6:
4195
3860
  _e = $ongoingTemplateResult;
4196
- return [4 /*yield*/, llmTools.callCompletionModel($deepFreeze($ongoingTemplateResult.$prompt))];
3861
+ return [4 /*yield*/, llmTools.callCompletionModel(
3862
+ // <- TODO: [🧁] Check that `callCompletionModel` is defined
3863
+ $deepFreeze($ongoingTemplateResult.$prompt))];
4197
3864
  case 7:
4198
3865
  _e.$completionResult = _t.sent();
4199
3866
  $ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
@@ -4657,12 +4324,12 @@
4657
4324
  */
4658
4325
  function executeTemplate(options) {
4659
4326
  return __awaiter(this, void 0, void 0, function () {
4660
- var currentTemplate, preparedPipeline, parametersToPass, tools, llmTools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4327
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4661
4328
  var e_1, _f, _g;
4662
4329
  return __generator(this, function (_h) {
4663
4330
  switch (_h.label) {
4664
4331
  case 0:
4665
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, llmTools = options.llmTools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4332
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4666
4333
  maxExecutionAttempts = settings.maxExecutionAttempts;
4667
4334
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4668
4335
  title = currentTemplate.title;
@@ -4745,7 +4412,6 @@
4745
4412
  template: currentTemplate,
4746
4413
  preparedPipeline: preparedPipeline,
4747
4414
  tools: tools,
4748
- llmTools: llmTools,
4749
4415
  settings: settings,
4750
4416
  $executionReport: $executionReport,
4751
4417
  pipelineIdentification: pipelineIdentification,
@@ -4827,7 +4493,7 @@
4827
4493
  */
4828
4494
  function executePipeline(options) {
4829
4495
  return __awaiter(this, void 0, void 0, function () {
4830
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, llmTools, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4496
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4831
4497
  var e_1, _f, e_2, _g;
4832
4498
  return __generator(this, function (_h) {
4833
4499
  switch (_h.label) {
@@ -4835,10 +4501,8 @@
4835
4501
  inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4836
4502
  maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4837
4503
  preparedPipeline = options.preparedPipeline;
4838
- llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
4839
4504
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4840
- return [4 /*yield*/, preparePipeline(pipeline, {
4841
- llmTools: llmTools,
4505
+ return [4 /*yield*/, preparePipeline(pipeline, tools, {
4842
4506
  rootDirname: rootDirname,
4843
4507
  isVerbose: isVerbose,
4844
4508
  maxParallelCount: maxParallelCount,
@@ -5025,7 +4689,6 @@
5025
4689
  preparedPipeline: preparedPipeline,
5026
4690
  parametersToPass: parametersToPass,
5027
4691
  tools: tools,
5028
- llmTools: llmTools,
5029
4692
  onProgress: function (progress) {
5030
4693
  if (isReturned) {
5031
4694
  throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
@@ -5197,631 +4860,412 @@
5197
4860
  */
5198
4861
 
5199
4862
  /**
5200
- * Scraper for markdown files
4863
+ * Prepares the persona for the pipeline
5201
4864
  *
5202
- * @see `documentationUrl` for more details
4865
+ * @see https://github.com/webgptorg/promptbook/discussions/22
5203
4866
  * @public exported from `@promptbook/core`
5204
4867
  */
5205
- var markdownScraper = {
5206
- /**
5207
- * Mime types that this scraper can handle
5208
- */
5209
- mimeTypes: ['text/markdown', 'text/plain'],
5210
- /**
5211
- * Link to documentation
5212
- */
5213
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5214
- /**
5215
- * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
5216
- */
5217
- scrape: function (source, options) {
5218
- return __awaiter(this, void 0, void 0, function () {
5219
- var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, knowledgeContent, result, outputParameters, knowledgePiecesRaw, knowledgeTextPieces, knowledge;
5220
- var _f, _g, _h;
5221
- var _this = this;
5222
- return __generator(this, function (_j) {
5223
- switch (_j.label) {
5224
- case 0:
5225
- llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5226
- if (llmTools === undefined) {
5227
- throw new MissingToolsError('LLM tools are required for scraping external files');
5228
- // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
5229
- }
5230
- TODO_USE(maxParallelCount); // <- [🪂]
5231
- collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
5232
- _c = createPipelineExecutor;
5233
- _f = {};
5234
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
5235
- case 1:
5236
- prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
5237
- _f.tools = {
5238
- llm: llmTools,
5239
- },
5240
- _f)]);
5241
- _d = createPipelineExecutor;
5242
- _g = {};
5243
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
5244
- case 2:
5245
- prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
5246
- _g.tools = {
5247
- llm: llmTools,
5248
- },
5249
- _g)]);
5250
- _e = createPipelineExecutor;
5251
- _h = {};
5252
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
5253
- case 3:
5254
- prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
5255
- _h.tools = {
5256
- llm: llmTools,
5257
- },
5258
- _h)]);
5259
- return [4 /*yield*/, source.asText()];
5260
- case 4:
5261
- knowledgeContent = _j.sent();
5262
- return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
5263
- case 5:
5264
- result = _j.sent();
5265
- assertsExecutionSuccessful(result);
5266
- outputParameters = result.outputParameters;
5267
- knowledgePiecesRaw = outputParameters.knowledgePieces;
5268
- knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
5269
- // <- TODO: [main] !!!!! Smarter split and filter out empty pieces
5270
- if (isVerbose) {
5271
- console.info('knowledgeTextPieces:', knowledgeTextPieces);
5272
- }
5273
- return [4 /*yield*/, Promise.all(
5274
- // TODO: [🪂] !! Do not send all at once but in chunks
5275
- knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
5276
- var name, title, knowledgePieceContent, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
5277
- return __generator(this, function (_c) {
5278
- switch (_c.label) {
5279
- case 0:
5280
- name = "piece-".concat(i);
5281
- title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
5282
- knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
5283
- keywords = [];
5284
- index = [];
5285
- _c.label = 1;
5286
- case 1:
5287
- _c.trys.push([1, 7, , 8]);
5288
- return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
5289
- case 2:
5290
- titleResult = _c.sent();
5291
- _a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
5292
- title = spaceTrim__default["default"](titleRaw) /* <- TODO: Maybe do in pipeline */;
5293
- name = titleToName(title);
5294
- return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
5295
- case 3:
5296
- keywordsResult = _c.sent();
5297
- _b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
5298
- keywords = (keywordsRaw || '')
5299
- .split(',')
5300
- .map(function (keyword) { return keyword.trim(); })
5301
- .filter(function (keyword) { return keyword !== ''; });
5302
- if (isVerbose) {
5303
- console.info("Keywords for \"".concat(title, "\":"), keywords);
5304
- }
5305
- if (!!llmTools.callEmbeddingModel) return [3 /*break*/, 4];
5306
- // TODO: [🟥] Detect browser / node and make it colorfull
5307
- console.error('No callEmbeddingModel function provided');
5308
- return [3 /*break*/, 6];
5309
- case 4: return [4 /*yield*/, llmTools.callEmbeddingModel({
5310
- title: "Embedding for ".concat(title) /* <- Note: No impact on embedding result itself, just for logging */,
5311
- parameters: {},
5312
- content: knowledgePieceContent,
5313
- modelRequirements: {
5314
- modelVariant: 'EMBEDDING',
5315
- },
5316
- })];
5317
- case 5:
5318
- embeddingResult = _c.sent();
5319
- index.push({
5320
- modelName: embeddingResult.modelName,
5321
- position: embeddingResult.content,
5322
- });
5323
- _c.label = 6;
5324
- case 6: return [3 /*break*/, 8];
5325
- case 7:
5326
- error_1 = _c.sent();
5327
- // Note: Here is expected error:
5328
- // > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
5329
- if (!(error_1 instanceof PipelineExecutionError)) {
5330
- throw error_1;
5331
- }
5332
- // TODO: [🟥] Detect browser / node and make it colorfull
5333
- console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
5334
- return [3 /*break*/, 8];
5335
- case 8: return [2 /*return*/, {
5336
- name: name,
5337
- title: title,
5338
- content: knowledgePieceContent,
5339
- keywords: keywords,
5340
- index: index,
5341
- // <- TODO: [☀] sources,
5342
- }];
5343
- }
5344
- });
5345
- }); }))];
5346
- case 6:
5347
- knowledge = _j.sent();
5348
- return [2 /*return*/, knowledge];
5349
- }
5350
- });
4868
+ function preparePersona(personaDescription, tools, options) {
4869
+ return __awaiter(this, void 0, void 0, function () {
4870
+ var _a, isVerbose, collection, preparePersonaExecutor, _b, _llms, llmTools, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
4871
+ var _c;
4872
+ return __generator(this, function (_d) {
4873
+ switch (_d.label) {
4874
+ case 0:
4875
+ _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4876
+ if (tools === undefined || tools.llm === undefined) {
4877
+ throw new MissingToolsError('LLM tools are required for preparing persona');
4878
+ }
4879
+ collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
4880
+ _b = createPipelineExecutor;
4881
+ _c = {};
4882
+ return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
4883
+ case 1:
4884
+ preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
4885
+ _c.tools = tools,
4886
+ _c)]);
4887
+ _llms = arrayableToArray(tools.llm);
4888
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
4889
+ return [4 /*yield*/, llmTools.listModels()];
4890
+ case 2:
4891
+ availableModels = _d.sent();
4892
+ availableModelNames = availableModels
4893
+ .filter(function (_a) {
4894
+ var modelVariant = _a.modelVariant;
4895
+ return modelVariant === 'CHAT';
4896
+ })
4897
+ .map(function (_a) {
4898
+ var modelName = _a.modelName;
4899
+ return modelName;
4900
+ })
4901
+ .join(',');
4902
+ return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
4903
+ case 3:
4904
+ result = _d.sent();
4905
+ assertsExecutionSuccessful(result);
4906
+ outputParameters = result.outputParameters;
4907
+ modelRequirementsRaw = outputParameters.modelRequirements;
4908
+ modelRequirements = JSON.parse(modelRequirementsRaw);
4909
+ if (isVerbose) {
4910
+ console.info("PERSONA ".concat(personaDescription), modelRequirements);
4911
+ }
4912
+ modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
4913
+ return [2 /*return*/, {
4914
+ modelVariant: 'CHAT',
4915
+ modelName: modelName,
4916
+ systemMessage: systemMessage,
4917
+ temperature: temperature,
4918
+ }];
4919
+ }
5351
4920
  });
5352
- },
5353
- } /* TODO: [🦷] as const */;
4921
+ });
4922
+ }
5354
4923
  /**
5355
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5356
- * TODO: [🪂] Do it in parallel 11:11
5357
- * TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
5358
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4924
+ * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
4925
+ * TODO: [🏢] !! Check validity of `modelName` in pipeline
4926
+ * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
4927
+ * TODO: [🏢] !! Check validity of `temperature` in pipeline
5359
4928
  */
5360
4929
 
5361
4930
  /**
5362
- * Scraper of .docx and .odt files
4931
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
5363
4932
  *
5364
- * @see `documentationUrl` for more details
5365
4933
  * @public exported from `@promptbook/core`
5366
4934
  */
5367
- var documentScraper = {
5368
- /**
5369
- * Mime types that this scraper can handle
5370
- */
5371
- mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
5372
- /**
5373
- * Link to documentation
5374
- */
5375
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5376
- /**
5377
- * Convert the `.docx` or `.odt` to `.md` file and returns intermediate source
5378
- *
5379
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5380
- */
5381
- $convert: function (source, options) {
5382
- return __awaiter(this, void 0, void 0, function () {
5383
- var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, command_1;
5384
- return __generator(this, function (_e) {
5385
- switch (_e.label) {
5386
- case 0:
5387
- _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5388
- if (!$isRunningInNode()) {
5389
- throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
5390
- }
5391
- if (externalProgramsPaths.pandocPath === undefined) {
5392
- throw new MissingToolsError('Pandoc is required for scraping .docx files');
5393
- }
5394
- if (source.filename === null) {
5395
- // TODO: [🧠] Maybe save file as temporary
5396
- throw new KnowledgeScrapeError('When parsing .docx file, it must be real file in the file system');
5397
- }
5398
- extension = getFileExtension(source.filename);
5399
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5400
- rootDirname: rootDirname,
5401
- cacheDirname: cacheDirname,
5402
- isCacheCleaned: isCacheCleaned,
5403
- extension: 'md',
5404
- isVerbose: isVerbose,
5405
- })];
5406
- case 1:
5407
- cacheFilehandler = _e.sent();
5408
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5409
- case 2:
5410
- if (!!(_e.sent())) return [3 /*break*/, 5];
5411
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
5412
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
5413
- return [4 /*yield*/, $execCommand(command_1)];
5414
- case 3:
5415
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
5416
- _e.sent();
5417
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5418
- case 4:
5419
- // Note: [0]
5420
- if (!(_e.sent())) {
5421
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n File that was supposed to be created by Pandoc does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
5422
- }
5423
- _e.label = 5;
5424
- case 5: return [2 /*return*/, cacheFilehandler];
5425
- }
5426
- });
5427
- });
5428
- },
5429
- /**
5430
- * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
5431
- */
5432
- scrape: function (source, options) {
5433
- return __awaiter(this, void 0, void 0, function () {
5434
- var cacheFilehandler, markdownSource, knowledge;
5435
- return __generator(this, function (_a) {
5436
- switch (_a.label) {
5437
- case 0: return [4 /*yield*/, documentScraper.$convert(source, options)];
5438
- case 1:
5439
- cacheFilehandler = _a.sent();
5440
- markdownSource = {
5441
- source: source.source,
5442
- filename: cacheFilehandler.filename,
5443
- url: null,
5444
- mimeType: 'text/markdown',
5445
- asText: function () {
5446
- return __awaiter(this, void 0, void 0, function () {
5447
- return __generator(this, function (_a) {
5448
- switch (_a.label) {
5449
- case 0: return [4 /*yield*/, promises.readFile(cacheFilehandler.filename, 'utf-8')];
5450
- case 1:
5451
- // Note: [0] In $convert we check that the file exists
5452
- return [2 /*return*/, _a.sent()];
5453
- }
5454
- });
5455
- });
5456
- },
5457
- asJson: function () {
5458
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5459
- },
5460
- asBlob: function () {
5461
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5462
- },
5463
- };
5464
- knowledge = markdownScraper.scrape(markdownSource, options);
5465
- return [4 /*yield*/, cacheFilehandler.destroy()];
5466
- case 2:
5467
- _a.sent();
5468
- return [2 /*return*/, knowledge];
5469
- }
5470
- });
5471
- });
5472
- },
5473
- } /* TODO: [🦷] as const */;
4935
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4936
+ __extends(KnowledgeScrapeError, _super);
4937
+ function KnowledgeScrapeError(message) {
4938
+ var _this = _super.call(this, message) || this;
4939
+ _this.name = 'KnowledgeScrapeError';
4940
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4941
+ return _this;
4942
+ }
4943
+ return KnowledgeScrapeError;
4944
+ }(Error));
4945
+
5474
4946
  /**
5475
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5476
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5477
- * TODO: [🪂] Do it in parallel 11:11
5478
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5479
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
4947
+ * @@@
4948
+ *
4949
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4950
+ *
4951
+ * @private internal function of `$Register`
5480
4952
  */
4953
+ function $getGlobalScope() {
4954
+ return Function('return this')();
4955
+ }
5481
4956
 
5482
4957
  /**
5483
- * Scraper for .docx files
4958
+ * @@@
5484
4959
  *
5485
- * @see `documentationUrl` for more details
5486
- * @public exported from `@promptbook/core`
4960
+ * @param text @@@
4961
+ * @returns @@@
4962
+ * @example 'HELLO_WORLD'
4963
+ * @example 'I_LOVE_PROMPTBOOK'
4964
+ * @public exported from `@promptbook/utils`
5487
4965
  */
5488
- var legacyDocumentScraper = {
5489
- /**
5490
- * Mime types that this scraper can handle
5491
- */
5492
- mimeTypes: ['application/msword', 'text/rtf'],
5493
- /**
5494
- * Link to documentation
5495
- */
5496
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5497
- /**
5498
- * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
5499
- *
5500
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5501
- */
5502
- $convert: function (source, options) {
5503
- return __awaiter(this, void 0, void 0, function () {
5504
- var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
5505
- return __generator(this, function (_e) {
5506
- switch (_e.label) {
5507
- case 0:
5508
- _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5509
- if (!$isRunningInNode()) {
5510
- throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
5511
- }
5512
- if (externalProgramsPaths.libreOfficePath === undefined) {
5513
- throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
5514
- }
5515
- if (source.filename === null) {
5516
- // TODO: [🧠] Maybe save file as temporary
5517
- throw new KnowledgeScrapeError('When parsing .doc or .rtf file, it must be real file in the file system');
5518
- }
5519
- extension = getFileExtension(source.filename);
5520
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5521
- rootDirname: rootDirname,
5522
- cacheDirname: cacheDirname,
5523
- isCacheCleaned: isCacheCleaned,
5524
- extension: 'docx',
5525
- isVerbose: isVerbose,
5526
- })];
5527
- case 1:
5528
- cacheFilehandler = _e.sent();
5529
- if (isVerbose) {
5530
- console.info("documentScraper: Converting .".concat(extension, " -> .docx"));
5531
- }
5532
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5533
- case 2:
5534
- if (!!(_e.sent())) return [3 /*break*/, 8];
5535
- documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
5536
- .split('\\')
5537
- .join('/');
5538
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
5539
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
5540
- return [4 /*yield*/, $execCommand(command_1)];
5541
- case 3:
5542
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
5543
- _e.sent();
5544
- return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
5545
- case 4:
5546
- files_1 = _e.sent();
5547
- if (files_1.length !== 1) {
5548
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n "); }));
5549
- }
5550
- file = files_1[0];
5551
- return [4 /*yield*/, promises.rename(path.join(documentSourceOutdirPathForLibreOffice_1, file), cacheFilehandler.filename)];
5552
- case 5:
5553
- _e.sent();
5554
- return [4 /*yield*/, promises.rmdir(documentSourceOutdirPathForLibreOffice_1)];
5555
- case 6:
5556
- _e.sent();
5557
- return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
5558
- case 7:
5559
- if (!(_e.sent())) {
5560
- throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n File that was supposed to be created by LibreOffice does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
5561
- }
5562
- _e.label = 8;
5563
- case 8: return [2 /*return*/, cacheFilehandler];
5564
- }
5565
- });
5566
- });
5567
- },
5568
- /**
5569
- * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
5570
- */
5571
- scrape: function (source, options) {
5572
- return __awaiter(this, void 0, void 0, function () {
5573
- var cacheFilehandler, markdownSource, knowledge;
5574
- return __generator(this, function (_a) {
5575
- switch (_a.label) {
5576
- case 0: return [4 /*yield*/, legacyDocumentScraper.$convert(source, options)];
5577
- case 1:
5578
- cacheFilehandler = _a.sent();
5579
- markdownSource = {
5580
- source: source.source,
5581
- filename: cacheFilehandler.filename,
5582
- url: null,
5583
- mimeType: 'text/markdown',
5584
- asText: function () {
5585
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asText`');
5586
- },
5587
- asJson: function () {
5588
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
5589
- },
5590
- asBlob: function () {
5591
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
5592
- },
5593
- };
5594
- knowledge = documentScraper.scrape(markdownSource, options);
5595
- return [4 /*yield*/, cacheFilehandler.destroy()];
5596
- case 2:
5597
- _a.sent();
5598
- return [2 /*return*/, knowledge];
5599
- }
5600
- });
5601
- });
5602
- },
5603
- } /* TODO: [🦷] as const */;
4966
+ function normalizeTo_SCREAMING_CASE(text) {
4967
+ var e_1, _a;
4968
+ var charType;
4969
+ var lastCharType = 'OTHER';
4970
+ var normalizedName = '';
4971
+ try {
4972
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4973
+ var char = text_1_1.value;
4974
+ var normalizedChar = void 0;
4975
+ if (/^[a-z]$/.test(char)) {
4976
+ charType = 'LOWERCASE';
4977
+ normalizedChar = char.toUpperCase();
4978
+ }
4979
+ else if (/^[A-Z]$/.test(char)) {
4980
+ charType = 'UPPERCASE';
4981
+ normalizedChar = char;
4982
+ }
4983
+ else if (/^[0-9]$/.test(char)) {
4984
+ charType = 'NUMBER';
4985
+ normalizedChar = char;
4986
+ }
4987
+ else {
4988
+ charType = 'OTHER';
4989
+ normalizedChar = '_';
4990
+ }
4991
+ if (charType !== lastCharType &&
4992
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4993
+ !(lastCharType === 'NUMBER') &&
4994
+ !(charType === 'NUMBER')) {
4995
+ normalizedName += '_';
4996
+ }
4997
+ normalizedName += normalizedChar;
4998
+ lastCharType = charType;
4999
+ }
5000
+ }
5001
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5002
+ finally {
5003
+ try {
5004
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5005
+ }
5006
+ finally { if (e_1) throw e_1.error; }
5007
+ }
5008
+ normalizedName = normalizedName.replace(/_+/g, '_');
5009
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
5010
+ normalizedName = normalizedName.replace(/^_/, '');
5011
+ normalizedName = normalizedName.replace(/_$/, '');
5012
+ return normalizedName;
5013
+ }
5604
5014
  /**
5605
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5606
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5607
- * TODO: [🪂] Do it in parallel 11:11
5608
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5609
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5015
+ * TODO: Tests
5016
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
5017
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
5018
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
5019
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
5020
+ * TODO: [🌺] Use some intermediate util splitWords
5610
5021
  */
5611
5022
 
5612
5023
  /**
5613
- * Scraper for .docx files
5024
+ * @@@
5614
5025
  *
5615
- * @see `documentationUrl` for more details
5616
- * @public exported from `@promptbook/core`
5026
+ * @param text @@@
5027
+ * @returns @@@
5028
+ * @example 'hello_world'
5029
+ * @example 'i_love_promptbook'
5030
+ * @public exported from `@promptbook/utils`
5617
5031
  */
5618
- var pdfScraper = {
5619
- /**
5620
- * Mime types that this scraper can handle
5621
- */
5622
- mimeTypes: ['application/pdf'],
5623
- /**
5624
- * Link to documentation
5625
- */
5626
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5627
- /**
5628
- * Converts the `.pdf` file to `.md` file and returns intermediate source
5629
- */
5630
- $convert: function (source, options) {
5631
- return __awaiter(this, void 0, void 0, function () {
5632
- return __generator(this, function (_a) {
5633
- TODO_USE(source);
5634
- TODO_USE(options);
5635
- throw new NotYetImplementedError('PDF conversion not yet implemented');
5636
- });
5637
- });
5638
- },
5639
- /**
5640
- * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
5641
- */
5642
- scrape: function (source, options) {
5643
- return __awaiter(this, void 0, void 0, function () {
5644
- return __generator(this, function (_a) {
5645
- TODO_USE(source);
5646
- TODO_USE(options);
5647
- /*
5648
- const {
5649
- externalProgramsPaths = {},
5650
- cacheDirname = SCRAPE_CACHE_DIRNAME,
5651
- isCacheCleaned = false,
5652
- isVerbose = IS_VERBOSE,
5653
- } = options;
5654
- */
5655
- throw new NotYetImplementedError('PDF scraping not yet implemented');
5656
- });
5657
- });
5658
- },
5659
- } /* TODO: [🦷] as const */;
5032
+ function normalizeTo_snake_case(text) {
5033
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
5034
+ }
5035
+
5660
5036
  /**
5661
- * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
5662
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5663
- * TODO: [🪂] Do it in parallel 11:11
5664
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5665
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5037
+ * Register is @@@
5038
+ *
5039
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
5040
+ *
5041
+ * @private internal utility, exported are only signleton instances of this class
5666
5042
  */
5043
+ var $Register = /** @class */ (function () {
5044
+ function $Register(registerName) {
5045
+ this.registerName = registerName;
5046
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
5047
+ var globalScope = $getGlobalScope();
5048
+ if (globalScope[storageName] === undefined) {
5049
+ globalScope[storageName] = [];
5050
+ }
5051
+ else if (!Array.isArray(globalScope[storageName])) {
5052
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
5053
+ }
5054
+ this.storage = globalScope[storageName];
5055
+ }
5056
+ $Register.prototype.list = function () {
5057
+ // <- TODO: ReadonlyDeep<Array<TRegistered>>
5058
+ return this.storage;
5059
+ };
5060
+ $Register.prototype.register = function (registered) {
5061
+ var packageName = registered.packageName, className = registered.className;
5062
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
5063
+ var existingRegistration = this.storage[existingRegistrationIndex];
5064
+ if (!existingRegistration) {
5065
+ this.storage.push(registered);
5066
+ }
5067
+ else {
5068
+ this.storage[existingRegistrationIndex] = registered;
5069
+ }
5070
+ return {
5071
+ registerName: this.registerName,
5072
+ packageName: packageName,
5073
+ className: className,
5074
+ get isDestroyed() {
5075
+ return false;
5076
+ },
5077
+ destroy: function () {
5078
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
5079
+ },
5080
+ };
5081
+ };
5082
+ return $Register;
5083
+ }());
5667
5084
 
5668
5085
  /**
5669
- * A converter instance that uses showdown and highlight extensions
5086
+ * @@@
5670
5087
  *
5671
- * @type {Converter}
5672
- * @private for markdown and html knowledge scrapers
5088
+ * Note: `$` is used to indicate that this interacts with the global scope
5089
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5090
+ * @public exported from `@promptbook/core`
5673
5091
  */
5674
- var markdownConverter = new showdown.Converter({
5675
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
5676
- /*
5677
- > import showdownHighlight from 'showdown-highlight';
5678
- > extensions: [
5679
- > showdownHighlight({
5680
- > // Whether to add the classes to the <pre> tag, default is false
5681
- > pre: true,
5682
- > // Whether to use hljs' auto language detection, default is true
5683
- > auto_detection: true,
5684
- > }),
5685
- > ],
5686
- */
5687
- });
5092
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
5688
5093
  /**
5689
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
5690
- * TODO: !!!!!! Lazy-make converter
5094
+ * TODO: [®] DRY Register logic
5691
5095
  */
5692
5096
 
5693
5097
  /**
5694
- * Scraper for .docx files
5098
+ * @@@
5695
5099
  *
5696
- * @see `documentationUrl` for more details
5100
+ * Note: `$` is used to indicate that this interacts with the global scope
5101
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5697
5102
  * @public exported from `@promptbook/core`
5698
5103
  */
5699
- var websiteScraper = {
5700
- /**
5701
- * Mime types that this scraper can handle
5702
- */
5703
- mimeTypes: ['text/html'],
5704
- /**
5705
- * Link to documentation
5706
- */
5707
- documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5104
+ var $scrapersRegister = new $Register('scraper_constructors');
5105
+ /**
5106
+ * TODO: [®] DRY Register logic
5107
+ */
5108
+
5109
+ // TODO: !!!!!! Maybe delete this function
5110
+ /**
5111
+ * Creates a message with all registered scrapers
5112
+ *
5113
+ * Note: This function is used to create a (error) message when there is no scraper for particular mime type
5114
+ *
5115
+ * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
5116
+ */
5117
+ function $registeredScrapersMessage() {
5118
+ var e_1, _a, e_2, _b;
5708
5119
  /**
5709
- * Convert the website to `.md` file and returns intermediate source
5710
- *
5711
- * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
5120
+ * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
5712
5121
  */
5713
- $convert: function (source, options) {
5714
- return __awaiter(this, void 0, void 0, function () {
5715
- var
5716
- // TODO: [🧠] Maybe in node use headless browser not just JSDOM
5717
- // externalProgramsPaths = {},
5718
- rootDirname, _a, cacheDirname, _b, isCacheCleaned, _c, isVerbose, jsdom$1, _d, reader, article, html, i, cacheFilehandler, markdown;
5719
- return __generator(this, function (_e) {
5720
- switch (_e.label) {
5721
- case 0:
5722
- rootDirname = options.rootDirname, _a = options.cacheDirname, cacheDirname = _a === void 0 ? SCRAPE_CACHE_DIRNAME : _a, _b = options.isCacheCleaned, isCacheCleaned = _b === void 0 ? false : _b, _c = options.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
5723
- // TODO: !!!!!! Does this work in browser? Make it work.
5724
- if (source.url === null) {
5725
- throw new KnowledgeScrapeError('Website scraper requires URL');
5726
- }
5727
- _d = jsdom.JSDOM.bind;
5728
- return [4 /*yield*/, source.asText()];
5729
- case 1:
5730
- jsdom$1 = new (_d.apply(jsdom.JSDOM, [void 0, _e.sent(), {
5731
- url: source.url,
5732
- }]))();
5733
- reader = new readability.Readability(jsdom$1.window.document);
5734
- article = reader.parse();
5735
- console.log(article);
5736
- return [4 /*yield*/, waitasecond.forTime(10000)];
5737
- case 2:
5738
- _e.sent();
5739
- html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
5740
- // Note: Unwrap html such as it is convertable by `markdownConverter`
5741
- for (i = 0; i < 2; i++) {
5742
- html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
5743
- }
5744
- if (html.includes('<div')) {
5745
- html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
5746
- }
5747
- return [4 /*yield*/, getScraperIntermediateSource(source, {
5748
- rootDirname: rootDirname,
5749
- cacheDirname: cacheDirname,
5750
- isCacheCleaned: isCacheCleaned,
5751
- extension: 'html',
5752
- isVerbose: isVerbose,
5753
- })];
5754
- case 3:
5755
- cacheFilehandler = _e.sent();
5756
- return [4 /*yield*/, promises.writeFile(cacheFilehandler.filename, html, 'utf-8')];
5757
- case 4:
5758
- _e.sent();
5759
- markdown = markdownConverter.makeMarkdown(html, jsdom$1.window.document);
5760
- return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
5761
- }
5762
- });
5122
+ var all = [];
5123
+ var _loop_1 = function (packageName, className) {
5124
+ if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
5125
+ return "continue";
5126
+ }
5127
+ all.push({ packageName: packageName, className: className });
5128
+ };
5129
+ try {
5130
+ for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
5131
+ var _e = _d.value, packageName = _e.packageName, className = _e.className;
5132
+ _loop_1(packageName, className);
5133
+ }
5134
+ }
5135
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5136
+ finally {
5137
+ try {
5138
+ if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
5139
+ }
5140
+ finally { if (e_1) throw e_1.error; }
5141
+ }
5142
+ var _loop_2 = function (packageName, className) {
5143
+ if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
5144
+ return "continue";
5145
+ }
5146
+ all.push({ packageName: packageName, className: className });
5147
+ };
5148
+ try {
5149
+ for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
5150
+ var _h = _g.value, packageName = _h.packageName, className = _h.className;
5151
+ _loop_2(packageName, className);
5152
+ }
5153
+ }
5154
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
5155
+ finally {
5156
+ try {
5157
+ if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
5158
+ }
5159
+ finally { if (e_2) throw e_2.error; }
5160
+ }
5161
+ var metadata = all.map(function (metadata) {
5162
+ var isMetadataAviailable = $scrapersMetadataRegister
5163
+ .list()
5164
+ .find(function (_a) {
5165
+ var packageName = _a.packageName, className = _a.className;
5166
+ return metadata.packageName === packageName && metadata.className === className;
5763
5167
  });
5764
- },
5765
- /**
5766
- * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
5767
- */
5768
- scrape: function (source, options) {
5769
- return __awaiter(this, void 0, void 0, function () {
5770
- var cacheFilehandler, markdownSource, knowledge;
5771
- return __generator(this, function (_a) {
5772
- switch (_a.label) {
5773
- case 0: return [4 /*yield*/, websiteScraper.$convert(source, options)];
5774
- case 1:
5775
- cacheFilehandler = _a.sent();
5776
- markdownSource = {
5777
- source: source.source,
5778
- filename: cacheFilehandler.filename,
5779
- url: null,
5780
- mimeType: 'text/markdown',
5781
- asText: function () {
5782
- return cacheFilehandler.markdown;
5783
- },
5784
- asJson: function () {
5785
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5786
- },
5787
- asBlob: function () {
5788
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5789
- },
5790
- };
5791
- knowledge = markdownScraper.scrape(markdownSource, options);
5792
- return [4 /*yield*/, cacheFilehandler.destroy()];
5793
- case 2:
5794
- _a.sent();
5795
- return [2 /*return*/, knowledge];
5796
- }
5797
- });
5168
+ var isInstalled = $scrapersRegister
5169
+ .list()
5170
+ .find(function (_a) {
5171
+ var packageName = _a.packageName, className = _a.className;
5172
+ return metadata.packageName === packageName && metadata.className === className;
5798
5173
  });
5799
- },
5800
- } /* TODO: [🦷] as const */;
5174
+ return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
5175
+ });
5176
+ if (metadata.length === 0) {
5177
+ return "No scrapers are available";
5178
+ }
5179
+ return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
5180
+ .map(function (_a, i) {
5181
+ var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
5182
+ var more;
5183
+ if (just(false)) {
5184
+ more = '';
5185
+ }
5186
+ else if (!isMetadataAviailable && !isInstalled) {
5187
+ // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
5188
+ more = "(not installed and no metadata, looks like a unexpected behavior)";
5189
+ }
5190
+ else if (isMetadataAviailable && !isInstalled) {
5191
+ // TODO: [�][�]
5192
+ more = "(not installed)";
5193
+ }
5194
+ else if (!isMetadataAviailable && isInstalled) {
5195
+ more = "(no metadata, looks like a unexpected behavior)";
5196
+ }
5197
+ else if (isMetadataAviailable && isInstalled) {
5198
+ more = "(installed)";
5199
+ }
5200
+ else {
5201
+ more = "(unknown state, looks like a unexpected behavior)";
5202
+ }
5203
+ return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(more);
5204
+ })
5205
+ .join('\n')), "\n "); });
5206
+ }
5801
5207
  /**
5802
- * TODO: !!!!!! Put into separate package
5803
- * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
5804
- * TODO: [🦖] Make some system for putting scrapers to separete packages
5805
- * TODO: [🪂] Do it in parallel 11:11
5806
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
5807
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
5208
+ * TODO: [®] DRY Register logic
5808
5209
  */
5809
5210
 
5810
- // TODO: [🦖] !!!!!! Pass scrapers as dependency,
5811
5211
  /**
5812
5212
  * @@@
5813
5213
  *
5814
- * @private because this will be replaced by a system of one scraper per package [🦖]
5815
- * TODO: [🦖] System for scrapers NOT public exported from `@promptbook/core`
5214
+ * @param text @@@
5215
+ * @returns @@@
5216
+ * @example 'hello-world'
5217
+ * @example 'i-love-promptbook'
5218
+ * @public exported from `@promptbook/utils`
5816
5219
  */
5817
- var SCRAPERS = [
5818
- markdownScraper,
5819
- documentScraper,
5820
- legacyDocumentScraper,
5821
- pdfScraper,
5822
- websiteScraper,
5823
- // <- Note: [♓️] This is the order of the scrapers for knowledge, BUT consider some better (more explicit) way to do this
5824
- ];
5220
+ function normalizeToKebabCase(text) {
5221
+ var e_1, _a;
5222
+ text = removeDiacritics(text);
5223
+ var charType;
5224
+ var lastCharType = 'OTHER';
5225
+ var normalizedName = '';
5226
+ try {
5227
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
5228
+ var char = text_1_1.value;
5229
+ var normalizedChar = void 0;
5230
+ if (/^[a-z]$/.test(char)) {
5231
+ charType = 'LOWERCASE';
5232
+ normalizedChar = char;
5233
+ }
5234
+ else if (/^[A-Z]$/.test(char)) {
5235
+ charType = 'UPPERCASE';
5236
+ normalizedChar = char.toLowerCase();
5237
+ }
5238
+ else if (/^[0-9]$/.test(char)) {
5239
+ charType = 'NUMBER';
5240
+ normalizedChar = char;
5241
+ }
5242
+ else {
5243
+ charType = 'OTHER';
5244
+ normalizedChar = '-';
5245
+ }
5246
+ if (charType !== lastCharType &&
5247
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
5248
+ !(lastCharType === 'NUMBER') &&
5249
+ !(charType === 'NUMBER')) {
5250
+ normalizedName += '-';
5251
+ }
5252
+ normalizedName += normalizedChar;
5253
+ lastCharType = charType;
5254
+ }
5255
+ }
5256
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
5257
+ finally {
5258
+ try {
5259
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
5260
+ }
5261
+ finally { if (e_1) throw e_1.error; }
5262
+ }
5263
+ normalizedName = normalizedName.split(/-+/g).join('-');
5264
+ normalizedName = normalizedName.split(/-?\/-?/g).join('/');
5265
+ normalizedName = normalizedName.replace(/^-/, '');
5266
+ normalizedName = normalizedName.replace(/-$/, '');
5267
+ return normalizedName;
5268
+ }
5825
5269
 
5826
5270
  /**
5827
5271
  * Creates unique name for the source
@@ -5854,15 +5298,89 @@
5854
5298
  return mimeTypes.lookup(value) || 'application/octet-stream';
5855
5299
  }
5856
5300
 
5301
+ /**
5302
+ * Get the file extension from a file name
5303
+ *
5304
+ * @private within the repository
5305
+ */
5306
+ function getFileExtension(value) {
5307
+ var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
5308
+ return match ? match[1].toLowerCase() : null;
5309
+ }
5310
+
5311
+ /**
5312
+ * Checks if the file exists
5313
+ *
5314
+ * @private within the repository
5315
+ */
5316
+ function isFileExisting(filename, fs) {
5317
+ return __awaiter(this, void 0, void 0, function () {
5318
+ var isReadAccessAllowed, isFile;
5319
+ return __generator(this, function (_a) {
5320
+ switch (_a.label) {
5321
+ case 0: return [4 /*yield*/, fs
5322
+ .access(filename, fs.constants.R_OK)
5323
+ .then(function () { return true; })
5324
+ .catch(function () { return false; })];
5325
+ case 1:
5326
+ isReadAccessAllowed = _a.sent();
5327
+ if (!isReadAccessAllowed) {
5328
+ return [2 /*return*/, false];
5329
+ }
5330
+ return [4 /*yield*/, fs
5331
+ .stat(filename)
5332
+ .then(function (fileStat) { return fileStat.isFile(); })
5333
+ .catch(function () { return false; })];
5334
+ case 2:
5335
+ isFile = _a.sent();
5336
+ return [2 /*return*/, isFile];
5337
+ }
5338
+ });
5339
+ });
5340
+ }
5341
+ /**
5342
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
5343
+ * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
5344
+ * TODO: [🖇] What about symlinks?
5345
+ */
5346
+
5347
+ /**
5348
+ * Tests if given string is valid URL.
5349
+ *
5350
+ * Note: This does not check if the file exists only if the path is valid
5351
+ * @public exported from `@promptbook/utils`
5352
+ */
5353
+ function isValidFilePath(filename) {
5354
+ if (typeof filename !== 'string') {
5355
+ return false;
5356
+ }
5357
+ var filenameSlashes = filename.split('\\').join('/');
5358
+ // Absolute Unix path: /hello.txt
5359
+ if (/^(\/)/i.test(filenameSlashes)) {
5360
+ return true;
5361
+ }
5362
+ // Absolute Windows path: /hello.txt
5363
+ if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
5364
+ return true;
5365
+ }
5366
+ // Relative path: ./hello.txt
5367
+ if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
5368
+ return true;
5369
+ }
5370
+ return false;
5371
+ }
5372
+
5857
5373
  /**
5858
5374
  * @@@
5859
5375
  *
5860
5376
  * @private for scraper utilities
5861
5377
  */
5862
- function makeKnowledgeSourceHandler(knowledgeSource, options) {
5378
+ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5863
5379
  var _a;
5864
5380
  return __awaiter(this, void 0, void 0, function () {
5865
- var sourceContent, name, _b, _c, rootDirname, _d, isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
5381
+ var sourceContent, name, _b, _c, rootDirname, _d,
5382
+ // <- TODO: process.cwd() if running in Node.js
5383
+ isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
5866
5384
  return __generator(this, function (_e) {
5867
5385
  switch (_e.label) {
5868
5386
  case 0:
@@ -5926,8 +5444,9 @@
5926
5444
  }];
5927
5445
  case 2:
5928
5446
  if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
5929
- if (!$isRunningInNode()) {
5930
- throw new EnvironmentMismatchError('Importing knowledge source file works only in Node.js environment');
5447
+ if (tools.fs === undefined) {
5448
+ throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
5449
+ // <- TODO: [🧠] What is the best error type here`
5931
5450
  }
5932
5451
  if (rootDirname === null) {
5933
5452
  throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
@@ -5936,7 +5455,7 @@
5936
5455
  filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
5937
5456
  fileExtension = getFileExtension(filename_1);
5938
5457
  mimeType_1 = extensionToMimeType(fileExtension || '');
5939
- return [4 /*yield*/, $isFileExisting(filename_1)];
5458
+ return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
5940
5459
  case 3:
5941
5460
  if (!(_e.sent())) {
5942
5461
  throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
@@ -5952,10 +5471,9 @@
5952
5471
  var content;
5953
5472
  return __generator(this, function (_a) {
5954
5473
  switch (_a.label) {
5955
- case 0: return [4 /*yield*/, promises.readFile(filename_1)];
5474
+ case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
5956
5475
  case 1:
5957
5476
  content = _a.sent();
5958
- // <- Note: Its OK to use sync in tooling for tests
5959
5477
  return [2 /*return*/, new Blob([
5960
5478
  content,
5961
5479
  // <- TODO: !!!!!! Maybe not working
@@ -5971,7 +5489,7 @@
5971
5489
  switch (_c.label) {
5972
5490
  case 0:
5973
5491
  _b = (_a = JSON).parse;
5974
- return [4 /*yield*/, promises.readFile(filename_1, 'utf-8')];
5492
+ return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
5975
5493
  case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
5976
5494
  }
5977
5495
  });
@@ -5981,7 +5499,7 @@
5981
5499
  return __awaiter(this, void 0, void 0, function () {
5982
5500
  return __generator(this, function (_a) {
5983
5501
  switch (_a.label) {
5984
- case 0: return [4 /*yield*/, promises.readFile(filename_1, 'utf-8')];
5502
+ case 0: return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
5985
5503
  case 1: return [2 /*return*/, _a.sent()];
5986
5504
  }
5987
5505
  });
@@ -6014,7 +5532,7 @@
6014
5532
  * @see https://github.com/webgptorg/promptbook/discussions/41
6015
5533
  * @public exported from `@promptbook/core`
6016
5534
  */
6017
- function prepareKnowledgePieces(knowledgeSources, options) {
5535
+ function prepareKnowledgePieces(knowledgeSources, tools, options) {
6018
5536
  return __awaiter(this, void 0, void 0, function () {
6019
5537
  var _a, maxParallelCount, rootDirname, _b, isVerbose, knowledgePreparedUnflatten, knowledgePrepared;
6020
5538
  var _this = this;
@@ -6024,55 +5542,53 @@
6024
5542
  _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
6025
5543
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
6026
5544
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
6027
- var partialPieces, sourceHandler, SCRAPERS_1, SCRAPERS_1_1, scraper, partialPiecesUnchecked, e_1_1, pieces;
6028
- var e_1, _a;
6029
- return __generator(this, function (_b) {
6030
- switch (_b.label) {
5545
+ var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
5546
+ var e_1, _c;
5547
+ return __generator(this, function (_d) {
5548
+ switch (_d.label) {
6031
5549
  case 0:
6032
5550
  partialPieces = null;
6033
- return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
5551
+ return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
6034
5552
  case 1:
6035
- sourceHandler = _b.sent();
6036
- _b.label = 2;
5553
+ sourceHandler = _d.sent();
5554
+ _d.label = 2;
6037
5555
  case 2:
6038
- _b.trys.push([2, 7, 8, 9]);
6039
- SCRAPERS_1 = __values(SCRAPERS), SCRAPERS_1_1 = SCRAPERS_1.next();
6040
- _b.label = 3;
5556
+ _d.trys.push([2, 7, 8, 9]);
5557
+ _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
5558
+ _d.label = 3;
6041
5559
  case 3:
6042
- if (!!SCRAPERS_1_1.done) return [3 /*break*/, 6];
6043
- scraper = SCRAPERS_1_1.value;
6044
- if (!scraper.mimeTypes.includes(sourceHandler.mimeType)
5560
+ if (!!_b.done) return [3 /*break*/, 6];
5561
+ scraper = _b.value;
5562
+ if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
6045
5563
  // <- TODO: [🦔] Implement mime-type wildcards
6046
5564
  ) {
6047
5565
  return [3 /*break*/, 5];
6048
5566
  }
6049
- return [4 /*yield*/, scraper.scrape(sourceHandler, options)];
5567
+ return [4 /*yield*/, scraper.scrape(sourceHandler)];
6050
5568
  case 4:
6051
- partialPiecesUnchecked = _b.sent();
5569
+ partialPiecesUnchecked = _d.sent();
6052
5570
  if (partialPiecesUnchecked !== null) {
6053
5571
  partialPieces = partialPiecesUnchecked;
6054
5572
  return [3 /*break*/, 6];
6055
5573
  }
6056
- _b.label = 5;
5574
+ _d.label = 5;
6057
5575
  case 5:
6058
- SCRAPERS_1_1 = SCRAPERS_1.next();
5576
+ _b = _a.next();
6059
5577
  return [3 /*break*/, 3];
6060
5578
  case 6: return [3 /*break*/, 9];
6061
5579
  case 7:
6062
- e_1_1 = _b.sent();
5580
+ e_1_1 = _d.sent();
6063
5581
  e_1 = { error: e_1_1 };
6064
5582
  return [3 /*break*/, 9];
6065
5583
  case 8:
6066
5584
  try {
6067
- if (SCRAPERS_1_1 && !SCRAPERS_1_1.done && (_a = SCRAPERS_1.return)) _a.call(SCRAPERS_1);
5585
+ if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
6068
5586
  }
6069
5587
  finally { if (e_1) throw e_1.error; }
6070
5588
  return [7 /*endfinally*/];
6071
5589
  case 9:
6072
5590
  if (partialPieces === null) {
6073
- throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n Available scrapers:\n ").concat(block(SCRAPERS.flatMap(function (scraper) { return scraper.mimeTypes; })
6074
- .map(function (mimeType) { return "- ".concat(mimeType); })
6075
- .join('\n')), "\n\n\n "); }));
5591
+ throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
6076
5592
  }
6077
5593
  pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
6078
5594
  {
@@ -6080,196 +5596,41 @@
6080
5596
  // line, column <- TODO: [☀]
6081
5597
  // <- TODO: [❎]
6082
5598
  },
6083
- ] })); });
6084
- knowledgePreparedUnflatten[index] = pieces;
6085
- return [2 /*return*/];
6086
- }
6087
- });
6088
- }); })];
6089
- case 1:
6090
- _c.sent();
6091
- knowledgePrepared = knowledgePreparedUnflatten.flat();
6092
- return [2 /*return*/, knowledgePrepared];
6093
- }
6094
- });
6095
- });
6096
- }
6097
- /*
6098
- TODO: [🧊] This is how it can look in future
6099
- > type PrepareKnowledgeKnowledge = {
6100
- > /**
6101
- > * Unprepared knowledge
6102
- > * /
6103
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
6104
- > };
6105
- >
6106
- > export async function prepareKnowledgePieces(
6107
- > knowledge: PrepareKnowledgeKnowledge,
6108
- > options: PrepareAndScrapeOptions,
6109
- > ):
6110
- */
6111
- /**
6112
- * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
6113
- * Put `knowledgePieces` into `PrepareKnowledgeOptions`
6114
- * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
6115
- * TODO: [🧠][❎] Do here propper M:N mapping
6116
- * [x] One source can make multiple pieces
6117
- * [ ] One piece can have multiple sources
6118
- */
6119
-
6120
- /**
6121
- * Intercepts LLM tools and counts total usage of the tools
6122
- *
6123
- * @param llmTools LLM tools to be intercepted with usage counting
6124
- * @returns LLM tools with same functionality with added total cost counting
6125
- * @public exported from `@promptbook/core`
6126
- */
6127
- function countTotalUsage(llmTools) {
6128
- var _this = this;
6129
- var totalUsage = ZERO_USAGE;
6130
- var proxyTools = {
6131
- get title() {
6132
- // TODO: [🧠] Maybe put here some suffix
6133
- return llmTools.title;
6134
- },
6135
- get description() {
6136
- // TODO: [🧠] Maybe put here some suffix
6137
- return llmTools.description;
6138
- },
6139
- checkConfiguration: function () {
6140
- return __awaiter(this, void 0, void 0, function () {
6141
- return __generator(this, function (_a) {
6142
- return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
6143
- });
6144
- });
6145
- },
6146
- listModels: function () {
6147
- return /* not await */ llmTools.listModels();
6148
- },
6149
- getTotalUsage: function () {
6150
- // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
6151
- return totalUsage;
6152
- },
6153
- };
6154
- if (llmTools.callChatModel !== undefined) {
6155
- proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6156
- var promptResult;
6157
- return __generator(this, function (_a) {
6158
- switch (_a.label) {
6159
- case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
6160
- case 1:
6161
- promptResult = _a.sent();
6162
- totalUsage = addUsage(totalUsage, promptResult.usage);
6163
- return [2 /*return*/, promptResult];
6164
- }
6165
- });
6166
- }); };
6167
- }
6168
- if (llmTools.callCompletionModel !== undefined) {
6169
- proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6170
- var promptResult;
6171
- return __generator(this, function (_a) {
6172
- switch (_a.label) {
6173
- case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
6174
- case 1:
6175
- promptResult = _a.sent();
6176
- totalUsage = addUsage(totalUsage, promptResult.usage);
6177
- return [2 /*return*/, promptResult];
6178
- }
6179
- });
6180
- }); };
6181
- }
6182
- if (llmTools.callEmbeddingModel !== undefined) {
6183
- proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
6184
- var promptResult;
6185
- return __generator(this, function (_a) {
6186
- switch (_a.label) {
6187
- case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
6188
- case 1:
6189
- promptResult = _a.sent();
6190
- totalUsage = addUsage(totalUsage, promptResult.usage);
6191
- return [2 /*return*/, promptResult];
6192
- }
6193
- });
6194
- }); };
6195
- }
6196
- // <- Note: [🤖]
6197
- return proxyTools;
6198
- }
6199
- /**
6200
- * TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
6201
- * TODO: [🧠] Is there some meaningfull way how to test this util
6202
- * TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
6203
- * > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
6204
- * TODO: [👷‍♂️] @@@ Manual about construction of llmTools
6205
- */
6206
-
6207
- /**
6208
- * Prepares the persona for the pipeline
6209
- *
6210
- * @see https://github.com/webgptorg/promptbook/discussions/22
6211
- * @public exported from `@promptbook/core`
6212
- */
6213
- function preparePersona(personaDescription, options) {
6214
- return __awaiter(this, void 0, void 0, function () {
6215
- var llmTools, _a, isVerbose, collection, preparePersonaExecutor, _b, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
6216
- var _c;
6217
- return __generator(this, function (_d) {
6218
- switch (_d.label) {
6219
- case 0:
6220
- llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
6221
- if (llmTools === undefined) {
6222
- throw new MissingToolsError('LLM tools are required for preparing persona');
6223
- }
6224
- collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
6225
- _b = createPipelineExecutor;
6226
- _c = {};
6227
- return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
6228
- case 1:
6229
- preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
6230
- _c.tools = {
6231
- llm: llmTools,
6232
- },
6233
- _c)]);
6234
- return [4 /*yield*/, llmTools.listModels()];
6235
- case 2:
6236
- availableModels = _d.sent();
6237
- availableModelNames = availableModels
6238
- .filter(function (_a) {
6239
- var modelVariant = _a.modelVariant;
6240
- return modelVariant === 'CHAT';
6241
- })
6242
- .map(function (_a) {
6243
- var modelName = _a.modelName;
6244
- return modelName;
6245
- })
6246
- .join(',');
6247
- return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
6248
- case 3:
6249
- result = _d.sent();
6250
- assertsExecutionSuccessful(result);
6251
- outputParameters = result.outputParameters;
6252
- modelRequirementsRaw = outputParameters.modelRequirements;
6253
- modelRequirements = JSON.parse(modelRequirementsRaw);
6254
- if (isVerbose) {
6255
- console.info("PERSONA ".concat(personaDescription), modelRequirements);
6256
- }
6257
- modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
6258
- return [2 /*return*/, {
6259
- modelVariant: 'CHAT',
6260
- modelName: modelName,
6261
- systemMessage: systemMessage,
6262
- temperature: temperature,
6263
- }];
5599
+ ] })); });
5600
+ knowledgePreparedUnflatten[index] = pieces;
5601
+ return [2 /*return*/];
5602
+ }
5603
+ });
5604
+ }); })];
5605
+ case 1:
5606
+ _c.sent();
5607
+ knowledgePrepared = knowledgePreparedUnflatten.flat();
5608
+ return [2 /*return*/, knowledgePrepared];
6264
5609
  }
6265
5610
  });
6266
5611
  });
6267
5612
  }
5613
+ /*
5614
+ TODO: [🧊] This is how it can look in future
5615
+ > type PrepareKnowledgeKnowledge = {
5616
+ > /**
5617
+ > * Unprepared knowledge
5618
+ > * /
5619
+ > readonly knowledgeSources: Array<KnowledgeSourceJson>;
5620
+ > };
5621
+ >
5622
+ > export async function prepareKnowledgePieces(
5623
+ > knowledge: PrepareKnowledgeKnowledge,
5624
+ > options: PrepareAndScrapeOptions,
5625
+ > ):
5626
+ */
6268
5627
  /**
6269
- * TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
6270
- * TODO: [🏢] !! Check validity of `modelName` in pipeline
6271
- * TODO: [🏢] !! Check validity of `systemMessage` in pipeline
6272
- * TODO: [🏢] !! Check validity of `temperature` in pipeline
5628
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
5629
+ * Put `knowledgePieces` into `PrepareKnowledgeOptions`
5630
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
5631
+ * TODO: [🧠][❎] Do here propper M:N mapping
5632
+ * [x] One source can make multiple pieces
5633
+ * [ ] One piece can have multiple sources
6273
5634
  */
6274
5635
 
6275
5636
  /**
@@ -6306,7 +5667,7 @@
6306
5667
  *
6307
5668
  * @public exported from `@promptbook/core`
6308
5669
  */
6309
- function prepareTemplates(pipeline, options) {
5670
+ function prepareTemplates(pipeline, tools, options) {
6310
5671
  return __awaiter(this, void 0, void 0, function () {
6311
5672
  var _a, maxParallelCount, templates, parameters, knowledgePiecesCount, templatesPrepared;
6312
5673
  var _this = this;
@@ -6317,9 +5678,7 @@
6317
5678
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
6318
5679
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
6319
5680
  TODO_USE(parameters);
6320
- templatesPrepared = new Array(
6321
- // <- TODO: [🧱] Implement in a functional (not new Class) way
6322
- templates.length);
5681
+ templatesPrepared = new Array(templates.length);
6323
5682
  return [4 /*yield*/, forEachAsync(templates, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (template, index) { return __awaiter(_this, void 0, void 0, function () {
6324
5683
  var dependentParameterNames, preparedContent, preparedTemplate;
6325
5684
  return __generator(this, function (_a) {
@@ -6362,14 +5721,14 @@
6362
5721
  * Note: When the pipeline is already prepared, it returns the same pipeline
6363
5722
  * @public exported from `@promptbook/core`
6364
5723
  */
6365
- function preparePipeline(pipeline, options) {
5724
+ function preparePipeline(pipeline, tools, options) {
6366
5725
  return __awaiter(this, void 0, void 0, function () {
6367
- var llmTools, rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
5726
+ var rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
6368
5727
  /*
6369
5728
  <- TODO: [🧠][🪑] `promptbookVersion` */
6370
5729
  knowledgeSources /*
6371
5730
  <- TODO: [🧊] `knowledgePieces` */, personas /*
6372
- <- TODO: [🧊] `preparations` */, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
5731
+ <- TODO: [🧊] `preparations` */, _llms, llmTools, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
6373
5732
  var _this = this;
6374
5733
  return __generator(this, function (_c) {
6375
5734
  switch (_c.label) {
@@ -6377,11 +5736,13 @@
6377
5736
  if (isPipelinePrepared(pipeline)) {
6378
5737
  return [2 /*return*/, pipeline];
6379
5738
  }
6380
- llmTools = options.llmTools, rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5739
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
6381
5740
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
6382
- if (llmTools === undefined) {
5741
+ if (tools === undefined || tools.llm === undefined) {
6383
5742
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
6384
5743
  }
5744
+ _llms = arrayableToArray(tools.llm);
5745
+ llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
6385
5746
  llmToolsWithUsage = countTotalUsage(llmTools);
6386
5747
  currentPreparation = {
6387
5748
  id: 1,
@@ -6394,15 +5755,12 @@
6394
5755
  // <- TODO: [🧊]
6395
5756
  currentPreparation,
6396
5757
  ];
6397
- preparedPersonas = new Array(
6398
- // <- TODO: [🧱] Implement in a functional (not new Class) way
6399
- personas.length);
5758
+ preparedPersonas = new Array(personas.length);
6400
5759
  return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () {
6401
5760
  var modelRequirements, preparedPersona;
6402
5761
  return __generator(this, function (_a) {
6403
5762
  switch (_a.label) {
6404
- case 0: return [4 /*yield*/, preparePersona(persona.description, {
6405
- llmTools: llmToolsWithUsage,
5763
+ case 0: return [4 /*yield*/, preparePersona(persona.description, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
6406
5764
  rootDirname: rootDirname,
6407
5765
  maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
6408
5766
  isVerbose: isVerbose,
@@ -6418,7 +5776,7 @@
6418
5776
  case 1:
6419
5777
  _c.sent();
6420
5778
  knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
6421
- return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, options), { llmTools: llmToolsWithUsage, rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
5779
+ return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), __assign(__assign({}, options), { rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
6422
5780
  case 2:
6423
5781
  partialknowledgePiecesPrepared = _c.sent();
6424
5782
  knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
@@ -6426,8 +5784,7 @@
6426
5784
  parameters: parameters,
6427
5785
  templates: templates,
6428
5786
  knowledgePiecesCount: knowledgePiecesPrepared.length,
6429
- }, {
6430
- llmTools: llmToolsWithUsage,
5787
+ }, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
6431
5788
  rootDirname: rootDirname,
6432
5789
  maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
6433
5790
  isVerbose: isVerbose,
@@ -6956,72 +6313,6 @@
6956
6313
  },
6957
6314
  };
6958
6315
 
6959
- /**
6960
- * @@@
6961
- *
6962
- * @param text @@@
6963
- * @returns @@@
6964
- * @example 'HELLO_WORLD'
6965
- * @example 'I_LOVE_PROMPTBOOK'
6966
- * @public exported from `@promptbook/utils`
6967
- */
6968
- function normalizeTo_SCREAMING_CASE(text) {
6969
- var e_1, _a;
6970
- var charType;
6971
- var lastCharType = 'OTHER';
6972
- var normalizedName = '';
6973
- try {
6974
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
6975
- var char = text_1_1.value;
6976
- var normalizedChar = void 0;
6977
- if (/^[a-z]$/.test(char)) {
6978
- charType = 'LOWERCASE';
6979
- normalizedChar = char.toUpperCase();
6980
- }
6981
- else if (/^[A-Z]$/.test(char)) {
6982
- charType = 'UPPERCASE';
6983
- normalizedChar = char;
6984
- }
6985
- else if (/^[0-9]$/.test(char)) {
6986
- charType = 'NUMBER';
6987
- normalizedChar = char;
6988
- }
6989
- else {
6990
- charType = 'OTHER';
6991
- normalizedChar = '_';
6992
- }
6993
- if (charType !== lastCharType &&
6994
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
6995
- !(lastCharType === 'NUMBER') &&
6996
- !(charType === 'NUMBER')) {
6997
- normalizedName += '_';
6998
- }
6999
- normalizedName += normalizedChar;
7000
- lastCharType = charType;
7001
- }
7002
- }
7003
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
7004
- finally {
7005
- try {
7006
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
7007
- }
7008
- finally { if (e_1) throw e_1.error; }
7009
- }
7010
- normalizedName = normalizedName.replace(/_+/g, '_');
7011
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
7012
- normalizedName = normalizedName.replace(/^_/, '');
7013
- normalizedName = normalizedName.replace(/_$/, '');
7014
- return normalizedName;
7015
- }
7016
- /**
7017
- * TODO: Tests
7018
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
7019
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
7020
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
7021
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
7022
- * TODO: [🌺] Use some intermediate util splitWords
7023
- */
7024
-
7025
6316
  /**
7026
6317
  * @@@
7027
6318
  *
@@ -7086,6 +6377,22 @@
7086
6377
  * TODO: [🌺] Use some intermediate util splitWords
7087
6378
  */
7088
6379
 
6380
+ /**
6381
+ * Removes emojis from a string and fix whitespaces
6382
+ *
6383
+ * @param text with emojis
6384
+ * @returns text without emojis
6385
+ * @public exported from `@promptbook/utils`
6386
+ */
6387
+ function removeEmojis(text) {
6388
+ // Replace emojis (and also ZWJ sequence) with hyphens
6389
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
6390
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
6391
+ text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
6392
+ text = text.replace(/\p{Extended_Pictographic}/gu, '');
6393
+ return text;
6394
+ }
6395
+
7089
6396
  /**
7090
6397
  * Removes quotes from a string
7091
6398
  *
@@ -8833,6 +8140,30 @@
8833
8140
  return spaceTrim.spaceTrim(content.replace(/<!--(.*?)-->/gs, ''));
8834
8141
  }
8835
8142
 
8143
+ /**
8144
+ * @@@
8145
+ *
8146
+ * @param value @@@
8147
+ * @returns @@@
8148
+ * @example @@@
8149
+ * @public exported from `@promptbook/utils`
8150
+ */
8151
+ function titleToName(value) {
8152
+ if (isValidUrl(value)) {
8153
+ value = value.replace(/^https?:\/\//, '');
8154
+ value = value.replace(/\.html$/, '');
8155
+ }
8156
+ else if (isValidFilePath(value)) {
8157
+ value = path.basename(value);
8158
+ // Note: Keeping extension in the name
8159
+ }
8160
+ value = value.split('/').join('-');
8161
+ value = removeEmojis(value);
8162
+ value = normalizeToKebabCase(value);
8163
+ // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
8164
+ return value;
8165
+ }
8166
+
8836
8167
  /**
8837
8168
  * Compile pipeline from string (markdown) format to JSON format synchronously
8838
8169
  *
@@ -9167,21 +8498,21 @@
9167
8498
  * Note: This function acts as compilation process
9168
8499
  *
9169
8500
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8501
+ * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
9170
8502
  * @param options - Options and tools for the compilation
9171
8503
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
9172
8504
  * @throws {ParseError} if the promptbook string is not valid
9173
8505
  * @public exported from `@promptbook/core`
9174
8506
  */
9175
- function pipelineStringToJson(pipelineString, options) {
8507
+ function pipelineStringToJson(pipelineString, tools, options) {
9176
8508
  return __awaiter(this, void 0, void 0, function () {
9177
- var llmTools, pipelineJson;
8509
+ var pipelineJson;
9178
8510
  return __generator(this, function (_a) {
9179
8511
  switch (_a.label) {
9180
8512
  case 0:
9181
- llmTools = (options || {}).llmTools;
9182
8513
  pipelineJson = pipelineStringToJsonSync(pipelineString);
9183
- if (!(llmTools !== undefined)) return [3 /*break*/, 2];
9184
- return [4 /*yield*/, preparePipeline(pipelineJson, options || {
8514
+ if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8515
+ return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
9185
8516
  rootDirname: null,
9186
8517
  })];
9187
8518
  case 1:
@@ -9498,56 +8829,6 @@
9498
8829
  * TODO: [🏛] Maybe make some markdown builder
9499
8830
  */
9500
8831
 
9501
- /**
9502
- * @@@
9503
- *
9504
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
9505
- *
9506
- * @private internal function of `$Register`
9507
- */
9508
- function $getGlobalScope() {
9509
- return Function('return this')();
9510
- }
9511
-
9512
- /**
9513
- * Register is @@@
9514
- *
9515
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
9516
- *
9517
- * @private internal utility, exported are only signleton instances of this class
9518
- */
9519
- var $Register = /** @class */ (function () {
9520
- function $Register(storageName) {
9521
- this.storageName = storageName;
9522
- storageName = "_promptbook_".concat(storageName);
9523
- var globalScope = $getGlobalScope();
9524
- if (globalScope[storageName] === undefined) {
9525
- globalScope[storageName] = [];
9526
- }
9527
- else if (!Array.isArray(globalScope[storageName])) {
9528
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
9529
- }
9530
- this.storage = globalScope[storageName];
9531
- }
9532
- $Register.prototype.list = function () {
9533
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
9534
- return this.storage;
9535
- };
9536
- $Register.prototype.register = function (registered) {
9537
- // <- TODO: What to return here
9538
- var packageName = registered.packageName, className = registered.className;
9539
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
9540
- var existingRegistration = this.storage[existingRegistrationIndex];
9541
- if (!existingRegistration) {
9542
- this.storage.push(registered);
9543
- }
9544
- else {
9545
- this.storage[existingRegistrationIndex] = registered;
9546
- }
9547
- };
9548
- return $Register;
9549
- }());
9550
-
9551
8832
  /**
9552
8833
  * @@@
9553
8834
  *
@@ -9556,6 +8837,9 @@
9556
8837
  * @public exported from `@promptbook/core`
9557
8838
  */
9558
8839
  var $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
8840
+ /**
8841
+ * TODO: [®] DRY Register logic
8842
+ */
9559
8843
 
9560
8844
  /**
9561
8845
  * @@@
@@ -9565,13 +8849,16 @@
9565
8849
  * @public exported from `@promptbook/core`
9566
8850
  */
9567
8851
  var $llmToolsRegister = new $Register('llm_execution_tools_constructors');
8852
+ /**
8853
+ * TODO: [®] DRY Register logic
8854
+ */
9568
8855
 
9569
8856
  /**
9570
8857
  * Creates a message with all registered LLM tools
9571
8858
  *
9572
8859
  * Note: This function is used to create a (error) message when there is no constructor for some LLM provider
9573
8860
  *
9574
- * @private internal function of `createLlmToolsFromConfiguration` and `createLlmToolsFromEnv`
8861
+ * @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv`
9575
8862
  */
9576
8863
  function $registeredLlmToolsMessage() {
9577
8864
  var e_1, _a, e_2, _b;
@@ -9663,6 +8950,9 @@
9663
8950
  })
9664
8951
  .join('\n')), "\n "); });
9665
8952
  }
8953
+ /**
8954
+ * TODO: [®] DRY Register logic
8955
+ */
9666
8956
 
9667
8957
  /**
9668
8958
  * @@@
@@ -9690,12 +8980,13 @@
9690
8980
  return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
9691
8981
  }
9692
8982
  /**
9693
- * TODO: [🎌] Togethere with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
8983
+ * TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
9694
8984
  * TODO: [🧠][🎌] Dynamically install required providers
9695
8985
  * TODO: @@@ write discussion about this - wizzard
9696
8986
  * TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
9697
8987
  * TODO: [🧠] Is there some meaningfull way how to test this util
9698
8988
  * TODO: This should be maybe not under `_common` but under `utils`
8989
+ * TODO: [®] DRY Register logic
9699
8990
  */
9700
8991
 
9701
8992
  /**
@@ -9774,11 +9065,7 @@
9774
9065
  function cacheLlmTools(llmTools, options) {
9775
9066
  var _this = this;
9776
9067
  if (options === void 0) { options = {}; }
9777
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a,
9778
- // <- TODO: [🧱] Implement in a functional (not new Class) way
9779
- _b = options.isReloaded,
9780
- // <- TODO: [🧱] Implement in a functional (not new Class) way
9781
- isReloaded = _b === void 0 ? false : _b;
9068
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isReloaded, isReloaded = _b === void 0 ? false : _b;
9782
9069
  var proxyTools = __assign(__assign({}, llmTools), {
9783
9070
  // <- Note: [🥫]
9784
9071
  get title() {
@@ -9928,9 +9215,9 @@
9928
9215
  */
9929
9216
 
9930
9217
  /**
9931
- * @@@ registration1 of default configuration for Anthropic Claude
9218
+ * Registration of LLM provider metadata
9932
9219
  *
9933
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9220
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
9934
9221
  *
9935
9222
  * @public exported from `@promptbook/core`
9936
9223
  * @public exported from `@promptbook/cli`
@@ -9969,9 +9256,9 @@
9969
9256
  });
9970
9257
 
9971
9258
  /**
9972
- * @@@ registration1 of default configuration for Azure Open AI
9259
+ * Registration of LLM provider metadata
9973
9260
  *
9974
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9261
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
9975
9262
  *
9976
9263
  * @public exported from `@promptbook/core`
9977
9264
  * @public exported from `@promptbook/cli`
@@ -10016,9 +9303,9 @@
10016
9303
  });
10017
9304
 
10018
9305
  /**
10019
- * @@@ registration1 of default configuration for Open AI
9306
+ * Registration of LLM provider metadata
10020
9307
  *
10021
- * Note: [🏐] Configurations registrations are done in @@@ BUT constructor @@@
9308
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
10022
9309
  *
10023
9310
  * @public exported from `@promptbook/core`
10024
9311
  * @public exported from `@promptbook/cli`
@@ -10097,6 +9384,126 @@
10097
9384
  },
10098
9385
  });
10099
9386
 
9387
+ /**
9388
+ * Metadata of the scraper
9389
+ *
9390
+ * @private within the scraper directory
9391
+ */
9392
+ var legacyDocumentScraperMetadata = $deepFreeze({
9393
+ title: 'LegacyDocument scraper',
9394
+ packageName: '@promptbook/legacy-documents',
9395
+ className: 'LegacyDocumentScraper',
9396
+ mimeTypes: ['application/msword', 'text/rtf'],
9397
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9398
+ isAvilableInBrowser: false,
9399
+ requiredExecutables: ['!!!!!!'],
9400
+ }); /* <- TODO: [🤛] */
9401
+ /**
9402
+ * Registration of known scraper metadata
9403
+ *
9404
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9405
+ *
9406
+ * @public exported from `@promptbook/core`
9407
+ * @public exported from `@promptbook/cli`
9408
+ */
9409
+ var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
9410
+
9411
+ /**
9412
+ * Metadata of the scraper
9413
+ *
9414
+ * @private within the scraper directory
9415
+ */
9416
+ var documentScraperMetadata = $deepFreeze({
9417
+ title: 'Document scraper',
9418
+ packageName: '@promptbook/documents',
9419
+ className: 'DocumentScraper',
9420
+ mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
9421
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9422
+ isAvilableInBrowser: false,
9423
+ requiredExecutables: ['!!!!!!'],
9424
+ }); /* <- TODO: [🤛] */
9425
+ /**
9426
+ * Registration of known scraper metadata
9427
+ *
9428
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9429
+ *
9430
+ * @public exported from `@promptbook/core`
9431
+ * @public exported from `@promptbook/cli`
9432
+ */
9433
+ var _DocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(documentScraperMetadata);
9434
+
9435
+ /**
9436
+ * Metadata of the scraper
9437
+ *
9438
+ * @private within the scraper directory
9439
+ */
9440
+ var markdownScraperMetadata = $deepFreeze({
9441
+ title: 'Markdown scraper',
9442
+ packageName: '@promptbook/markdown-utils',
9443
+ className: 'MarkdownScraper',
9444
+ mimeTypes: ['text/markdown', 'text/plain'],
9445
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9446
+ isAvilableInBrowser: true,
9447
+ requiredExecutables: ['!!!!!!'],
9448
+ }); /* <- TODO: [🤛] */
9449
+ /**
9450
+ * Registration of known scraper metadata
9451
+ *
9452
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9453
+ *
9454
+ * @public exported from `@promptbook/core`
9455
+ * @public exported from `@promptbook/cli`
9456
+ */
9457
+ var _MarkdownScraperMetadataRegistration = $scrapersMetadataRegister.register(markdownScraperMetadata);
9458
+
9459
+ /**
9460
+ * Metadata of the scraper
9461
+ *
9462
+ * @private within the scraper directory
9463
+ */
9464
+ var pdfScraperMetadata = $deepFreeze({
9465
+ title: 'Pdf scraper',
9466
+ packageName: '@promptbook/pdf',
9467
+ className: 'PdfScraper',
9468
+ mimeTypes: ['application/pdf'],
9469
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9470
+ isAvilableInBrowser: true,
9471
+ requiredExecutables: ['!!!!!!'],
9472
+ }); /* <- TODO: [🤛] */
9473
+ /**
9474
+ * Registration of known scraper metadata
9475
+ *
9476
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9477
+ *
9478
+ * @public exported from `@promptbook/core`
9479
+ * @public exported from `@promptbook/cli`
9480
+ */
9481
+ var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
9482
+
9483
+ /**
9484
+ * Metadata of the scraper
9485
+ *
9486
+ * @private within the scraper directory
9487
+ */
9488
+ var websiteScraperMetadata = $deepFreeze({
9489
+ title: 'Website scraper',
9490
+ packageName: '@promptbook/website-crawler',
9491
+ className: 'WebsiteScraper',
9492
+ mimeTypes: ['text/html'],
9493
+ documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
9494
+ isAvilableInBrowser: false,
9495
+ requiredExecutables: ['!!!!!!'],
9496
+ }); /* <- TODO: [🤛] */
9497
+ /**
9498
+ * Registration of known scraper metadata
9499
+ *
9500
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
9501
+ *
9502
+ * @public exported from `@promptbook/core`
9503
+ * @public exported from `@promptbook/cli`
9504
+ */
9505
+ var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
9506
+
10100
9507
  /**
10101
9508
  * This class behaves like LocalStorage but separates keys by prefix
10102
9509
  *
@@ -10449,6 +9856,8 @@
10449
9856
 
10450
9857
  exports.$llmToolsMetadataRegister = $llmToolsMetadataRegister;
10451
9858
  exports.$llmToolsRegister = $llmToolsRegister;
9859
+ exports.$scrapersMetadataRegister = $scrapersMetadataRegister;
9860
+ exports.$scrapersRegister = $scrapersRegister;
10452
9861
  exports.AbstractFormatError = AbstractFormatError;
10453
9862
  exports.CLAIM = CLAIM;
10454
9863
  exports.CallbackInterfaceTools = CallbackInterfaceTools;
@@ -10464,6 +9873,7 @@
10464
9873
  exports.EnvironmentMismatchError = EnvironmentMismatchError;
10465
9874
  exports.ExecutionReportStringOptionsDefaults = ExecutionReportStringOptionsDefaults;
10466
9875
  exports.ExpectError = ExpectError;
9876
+ exports.IS_AUTO_INSTALLED = IS_AUTO_INSTALLED;
10467
9877
  exports.IS_VERBOSE = IS_VERBOSE;
10468
9878
  exports.KnowledgeScrapeError = KnowledgeScrapeError;
10469
9879
  exports.LimitReachedError = LimitReachedError;
@@ -10494,8 +9904,13 @@
10494
9904
  exports.ZERO_USAGE = ZERO_USAGE;
10495
9905
  exports._AnthropicClaudeMetadataRegistration = _AnthropicClaudeMetadataRegistration;
10496
9906
  exports._AzureOpenAiMetadataRegistration = _AzureOpenAiMetadataRegistration;
9907
+ exports._DocumentScraperMetadataRegistration = _DocumentScraperMetadataRegistration;
9908
+ exports._LegacyDocumentScraperMetadataRegistration = _LegacyDocumentScraperMetadataRegistration;
9909
+ exports._MarkdownScraperMetadataRegistration = _MarkdownScraperMetadataRegistration;
10497
9910
  exports._OpenAiAssistantMetadataRegistration = _OpenAiAssistantMetadataRegistration;
10498
9911
  exports._OpenAiMetadataRegistration = _OpenAiMetadataRegistration;
9912
+ exports._PdfScraperMetadataRegistration = _PdfScraperMetadataRegistration;
9913
+ exports._WebsiteScraperMetadataRegistration = _WebsiteScraperMetadataRegistration;
10499
9914
  exports.addUsage = addUsage;
10500
9915
  exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
10501
9916
  exports.cacheLlmTools = cacheLlmTools;
@@ -10507,16 +9922,12 @@
10507
9922
  exports.createLlmToolsFromConfiguration = createLlmToolsFromConfiguration;
10508
9923
  exports.createPipelineExecutor = createPipelineExecutor;
10509
9924
  exports.createSubcollection = createSubcollection;
10510
- exports.documentScraper = documentScraper;
10511
9925
  exports.embeddingVectorToString = embeddingVectorToString;
10512
9926
  exports.executionReportJsonToString = executionReportJsonToString;
10513
9927
  exports.isPassingExpectations = isPassingExpectations;
10514
9928
  exports.isPipelinePrepared = isPipelinePrepared;
10515
9929
  exports.joinLlmExecutionTools = joinLlmExecutionTools;
10516
- exports.legacyDocumentScraper = legacyDocumentScraper;
10517
9930
  exports.limitTotalUsage = limitTotalUsage;
10518
- exports.markdownScraper = markdownScraper;
10519
- exports.pdfScraper = pdfScraper;
10520
9931
  exports.pipelineJsonToString = pipelineJsonToString;
10521
9932
  exports.pipelineStringToJson = pipelineStringToJson;
10522
9933
  exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
@@ -10530,7 +9941,6 @@
10530
9941
  exports.usageToHuman = usageToHuman;
10531
9942
  exports.usageToWorktime = usageToWorktime;
10532
9943
  exports.validatePipeline = validatePipeline;
10533
- exports.websiteScraper = websiteScraper;
10534
9944
 
10535
9945
  Object.defineProperty(exports, '__esModule', { value: true });
10536
9946