@promptbook/cli 0.72.0-9 → 0.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +4 -8
  2. package/esm/index.es.js +987 -649
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +24 -18
  5. package/esm/typings/src/_packages/node.index.d.ts +2 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +18 -12
  7. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  8. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  10. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  11. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  12. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  14. package/esm/typings/src/config.d.ts +21 -14
  15. package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
  16. package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
  17. package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
  18. package/esm/typings/src/executables/locateApp.d.ts +33 -0
  19. package/esm/typings/src/executables/locateApp.test.d.ts +1 -0
  20. package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
  21. package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
  22. package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
  23. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  24. package/esm/typings/src/execution/Executables.d.ts +18 -0
  25. package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
  26. package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
  27. package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
  28. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  29. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
  30. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  31. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
  32. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
  33. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
  34. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  35. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  36. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  37. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  38. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  39. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
  40. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
  41. package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +7 -0
  42. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  43. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +2 -2
  45. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  46. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  47. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -1
  48. package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
  49. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  50. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
  51. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
  52. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
  53. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  54. package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
  55. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
  56. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
  57. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  58. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
  59. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +15 -6
  60. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
  61. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +6 -12
  62. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
  63. package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +9 -14
  64. package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +23 -8
  65. package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
  66. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
  67. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  68. package/esm/typings/src/scrapers/_common/Scraper.d.ts +2 -9
  69. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
  70. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
  71. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
  72. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
  73. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
  74. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
  75. package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
  76. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  77. package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
  78. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
  79. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
  80. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
  81. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
  82. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
  83. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
  84. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
  85. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
  86. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
  87. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
  88. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
  89. package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
  90. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  91. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.test.d.ts +1 -0
  92. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  93. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  94. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  95. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  96. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  97. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  98. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  99. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  100. package/esm/typings/src/types/Prompt.d.ts +2 -1
  101. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  102. package/esm/typings/src/types/typeAliases.d.ts +11 -8
  103. package/esm/typings/src/utils/$Register.d.ts +1 -1
  104. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  105. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  106. package/esm/typings/src/utils/emojis.d.ts +1 -1
  107. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +4 -2
  108. package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
  109. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -0
  110. package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
  111. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  112. package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
  113. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  114. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  115. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  116. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  117. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  118. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  119. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  120. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  121. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  122. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  123. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  124. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  125. package/package.json +3 -2
  126. package/umd/index.umd.js +988 -651
  127. package/umd/index.umd.js.map +1 -1
  128. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  129. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  130. package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
  131. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
  132. /package/esm/typings/src/{scrapers/website/WebsiteScraper.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
  133. /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
package/umd/index.umd.js CHANGED
@@ -1,8 +1,8 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('commander'), require('spacetrim'), require('colors'), require('waitasecond'), require('fs/promises'), require('path'), require('prettier'), require('prettier/parser-html'), require('papaparse'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('dotenv'), require('crypto-js/sha256'), require('glob-promise'), require('socket.io-client'), require('@anthropic-ai/sdk'), require('@azure/openai'), require('openai'), require('child_process'), require('@mozilla/readability'), require('jsdom'), require('showdown')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'commander', 'spacetrim', 'colors', 'waitasecond', 'fs/promises', 'path', 'prettier', 'prettier/parser-html', 'papaparse', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'dotenv', 'crypto-js/sha256', 'glob-promise', 'socket.io-client', '@anthropic-ai/sdk', '@azure/openai', 'openai', 'child_process', '@mozilla/readability', 'jsdom', 'showdown'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-cli"] = {}, global.commander, global.spaceTrim, global.colors, global.waitasecond, global.promises, global.path, global.prettier, global.parserHtml, global.papaparse, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.dotenv, global.sha256, global.glob, global.socket_ioClient, global.Anthropic, global.openai, global.OpenAI, global.child_process, global.readability, global.jsdom, global.showdown));
5
- })(this, (function (exports, commander, spaceTrim, colors, waitasecond, promises, path, prettier, parserHtml, papaparse, cryptoJs, hexEncoder, mimeTypes, dotenv, sha256, glob, socket_ioClient, Anthropic, openai, OpenAI, child_process, readability, jsdom, showdown) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('commander'), require('spacetrim'), require('colors'), require('waitasecond'), require('fs/promises'), require('path'), require('prettier'), require('prettier/parser-html'), require('papaparse'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('child_process'), require('util'), require('dotenv'), require('crypto-js/sha256'), require('glob-promise'), require('socket.io-client'), require('@anthropic-ai/sdk'), require('@azure/openai'), require('openai'), require('@mozilla/readability'), require('jsdom'), require('showdown')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'commander', 'spacetrim', 'colors', 'waitasecond', 'fs/promises', 'path', 'prettier', 'prettier/parser-html', 'papaparse', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'child_process', 'util', 'dotenv', 'crypto-js/sha256', 'glob-promise', 'socket.io-client', '@anthropic-ai/sdk', '@azure/openai', 'openai', '@mozilla/readability', 'jsdom', 'showdown'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-cli"] = {}, global.commander, global.spaceTrim, global.colors, global.waitasecond, global.promises, global.path, global.prettier, global.parserHtml, global.papaparse, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.child_process, global.util, global.dotenv, global.sha256, global.glob, global.socket_ioClient, global.Anthropic, global.openai, global.OpenAI, global.readability, global.jsdom, global.showdown));
5
+ })(this, (function (exports, commander, spaceTrim, colors, waitasecond, promises, path, prettier, parserHtml, papaparse, cryptoJs, hexEncoder, mimeTypes, child_process, util, dotenv, sha256, glob, socket_ioClient, Anthropic, openai, OpenAI, readability, jsdom, showdown) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
@@ -39,7 +39,7 @@
39
39
  /**
40
40
  * The version of the Promptbook library
41
41
  */
42
- var PROMPTBOOK_VERSION = '0.72.0-8';
42
+ var PROMPTBOOK_VERSION = '0.72.0-34';
43
43
  // TODO: [main] !!!! List here all the versions and annotate + put into script
44
44
 
45
45
  /*! *****************************************************************************
@@ -412,7 +412,8 @@
412
412
  *
413
413
  * @public exported from `@promptbook/core`
414
414
  */
415
- var CLAIM = "Supercharge LLM models with Promptbook";
415
+ var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
416
+ // <- TODO: [🐊] Pick the best claim
416
417
  /**
417
418
  * Warning message for the generated sections and files files
418
419
  *
@@ -432,6 +433,7 @@
432
433
  * @private within the repository - too low-level in comparison with other `MAX_...`
433
434
  */
434
435
  var CONNECTION_TIMEOUT_MS = 7 * 1000;
436
+ // <- TODO: [⏳] Standartize timeouts, Make DEFAULT_TIMEOUT_MS as global constant
435
437
  /**
436
438
  * How many times to retry the connections
437
439
  *
@@ -445,23 +447,30 @@
445
447
  */
446
448
  var IMMEDIATE_TIME = 10;
447
449
  /**
448
- * The maximum number of (LLM) tasks running in parallel
450
+ * The maximum length of the (generated) filename
449
451
  *
450
452
  * @public exported from `@promptbook/core`
451
453
  */
452
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
454
+ var MAX_FILENAME_LENGTH = 30;
453
455
  /**
454
- * The maximum number of attempts to execute LLM task before giving up
456
+ * Strategy for caching the intermediate results for knowledge sources
455
457
  *
456
458
  * @public exported from `@promptbook/core`
457
459
  */
458
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
460
+ var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
461
+ // <- TODO: [😡] Change to 'VISIBLE'
459
462
  /**
460
- * The maximum length of the (generated) filename
463
+ * The maximum number of (LLM) tasks running in parallel
461
464
  *
462
465
  * @public exported from `@promptbook/core`
463
466
  */
464
- var MAX_FILENAME_LENGTH = 30;
467
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
468
+ /**
469
+ * The maximum number of attempts to execute LLM task before giving up
470
+ *
471
+ * @public exported from `@promptbook/core`
472
+ */
473
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
465
474
  /**
466
475
  * Where to store the cache of executions for promptbook CLI
467
476
  *
@@ -469,7 +478,7 @@
469
478
  *
470
479
  * @public exported from `@promptbook/core`
471
480
  */
472
- var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
481
+ var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
473
482
  /**
474
483
  * Where to store the scrape cache
475
484
  *
@@ -477,13 +486,13 @@
477
486
  *
478
487
  * @public exported from `@promptbook/core`
479
488
  */
480
- var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
489
+ var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
481
490
  /**
482
491
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
483
492
  *
484
493
  * @public exported from `@promptbook/core`
485
494
  */
486
- var PIPELINE_COLLECTION_BASE_FILENAME = "index";
495
+ var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
487
496
  /**
488
497
  * Nonce which is used for replacing things in strings
489
498
  *
@@ -547,13 +556,13 @@
547
556
  *
548
557
  * @public exported from `@promptbook/core`
549
558
  */
550
- var IS_VERBOSE = false;
559
+ var DEFAULT_IS_VERBOSE = false;
551
560
  /**
552
561
  * @@@
553
562
  *
554
563
  * @public exported from `@promptbook/core`
555
564
  */
556
- var IS_AUTO_INSTALLED = false;
565
+ var DEFAULT_IS_AUTO_INSTALLED = false;
557
566
  /**
558
567
  * @@@
559
568
  *
@@ -626,271 +635,6 @@
626
635
  * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
627
636
  */
628
637
 
629
- /**
630
- * Just marks a place of place where should be something implemented
631
- * No side effects.
632
- *
633
- * Note: It can be usefull suppressing eslint errors of unused variables
634
- *
635
- * @param value any values
636
- * @returns void
637
- * @private within the repository
638
- */
639
- function TODO_USE() {
640
- var value = [];
641
- for (var _i = 0; _i < arguments.length; _i++) {
642
- value[_i] = arguments[_i];
643
- }
644
- }
645
-
646
- /**
647
- * @@@
648
- *
649
- * @public exported from `@promptbook/node`
650
- */
651
- function $provideFilesystemForNode(options) {
652
- if (!$isRunningInNode()) {
653
- throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
654
- }
655
- var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
656
- TODO_USE(isVerbose);
657
- return {
658
- stat: promises.stat,
659
- access: promises.access,
660
- constants: promises.constants,
661
- readFile: promises.readFile,
662
- readdir: promises.readdir,
663
- };
664
- }
665
- /**
666
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
667
- */
668
-
669
- /**
670
- * This error type indicates that some part of the code is not implemented yet
671
- *
672
- * @public exported from `@promptbook/core`
673
- */
674
- var NotYetImplementedError = /** @class */ (function (_super) {
675
- __extends(NotYetImplementedError, _super);
676
- function NotYetImplementedError(message) {
677
- var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
678
- _this.name = 'NotYetImplementedError';
679
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
680
- return _this;
681
- }
682
- return NotYetImplementedError;
683
- }(Error));
684
-
685
- /**
686
- * @@@
687
- *
688
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
689
- *
690
- * @private internal function of `$Register`
691
- */
692
- function $getGlobalScope() {
693
- return Function('return this')();
694
- }
695
-
696
- /**
697
- * @@@
698
- *
699
- * @param text @@@
700
- * @returns @@@
701
- * @example 'HELLO_WORLD'
702
- * @example 'I_LOVE_PROMPTBOOK'
703
- * @public exported from `@promptbook/utils`
704
- */
705
- function normalizeTo_SCREAMING_CASE(text) {
706
- var e_1, _a;
707
- var charType;
708
- var lastCharType = 'OTHER';
709
- var normalizedName = '';
710
- try {
711
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
712
- var char = text_1_1.value;
713
- var normalizedChar = void 0;
714
- if (/^[a-z]$/.test(char)) {
715
- charType = 'LOWERCASE';
716
- normalizedChar = char.toUpperCase();
717
- }
718
- else if (/^[A-Z]$/.test(char)) {
719
- charType = 'UPPERCASE';
720
- normalizedChar = char;
721
- }
722
- else if (/^[0-9]$/.test(char)) {
723
- charType = 'NUMBER';
724
- normalizedChar = char;
725
- }
726
- else {
727
- charType = 'OTHER';
728
- normalizedChar = '_';
729
- }
730
- if (charType !== lastCharType &&
731
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
732
- !(lastCharType === 'NUMBER') &&
733
- !(charType === 'NUMBER')) {
734
- normalizedName += '_';
735
- }
736
- normalizedName += normalizedChar;
737
- lastCharType = charType;
738
- }
739
- }
740
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
741
- finally {
742
- try {
743
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
744
- }
745
- finally { if (e_1) throw e_1.error; }
746
- }
747
- normalizedName = normalizedName.replace(/_+/g, '_');
748
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
749
- normalizedName = normalizedName.replace(/^_/, '');
750
- normalizedName = normalizedName.replace(/_$/, '');
751
- return normalizedName;
752
- }
753
- /**
754
- * TODO: Tests
755
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
756
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
757
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
758
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
759
- * TODO: [🌺] Use some intermediate util splitWords
760
- */
761
-
762
- /**
763
- * @@@
764
- *
765
- * @param text @@@
766
- * @returns @@@
767
- * @example 'hello_world'
768
- * @example 'i_love_promptbook'
769
- * @public exported from `@promptbook/utils`
770
- */
771
- function normalizeTo_snake_case(text) {
772
- return normalizeTo_SCREAMING_CASE(text).toLowerCase();
773
- }
774
-
775
- /**
776
- * Register is @@@
777
- *
778
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
779
- *
780
- * @private internal utility, exported are only signleton instances of this class
781
- */
782
- var $Register = /** @class */ (function () {
783
- function $Register(registerName) {
784
- this.registerName = registerName;
785
- var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
786
- var globalScope = $getGlobalScope();
787
- if (globalScope[storageName] === undefined) {
788
- globalScope[storageName] = [];
789
- }
790
- else if (!Array.isArray(globalScope[storageName])) {
791
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
792
- }
793
- this.storage = globalScope[storageName];
794
- }
795
- $Register.prototype.list = function () {
796
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
797
- return this.storage;
798
- };
799
- $Register.prototype.register = function (registered) {
800
- var packageName = registered.packageName, className = registered.className;
801
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
802
- var existingRegistration = this.storage[existingRegistrationIndex];
803
- if (!existingRegistration) {
804
- this.storage.push(registered);
805
- }
806
- else {
807
- this.storage[existingRegistrationIndex] = registered;
808
- }
809
- return {
810
- registerName: this.registerName,
811
- packageName: packageName,
812
- className: className,
813
- get isDestroyed() {
814
- return false;
815
- },
816
- destroy: function () {
817
- throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
818
- },
819
- };
820
- };
821
- return $Register;
822
- }());
823
-
824
- /**
825
- * @@@
826
- *
827
- * Note: `$` is used to indicate that this interacts with the global scope
828
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
829
- * @public exported from `@promptbook/core`
830
- */
831
- var $scrapersRegister = new $Register('scraper_constructors');
832
- /**
833
- * TODO: [®] DRY Register logic
834
- */
835
-
836
- /**
837
- * !!!!!!
838
- *
839
- * 1) @@@
840
- * 2) @@@
841
- *
842
- * @public exported from `@promptbook/node`
843
- */
844
- function $provideScrapersForNode(tools, options) {
845
- return __awaiter(this, void 0, void 0, function () {
846
- var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
847
- var e_1, _f;
848
- return __generator(this, function (_g) {
849
- switch (_g.label) {
850
- case 0:
851
- if (!$isRunningInNode()) {
852
- throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
853
- }
854
- _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
855
- TODO_USE(isAutoInstalled);
856
- TODO_USE(isVerbose);
857
- scrapers = [];
858
- _g.label = 1;
859
- case 1:
860
- _g.trys.push([1, 6, 7, 8]);
861
- _d = __values($scrapersRegister.list()), _e = _d.next();
862
- _g.label = 2;
863
- case 2:
864
- if (!!_e.done) return [3 /*break*/, 5];
865
- scraperFactory = _e.value;
866
- return [4 /*yield*/, scraperFactory(tools, options || {})];
867
- case 3:
868
- scraper = _g.sent();
869
- scrapers.push(scraper);
870
- _g.label = 4;
871
- case 4:
872
- _e = _d.next();
873
- return [3 /*break*/, 2];
874
- case 5: return [3 /*break*/, 8];
875
- case 6:
876
- e_1_1 = _g.sent();
877
- e_1 = { error: e_1_1 };
878
- return [3 /*break*/, 8];
879
- case 7:
880
- try {
881
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
882
- }
883
- finally { if (e_1) throw e_1.error; }
884
- return [7 /*endfinally*/];
885
- case 8: return [2 /*return*/, scrapers];
886
- }
887
- });
888
- });
889
- }
890
- /**
891
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
892
- */
893
-
894
638
  /**
895
639
  * Converts PipelineCollection to serialized JSON
896
640
  *
@@ -976,7 +720,6 @@
976
720
  pipelineString += '\n\n';
977
721
  pipelineString += description;
978
722
  }
979
- // TODO:> const commands: Array<Command>
980
723
  var commands = [];
981
724
  if (pipelineUrl) {
982
725
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -1032,7 +775,6 @@
1032
775
  pipelineString += '\n\n';
1033
776
  pipelineString += description_1;
1034
777
  }
1035
- // TODO:> const commands: Array<Command>
1036
778
  var commands_1 = [];
1037
779
  var contentLanguage = 'text';
1038
780
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -1331,8 +1073,7 @@
1331
1073
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1332
1074
  *
1333
1075
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1334
- * @public exported from `@promptbook/types`
1335
- * TODO: !!!!!! Export as runtime class not just type
1076
+ * @public exported from `@promptbook/core`
1336
1077
  */
1337
1078
  var MultipleLlmExecutionTools = /** @class */ (function () {
1338
1079
  /**
@@ -2491,6 +2232,22 @@
2491
2232
  return LimitReachedError;
2492
2233
  }(Error));
2493
2234
 
2235
+ /**
2236
+ * This error type indicates that some part of the code is not implemented yet
2237
+ *
2238
+ * @public exported from `@promptbook/core`
2239
+ */
2240
+ var NotYetImplementedError = /** @class */ (function (_super) {
2241
+ __extends(NotYetImplementedError, _super);
2242
+ function NotYetImplementedError(message) {
2243
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2244
+ _this.name = 'NotYetImplementedError';
2245
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2246
+ return _this;
2247
+ }
2248
+ return NotYetImplementedError;
2249
+ }(Error));
2250
+
2494
2251
  /**
2495
2252
  * Index of all custom errors
2496
2253
  *
@@ -2552,6 +2309,7 @@
2552
2309
  }
2553
2310
  }
2554
2311
  /**
2312
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2555
2313
  * TODO: [🧠] Can this return type be better typed than void
2556
2314
  */
2557
2315
 
@@ -2809,6 +2567,23 @@
2809
2567
  return union;
2810
2568
  }
2811
2569
 
2570
+ /**
2571
+ * Just marks a place of place where should be something implemented
2572
+ * No side effects.
2573
+ *
2574
+ * Note: It can be usefull suppressing eslint errors of unused variables
2575
+ *
2576
+ * @param value any values
2577
+ * @returns void
2578
+ * @private within the repository
2579
+ */
2580
+ function TODO_USE() {
2581
+ var value = [];
2582
+ for (var _i = 0; _i < arguments.length; _i++) {
2583
+ value[_i] = arguments[_i];
2584
+ }
2585
+ }
2586
+
2812
2587
  /**
2813
2588
  * This error indicates problems parsing the format value
2814
2589
  *
@@ -3841,12 +3616,11 @@
3841
3616
  */
3842
3617
  function executeAttempts(options) {
3843
3618
  return __awaiter(this, void 0, void 0, function () {
3844
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3619
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3845
3620
  return __generator(this, function (_a) {
3846
3621
  switch (_a.label) {
3847
3622
  case 0:
3848
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3849
- maxExecutionAttempts = settings.maxExecutionAttempts;
3623
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
3850
3624
  $ongoingTemplateResult = {
3851
3625
  $result: null,
3852
3626
  $resultString: null,
@@ -4212,12 +3986,12 @@
4212
3986
  */
4213
3987
  function executeFormatSubvalues(options) {
4214
3988
  return __awaiter(this, void 0, void 0, function () {
4215
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3989
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4216
3990
  var _this = this;
4217
3991
  return __generator(this, function (_a) {
4218
3992
  switch (_a.label) {
4219
3993
  case 0:
4220
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3994
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4221
3995
  if (template.foreach === undefined) {
4222
3996
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4223
3997
  }
@@ -4247,7 +4021,7 @@
4247
4021
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4248
4022
  }
4249
4023
  if (formatDefinition.formatName === 'CSV') {
4250
- formatSettings = settings.csvSettings;
4024
+ formatSettings = csvSettings;
4251
4025
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4252
4026
  }
4253
4027
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4400,13 +4174,12 @@
4400
4174
  */
4401
4175
  function executeTemplate(options) {
4402
4176
  return __awaiter(this, void 0, void 0, function () {
4403
- var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4404
- var e_1, _f, _g;
4405
- return __generator(this, function (_h) {
4406
- switch (_h.label) {
4177
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4178
+ var e_1, _g, _h;
4179
+ return __generator(this, function (_j) {
4180
+ switch (_j.label) {
4407
4181
  case 0:
4408
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4409
- maxExecutionAttempts = settings.maxExecutionAttempts;
4182
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4410
4183
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4411
4184
  title = currentTemplate.title;
4412
4185
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -4421,7 +4194,7 @@
4421
4194
  // <- [🍸]
4422
4195
  })];
4423
4196
  case 1:
4424
- _h.sent();
4197
+ _j.sent();
4425
4198
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
4426
4199
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
4427
4200
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4432,15 +4205,15 @@
4432
4205
  .map(function (name) { return "{".concat(name, "}"); })
4433
4206
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4434
4207
  }
4435
- _b = (_a = Object).freeze;
4436
- _c = [{}];
4208
+ _c = (_b = Object).freeze;
4209
+ _d = [{}];
4437
4210
  return [4 /*yield*/, getReservedParametersForTemplate({
4438
4211
  preparedPipeline: preparedPipeline,
4439
4212
  template: currentTemplate,
4440
4213
  pipelineIdentification: pipelineIdentification,
4441
4214
  })];
4442
4215
  case 2:
4443
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4216
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
4444
4217
  definedParameterNames = new Set(Object.keys(definedParameters));
4445
4218
  parameters = {};
4446
4219
  _loop_1 = function (parameterName) {
@@ -4460,15 +4233,15 @@
4460
4233
  try {
4461
4234
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4462
4235
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4463
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4464
- parameterName = _e.value;
4236
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
4237
+ parameterName = _f.value;
4465
4238
  _loop_1(parameterName);
4466
4239
  }
4467
4240
  }
4468
4241
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4469
4242
  finally {
4470
4243
  try {
4471
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4244
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
4472
4245
  }
4473
4246
  finally { if (e_1) throw e_1.error; }
4474
4247
  }
@@ -4488,12 +4261,11 @@
4488
4261
  template: currentTemplate,
4489
4262
  preparedPipeline: preparedPipeline,
4490
4263
  tools: tools,
4491
- settings: settings,
4492
4264
  $executionReport: $executionReport,
4493
4265
  pipelineIdentification: pipelineIdentification,
4494
4266
  })];
4495
4267
  case 3:
4496
- resultString = _h.sent();
4268
+ resultString = _j.sent();
4497
4269
  return [4 /*yield*/, onProgress({
4498
4270
  name: name,
4499
4271
  title: title,
@@ -4505,12 +4277,12 @@
4505
4277
  // <- [🍸]
4506
4278
  })];
4507
4279
  case 4:
4508
- _h.sent();
4509
- return [2 /*return*/, Object.freeze((_g = {},
4510
- _g[currentTemplate.resultingParameterName] =
4280
+ _j.sent();
4281
+ return [2 /*return*/, Object.freeze((_h = {},
4282
+ _h[currentTemplate.resultingParameterName] =
4511
4283
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4512
4284
  resultString,
4513
- _g))];
4285
+ _h))];
4514
4286
  }
4515
4287
  });
4516
4288
  });
@@ -4569,13 +4341,12 @@
4569
4341
  */
4570
4342
  function executePipeline(options) {
4571
4343
  return __awaiter(this, void 0, void 0, function () {
4572
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4344
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4573
4345
  var e_1, _f, e_2, _g;
4574
4346
  return __generator(this, function (_h) {
4575
4347
  switch (_h.label) {
4576
4348
  case 0:
4577
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4578
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4349
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4579
4350
  preparedPipeline = options.preparedPipeline;
4580
4351
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4581
4352
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -4760,12 +4531,7 @@
4760
4531
  return [3 /*break*/, 4];
4761
4532
  case 3:
4762
4533
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
4763
- work_1 = executeTemplate({
4764
- currentTemplate: currentTemplate,
4765
- preparedPipeline: preparedPipeline,
4766
- parametersToPass: parametersToPass,
4767
- tools: tools,
4768
- onProgress: function (progress) {
4534
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
4769
4535
  if (isReturned) {
4770
4536
  throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4771
4537
  .split('\n')
@@ -4775,11 +4541,7 @@
4775
4541
  if (onProgress) {
4776
4542
  onProgress(progress);
4777
4543
  }
4778
- },
4779
- settings: settings,
4780
- $executionReport: executionReport,
4781
- pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
4782
- })
4544
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
4783
4545
  .then(function (newParametersToPass) {
4784
4546
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
4785
4547
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -4883,8 +4645,7 @@
4883
4645
  */
4884
4646
  function createPipelineExecutor(options) {
4885
4647
  var _this = this;
4886
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
4887
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
4648
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
4888
4649
  validatePipeline(pipeline);
4889
4650
  var pipelineIdentification = (function () {
4890
4651
  // Note: This is a 😐 implementation of [🚞]
@@ -4918,14 +4679,12 @@
4918
4679
  tools: tools,
4919
4680
  onProgress: onProgress,
4920
4681
  pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
4921
- settings: {
4922
- maxExecutionAttempts: maxExecutionAttempts,
4923
- maxParallelCount: maxParallelCount,
4924
- csvSettings: csvSettings,
4925
- isVerbose: isVerbose,
4926
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4927
- rootDirname: rootDirname,
4928
- },
4682
+ maxExecutionAttempts: maxExecutionAttempts,
4683
+ maxParallelCount: maxParallelCount,
4684
+ csvSettings: csvSettings,
4685
+ isVerbose: isVerbose,
4686
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4687
+ rootDirname: rootDirname,
4929
4688
  })];
4930
4689
  });
4931
4690
  }); };
@@ -4948,7 +4707,7 @@
4948
4707
  return __generator(this, function (_d) {
4949
4708
  switch (_d.label) {
4950
4709
  case 0:
4951
- _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4710
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4952
4711
  if (tools === undefined || tools.llm === undefined) {
4953
4712
  throw new MissingToolsError('LLM tools are required for preparing persona');
4954
4713
  }
@@ -5004,20 +4763,171 @@
5004
4763
  */
5005
4764
 
5006
4765
  /**
5007
- * This error indicates that the promptbook can not retrieve knowledge from external sources
4766
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
4767
+ *
4768
+ * @public exported from `@promptbook/core`
4769
+ */
4770
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4771
+ __extends(KnowledgeScrapeError, _super);
4772
+ function KnowledgeScrapeError(message) {
4773
+ var _this = _super.call(this, message) || this;
4774
+ _this.name = 'KnowledgeScrapeError';
4775
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4776
+ return _this;
4777
+ }
4778
+ return KnowledgeScrapeError;
4779
+ }(Error));
4780
+
4781
+ /**
4782
+ * @@@
4783
+ *
4784
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4785
+ *
4786
+ * @private internal function of `$Register`
4787
+ */
4788
+ function $getGlobalScope() {
4789
+ return Function('return this')();
4790
+ }
4791
+
4792
+ /**
4793
+ * @@@
4794
+ *
4795
+ * @param text @@@
4796
+ * @returns @@@
4797
+ * @example 'HELLO_WORLD'
4798
+ * @example 'I_LOVE_PROMPTBOOK'
4799
+ * @public exported from `@promptbook/utils`
4800
+ */
4801
+ function normalizeTo_SCREAMING_CASE(text) {
4802
+ var e_1, _a;
4803
+ var charType;
4804
+ var lastCharType = 'OTHER';
4805
+ var normalizedName = '';
4806
+ try {
4807
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4808
+ var char = text_1_1.value;
4809
+ var normalizedChar = void 0;
4810
+ if (/^[a-z]$/.test(char)) {
4811
+ charType = 'LOWERCASE';
4812
+ normalizedChar = char.toUpperCase();
4813
+ }
4814
+ else if (/^[A-Z]$/.test(char)) {
4815
+ charType = 'UPPERCASE';
4816
+ normalizedChar = char;
4817
+ }
4818
+ else if (/^[0-9]$/.test(char)) {
4819
+ charType = 'NUMBER';
4820
+ normalizedChar = char;
4821
+ }
4822
+ else {
4823
+ charType = 'OTHER';
4824
+ normalizedChar = '_';
4825
+ }
4826
+ if (charType !== lastCharType &&
4827
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4828
+ !(lastCharType === 'NUMBER') &&
4829
+ !(charType === 'NUMBER')) {
4830
+ normalizedName += '_';
4831
+ }
4832
+ normalizedName += normalizedChar;
4833
+ lastCharType = charType;
4834
+ }
4835
+ }
4836
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4837
+ finally {
4838
+ try {
4839
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
4840
+ }
4841
+ finally { if (e_1) throw e_1.error; }
4842
+ }
4843
+ normalizedName = normalizedName.replace(/_+/g, '_');
4844
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
4845
+ normalizedName = normalizedName.replace(/^_/, '');
4846
+ normalizedName = normalizedName.replace(/_$/, '');
4847
+ return normalizedName;
4848
+ }
4849
+ /**
4850
+ * TODO: Tests
4851
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
4852
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
4853
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
4854
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
4855
+ * TODO: [🌺] Use some intermediate util splitWords
4856
+ */
4857
+
4858
+ /**
4859
+ * @@@
4860
+ *
4861
+ * @param text @@@
4862
+ * @returns @@@
4863
+ * @example 'hello_world'
4864
+ * @example 'i_love_promptbook'
4865
+ * @public exported from `@promptbook/utils`
4866
+ */
4867
+ function normalizeTo_snake_case(text) {
4868
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
4869
+ }
4870
+
4871
+ /**
4872
+ * Register is @@@
4873
+ *
4874
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
4875
+ *
4876
+ * @private internal utility, exported are only signleton instances of this class
4877
+ */
4878
+ var $Register = /** @class */ (function () {
4879
+ function $Register(registerName) {
4880
+ this.registerName = registerName;
4881
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
4882
+ var globalScope = $getGlobalScope();
4883
+ if (globalScope[storageName] === undefined) {
4884
+ globalScope[storageName] = [];
4885
+ }
4886
+ else if (!Array.isArray(globalScope[storageName])) {
4887
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
4888
+ }
4889
+ this.storage = globalScope[storageName];
4890
+ }
4891
+ $Register.prototype.list = function () {
4892
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
4893
+ return this.storage;
4894
+ };
4895
+ $Register.prototype.register = function (registered) {
4896
+ var packageName = registered.packageName, className = registered.className;
4897
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
4898
+ var existingRegistration = this.storage[existingRegistrationIndex];
4899
+ if (!existingRegistration) {
4900
+ this.storage.push(registered);
4901
+ }
4902
+ else {
4903
+ this.storage[existingRegistrationIndex] = registered;
4904
+ }
4905
+ return {
4906
+ registerName: this.registerName,
4907
+ packageName: packageName,
4908
+ className: className,
4909
+ get isDestroyed() {
4910
+ return false;
4911
+ },
4912
+ destroy: function () {
4913
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
4914
+ },
4915
+ };
4916
+ };
4917
+ return $Register;
4918
+ }());
4919
+
4920
+ /**
4921
+ * @@@
5008
4922
  *
4923
+ * Note: `$` is used to indicate that this interacts with the global scope
4924
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5009
4925
  * @public exported from `@promptbook/core`
5010
4926
  */
5011
- var KnowledgeScrapeError = /** @class */ (function (_super) {
5012
- __extends(KnowledgeScrapeError, _super);
5013
- function KnowledgeScrapeError(message) {
5014
- var _this = _super.call(this, message) || this;
5015
- _this.name = 'KnowledgeScrapeError';
5016
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
5017
- return _this;
5018
- }
5019
- return KnowledgeScrapeError;
5020
- }(Error));
4927
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4928
+ /**
4929
+ * TODO: [®] DRY Register logic
4930
+ */
5021
4931
 
5022
4932
  /**
5023
4933
  * @@@
@@ -5026,12 +4936,11 @@
5026
4936
  * @singleton Only one instance of each register is created per build, but thare can be more @@@
5027
4937
  * @public exported from `@promptbook/core`
5028
4938
  */
5029
- var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4939
+ var $scrapersRegister = new $Register('scraper_constructors');
5030
4940
  /**
5031
4941
  * TODO: [®] DRY Register logic
5032
4942
  */
5033
4943
 
5034
- // TODO: !!!!!! Maybe delete this function
5035
4944
  /**
5036
4945
  * Creates a message with all registered scrapers
5037
4946
  *
@@ -5198,7 +5107,6 @@
5198
5107
  * @private within the repository
5199
5108
  */
5200
5109
  function sourceContentToName(sourceContent) {
5201
- // TODO: !!!!!! Better name for source than gibberish hash
5202
5110
  var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
5203
5111
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5204
5112
  .toString( /* hex */)
@@ -5298,20 +5206,20 @@
5298
5206
  /**
5299
5207
  * @@@
5300
5208
  *
5301
- * @private for scraper utilities
5209
+ * @public exported from `@promptbook/core`
5302
5210
  */
5303
5211
  function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5304
5212
  var _a;
5305
5213
  return __awaiter(this, void 0, void 0, function () {
5306
5214
  var sourceContent, name, _b, _c, rootDirname, _d,
5307
5215
  // <- TODO: process.cwd() if running in Node.js
5308
- isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
5216
+ isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
5309
5217
  return __generator(this, function (_e) {
5310
5218
  switch (_e.label) {
5311
5219
  case 0:
5312
5220
  sourceContent = knowledgeSource.sourceContent;
5313
5221
  name = knowledgeSource.name;
5314
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5222
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
5315
5223
  TODO_USE(isVerbose);
5316
5224
  if (!name) {
5317
5225
  name = sourceContentToName(sourceContent);
@@ -5327,19 +5235,14 @@
5327
5235
  filename: null,
5328
5236
  url: url,
5329
5237
  mimeType: mimeType,
5330
- asBlob: function () {
5331
- return __awaiter(this, void 0, void 0, function () {
5332
- var content;
5333
- return __generator(this, function (_a) {
5334
- switch (_a.label) {
5335
- case 0: return [4 /*yield*/, response_1.blob()];
5336
- case 1:
5337
- content = _a.sent();
5338
- return [2 /*return*/, content];
5339
- }
5340
- });
5341
- });
5342
- },
5238
+ /*
5239
+ TODO: [🥽]
5240
+ > async asBlob() {
5241
+ > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
5242
+ > const content = await response.blob();
5243
+ > return content;
5244
+ > },
5245
+ */
5343
5246
  asJson: function () {
5344
5247
  return __awaiter(this, void 0, void 0, function () {
5345
5248
  var content;
@@ -5379,34 +5282,31 @@
5379
5282
  }
5380
5283
  filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
5381
5284
  fileExtension = getFileExtension(filename_1);
5382
- mimeType_1 = extensionToMimeType(fileExtension || '');
5285
+ mimeType = extensionToMimeType(fileExtension || '');
5383
5286
  return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
5384
5287
  case 3:
5385
5288
  if (!(_e.sent())) {
5386
5289
  throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
5387
5290
  }
5388
- // TODO: !!!!!! Test security file - file is scoped to the project (maybe do this in `filesystemTools`)
5291
+ // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
5389
5292
  return [2 /*return*/, {
5390
5293
  source: name,
5391
5294
  filename: filename_1,
5392
5295
  url: null,
5393
- mimeType: mimeType_1,
5394
- asBlob: function () {
5395
- return __awaiter(this, void 0, void 0, function () {
5396
- var content;
5397
- return __generator(this, function (_a) {
5398
- switch (_a.label) {
5399
- case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
5400
- case 1:
5401
- content = _a.sent();
5402
- return [2 /*return*/, new Blob([
5403
- content,
5404
- // <- TODO: !!!!!! Maybe not working
5405
- ], { type: mimeType_1 })];
5406
- }
5407
- });
5408
- });
5409
- },
5296
+ mimeType: mimeType,
5297
+ /*
5298
+ TODO: [🥽]
5299
+ > async asBlob() {
5300
+ > const content = await tools.fs!.readFile(filename);
5301
+ > return new Blob(
5302
+ > [
5303
+ > content,
5304
+ > // <- TODO: [🥽] This is NOT tested, test it
5305
+ > ],
5306
+ > { type: mimeType },
5307
+ > );
5308
+ > },
5309
+ */
5410
5310
  asJson: function () {
5411
5311
  return __awaiter(this, void 0, void 0, function () {
5412
5312
  var _a, _b;
@@ -5442,9 +5342,14 @@
5442
5342
  asJson: function () {
5443
5343
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5444
5344
  },
5445
- asBlob: function () {
5446
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5447
- },
5345
+ /*
5346
+ TODO: [🥽]
5347
+ > asBlob() {
5348
+ > throw new UnexpectedError(
5349
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
5350
+ > );
5351
+ > },
5352
+ */
5448
5353
  }];
5449
5354
  }
5450
5355
  });
@@ -5464,7 +5369,7 @@
5464
5369
  return __generator(this, function (_c) {
5465
5370
  switch (_c.label) {
5466
5371
  case 0:
5467
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5372
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5468
5373
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
5469
5374
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
5470
5375
  var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
@@ -5493,7 +5398,8 @@
5493
5398
  case 4:
5494
5399
  partialPiecesUnchecked = _d.sent();
5495
5400
  if (partialPiecesUnchecked !== null) {
5496
- partialPieces = partialPiecesUnchecked;
5401
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
5402
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
5497
5403
  return [3 /*break*/, 6];
5498
5404
  }
5499
5405
  _d.label = 5;
@@ -5541,7 +5447,7 @@
5541
5447
  > /**
5542
5448
  > * Unprepared knowledge
5543
5449
  > * /
5544
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
5450
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
5545
5451
  > };
5546
5452
  >
5547
5453
  > export async function prepareKnowledgePieces(
@@ -5599,7 +5505,7 @@
5599
5505
  return __generator(this, function (_b) {
5600
5506
  switch (_b.label) {
5601
5507
  case 0:
5602
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
5508
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
5603
5509
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
5604
5510
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
5605
5511
  TODO_USE(parameters);
@@ -5661,7 +5567,7 @@
5661
5567
  if (isPipelinePrepared(pipeline)) {
5662
5568
  return [2 /*return*/, pipeline];
5663
5569
  }
5664
- rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5570
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5665
5571
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
5666
5572
  if (tools === undefined || tools.llm === undefined) {
5667
5573
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
@@ -5719,7 +5625,9 @@
5719
5625
  // ----- /Templates preparation -----
5720
5626
  // Note: Count total usage
5721
5627
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
5722
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
5628
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
5629
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
5630
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
5723
5631
  }
5724
5632
  });
5725
5633
  });
@@ -5766,6 +5674,7 @@
5766
5674
  'KNOWLEDGE ./hejny-cv.md',
5767
5675
  'KNOWLEDGE ./hejny-cv.pdf',
5768
5676
  'KNOWLEDGE ./hejny-cv.docx',
5677
+ // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
5769
5678
  ],
5770
5679
  /**
5771
5680
  * Parses the KNOWLEDGE command
@@ -8440,38 +8349,359 @@
8440
8349
  * Note: This function does not validate logic of the pipeline only the parsing
8441
8350
  * Note: This function acts as compilation process
8442
8351
  *
8443
- * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8444
- * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
8445
- * @param options - Options and tools for the compilation
8446
- * @returns {Promptbook} compiled in JSON format (.ptbk.json)
8447
- * @throws {ParseError} if the promptbook string is not valid
8448
- * @public exported from `@promptbook/core`
8352
+ * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8353
+ * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
8354
+ * @param options - Options and tools for the compilation
8355
+ * @returns {Promptbook} compiled in JSON format (.ptbk.json)
8356
+ * @throws {ParseError} if the promptbook string is not valid
8357
+ * @public exported from `@promptbook/core`
8358
+ */
8359
+ function pipelineStringToJson(pipelineString, tools, options) {
8360
+ return __awaiter(this, void 0, void 0, function () {
8361
+ var pipelineJson;
8362
+ return __generator(this, function (_a) {
8363
+ switch (_a.label) {
8364
+ case 0:
8365
+ pipelineJson = pipelineStringToJsonSync(pipelineString);
8366
+ if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8367
+ return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
8368
+ rootDirname: null,
8369
+ })];
8370
+ case 1:
8371
+ pipelineJson = _a.sent();
8372
+ _a.label = 2;
8373
+ case 2:
8374
+ // Note: No need to use `$asDeeplyFrozenSerializableJson` because `pipelineStringToJsonSync` and `preparePipeline` already do that
8375
+ return [2 /*return*/, pipelineJson];
8376
+ }
8377
+ });
8378
+ });
8379
+ }
8380
+ /**
8381
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
8382
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
8383
+ * TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
8384
+ */
8385
+
8386
+ // Note: We want to use the `exec` as async function
8387
+ var exec$1 = util.promisify(child_process.exec);
8388
+ /**
8389
+ * @@@
8390
+ *
8391
+ * @private within the repository
8392
+ */
8393
+ function locateAppOnLinux(_a) {
8394
+ var appName = _a.appName, linuxWhich = _a.linuxWhich;
8395
+ return __awaiter(this, void 0, void 0, function () {
8396
+ var _b, stderr, stdout, error_1;
8397
+ return __generator(this, function (_c) {
8398
+ switch (_c.label) {
8399
+ case 0:
8400
+ _c.trys.push([0, 2, , 3]);
8401
+ return [4 /*yield*/, exec$1("which ".concat(linuxWhich))];
8402
+ case 1:
8403
+ _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
8404
+ if (!stderr && stdout) {
8405
+ return [2 /*return*/, stdout.trim()];
8406
+ }
8407
+ throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr));
8408
+ case 2:
8409
+ error_1 = _c.sent();
8410
+ if (!(error_1 instanceof Error)) {
8411
+ throw error_1;
8412
+ }
8413
+ return [2 /*return*/, null];
8414
+ case 3: return [2 /*return*/];
8415
+ }
8416
+ });
8417
+ });
8418
+ }
8419
+ /**
8420
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8421
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8422
+ */
8423
+
8424
+ /**
8425
+ * @@@
8426
+ *
8427
+ * @public exported from `@promptbook/node`
8428
+ */
8429
+ function $provideFilesystemForNode(options) {
8430
+ if (!$isRunningInNode()) {
8431
+ throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
8432
+ }
8433
+ var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
8434
+ TODO_USE(isVerbose);
8435
+ return {
8436
+ stat: promises.stat,
8437
+ access: promises.access,
8438
+ constants: promises.constants,
8439
+ readFile: promises.readFile,
8440
+ writeFile: promises.writeFile,
8441
+ readdir: promises.readdir,
8442
+ };
8443
+ }
8444
+ /**
8445
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8446
+ */
8447
+
8448
+ /**
8449
+ * Checks if the file is executable
8450
+ *
8451
+ * @private within the repository
8452
+ */
8453
+ function isExecutable(path, fs) {
8454
+ return __awaiter(this, void 0, void 0, function () {
8455
+ return __generator(this, function (_a) {
8456
+ switch (_a.label) {
8457
+ case 0:
8458
+ _a.trys.push([0, 2, , 3]);
8459
+ return [4 /*yield*/, fs.access(path, fs.constants.X_OK)];
8460
+ case 1:
8461
+ _a.sent();
8462
+ return [2 /*return*/, true];
8463
+ case 2:
8464
+ _a.sent();
8465
+ return [2 /*return*/, false];
8466
+ case 3: return [2 /*return*/];
8467
+ }
8468
+ });
8469
+ });
8470
+ }
8471
+ /**
8472
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
8473
+ * TODO: [🖇] What about symlinks?
8474
+ */
8475
+
8476
+ // Note: Module `userhome` has no types available, so it is imported using `require`
8477
+ // @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions
8478
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
8479
+ var userhome = require('userhome');
8480
+ // Note: We want to use the `exec` as async function
8481
+ var exec = util.promisify(child_process.exec);
8482
+ /**
8483
+ * @@@
8484
+ *
8485
+ * @private within the repository
8486
+ */
8487
+ function locateAppOnMacOs(_a) {
8488
+ var appName = _a.appName, macOsName = _a.macOsName;
8489
+ return __awaiter(this, void 0, void 0, function () {
8490
+ var toExec, regPath, altPath, _b, stderr, stdout, error_1;
8491
+ return __generator(this, function (_c) {
8492
+ switch (_c.label) {
8493
+ case 0:
8494
+ _c.trys.push([0, 6, , 7]);
8495
+ toExec = "/Contents/MacOS/".concat(macOsName);
8496
+ regPath = "/Applications/".concat(macOsName, ".app") + toExec;
8497
+ altPath = userhome(regPath.slice(1));
8498
+ return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())];
8499
+ case 1:
8500
+ if (!_c.sent()) return [3 /*break*/, 2];
8501
+ return [2 /*return*/, regPath];
8502
+ case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())];
8503
+ case 3:
8504
+ if (_c.sent()) {
8505
+ return [2 /*return*/, altPath];
8506
+ }
8507
+ _c.label = 4;
8508
+ case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))];
8509
+ case 5:
8510
+ _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
8511
+ if (!stderr && stdout) {
8512
+ return [2 /*return*/, stdout.trim() + toExec];
8513
+ }
8514
+ throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr));
8515
+ case 6:
8516
+ error_1 = _c.sent();
8517
+ if (!(error_1 instanceof Error)) {
8518
+ throw error_1;
8519
+ }
8520
+ return [2 /*return*/, null];
8521
+ case 7: return [2 /*return*/];
8522
+ }
8523
+ });
8524
+ });
8525
+ }
8526
+ /**
8527
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8528
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8529
+ */
8530
+
8531
+ /**
8532
+ * @@@
8533
+ *
8534
+ * @private within the repository
8535
+ */
8536
+ function locateAppOnWindows(_a) {
8537
+ var appName = _a.appName, windowsSuffix = _a.windowsSuffix;
8538
+ return __awaiter(this, void 0, void 0, function () {
8539
+ var prefixes, prefixes_1, prefixes_1_1, prefix, path$1, e_1_1, error_1;
8540
+ var e_1, _b;
8541
+ return __generator(this, function (_c) {
8542
+ switch (_c.label) {
8543
+ case 0:
8544
+ _c.trys.push([0, 9, , 10]);
8545
+ prefixes = [
8546
+ process.env.LOCALAPPDATA,
8547
+ path.join(process.env.LOCALAPPDATA || '', 'Programs'),
8548
+ process.env.PROGRAMFILES,
8549
+ process.env['PROGRAMFILES(X86)'],
8550
+ ];
8551
+ _c.label = 1;
8552
+ case 1:
8553
+ _c.trys.push([1, 6, 7, 8]);
8554
+ prefixes_1 = __values(prefixes), prefixes_1_1 = prefixes_1.next();
8555
+ _c.label = 2;
8556
+ case 2:
8557
+ if (!!prefixes_1_1.done) return [3 /*break*/, 5];
8558
+ prefix = prefixes_1_1.value;
8559
+ path$1 = prefix + windowsSuffix;
8560
+ return [4 /*yield*/, isExecutable(path$1, $provideFilesystemForNode())];
8561
+ case 3:
8562
+ if (_c.sent()) {
8563
+ return [2 /*return*/, path$1];
8564
+ }
8565
+ _c.label = 4;
8566
+ case 4:
8567
+ prefixes_1_1 = prefixes_1.next();
8568
+ return [3 /*break*/, 2];
8569
+ case 5: return [3 /*break*/, 8];
8570
+ case 6:
8571
+ e_1_1 = _c.sent();
8572
+ e_1 = { error: e_1_1 };
8573
+ return [3 /*break*/, 8];
8574
+ case 7:
8575
+ try {
8576
+ if (prefixes_1_1 && !prefixes_1_1.done && (_b = prefixes_1.return)) _b.call(prefixes_1);
8577
+ }
8578
+ finally { if (e_1) throw e_1.error; }
8579
+ return [7 /*endfinally*/];
8580
+ case 8: throw new Error("Can not locate app ".concat(appName, " on Windows."));
8581
+ case 9:
8582
+ error_1 = _c.sent();
8583
+ if (!(error_1 instanceof Error)) {
8584
+ throw error_1;
8585
+ }
8586
+ return [2 /*return*/, null];
8587
+ case 10: return [2 /*return*/];
8588
+ }
8589
+ });
8590
+ });
8591
+ }
8592
+ /**
8593
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8594
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8595
+ */
8596
+
8597
+ /**
8598
+ * Locates an application on the system
8599
+ *
8600
+ * @private within the repository
8601
+ */
8602
+ function locateApp(options) {
8603
+ if (!$isRunningInNode()) {
8604
+ throw new EnvironmentMismatchError('Locating apps works only in Node.js environment');
8605
+ }
8606
+ var appName = options.appName, linuxWhich = options.linuxWhich, windowsSuffix = options.windowsSuffix, macOsName = options.macOsName;
8607
+ if (process.platform === 'win32') {
8608
+ if (windowsSuffix) {
8609
+ return locateAppOnWindows({ appName: appName, windowsSuffix: windowsSuffix });
8610
+ }
8611
+ else {
8612
+ throw new Error("".concat(appName, " is not available on Windows."));
8613
+ }
8614
+ }
8615
+ else if (process.platform === 'darwin') {
8616
+ if (macOsName) {
8617
+ return locateAppOnMacOs({ appName: appName, macOsName: macOsName });
8618
+ }
8619
+ else {
8620
+ throw new Error("".concat(appName, " is not available on macOS."));
8621
+ }
8622
+ }
8623
+ else {
8624
+ if (linuxWhich) {
8625
+ return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich });
8626
+ }
8627
+ else {
8628
+ throw new Error("".concat(appName, " is not available on Linux."));
8629
+ }
8630
+ }
8631
+ }
8632
+ /**
8633
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8634
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8635
+ */
8636
+
8637
+ /**
8638
+ * @@@
8639
+ *
8640
+ * @private within the repository
8641
+ */
8642
+ function locateLibreoffice() {
8643
+ return locateApp({
8644
+ appName: 'Libreoffice',
8645
+ linuxWhich: 'libreoffice',
8646
+ windowsSuffix: '\\LibreOffice\\program\\soffice.exe',
8647
+ macOsName: 'LibreOffice',
8648
+ });
8649
+ }
8650
+ /**
8651
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents`
8652
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8653
+ */
8654
+
8655
+ /**
8656
+ * @@@
8657
+ *
8658
+ * @private within the repository
8659
+ */
8660
+ function locatePandoc() {
8661
+ return locateApp({
8662
+ appName: 'Pandoc',
8663
+ linuxWhich: 'pandoc',
8664
+ windowsSuffix: '\\Pandoc\\pandoc.exe',
8665
+ macOsName: 'Pandoc',
8666
+ });
8667
+ }
8668
+ /**
8669
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents`
8670
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8671
+ */
8672
+
8673
+ /**
8674
+ * @@@
8675
+ *
8676
+ * @public exported from `@promptbook/node`
8449
8677
  */
8450
- function pipelineStringToJson(pipelineString, tools, options) {
8678
+ function $provideExecutablesForNode(options) {
8451
8679
  return __awaiter(this, void 0, void 0, function () {
8452
- var pipelineJson;
8453
- return __generator(this, function (_a) {
8454
- switch (_a.label) {
8680
+ var _a, _b, isAutoInstalled, _c, isVerbose;
8681
+ var _d;
8682
+ return __generator(this, function (_e) {
8683
+ switch (_e.label) {
8455
8684
  case 0:
8456
- pipelineJson = pipelineStringToJsonSync(pipelineString);
8457
- if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8458
- return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
8459
- rootDirname: null,
8460
- })];
8685
+ if (!$isRunningInNode()) {
8686
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8687
+ }
8688
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8689
+ TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables
8690
+ TODO_USE(isVerbose);
8691
+ _d = {};
8692
+ return [4 /*yield*/, locatePandoc()];
8461
8693
  case 1:
8462
- pipelineJson = _a.sent();
8463
- _a.label = 2;
8464
- case 2:
8465
- // Note: No need to use `$asDeeplyFrozenSerializableJson` because `pipelineStringToJsonSync` and `preparePipeline` already do that
8466
- return [2 /*return*/, pipelineJson];
8694
+ _d.pandocPath = (_e.sent()) || undefined;
8695
+ return [4 /*yield*/, locateLibreoffice()];
8696
+ case 2: return [2 /*return*/, (_d.libreOfficePath = (_e.sent()) || undefined,
8697
+ _d)];
8467
8698
  }
8468
8699
  });
8469
8700
  });
8470
8701
  }
8471
8702
  /**
8472
- * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
8473
- * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
8474
- * TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
8703
+ * TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided
8704
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8475
8705
  */
8476
8706
 
8477
8707
  /**
@@ -8643,7 +8873,7 @@
8643
8873
  */
8644
8874
  function createLlmToolsFromConfiguration(configuration, options) {
8645
8875
  if (options === void 0) { options = {}; }
8646
- var _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
8876
+ var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
8647
8877
  var llmTools = configuration.map(function (llmConfiguration) {
8648
8878
  var registeredItem = $llmToolsRegister
8649
8879
  .list()
@@ -8654,7 +8884,7 @@
8654
8884
  if (registeredItem === undefined) {
8655
8885
  throw new Error(spaceTrim__default["default"](function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
8656
8886
  }
8657
- return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
8887
+ return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
8658
8888
  });
8659
8889
  return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
8660
8890
  }
@@ -8704,6 +8934,64 @@
8704
8934
  * TODO: [®] DRY Register logic
8705
8935
  */
8706
8936
 
8937
+ /**
8938
+ * @@@
8939
+ *
8940
+ * 1) @@@
8941
+ * 2) @@@
8942
+ *
8943
+ * @public exported from `@promptbook/node`
8944
+ */
8945
+ function $provideScrapersForNode(tools, options) {
8946
+ return __awaiter(this, void 0, void 0, function () {
8947
+ var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
8948
+ var e_1, _f;
8949
+ return __generator(this, function (_g) {
8950
+ switch (_g.label) {
8951
+ case 0:
8952
+ if (!$isRunningInNode()) {
8953
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8954
+ }
8955
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8956
+ TODO_USE(isAutoInstalled);
8957
+ TODO_USE(isVerbose);
8958
+ scrapers = [];
8959
+ _g.label = 1;
8960
+ case 1:
8961
+ _g.trys.push([1, 6, 7, 8]);
8962
+ _d = __values($scrapersRegister.list()), _e = _d.next();
8963
+ _g.label = 2;
8964
+ case 2:
8965
+ if (!!_e.done) return [3 /*break*/, 5];
8966
+ scraperFactory = _e.value;
8967
+ return [4 /*yield*/, scraperFactory(tools, options || {})];
8968
+ case 3:
8969
+ scraper = _g.sent();
8970
+ scrapers.push(scraper);
8971
+ _g.label = 4;
8972
+ case 4:
8973
+ _e = _d.next();
8974
+ return [3 /*break*/, 2];
8975
+ case 5: return [3 /*break*/, 8];
8976
+ case 6:
8977
+ e_1_1 = _g.sent();
8978
+ e_1 = { error: e_1_1 };
8979
+ return [3 /*break*/, 8];
8980
+ case 7:
8981
+ try {
8982
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
8983
+ }
8984
+ finally { if (e_1) throw e_1.error; }
8985
+ return [7 /*endfinally*/];
8986
+ case 8: return [2 /*return*/, scrapers];
8987
+ }
8988
+ });
8989
+ });
8990
+ }
8991
+ /**
8992
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8993
+ */
8994
+
8707
8995
  /**
8708
8996
  * Extracts code block from markdown.
8709
8997
  *
@@ -9158,7 +9446,7 @@
9158
9446
  */
9159
9447
  function $provideExecutionToolsForNode(options) {
9160
9448
  return __awaiter(this, void 0, void 0, function () {
9161
- var fs, llm, tools;
9449
+ var fs, llm, executables, tools;
9162
9450
  var _a;
9163
9451
  return __generator(this, function (_b) {
9164
9452
  switch (_b.label) {
@@ -9168,12 +9456,16 @@
9168
9456
  }
9169
9457
  fs = $provideFilesystemForNode();
9170
9458
  llm = $provideLlmToolsFromEnv(options);
9459
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
9460
+ case 1:
9461
+ executables = _b.sent();
9171
9462
  _a = {
9172
9463
  llm: llm,
9173
- fs: fs
9464
+ fs: fs,
9465
+ executables: executables
9174
9466
  };
9175
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
9176
- case 1:
9467
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
9468
+ case 2:
9177
9469
  tools = (_a.scrapers = _b.sent(),
9178
9470
  _a.script = [new JavascriptExecutionTools(options)],
9179
9471
  _a);
@@ -9411,7 +9703,9 @@
9411
9703
  throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
9412
9704
  // <- TODO: [🧠] What is the best error type here`
9413
9705
  }
9414
- makedLibraryFilePath = path.join(path$1, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".json"));
9706
+ makedLibraryFilePath = path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
9707
+ // <- TODO: [🦒] Allow to override (pass different value into the function)
9708
+ , ".json"));
9415
9709
  return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
9416
9710
  case 3:
9417
9711
  if (!(_f.sent())) {
@@ -9422,7 +9716,7 @@
9422
9716
  // TODO: !! Implement;
9423
9717
  // TODO: [🌗]
9424
9718
  }
9425
- _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9719
+ _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9426
9720
  collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
9427
9721
  var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
9428
9722
  var e_1, _a;
@@ -9894,7 +10188,7 @@
9894
10188
  function cacheLlmTools(llmTools, options) {
9895
10189
  var _this = this;
9896
10190
  if (options === void 0) { options = {}; }
9897
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isReloaded, isReloaded = _b === void 0 ? false : _b;
10191
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
9898
10192
  var proxyTools = __assign(__assign({}, llmTools), {
9899
10193
  // <- Note: [🥫]
9900
10194
  get title() {
@@ -9916,7 +10210,7 @@
9916
10210
  key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
9917
10211
  '-' +
9918
10212
  sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
9919
- if (!!isReloaded) return [3 /*break*/, 2];
10213
+ if (!!isCacheReloaded) return [3 /*break*/, 2];
9920
10214
  return [4 /*yield*/, storage.getItem(key)];
9921
10215
  case 1:
9922
10216
  _a = _c.sent();
@@ -10002,12 +10296,14 @@
10002
10296
  if (!$isRunningInNode()) {
10003
10297
  throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
10004
10298
  }
10005
- var _a = (options !== null && options !== void 0 ? options : {}).isCacheCleaned, isCacheCleaned = _a === void 0 ? false : _a;
10299
+ var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
10006
10300
  return cacheLlmTools(countTotalUsage(
10007
10301
  // <- Note: for example here we don`t want the [🌯]
10008
10302
  $provideLlmToolsFromEnv()), {
10009
- storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, { rootFolderPath: path.join(process.cwd(), EXECUTIONS_CACHE_DIRNAME) }),
10010
- isReloaded: isCacheCleaned,
10303
+ storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
10304
+ rootFolderPath: path.join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
10305
+ }),
10306
+ isCacheReloaded: isCacheReloaded,
10011
10307
  });
10012
10308
  }
10013
10309
  /**
@@ -10034,13 +10330,13 @@
10034
10330
  makeCommand.option('-f, --format <format>', spaceTrim__default["default"]("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳‍🌈] */);
10035
10331
  makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
10036
10332
  makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
10037
- makeCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache", false);
10333
+ makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
10038
10334
  makeCommand.option('--verbose', "Is output verbose", false);
10039
- makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), PIPELINE_COLLECTION_BASE_FILENAME);
10335
+ makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
10040
10336
  makeCommand.action(function (path$1, _a) {
10041
- var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10337
+ var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10042
10338
  return __awaiter(_this, void 0, void 0, function () {
10043
- var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10339
+ var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10044
10340
  var _d, e_2, _e, e_1, _f;
10045
10341
  var _this = this;
10046
10342
  return __generator(this, function (_g) {
@@ -10054,22 +10350,25 @@
10054
10350
  .split(',')
10055
10351
  .map(function (_) { return _.trim(); })
10056
10352
  .filter(function (_) { return _ !== ''; });
10057
- if (outFile !== PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10353
+ if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10058
10354
  console.error(colors__default["default"].red("You can only use one format if you specify --out-file"));
10059
10355
  process.exit(1);
10060
10356
  }
10061
10357
  options = {
10062
10358
  isVerbose: isVerbose,
10063
- isCacheCleaned: isCacheCleaned,
10359
+ isCacheReloaded: isCacheReloaded,
10064
10360
  };
10065
10361
  fs = $provideFilesystemForNode(options);
10066
10362
  llm = $provideLlmToolsForCli(options);
10363
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10364
+ case 1:
10365
+ executables = _g.sent();
10067
10366
  _d = {
10068
10367
  llm: llm,
10069
10368
  fs: fs
10070
10369
  };
10071
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10072
- case 1:
10370
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10371
+ case 2:
10073
10372
  tools = (_d.scrapers = _g.sent(),
10074
10373
  _d.script = [
10075
10374
  /*new JavascriptExecutionTools(options)*/
@@ -10078,31 +10377,31 @@
10078
10377
  return [4 /*yield*/, createCollectionFromDirectory(path$1, tools, {
10079
10378
  isVerbose: isVerbose,
10080
10379
  isRecursive: true,
10081
- // <- TODO: [🍖] isCacheReloaded
10380
+ // <- TODO: [🍖] Add `intermediateFilesStrategy`
10082
10381
  })];
10083
- case 2:
10084
- collection = _g.sent();
10085
- _g.label = 3;
10086
10382
  case 3:
10087
- _g.trys.push([3, 15, 16, 17]);
10088
- validations_1 = __values(validations), validations_1_1 = validations_1.next();
10383
+ collection = _g.sent();
10089
10384
  _g.label = 4;
10090
10385
  case 4:
10091
- if (!!validations_1_1.done) return [3 /*break*/, 14];
10092
- validation_1 = validations_1_1.value;
10386
+ _g.trys.push([4, 16, 17, 18]);
10387
+ validations_1 = __values(validations), validations_1_1 = validations_1.next();
10093
10388
  _g.label = 5;
10094
10389
  case 5:
10095
- _g.trys.push([5, 11, 12, 13]);
10390
+ if (!!validations_1_1.done) return [3 /*break*/, 15];
10391
+ validation_1 = validations_1_1.value;
10392
+ _g.label = 6;
10393
+ case 6:
10394
+ _g.trys.push([6, 12, 13, 14]);
10096
10395
  e_1 = void 0;
10097
10396
  return [4 /*yield*/, collection.listPipelines()];
10098
- case 6:
10099
- _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10100
- _g.label = 7;
10101
10397
  case 7:
10102
- if (!!_c.done) return [3 /*break*/, 10];
10398
+ _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10399
+ _g.label = 8;
10400
+ case 8:
10401
+ if (!!_c.done) return [3 /*break*/, 11];
10103
10402
  pipelineUrl = _c.value;
10104
10403
  return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
10105
- case 8:
10404
+ case 9:
10106
10405
  pipeline = _g.sent();
10107
10406
  if (validation_1 === 'logic') {
10108
10407
  validatePipeline(pipeline);
@@ -10110,37 +10409,37 @@
10110
10409
  console.info(colors__default["default"].cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
10111
10410
  }
10112
10411
  }
10113
- _g.label = 9;
10114
- case 9:
10412
+ _g.label = 10;
10413
+ case 10:
10115
10414
  _c = _b.next();
10116
- return [3 /*break*/, 7];
10117
- case 10: return [3 /*break*/, 13];
10118
- case 11:
10415
+ return [3 /*break*/, 8];
10416
+ case 11: return [3 /*break*/, 14];
10417
+ case 12:
10119
10418
  e_1_1 = _g.sent();
10120
10419
  e_1 = { error: e_1_1 };
10121
- return [3 /*break*/, 13];
10122
- case 12:
10420
+ return [3 /*break*/, 14];
10421
+ case 13:
10123
10422
  try {
10124
10423
  if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
10125
10424
  }
10126
10425
  finally { if (e_1) throw e_1.error; }
10127
10426
  return [7 /*endfinally*/];
10128
- case 13:
10427
+ case 14:
10129
10428
  validations_1_1 = validations_1.next();
10130
- return [3 /*break*/, 4];
10131
- case 14: return [3 /*break*/, 17];
10132
- case 15:
10429
+ return [3 /*break*/, 5];
10430
+ case 15: return [3 /*break*/, 18];
10431
+ case 16:
10133
10432
  e_2_1 = _g.sent();
10134
10433
  e_2 = { error: e_2_1 };
10135
- return [3 /*break*/, 17];
10136
- case 16:
10434
+ return [3 /*break*/, 18];
10435
+ case 17:
10137
10436
  try {
10138
10437
  if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
10139
10438
  }
10140
10439
  finally { if (e_2) throw e_2.error; }
10141
10440
  return [7 /*endfinally*/];
10142
- case 17: return [4 /*yield*/, collectionToJson(collection)];
10143
- case 18:
10441
+ case 18: return [4 /*yield*/, collectionToJson(collection)];
10442
+ case 19:
10144
10443
  collectionJson = _g.sent();
10145
10444
  collectionJsonString = stringifyPipelineJson(collectionJson).trim();
10146
10445
  collectionJsonItems = (function () {
@@ -10159,9 +10458,9 @@
10159
10458
  return __generator(this, function (_a) {
10160
10459
  switch (_a.label) {
10161
10460
  case 0:
10162
- filename = outFile !== PIPELINE_COLLECTION_BASE_FILENAME
10461
+ filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
10163
10462
  ? outFile
10164
- : path.join(path$1, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10463
+ : path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10165
10464
  if (!outFile.endsWith(".".concat(extension))) {
10166
10465
  console.warn(colors__default["default"].yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
10167
10466
  }
@@ -10177,27 +10476,27 @@
10177
10476
  }
10178
10477
  });
10179
10478
  }); };
10180
- if (!formats.includes('json')) return [3 /*break*/, 20];
10479
+ if (!formats.includes('json')) return [3 /*break*/, 21];
10181
10480
  formats = formats.filter(function (format) { return format !== 'json'; });
10182
10481
  return [4 /*yield*/, saveFile('json', collectionJsonString)];
10183
- case 19:
10184
- _g.sent();
10185
- _g.label = 20;
10186
10482
  case 20:
10187
- if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 22];
10483
+ _g.sent();
10484
+ _g.label = 21;
10485
+ case 21:
10486
+ if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
10188
10487
  formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
10189
10488
  return [4 /*yield*/, saveFile('js', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
10190
- case 21:
10191
- (_g.sent()) + '\n';
10192
- _g.label = 22;
10193
10489
  case 22:
10194
- if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 24];
10490
+ (_g.sent()) + '\n';
10491
+ _g.label = 23;
10492
+ case 23:
10493
+ if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
10195
10494
  formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
10196
10495
  return [4 /*yield*/, saveFile('ts', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
10197
- case 23:
10198
- _g.sent();
10199
- _g.label = 24;
10200
10496
  case 24:
10497
+ _g.sent();
10498
+ _g.label = 25;
10499
+ case 25:
10201
10500
  if (formats.length > 0) {
10202
10501
  console.warn(colors__default["default"].yellow("Format ".concat(formats.join(' and '), " is not supported")));
10203
10502
  }
@@ -10459,76 +10758,79 @@
10459
10758
  // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument
10460
10759
  'Pipelines to test as glob pattern');
10461
10760
  testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
10462
- testCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache ", false);
10761
+ testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
10463
10762
  testCommand.option('-v, --verbose', "Is output verbose", false);
10464
10763
  testCommand.action(function (filesGlob, _a) {
10465
- var ignore = _a.ignore, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose;
10764
+ var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
10466
10765
  return __awaiter(_this, void 0, void 0, function () {
10467
- var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10766
+ var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10468
10767
  var _d, e_1, _e;
10469
10768
  return __generator(this, function (_f) {
10470
10769
  switch (_f.label) {
10471
10770
  case 0:
10472
10771
  options = {
10473
10772
  isVerbose: isVerbose,
10474
- isCacheCleaned: isCacheCleaned,
10773
+ isCacheReloaded: isCacheReloaded,
10475
10774
  };
10476
10775
  fs = $provideFilesystemForNode(options);
10477
10776
  llm = $provideLlmToolsForCli(options);
10777
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10778
+ case 1:
10779
+ executables = _f.sent();
10478
10780
  _d = {
10479
10781
  llm: llm,
10480
10782
  fs: fs
10481
10783
  };
10482
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10483
- case 1:
10784
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10785
+ case 2:
10484
10786
  tools = (_d.scrapers = _f.sent(),
10485
10787
  _d.script = [
10486
10788
  /*new JavascriptExecutionTools(options)*/
10487
10789
  ],
10488
10790
  _d);
10489
10791
  return [4 /*yield*/, glob__default["default"](filesGlob, { ignore: ignore })];
10490
- case 2:
10491
- filenames = _f.sent();
10492
- _f.label = 3;
10493
10792
  case 3:
10494
- _f.trys.push([3, 15, 16, 17]);
10495
- filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10793
+ filenames = _f.sent();
10496
10794
  _f.label = 4;
10497
10795
  case 4:
10498
- if (!!filenames_1_1.done) return [3 /*break*/, 14];
10499
- filename = filenames_1_1.value;
10796
+ _f.trys.push([4, 16, 17, 18]);
10797
+ filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10500
10798
  _f.label = 5;
10501
10799
  case 5:
10502
- _f.trys.push([5, 12, , 13]);
10800
+ if (!!filenames_1_1.done) return [3 /*break*/, 15];
10801
+ filename = filenames_1_1.value;
10802
+ _f.label = 6;
10803
+ case 6:
10804
+ _f.trys.push([6, 13, , 14]);
10503
10805
  pipeline = void 0;
10504
- if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 8];
10806
+ if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
10505
10807
  return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
10506
- case 6:
10808
+ case 7:
10507
10809
  pipelineMarkdown = (_f.sent());
10508
10810
  return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
10509
- case 7:
10811
+ case 8:
10510
10812
  pipeline = _f.sent();
10511
10813
  if (isVerbose) {
10512
10814
  console.info(colors__default["default"].green("Parsed ".concat(filename)));
10513
10815
  }
10514
- _f.label = 8;
10515
- case 8:
10516
- if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 10];
10816
+ _f.label = 9;
10817
+ case 9:
10818
+ if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
10517
10819
  _c = (_b = JSON).parse;
10518
10820
  return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
10519
- case 9:
10520
- pipeline = _c.apply(_b, [_f.sent()]);
10521
- return [3 /*break*/, 11];
10522
10821
  case 10:
10822
+ pipeline = _c.apply(_b, [_f.sent()]);
10823
+ return [3 /*break*/, 12];
10824
+ case 11:
10523
10825
  if (isVerbose) {
10524
10826
  console.info(colors__default["default"].gray("Skipping ".concat(filename)));
10525
10827
  }
10526
- return [3 /*break*/, 13];
10527
- case 11:
10828
+ return [3 /*break*/, 14];
10829
+ case 12:
10528
10830
  validatePipeline(pipeline);
10529
10831
  console.info(colors__default["default"].green("Validated ".concat(filename)));
10530
- return [3 /*break*/, 13];
10531
- case 12:
10832
+ return [3 /*break*/, 14];
10833
+ case 13:
10532
10834
  error_1 = _f.sent();
10533
10835
  if (!(error_1 instanceof Error)) {
10534
10836
  throw error_1;
@@ -10537,22 +10839,22 @@
10537
10839
  console.error(colors__default["default"].bgRed(error_1.name /* <- 11:11 */));
10538
10840
  console.error(colors__default["default"].red(error_1.stack || error_1.message));
10539
10841
  process.exit(1);
10540
- return [3 /*break*/, 13];
10541
- case 13:
10842
+ return [3 /*break*/, 14];
10843
+ case 14:
10542
10844
  filenames_1_1 = filenames_1.next();
10543
- return [3 /*break*/, 4];
10544
- case 14: return [3 /*break*/, 17];
10545
- case 15:
10845
+ return [3 /*break*/, 5];
10846
+ case 15: return [3 /*break*/, 18];
10847
+ case 16:
10546
10848
  e_1_1 = _f.sent();
10547
10849
  e_1 = { error: e_1_1 };
10548
- return [3 /*break*/, 17];
10549
- case 16:
10850
+ return [3 /*break*/, 18];
10851
+ case 17:
10550
10852
  try {
10551
10853
  if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
10552
10854
  }
10553
10855
  finally { if (e_1) throw e_1.error; }
10554
10856
  return [7 /*endfinally*/];
10555
- case 17:
10857
+ case 18:
10556
10858
  console.info(colors__default["default"].green("All pipelines are valid"));
10557
10859
  process.exit(0);
10558
10860
  return [2 /*return*/];
@@ -10666,6 +10968,7 @@
10666
10968
  * @public exported from `@promptbook/remote-client`
10667
10969
  */
10668
10970
  var RemoteLlmExecutionTools = /** @class */ (function () {
10971
+ /* <- TODO: [🍚] `, Destroyable` */
10669
10972
  function RemoteLlmExecutionTools(options) {
10670
10973
  this.options = options;
10671
10974
  }
@@ -10715,12 +11018,16 @@
10715
11018
  if (this.options.isAnonymous) {
10716
11019
  socket.emit('listModels-request', {
10717
11020
  isAnonymous: true,
11021
+ userId: this.options.userId,
10718
11022
  llmToolsConfiguration: this.options.llmToolsConfiguration,
10719
11023
  } /* <- TODO: [🤛] */);
10720
11024
  }
10721
11025
  else {
10722
11026
  socket.emit('listModels-request', {
10723
11027
  isAnonymous: false,
11028
+ appId: this.options.appId,
11029
+ userId: this.options.userId,
11030
+ customOptions: this.options.customOptions,
10724
11031
  } /* <- TODO: [🤛] */);
10725
11032
  }
10726
11033
  return [4 /*yield*/, new Promise(function (resolve, reject) {
@@ -10814,7 +11121,9 @@
10814
11121
  else {
10815
11122
  socket.emit('prompt-request', {
10816
11123
  isAnonymous: false,
11124
+ appId: this.options.appId,
10817
11125
  userId: this.options.userId,
11126
+ customOptions: this.options.customOptions,
10818
11127
  prompt: prompt,
10819
11128
  } /* <- TODO: [🤛] */);
10820
11129
  }
@@ -11296,7 +11605,7 @@
11296
11605
  */
11297
11606
  var createAnthropicClaudeExecutionTools = Object.assign(function (options) {
11298
11607
  if (options.isProxied) {
11299
- return new RemoteLlmExecutionTools(__assign(__assign({}, options), { isAnonymous: true, llmToolsConfiguration: [
11608
+ return new RemoteLlmExecutionTools(__assign(__assign({}, options), { userId: null, isAnonymous: true, llmToolsConfiguration: [
11300
11609
  {
11301
11610
  title: 'Anthropic Claude (proxied)',
11302
11611
  packageName: '@promptbook/anthropic-claude',
@@ -11856,12 +12165,12 @@
11856
12165
  * Calls OpenAI API to use a chat model.
11857
12166
  */
11858
12167
  AzureOpenAiExecutionTools.prototype.callChatModel = function (prompt) {
11859
- var _a, _b;
12168
+ var _a, _b, _c;
11860
12169
  return __awaiter(this, void 0, void 0, function () {
11861
12170
  var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, messages, start, complete, rawRequest, rawResponse, resultContent, usage, error_1;
11862
12171
  var _this = this;
11863
- return __generator(this, function (_c) {
11864
- switch (_c.label) {
12172
+ return __generator(this, function (_d) {
12173
+ switch (_d.label) {
11865
12174
  case 0:
11866
12175
  if (this.options.isVerbose) {
11867
12176
  console.info('💬 OpenAI callChatModel call');
@@ -11869,20 +12178,20 @@
11869
12178
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
11870
12179
  return [4 /*yield*/, this.getClient()];
11871
12180
  case 1:
11872
- client = _c.sent();
12181
+ client = _d.sent();
11873
12182
  // TODO: [☂] Use here more modelRequirements
11874
12183
  if (modelRequirements.modelVariant !== 'CHAT') {
11875
12184
  throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
11876
12185
  }
11877
- _c.label = 2;
12186
+ _d.label = 2;
11878
12187
  case 2:
11879
- _c.trys.push([2, 4, , 5]);
12188
+ _d.trys.push([2, 4, , 5]);
11880
12189
  modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
11881
12190
  modelSettings = {
11882
12191
  maxTokens: modelRequirements.maxTokens,
11883
12192
  // <- TODO: [🌾] Make some global max cap for maxTokens
11884
12193
  temperature: modelRequirements.temperature,
11885
- user: this.options.user,
12194
+ user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
11886
12195
  // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
11887
12196
  // <- Note: [🧆]
11888
12197
  };
@@ -11913,7 +12222,7 @@
11913
12222
  throw error;
11914
12223
  })];
11915
12224
  case 3:
11916
- rawResponse = _c.sent();
12225
+ rawResponse = _d.sent();
11917
12226
  if (this.options.isVerbose) {
11918
12227
  console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
11919
12228
  }
@@ -11932,8 +12241,8 @@
11932
12241
  complete = getCurrentIsoDate();
11933
12242
  usage = {
11934
12243
  price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
11935
- input: __assign({ tokensCount: uncertainNumber((_a = rawResponse.usage) === null || _a === void 0 ? void 0 : _a.promptTokens) }, computeUsageCounts(prompt.content)),
11936
- output: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completionTokens) }, computeUsageCounts(prompt.content)),
12244
+ input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
12245
+ output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
11937
12246
  };
11938
12247
  return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools ChatPromptResult', {
11939
12248
  content: resultContent,
@@ -11949,7 +12258,7 @@
11949
12258
  // <- [🗯]
11950
12259
  })];
11951
12260
  case 4:
11952
- error_1 = _c.sent();
12261
+ error_1 = _d.sent();
11953
12262
  throw this.transformAzureError(error_1);
11954
12263
  case 5: return [2 /*return*/];
11955
12264
  }
@@ -11960,12 +12269,12 @@
11960
12269
  * Calls Azure OpenAI API to use a complete model.
11961
12270
  */
11962
12271
  AzureOpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
11963
- var _a, _b;
12272
+ var _a, _b, _c;
11964
12273
  return __awaiter(this, void 0, void 0, function () {
11965
12274
  var content, parameters, modelRequirements, client, modelName, modelSettings, start, complete, rawPromptContent, rawRequest, rawResponse, resultContent, usage, error_2;
11966
12275
  var _this = this;
11967
- return __generator(this, function (_c) {
11968
- switch (_c.label) {
12276
+ return __generator(this, function (_d) {
12277
+ switch (_d.label) {
11969
12278
  case 0:
11970
12279
  if (this.options.isVerbose) {
11971
12280
  console.info('🖋 OpenAI callCompletionModel call');
@@ -11973,20 +12282,20 @@
11973
12282
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
11974
12283
  return [4 /*yield*/, this.getClient()];
11975
12284
  case 1:
11976
- client = _c.sent();
12285
+ client = _d.sent();
11977
12286
  // TODO: [☂] Use here more modelRequirements
11978
12287
  if (modelRequirements.modelVariant !== 'COMPLETION') {
11979
12288
  throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
11980
12289
  }
11981
- _c.label = 2;
12290
+ _d.label = 2;
11982
12291
  case 2:
11983
- _c.trys.push([2, 4, , 5]);
12292
+ _d.trys.push([2, 4, , 5]);
11984
12293
  modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
11985
12294
  modelSettings = {
11986
12295
  maxTokens: modelRequirements.maxTokens || 2000,
11987
12296
  // <- TODO: [🌾] Make some global max cap for maxTokens
11988
12297
  temperature: modelRequirements.temperature,
11989
- user: this.options.user,
12298
+ user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
11990
12299
  // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
11991
12300
  // <- Note: [🧆]
11992
12301
  };
@@ -12009,7 +12318,7 @@
12009
12318
  throw error;
12010
12319
  })];
12011
12320
  case 3:
12012
- rawResponse = _c.sent();
12321
+ rawResponse = _d.sent();
12013
12322
  if (this.options.isVerbose) {
12014
12323
  console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
12015
12324
  }
@@ -12025,8 +12334,8 @@
12025
12334
  complete = getCurrentIsoDate();
12026
12335
  usage = {
12027
12336
  price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
12028
- input: __assign({ tokensCount: uncertainNumber((_a = rawResponse.usage) === null || _a === void 0 ? void 0 : _a.promptTokens) }, computeUsageCounts(prompt.content)),
12029
- output: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completionTokens) }, computeUsageCounts(prompt.content)),
12337
+ input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
12338
+ output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
12030
12339
  };
12031
12340
  return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools CompletionPromptResult', {
12032
12341
  content: resultContent,
@@ -12042,7 +12351,7 @@
12042
12351
  // <- [🗯]
12043
12352
  })];
12044
12353
  case 4:
12045
- error_2 = _c.sent();
12354
+ error_2 = _d.sent();
12046
12355
  throw this.transformAzureError(error_2);
12047
12356
  case 5: return [2 /*return*/];
12048
12357
  }
@@ -12267,7 +12576,6 @@
12267
12576
  * @param options which are relevant are directly passed to the OpenAI client
12268
12577
  */
12269
12578
  function OpenAiExecutionTools(options) {
12270
- if (options === void 0) { options = {}; }
12271
12579
  this.options = options;
12272
12580
  /**
12273
12581
  * OpenAI API client.
@@ -12295,8 +12603,8 @@
12295
12603
  if (this.client === null) {
12296
12604
  openAiOptions = __assign({}, this.options);
12297
12605
  delete openAiOptions.isVerbose;
12298
- delete openAiOptions.user;
12299
- this.client = new OpenAI__default["default"](__assign({}, openAiOptions));
12606
+ delete openAiOptions.userId;
12607
+ this.client = new OpenAI__default["default"](openAiOptions);
12300
12608
  }
12301
12609
  return [2 /*return*/, this.client];
12302
12610
  });
@@ -12343,11 +12651,12 @@
12343
12651
  * Calls OpenAI API to use a chat model.
12344
12652
  */
12345
12653
  OpenAiExecutionTools.prototype.callChatModel = function (prompt) {
12654
+ var _a;
12346
12655
  return __awaiter(this, void 0, void 0, function () {
12347
12656
  var content, parameters, modelRequirements, format, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
12348
12657
  var _this = this;
12349
- return __generator(this, function (_a) {
12350
- switch (_a.label) {
12658
+ return __generator(this, function (_b) {
12659
+ switch (_b.label) {
12351
12660
  case 0:
12352
12661
  if (this.options.isVerbose) {
12353
12662
  console.info('💬 OpenAI callChatModel call', { prompt: prompt });
@@ -12355,7 +12664,7 @@
12355
12664
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements, format = prompt.format;
12356
12665
  return [4 /*yield*/, this.getClient()];
12357
12666
  case 1:
12358
- client = _a.sent();
12667
+ client = _b.sent();
12359
12668
  // TODO: [☂] Use here more modelRequirements
12360
12669
  if (modelRequirements.modelVariant !== 'CHAT') {
12361
12670
  throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
@@ -12387,7 +12696,7 @@
12387
12696
  role: 'user',
12388
12697
  content: rawPromptContent,
12389
12698
  },
12390
- ], false), user: this.options.user });
12699
+ ], false), user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
12391
12700
  start = getCurrentIsoDate();
12392
12701
  if (this.options.isVerbose) {
12393
12702
  console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
@@ -12399,7 +12708,7 @@
12399
12708
  throw error;
12400
12709
  })];
12401
12710
  case 2:
12402
- rawResponse = _a.sent();
12711
+ rawResponse = _b.sent();
12403
12712
  if (this.options.isVerbose) {
12404
12713
  console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
12405
12714
  }
@@ -12438,11 +12747,12 @@
12438
12747
  * Calls OpenAI API to use a complete model.
12439
12748
  */
12440
12749
  OpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
12750
+ var _a;
12441
12751
  return __awaiter(this, void 0, void 0, function () {
12442
12752
  var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
12443
12753
  var _this = this;
12444
- return __generator(this, function (_a) {
12445
- switch (_a.label) {
12754
+ return __generator(this, function (_b) {
12755
+ switch (_b.label) {
12446
12756
  case 0:
12447
12757
  if (this.options.isVerbose) {
12448
12758
  console.info('🖋 OpenAI callCompletionModel call', { prompt: prompt });
@@ -12450,7 +12760,7 @@
12450
12760
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
12451
12761
  return [4 /*yield*/, this.getClient()];
12452
12762
  case 1:
12453
- client = _a.sent();
12763
+ client = _b.sent();
12454
12764
  // TODO: [☂] Use here more modelRequirements
12455
12765
  if (modelRequirements.modelVariant !== 'COMPLETION') {
12456
12766
  throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
@@ -12465,7 +12775,7 @@
12465
12775
  // <- Note: [🧆]
12466
12776
  };
12467
12777
  rawPromptContent = replaceParameters(content, __assign(__assign({}, parameters), { modelName: modelName }));
12468
- rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: this.options.user });
12778
+ rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
12469
12779
  start = getCurrentIsoDate();
12470
12780
  if (this.options.isVerbose) {
12471
12781
  console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
@@ -12477,7 +12787,7 @@
12477
12787
  throw error;
12478
12788
  })];
12479
12789
  case 2:
12480
- rawResponse = _a.sent();
12790
+ rawResponse = _b.sent();
12481
12791
  if (this.options.isVerbose) {
12482
12792
  console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
12483
12793
  }
@@ -12703,17 +13013,16 @@
12703
13013
  assistant_id: this.assistantId,
12704
13014
  thread: {
12705
13015
  messages: [
12706
- // TODO: !!!!!! Allow threads to be passed
13016
+ // TODO: [🗯] !! Allow threads to be passed
12707
13017
  { role: 'user', content: rawPromptContent },
12708
13018
  ],
12709
13019
  },
12710
- // !!!!!! user: this.options.user,
13020
+ // <- TODO: Add user identification here> user: this.options.user,
12711
13021
  };
12712
13022
  start = getCurrentIsoDate();
12713
13023
  if (this.options.isVerbose) {
12714
13024
  console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
12715
13025
  }
12716
- console.log('!!!!!! OpenAI client', client);
12717
13026
  return [4 /*yield*/, client.beta.threads.createAndRunStream(rawRequest)];
12718
13027
  case 2:
12719
13028
  stream = _g.sent();
@@ -12731,7 +13040,7 @@
12731
13040
  messageDelta.content[0].type === 'text') {
12732
13041
  console.info('messageDelta', (_a = messageDelta.content[0].text) === null || _a === void 0 ? void 0 : _a.value);
12733
13042
  }
12734
- // TODO: !!!!!! report progress
13043
+ // <- TODO: [🐚] Make streaming and running tasks working
12735
13044
  });
12736
13045
  stream.on('messageCreated', function (message) {
12737
13046
  if (_this.options.isVerbose) {
@@ -12759,18 +13068,20 @@
12759
13068
  throw new PipelineExecutionError("There is NOT 'text' BUT ".concat((_b = rawResponse[0].content[0]) === null || _b === void 0 ? void 0 : _b.type, " finalMessages content type from OpenAI"));
12760
13069
  }
12761
13070
  resultContent = (_c = rawResponse[0].content[0]) === null || _c === void 0 ? void 0 : _c.text.value;
12762
- // <- TODO: !!!!!! There are also annotations, maybe use them
13071
+ // <- TODO: [🧠] There are also annotations, maybe use them
12763
13072
  // eslint-disable-next-line prefer-const
12764
13073
  complete = getCurrentIsoDate();
12765
13074
  usage = UNCERTAIN_USAGE;
12766
- // TODO: !!!!!!> = computeOpenAiUsage(content, resultContent || '', rawResponse);
13075
+ // <- TODO: [🥘] Compute real usage for assistant
13076
+ // ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse);
12767
13077
  if (resultContent === null) {
12768
13078
  throw new PipelineExecutionError('No response message from OpenAI');
12769
13079
  }
12770
13080
  return [2 /*return*/, $asDeeplyFrozenSerializableJson('OpenAiAssistantExecutionTools ChatPromptResult', {
12771
13081
  content: resultContent,
12772
13082
  modelName: 'assistant',
12773
- // <- TODO: !!!!!! Can we detect really used model: rawResponse.model || modelName,
13083
+ // <- TODO: [🥘] Detect used model in assistant
13084
+ // ?> model: rawResponse.model || modelName,
12774
13085
  timing: {
12775
13086
  start: start,
12776
13087
  complete: complete,
@@ -12788,7 +13099,6 @@
12788
13099
  return OpenAiAssistantExecutionTools;
12789
13100
  }(OpenAiExecutionTools));
12790
13101
  /**
12791
- * TODO: !!!!!! DO not use colors - can be used in browser
12792
13102
  * TODO: [🧠][🧙‍♂️] Maybe there can be some wizzard for thoose who want to use just OpenAI
12793
13103
  * TODO: Maybe make custom OpenAiError
12794
13104
  * TODO: [🧠][🈁] Maybe use `isDeterministic` from options
@@ -12860,22 +13170,26 @@
12860
13170
  /**
12861
13171
  * Normalize options for `execCommand` and `execCommands`
12862
13172
  *
13173
+ * Note: `$` is used to indicate that this function behaves differently according to `process.platform`
13174
+ *
12863
13175
  * @private internal utility of `execCommand` and `execCommands`
12864
13176
  */
12865
- function execCommandNormalizeOptions(options) {
13177
+ function $execCommandNormalizeOptions(options) {
12866
13178
  var _a;
12867
- var _b, _c, _d;
13179
+ var _b, _c, _d, _e;
12868
13180
  var command;
12869
13181
  var cwd;
12870
13182
  var crashOnError;
12871
13183
  var args = [];
12872
13184
  var timeout;
13185
+ var isVerbose;
12873
13186
  if (typeof options === 'string') {
12874
13187
  // TODO: [1] DRY default values
12875
13188
  command = options;
12876
13189
  cwd = process.cwd();
12877
13190
  crashOnError = true;
12878
- timeout = Infinity;
13191
+ timeout = Infinity; // <- TODO: [⏳]
13192
+ isVerbose = DEFAULT_IS_VERBOSE;
12879
13193
  }
12880
13194
  else {
12881
13195
  /*
@@ -12891,6 +13205,7 @@
12891
13205
  cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
12892
13206
  crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
12893
13207
  timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
13208
+ isVerbose = (_e = options.isVerbose) !== null && _e !== void 0 ? _e : DEFAULT_IS_VERBOSE;
12894
13209
  }
12895
13210
  // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
12896
13211
  var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
@@ -12912,16 +13227,21 @@
12912
13227
  if (['ts-node'].includes(humanReadableCommand)) {
12913
13228
  humanReadableCommand += " ".concat(args[1]);
12914
13229
  }
12915
- return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
13230
+ if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
13231
+ command = "".concat(command, ".cmd");
13232
+ }
13233
+ return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout, isVerbose: isVerbose };
12916
13234
  }
12917
13235
  // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
12918
13236
 
12919
13237
  /**
12920
13238
  * Run one command in a shell
12921
13239
  *
13240
+ *
12922
13241
  * Note: There are 2 similar functions in the codebase:
12923
13242
  * - `$execCommand` which runs a single command
12924
13243
  * - `$execCommands` which runs multiple commands
13244
+ * Note: `$` is used to indicate that this function is not a pure function - it runs a command in a shell
12925
13245
  *
12926
13246
  * @public exported from `@promptbook/node`
12927
13247
  */
@@ -12931,7 +13251,7 @@
12931
13251
  }
12932
13252
  return new Promise(function (resolve, reject) {
12933
13253
  // eslint-disable-next-line prefer-const
12934
- var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
13254
+ var _a = $execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
12935
13255
  if (timeout !== Infinity) {
12936
13256
  // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
12937
13257
  waitasecond.forTime(timeout).then(function () {
@@ -12944,24 +13264,26 @@
12944
13264
  }
12945
13265
  });
12946
13266
  }
12947
- if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
12948
- command = "".concat(command, ".cmd");
13267
+ if (isVerbose) {
13268
+ console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
12949
13269
  }
12950
- // !!!!!! Verbose mode - to all consoles
12951
- console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
12952
13270
  try {
12953
13271
  var commandProcess = child_process.spawn(command, args, { cwd: cwd, shell: true });
12954
- commandProcess.on('message', function (message) {
12955
- console.info({ message: message });
12956
- });
13272
+ if (isVerbose) {
13273
+ commandProcess.on('message', function (message) {
13274
+ console.info({ message: message });
13275
+ });
13276
+ }
12957
13277
  var output_1 = [];
12958
13278
  commandProcess.stdout.on('data', function (stdout) {
12959
13279
  output_1.push(stdout.toString());
12960
- console.info(stdout.toString());
13280
+ if (isVerbose) {
13281
+ console.info(stdout.toString());
13282
+ }
12961
13283
  });
12962
13284
  commandProcess.stderr.on('data', function (stderr) {
12963
13285
  output_1.push(stderr.toString());
12964
- if (stderr.toString().trim()) {
13286
+ if (isVerbose && stderr.toString().trim()) {
12965
13287
  console.warn(stderr.toString());
12966
13288
  }
12967
13289
  });
@@ -12972,7 +13294,9 @@
12972
13294
  "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
12973
13295
  }
12974
13296
  else {
12975
- console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
13297
+ if (isVerbose) {
13298
+ console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
13299
+ }
12976
13300
  resolve(spaceTrim.spaceTrim(output_1.join('\n')));
12977
13301
  }
12978
13302
  }
@@ -12991,7 +13315,9 @@
12991
13315
  reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
12992
13316
  }
12993
13317
  else {
12994
- console.warn(error);
13318
+ if (isVerbose) {
13319
+ console.warn(error);
13320
+ }
12995
13321
  resolve(spaceTrim.spaceTrim(output_1.join('\n')));
12996
13322
  }
12997
13323
  });
@@ -13015,12 +13341,12 @@
13015
13341
  */
13016
13342
  function getScraperIntermediateSource(source, options) {
13017
13343
  return __awaiter(this, void 0, void 0, function () {
13018
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13344
+ var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13019
13345
  return __generator(this, function (_a) {
13020
13346
  switch (_a.label) {
13021
13347
  case 0:
13022
13348
  sourceFilename = source.filename, url = source.url;
13023
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
13349
+ rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
13024
13350
  hash = cryptoJs.SHA256(
13025
13351
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
13026
13352
  hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
@@ -13030,7 +13356,7 @@
13030
13356
  pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
13031
13357
  name = pieces.join('-').split('--').join('-');
13032
13358
  // <- TODO: Use MAX_FILENAME_LENGTH
13033
- TODO_USE(rootDirname); // <- TODO: !!!!!!
13359
+ TODO_USE(rootDirname); // <- TODO: [😡]
13034
13360
  cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
13035
13361
  cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
13036
13362
  .join('/') +
@@ -13050,7 +13376,7 @@
13050
13376
  return __generator(this, function (_a) {
13051
13377
  switch (_a.label) {
13052
13378
  case 0:
13053
- if (!isCacheCleaned) return [3 /*break*/, 2];
13379
+ if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
13054
13380
  if (isVerbose) {
13055
13381
  console.info('legacyDocumentScraper: Clening cache');
13056
13382
  }
@@ -13074,7 +13400,7 @@
13074
13400
  /**
13075
13401
  * Note: Not using `FileCacheStorage` for two reasons:
13076
13402
  * 1) Need to store more than serialized JSONs
13077
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
13403
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
13078
13404
  * TODO: [🐱‍🐉][🧠] Make some smart crop
13079
13405
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
13080
13406
  */
@@ -13091,7 +13417,7 @@
13091
13417
  mimeTypes: ['text/markdown', 'text/plain'],
13092
13418
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13093
13419
  isAvilableInBrowser: true,
13094
- requiredExecutables: ['!!!!!!'],
13420
+ requiredExecutables: [],
13095
13421
  }); /* <- TODO: [🤛] */
13096
13422
  /**
13097
13423
  * Registration of known scraper metadata
@@ -13135,7 +13461,7 @@
13135
13461
  return __generator(this, function (_k) {
13136
13462
  switch (_k.label) {
13137
13463
  case 0:
13138
- _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
13464
+ _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
13139
13465
  llm = this.tools.llm;
13140
13466
  if (llm === undefined) {
13141
13467
  throw new MissingToolsError('LLM tools are required for scraping external files');
@@ -13234,7 +13560,8 @@
13234
13560
  embeddingResult = _c.sent();
13235
13561
  index.push({
13236
13562
  modelName: embeddingResult.modelName,
13237
- position: embeddingResult.content,
13563
+ position: __spreadArray([], __read(embeddingResult.content), false),
13564
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
13238
13565
  });
13239
13566
  _c.label = 6;
13240
13567
  case 6: return [3 /*break*/, 8];
@@ -13285,7 +13612,7 @@
13285
13612
  mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
13286
13613
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13287
13614
  isAvilableInBrowser: false,
13288
- requiredExecutables: ['!!!!!!'],
13615
+ requiredExecutables: ['Pandoc'],
13289
13616
  }); /* <- TODO: [🤛] */
13290
13617
  /**
13291
13618
  * Registration of known scraper metadata
@@ -13325,12 +13652,13 @@
13325
13652
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13326
13653
  */
13327
13654
  DocumentScraper.prototype.$convert = function (source) {
13655
+ var _a;
13328
13656
  return __awaiter(this, void 0, void 0, function () {
13329
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, command_1;
13657
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
13330
13658
  return __generator(this, function (_g) {
13331
13659
  switch (_g.label) {
13332
13660
  case 0:
13333
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13661
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13334
13662
  if (!$isRunningInNode()) {
13335
13663
  throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
13336
13664
  }
@@ -13338,7 +13666,7 @@
13338
13666
  throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
13339
13667
  // <- TODO: [🧠] What is the best error type here`
13340
13668
  }
13341
- if (externalProgramsPaths.pandocPath === undefined) {
13669
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
13342
13670
  throw new MissingToolsError('Pandoc is required for scraping .docx files');
13343
13671
  }
13344
13672
  if (source.filename === null) {
@@ -13349,7 +13677,7 @@
13349
13677
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13350
13678
  rootDirname: rootDirname,
13351
13679
  cacheDirname: cacheDirname,
13352
- isCacheCleaned: isCacheCleaned,
13680
+ intermediateFilesStrategy: intermediateFilesStrategy,
13353
13681
  extension: 'md',
13354
13682
  isVerbose: isVerbose,
13355
13683
  })];
@@ -13358,11 +13686,9 @@
13358
13686
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13359
13687
  case 2:
13360
13688
  if (!!(_g.sent())) return [3 /*break*/, 5];
13361
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13362
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13689
+ command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13363
13690
  return [4 /*yield*/, $execCommand(command_1)];
13364
13691
  case 3:
13365
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13366
13692
  _g.sent();
13367
13693
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13368
13694
  case 4:
@@ -13407,9 +13733,14 @@
13407
13733
  asJson: function () {
13408
13734
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
13409
13735
  },
13410
- asBlob: function () {
13411
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
13412
- },
13736
+ /*
13737
+ TODO: [🥽]
13738
+ > asBlob() {
13739
+ > throw new UnexpectedError(
13740
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
13741
+ > );
13742
+ > },
13743
+ */
13413
13744
  };
13414
13745
  knowledge = this.markdownScraper.scrape(markdownSource);
13415
13746
  return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13441,7 +13772,11 @@
13441
13772
  mimeTypes: ['application/msword', 'text/rtf'],
13442
13773
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13443
13774
  isAvilableInBrowser: false,
13444
- requiredExecutables: ['!!!!!!'],
13775
+ requiredExecutables: [
13776
+ 'Pandoc',
13777
+ 'LibreOffice',
13778
+ // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
13779
+ ],
13445
13780
  }); /* <- TODO: [🤛] */
13446
13781
  /**
13447
13782
  * Registration of known scraper metadata
@@ -13454,7 +13789,7 @@
13454
13789
  var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
13455
13790
 
13456
13791
  /**
13457
- * Scraper for .docx files
13792
+ * Scraper for old document files (like .doc and .rtf)
13458
13793
  *
13459
13794
  * @see `documentationUrl` for more details
13460
13795
  * @public exported from `@promptbook/legacy-documents`
@@ -13481,12 +13816,13 @@
13481
13816
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13482
13817
  */
13483
13818
  LegacyDocumentScraper.prototype.$convert = function (source) {
13819
+ var _a;
13484
13820
  return __awaiter(this, void 0, void 0, function () {
13485
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13821
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13486
13822
  return __generator(this, function (_g) {
13487
13823
  switch (_g.label) {
13488
13824
  case 0:
13489
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13825
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13490
13826
  if (!$isRunningInNode()) {
13491
13827
  throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
13492
13828
  }
@@ -13494,7 +13830,7 @@
13494
13830
  throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
13495
13831
  // <- TODO: [🧠] What is the best error type here`
13496
13832
  }
13497
- if (externalProgramsPaths.libreOfficePath === undefined) {
13833
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
13498
13834
  throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
13499
13835
  }
13500
13836
  if (source.filename === null) {
@@ -13505,7 +13841,7 @@
13505
13841
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13506
13842
  rootDirname: rootDirname,
13507
13843
  cacheDirname: cacheDirname,
13508
- isCacheCleaned: isCacheCleaned,
13844
+ intermediateFilesStrategy: intermediateFilesStrategy,
13509
13845
  extension: 'docx',
13510
13846
  isVerbose: isVerbose,
13511
13847
  })];
@@ -13520,11 +13856,9 @@
13520
13856
  documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
13521
13857
  .split('\\')
13522
13858
  .join('/');
13523
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13524
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13859
+ command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13525
13860
  return [4 /*yield*/, $execCommand(command_1)];
13526
13861
  case 3:
13527
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13528
13862
  _g.sent();
13529
13863
  return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
13530
13864
  case 4:
@@ -13572,9 +13906,14 @@
13572
13906
  asJson: function () {
13573
13907
  throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
13574
13908
  },
13575
- asBlob: function () {
13576
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
13577
- },
13909
+ /*
13910
+ TODO: [🥽]
13911
+ > asBlob() {
13912
+ > throw new UnexpectedError(
13913
+ > 'Did not expect that `documentScraper` would need to get the content `asBlob`',
13914
+ > );
13915
+ > },
13916
+ */
13578
13917
  };
13579
13918
  knowledge = this.documentScraper.scrape(markdownSource);
13580
13919
  return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13681,7 +14020,7 @@
13681
14020
  mimeTypes: ['application/pdf'],
13682
14021
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13683
14022
  isAvilableInBrowser: true,
13684
- requiredExecutables: ['!!!!!!'],
14023
+ requiredExecutables: [],
13685
14024
  }); /* <- TODO: [🤛] */
13686
14025
  /**
13687
14026
  * Registration of known scraper metadata
@@ -13694,7 +14033,7 @@
13694
14033
  var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
13695
14034
 
13696
14035
  /**
13697
- * Scraper for .docx files
14036
+ * Scraper for .pdf files
13698
14037
  *
13699
14038
  * @see `documentationUrl` for more details
13700
14039
  * @public exported from `@promptbook/pdf`
@@ -13737,9 +14076,8 @@
13737
14076
  TODO_USE(this.options);
13738
14077
  /*
13739
14078
  const {
13740
- externalProgramsPaths = {},
13741
14079
  cacheDirname = SCRAPE_CACHE_DIRNAME,
13742
- isCacheCleaned = false,
14080
+ intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
13743
14081
  isVerbose = IS_VERBOSE,
13744
14082
  } = options;
13745
14083
  */
@@ -13792,7 +14130,7 @@
13792
14130
  mimeTypes: ['text/html'],
13793
14131
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13794
14132
  isAvilableInBrowser: false,
13795
- requiredExecutables: ['!!!!!!'],
14133
+ requiredExecutables: [],
13796
14134
  }); /* <- TODO: [🤛] */
13797
14135
  /**
13798
14136
  * Registration of known scraper metadata
@@ -13805,32 +14143,29 @@
13805
14143
  var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
13806
14144
 
13807
14145
  /**
13808
- * A converter instance that uses showdown and highlight extensions
14146
+ * Create a new showdown converter instance
13809
14147
  *
13810
- * @type {Converter}
13811
- * @private for markdown and html knowledge scrapers
13812
- */
13813
- var markdownConverter = new showdown.Converter({
13814
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
13815
- /*
13816
- > import showdownHighlight from 'showdown-highlight';
13817
- > extensions: [
13818
- > showdownHighlight({
13819
- > // Whether to add the classes to the <pre> tag, default is false
13820
- > pre: true,
13821
- > // Whether to use hljs' auto language detection, default is true
13822
- > auto_detection: true,
13823
- > }),
13824
- > ],
13825
- */
13826
- });
13827
- /**
13828
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
13829
- * TODO: !!!!!! Lazy-make converter
14148
+ * @private utility of `WebsiteScraper`
13830
14149
  */
14150
+ function createShowdownConverter() {
14151
+ return new showdown.Converter({
14152
+ flavor: 'github',
14153
+ /*
14154
+ > import showdownHighlight from 'showdown-highlight';
14155
+ > extensions: [
14156
+ > showdownHighlight({
14157
+ > // Whether to add the classes to the <pre> tag, default is false
14158
+ > pre: true,
14159
+ > // Whether to use hljs' auto language detection, default is true
14160
+ > auto_detection: true,
14161
+ > }),
14162
+ > ],
14163
+ */
14164
+ });
14165
+ }
13831
14166
 
13832
14167
  /**
13833
- * Scraper for .docx files
14168
+ * Scraper for websites
13834
14169
  *
13835
14170
  * @see `documentationUrl` for more details
13836
14171
  * @public exported from `@promptbook/website-crawler`
@@ -13840,6 +14175,7 @@
13840
14175
  this.tools = tools;
13841
14176
  this.options = options;
13842
14177
  this.markdownScraper = new MarkdownScraper(tools, options);
14178
+ this.showdownConverter = createShowdownConverter();
13843
14179
  }
13844
14180
  Object.defineProperty(WebsiteScraper.prototype, "metadata", {
13845
14181
  /**
@@ -13860,16 +14196,17 @@
13860
14196
  return __awaiter(this, void 0, void 0, function () {
13861
14197
  var _a, _b,
13862
14198
  // TODO: [🧠] Maybe in node use headless browser not just JSDOM
13863
- // externalProgramsPaths = {},
13864
- rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
14199
+ rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
13865
14200
  return __generator(this, function (_g) {
13866
14201
  switch (_g.label) {
13867
14202
  case 0:
13868
- _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? SCRAPE_CACHE_DIRNAME : _c, _d = _a.isCacheCleaned, isCacheCleaned = _d === void 0 ? false : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e;
13869
- // TODO: !!!!!! Does this work in browser? Make it work.
14203
+ _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
13870
14204
  if (source.url === null) {
13871
14205
  throw new KnowledgeScrapeError('Website scraper requires URL');
13872
14206
  }
14207
+ if (this.tools.fs === undefined) {
14208
+ throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
14209
+ }
13873
14210
  _f = jsdom.JSDOM.bind;
13874
14211
  return [4 /*yield*/, source.asText()];
13875
14212
  case 1:
@@ -13878,10 +14215,6 @@
13878
14215
  }]))();
13879
14216
  reader = new readability.Readability(jsdom$1.window.document);
13880
14217
  article = reader.parse();
13881
- console.log(article);
13882
- return [4 /*yield*/, waitasecond.forTime(10000)];
13883
- case 2:
13884
- _g.sent();
13885
14218
  html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
13886
14219
  // Note: Unwrap html such as it is convertable by `markdownConverter`
13887
14220
  for (i = 0; i < 2; i++) {
@@ -13893,16 +14226,16 @@
13893
14226
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13894
14227
  rootDirname: rootDirname,
13895
14228
  cacheDirname: cacheDirname,
13896
- isCacheCleaned: isCacheCleaned,
14229
+ intermediateFilesStrategy: intermediateFilesStrategy,
13897
14230
  extension: 'html',
13898
14231
  isVerbose: isVerbose,
13899
14232
  })];
13900
- case 3:
14233
+ case 2:
13901
14234
  cacheFilehandler = _g.sent();
13902
- return [4 /*yield*/, promises.writeFile(cacheFilehandler.filename, html, 'utf-8')];
13903
- case 4:
14235
+ return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8')];
14236
+ case 3:
13904
14237
  _g.sent();
13905
- markdown = markdownConverter.makeMarkdown(html, jsdom$1.window.document);
14238
+ markdown = this.showdownConverter.makeMarkdown(html, jsdom$1.window.document);
13906
14239
  return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
13907
14240
  }
13908
14241
  });
@@ -13930,9 +14263,14 @@
13930
14263
  asJson: function () {
13931
14264
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
13932
14265
  },
13933
- asBlob: function () {
13934
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
13935
- },
14266
+ /*
14267
+ TODO: [🥽]
14268
+ > asBlob() {
14269
+ > throw new UnexpectedError(
14270
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
14271
+ > );
14272
+ > },
14273
+ */
13936
14274
  };
13937
14275
  knowledge = this.markdownScraper.scrape(markdownSource);
13938
14276
  return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13946,7 +14284,6 @@
13946
14284
  return WebsiteScraper;
13947
14285
  }());
13948
14286
  /**
13949
- * TODO: !!!!!! Put into separate package
13950
14287
  * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
13951
14288
  * TODO: [🪂] Do it in parallel 11:11
13952
14289
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools