@promptbook/cli 0.72.0-14 → 0.72.0-15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +1 -1
  2. package/esm/index.es.js +544 -516
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +22 -18
  5. package/esm/typings/src/_packages/node.index.d.ts +2 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +10 -10
  7. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  8. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  10. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  11. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  12. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  14. package/esm/typings/src/config.d.ts +21 -14
  15. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  16. package/esm/typings/src/execution/Executables.d.ts +18 -0
  17. package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
  18. package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
  19. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  20. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
  21. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  22. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
  23. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
  24. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
  25. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  26. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  27. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  28. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  29. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  30. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
  31. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
  32. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  33. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  36. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
  38. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
  39. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
  40. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  41. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +3 -3
  45. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +2 -2
  46. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
  47. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  48. package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -5
  49. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
  50. package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
  51. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
  52. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
  53. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
  54. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
  55. package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
  56. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  57. package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
  58. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
  59. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
  60. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
  61. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
  62. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
  63. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
  64. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
  65. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
  66. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
  67. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
  68. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
  69. package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
  70. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  71. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  72. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  73. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  74. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  75. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  76. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  77. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  78. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  79. package/esm/typings/src/types/Prompt.d.ts +1 -1
  80. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  81. package/esm/typings/src/utils/$Register.d.ts +1 -1
  82. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  83. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  84. package/esm/typings/src/utils/emojis.d.ts +1 -1
  85. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
  86. package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
  87. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
  88. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  89. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  90. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  91. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  92. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  93. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  94. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  95. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  96. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  97. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  98. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  99. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  100. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  101. package/package.json +3 -2
  102. package/umd/index.umd.js +544 -516
  103. package/umd/index.umd.js.map +1 -1
  104. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  105. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  106. /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/esm/index.es.js CHANGED
@@ -26,7 +26,7 @@ import { Converter } from 'showdown';
26
26
  /**
27
27
  * The version of the Promptbook library
28
28
  */
29
- var PROMPTBOOK_VERSION = '0.72.0-13';
29
+ var PROMPTBOOK_VERSION = '0.72.0-14';
30
30
  // TODO: [main] !!!! List here all the versions and annotate + put into script
31
31
 
32
32
  /*! *****************************************************************************
@@ -399,7 +399,8 @@ var GENERATOR_WARNING = "\u26A0\uFE0F WARNING: This code has been generated so t
399
399
  *
400
400
  * @public exported from `@promptbook/core`
401
401
  */
402
- var CLAIM = "Supercharge LLM models with Promptbook";
402
+ var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
403
+ // <- TODO: [🐊] Pick the best claim
403
404
  /**
404
405
  * Warning message for the generated sections and files files
405
406
  *
@@ -432,23 +433,30 @@ var CONNECTION_RETRIES_LIMIT = 5;
432
433
  */
433
434
  var IMMEDIATE_TIME = 10;
434
435
  /**
435
- * The maximum number of (LLM) tasks running in parallel
436
+ * The maximum length of the (generated) filename
436
437
  *
437
438
  * @public exported from `@promptbook/core`
438
439
  */
439
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
440
+ var MAX_FILENAME_LENGTH = 30;
440
441
  /**
441
- * The maximum number of attempts to execute LLM task before giving up
442
+ * Strategy for caching the intermediate results for knowledge sources
442
443
  *
443
444
  * @public exported from `@promptbook/core`
444
445
  */
445
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
446
+ var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
447
+ // <- TODO: [😡] Change to 'VISIBLE'
446
448
  /**
447
- * The maximum length of the (generated) filename
449
+ * The maximum number of (LLM) tasks running in parallel
448
450
  *
449
451
  * @public exported from `@promptbook/core`
450
452
  */
451
- var MAX_FILENAME_LENGTH = 30;
453
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
454
+ /**
455
+ * The maximum number of attempts to execute LLM task before giving up
456
+ *
457
+ * @public exported from `@promptbook/core`
458
+ */
459
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
452
460
  /**
453
461
  * Where to store the cache of executions for promptbook CLI
454
462
  *
@@ -456,7 +464,7 @@ var MAX_FILENAME_LENGTH = 30;
456
464
  *
457
465
  * @public exported from `@promptbook/core`
458
466
  */
459
- var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
467
+ var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
460
468
  /**
461
469
  * Where to store the scrape cache
462
470
  *
@@ -464,13 +472,13 @@ var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
464
472
  *
465
473
  * @public exported from `@promptbook/core`
466
474
  */
467
- var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
475
+ var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
468
476
  /**
469
477
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
470
478
  *
471
479
  * @public exported from `@promptbook/core`
472
480
  */
473
- var PIPELINE_COLLECTION_BASE_FILENAME = "index";
481
+ var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
474
482
  /**
475
483
  * Nonce which is used for replacing things in strings
476
484
  *
@@ -534,13 +542,13 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
534
542
  *
535
543
  * @public exported from `@promptbook/core`
536
544
  */
537
- var IS_VERBOSE = false;
545
+ var DEFAULT_IS_VERBOSE = false;
538
546
  /**
539
547
  * @@@
540
548
  *
541
549
  * @public exported from `@promptbook/core`
542
550
  */
543
- var IS_AUTO_INSTALLED = false;
551
+ var DEFAULT_IS_AUTO_INSTALLED = false;
544
552
  /**
545
553
  * @@@
546
554
  *
@@ -613,271 +621,6 @@ function initializeHelloCommand(program) {
613
621
  * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
614
622
  */
615
623
 
616
- /**
617
- * Just marks a place of place where should be something implemented
618
- * No side effects.
619
- *
620
- * Note: It can be usefull suppressing eslint errors of unused variables
621
- *
622
- * @param value any values
623
- * @returns void
624
- * @private within the repository
625
- */
626
- function TODO_USE() {
627
- var value = [];
628
- for (var _i = 0; _i < arguments.length; _i++) {
629
- value[_i] = arguments[_i];
630
- }
631
- }
632
-
633
- /**
634
- * @@@
635
- *
636
- * @public exported from `@promptbook/node`
637
- */
638
- function $provideFilesystemForNode(options) {
639
- if (!$isRunningInNode()) {
640
- throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
641
- }
642
- var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
643
- TODO_USE(isVerbose);
644
- return {
645
- stat: stat,
646
- access: access,
647
- constants: constants,
648
- readFile: readFile,
649
- readdir: readdir,
650
- };
651
- }
652
- /**
653
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
654
- */
655
-
656
- /**
657
- * This error type indicates that some part of the code is not implemented yet
658
- *
659
- * @public exported from `@promptbook/core`
660
- */
661
- var NotYetImplementedError = /** @class */ (function (_super) {
662
- __extends(NotYetImplementedError, _super);
663
- function NotYetImplementedError(message) {
664
- var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
665
- _this.name = 'NotYetImplementedError';
666
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
667
- return _this;
668
- }
669
- return NotYetImplementedError;
670
- }(Error));
671
-
672
- /**
673
- * @@@
674
- *
675
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
676
- *
677
- * @private internal function of `$Register`
678
- */
679
- function $getGlobalScope() {
680
- return Function('return this')();
681
- }
682
-
683
- /**
684
- * @@@
685
- *
686
- * @param text @@@
687
- * @returns @@@
688
- * @example 'HELLO_WORLD'
689
- * @example 'I_LOVE_PROMPTBOOK'
690
- * @public exported from `@promptbook/utils`
691
- */
692
- function normalizeTo_SCREAMING_CASE(text) {
693
- var e_1, _a;
694
- var charType;
695
- var lastCharType = 'OTHER';
696
- var normalizedName = '';
697
- try {
698
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
699
- var char = text_1_1.value;
700
- var normalizedChar = void 0;
701
- if (/^[a-z]$/.test(char)) {
702
- charType = 'LOWERCASE';
703
- normalizedChar = char.toUpperCase();
704
- }
705
- else if (/^[A-Z]$/.test(char)) {
706
- charType = 'UPPERCASE';
707
- normalizedChar = char;
708
- }
709
- else if (/^[0-9]$/.test(char)) {
710
- charType = 'NUMBER';
711
- normalizedChar = char;
712
- }
713
- else {
714
- charType = 'OTHER';
715
- normalizedChar = '_';
716
- }
717
- if (charType !== lastCharType &&
718
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
719
- !(lastCharType === 'NUMBER') &&
720
- !(charType === 'NUMBER')) {
721
- normalizedName += '_';
722
- }
723
- normalizedName += normalizedChar;
724
- lastCharType = charType;
725
- }
726
- }
727
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
728
- finally {
729
- try {
730
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
731
- }
732
- finally { if (e_1) throw e_1.error; }
733
- }
734
- normalizedName = normalizedName.replace(/_+/g, '_');
735
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
736
- normalizedName = normalizedName.replace(/^_/, '');
737
- normalizedName = normalizedName.replace(/_$/, '');
738
- return normalizedName;
739
- }
740
- /**
741
- * TODO: Tests
742
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
743
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
744
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
745
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
746
- * TODO: [🌺] Use some intermediate util splitWords
747
- */
748
-
749
- /**
750
- * @@@
751
- *
752
- * @param text @@@
753
- * @returns @@@
754
- * @example 'hello_world'
755
- * @example 'i_love_promptbook'
756
- * @public exported from `@promptbook/utils`
757
- */
758
- function normalizeTo_snake_case(text) {
759
- return normalizeTo_SCREAMING_CASE(text).toLowerCase();
760
- }
761
-
762
- /**
763
- * Register is @@@
764
- *
765
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
766
- *
767
- * @private internal utility, exported are only signleton instances of this class
768
- */
769
- var $Register = /** @class */ (function () {
770
- function $Register(registerName) {
771
- this.registerName = registerName;
772
- var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
773
- var globalScope = $getGlobalScope();
774
- if (globalScope[storageName] === undefined) {
775
- globalScope[storageName] = [];
776
- }
777
- else if (!Array.isArray(globalScope[storageName])) {
778
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
779
- }
780
- this.storage = globalScope[storageName];
781
- }
782
- $Register.prototype.list = function () {
783
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
784
- return this.storage;
785
- };
786
- $Register.prototype.register = function (registered) {
787
- var packageName = registered.packageName, className = registered.className;
788
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
789
- var existingRegistration = this.storage[existingRegistrationIndex];
790
- if (!existingRegistration) {
791
- this.storage.push(registered);
792
- }
793
- else {
794
- this.storage[existingRegistrationIndex] = registered;
795
- }
796
- return {
797
- registerName: this.registerName,
798
- packageName: packageName,
799
- className: className,
800
- get isDestroyed() {
801
- return false;
802
- },
803
- destroy: function () {
804
- throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
805
- },
806
- };
807
- };
808
- return $Register;
809
- }());
810
-
811
- /**
812
- * @@@
813
- *
814
- * Note: `$` is used to indicate that this interacts with the global scope
815
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
816
- * @public exported from `@promptbook/core`
817
- */
818
- var $scrapersRegister = new $Register('scraper_constructors');
819
- /**
820
- * TODO: [®] DRY Register logic
821
- */
822
-
823
- /**
824
- * !!!!!!
825
- *
826
- * 1) @@@
827
- * 2) @@@
828
- *
829
- * @public exported from `@promptbook/node`
830
- */
831
- function $provideScrapersForNode(tools, options) {
832
- return __awaiter(this, void 0, void 0, function () {
833
- var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
834
- var e_1, _f;
835
- return __generator(this, function (_g) {
836
- switch (_g.label) {
837
- case 0:
838
- if (!$isRunningInNode()) {
839
- throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
840
- }
841
- _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
842
- TODO_USE(isAutoInstalled);
843
- TODO_USE(isVerbose);
844
- scrapers = [];
845
- _g.label = 1;
846
- case 1:
847
- _g.trys.push([1, 6, 7, 8]);
848
- _d = __values($scrapersRegister.list()), _e = _d.next();
849
- _g.label = 2;
850
- case 2:
851
- if (!!_e.done) return [3 /*break*/, 5];
852
- scraperFactory = _e.value;
853
- return [4 /*yield*/, scraperFactory(tools, options || {})];
854
- case 3:
855
- scraper = _g.sent();
856
- scrapers.push(scraper);
857
- _g.label = 4;
858
- case 4:
859
- _e = _d.next();
860
- return [3 /*break*/, 2];
861
- case 5: return [3 /*break*/, 8];
862
- case 6:
863
- e_1_1 = _g.sent();
864
- e_1 = { error: e_1_1 };
865
- return [3 /*break*/, 8];
866
- case 7:
867
- try {
868
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
869
- }
870
- finally { if (e_1) throw e_1.error; }
871
- return [7 /*endfinally*/];
872
- case 8: return [2 /*return*/, scrapers];
873
- }
874
- });
875
- });
876
- }
877
- /**
878
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
879
- */
880
-
881
624
  /**
882
625
  * Converts PipelineCollection to serialized JSON
883
626
  *
@@ -963,7 +706,6 @@ function pipelineJsonToString(pipelineJson) {
963
706
  pipelineString += '\n\n';
964
707
  pipelineString += description;
965
708
  }
966
- // TODO:> const commands: Array<Command>
967
709
  var commands = [];
968
710
  if (pipelineUrl) {
969
711
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -1019,7 +761,6 @@ function pipelineJsonToString(pipelineJson) {
1019
761
  pipelineString += '\n\n';
1020
762
  pipelineString += description_1;
1021
763
  }
1022
- // TODO:> const commands: Array<Command>
1023
764
  var commands_1 = [];
1024
765
  var contentLanguage = 'text';
1025
766
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -1318,8 +1059,7 @@ var PipelineExecutionError = /** @class */ (function (_super) {
1318
1059
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1319
1060
  *
1320
1061
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1321
- * @public exported from `@promptbook/types`
1322
- * TODO: !!!!!! Export as runtime class not just type
1062
+ * @public exported from `@promptbook/core`
1323
1063
  */
1324
1064
  var MultipleLlmExecutionTools = /** @class */ (function () {
1325
1065
  /**
@@ -2478,6 +2218,22 @@ var LimitReachedError = /** @class */ (function (_super) {
2478
2218
  return LimitReachedError;
2479
2219
  }(Error));
2480
2220
 
2221
+ /**
2222
+ * This error type indicates that some part of the code is not implemented yet
2223
+ *
2224
+ * @public exported from `@promptbook/core`
2225
+ */
2226
+ var NotYetImplementedError = /** @class */ (function (_super) {
2227
+ __extends(NotYetImplementedError, _super);
2228
+ function NotYetImplementedError(message) {
2229
+ var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2230
+ _this.name = 'NotYetImplementedError';
2231
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2232
+ return _this;
2233
+ }
2234
+ return NotYetImplementedError;
2235
+ }(Error));
2236
+
2481
2237
  /**
2482
2238
  * Index of all custom errors
2483
2239
  *
@@ -2539,6 +2295,7 @@ function assertsExecutionSuccessful(executionResult) {
2539
2295
  }
2540
2296
  }
2541
2297
  /**
2298
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2542
2299
  * TODO: [🧠] Can this return type be better typed than void
2543
2300
  */
2544
2301
 
@@ -2796,6 +2553,23 @@ function union() {
2796
2553
  return union;
2797
2554
  }
2798
2555
 
2556
+ /**
2557
+ * Just marks a place of place where should be something implemented
2558
+ * No side effects.
2559
+ *
2560
+ * Note: It can be usefull suppressing eslint errors of unused variables
2561
+ *
2562
+ * @param value any values
2563
+ * @returns void
2564
+ * @private within the repository
2565
+ */
2566
+ function TODO_USE() {
2567
+ var value = [];
2568
+ for (var _i = 0; _i < arguments.length; _i++) {
2569
+ value[_i] = arguments[_i];
2570
+ }
2571
+ }
2572
+
2799
2573
  /**
2800
2574
  * This error indicates problems parsing the format value
2801
2575
  *
@@ -3828,12 +3602,11 @@ function checkExpectations(expectations, value) {
3828
3602
  */
3829
3603
  function executeAttempts(options) {
3830
3604
  return __awaiter(this, void 0, void 0, function () {
3831
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3605
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3832
3606
  return __generator(this, function (_a) {
3833
3607
  switch (_a.label) {
3834
3608
  case 0:
3835
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3836
- maxExecutionAttempts = settings.maxExecutionAttempts;
3609
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
3837
3610
  $ongoingTemplateResult = {
3838
3611
  $result: null,
3839
3612
  $resultString: null,
@@ -4199,12 +3972,12 @@ function executeAttempts(options) {
4199
3972
  */
4200
3973
  function executeFormatSubvalues(options) {
4201
3974
  return __awaiter(this, void 0, void 0, function () {
4202
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3975
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4203
3976
  var _this = this;
4204
3977
  return __generator(this, function (_a) {
4205
3978
  switch (_a.label) {
4206
3979
  case 0:
4207
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3980
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4208
3981
  if (template.foreach === undefined) {
4209
3982
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4210
3983
  }
@@ -4234,7 +4007,7 @@ function executeFormatSubvalues(options) {
4234
4007
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4235
4008
  }
4236
4009
  if (formatDefinition.formatName === 'CSV') {
4237
- formatSettings = settings.csvSettings;
4010
+ formatSettings = csvSettings;
4238
4011
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4239
4012
  }
4240
4013
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4387,13 +4160,12 @@ function getReservedParametersForTemplate(options) {
4387
4160
  */
4388
4161
  function executeTemplate(options) {
4389
4162
  return __awaiter(this, void 0, void 0, function () {
4390
- var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4391
- var e_1, _f, _g;
4392
- return __generator(this, function (_h) {
4393
- switch (_h.label) {
4163
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4164
+ var e_1, _g, _h;
4165
+ return __generator(this, function (_j) {
4166
+ switch (_j.label) {
4394
4167
  case 0:
4395
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4396
- maxExecutionAttempts = settings.maxExecutionAttempts;
4168
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4397
4169
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4398
4170
  title = currentTemplate.title;
4399
4171
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -4408,7 +4180,7 @@ function executeTemplate(options) {
4408
4180
  // <- [🍸]
4409
4181
  })];
4410
4182
  case 1:
4411
- _h.sent();
4183
+ _j.sent();
4412
4184
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
4413
4185
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
4414
4186
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4419,15 +4191,15 @@ function executeTemplate(options) {
4419
4191
  .map(function (name) { return "{".concat(name, "}"); })
4420
4192
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4421
4193
  }
4422
- _b = (_a = Object).freeze;
4423
- _c = [{}];
4194
+ _c = (_b = Object).freeze;
4195
+ _d = [{}];
4424
4196
  return [4 /*yield*/, getReservedParametersForTemplate({
4425
4197
  preparedPipeline: preparedPipeline,
4426
4198
  template: currentTemplate,
4427
4199
  pipelineIdentification: pipelineIdentification,
4428
4200
  })];
4429
4201
  case 2:
4430
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4202
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
4431
4203
  definedParameterNames = new Set(Object.keys(definedParameters));
4432
4204
  parameters = {};
4433
4205
  _loop_1 = function (parameterName) {
@@ -4447,15 +4219,15 @@ function executeTemplate(options) {
4447
4219
  try {
4448
4220
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4449
4221
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4450
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4451
- parameterName = _e.value;
4222
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
4223
+ parameterName = _f.value;
4452
4224
  _loop_1(parameterName);
4453
4225
  }
4454
4226
  }
4455
4227
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4456
4228
  finally {
4457
4229
  try {
4458
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4230
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
4459
4231
  }
4460
4232
  finally { if (e_1) throw e_1.error; }
4461
4233
  }
@@ -4475,12 +4247,11 @@ function executeTemplate(options) {
4475
4247
  template: currentTemplate,
4476
4248
  preparedPipeline: preparedPipeline,
4477
4249
  tools: tools,
4478
- settings: settings,
4479
4250
  $executionReport: $executionReport,
4480
4251
  pipelineIdentification: pipelineIdentification,
4481
4252
  })];
4482
4253
  case 3:
4483
- resultString = _h.sent();
4254
+ resultString = _j.sent();
4484
4255
  return [4 /*yield*/, onProgress({
4485
4256
  name: name,
4486
4257
  title: title,
@@ -4492,12 +4263,12 @@ function executeTemplate(options) {
4492
4263
  // <- [🍸]
4493
4264
  })];
4494
4265
  case 4:
4495
- _h.sent();
4496
- return [2 /*return*/, Object.freeze((_g = {},
4497
- _g[currentTemplate.resultingParameterName] =
4266
+ _j.sent();
4267
+ return [2 /*return*/, Object.freeze((_h = {},
4268
+ _h[currentTemplate.resultingParameterName] =
4498
4269
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4499
4270
  resultString,
4500
- _g))];
4271
+ _h))];
4501
4272
  }
4502
4273
  });
4503
4274
  });
@@ -4556,13 +4327,12 @@ function filterJustOutputParameters(options) {
4556
4327
  */
4557
4328
  function executePipeline(options) {
4558
4329
  return __awaiter(this, void 0, void 0, function () {
4559
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4330
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4560
4331
  var e_1, _f, e_2, _g;
4561
4332
  return __generator(this, function (_h) {
4562
4333
  switch (_h.label) {
4563
4334
  case 0:
4564
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4565
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4335
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4566
4336
  preparedPipeline = options.preparedPipeline;
4567
4337
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4568
4338
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -4747,12 +4517,7 @@ function executePipeline(options) {
4747
4517
  return [3 /*break*/, 4];
4748
4518
  case 3:
4749
4519
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
4750
- work_1 = executeTemplate({
4751
- currentTemplate: currentTemplate,
4752
- preparedPipeline: preparedPipeline,
4753
- parametersToPass: parametersToPass,
4754
- tools: tools,
4755
- onProgress: function (progress) {
4520
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
4756
4521
  if (isReturned) {
4757
4522
  throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4758
4523
  .split('\n')
@@ -4762,11 +4527,7 @@ function executePipeline(options) {
4762
4527
  if (onProgress) {
4763
4528
  onProgress(progress);
4764
4529
  }
4765
- },
4766
- settings: settings,
4767
- $executionReport: executionReport,
4768
- pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
4769
- })
4530
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
4770
4531
  .then(function (newParametersToPass) {
4771
4532
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
4772
4533
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -4870,8 +4631,7 @@ function executePipeline(options) {
4870
4631
  */
4871
4632
  function createPipelineExecutor(options) {
4872
4633
  var _this = this;
4873
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
4874
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
4634
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
4875
4635
  validatePipeline(pipeline);
4876
4636
  var pipelineIdentification = (function () {
4877
4637
  // Note: This is a 😐 implementation of [🚞]
@@ -4905,14 +4665,12 @@ function createPipelineExecutor(options) {
4905
4665
  tools: tools,
4906
4666
  onProgress: onProgress,
4907
4667
  pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
4908
- settings: {
4909
- maxExecutionAttempts: maxExecutionAttempts,
4910
- maxParallelCount: maxParallelCount,
4911
- csvSettings: csvSettings,
4912
- isVerbose: isVerbose,
4913
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4914
- rootDirname: rootDirname,
4915
- },
4668
+ maxExecutionAttempts: maxExecutionAttempts,
4669
+ maxParallelCount: maxParallelCount,
4670
+ csvSettings: csvSettings,
4671
+ isVerbose: isVerbose,
4672
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4673
+ rootDirname: rootDirname,
4916
4674
  })];
4917
4675
  });
4918
4676
  }); };
@@ -4935,7 +4693,7 @@ function preparePersona(personaDescription, tools, options) {
4935
4693
  return __generator(this, function (_d) {
4936
4694
  switch (_d.label) {
4937
4695
  case 0:
4938
- _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4696
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4939
4697
  if (tools === undefined || tools.llm === undefined) {
4940
4698
  throw new MissingToolsError('LLM tools are required for preparing persona');
4941
4699
  }
@@ -4991,20 +4749,171 @@ function preparePersona(personaDescription, tools, options) {
4991
4749
  */
4992
4750
 
4993
4751
  /**
4994
- * This error indicates that the promptbook can not retrieve knowledge from external sources
4752
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
4753
+ *
4754
+ * @public exported from `@promptbook/core`
4755
+ */
4756
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4757
+ __extends(KnowledgeScrapeError, _super);
4758
+ function KnowledgeScrapeError(message) {
4759
+ var _this = _super.call(this, message) || this;
4760
+ _this.name = 'KnowledgeScrapeError';
4761
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4762
+ return _this;
4763
+ }
4764
+ return KnowledgeScrapeError;
4765
+ }(Error));
4766
+
4767
+ /**
4768
+ * @@@
4769
+ *
4770
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4771
+ *
4772
+ * @private internal function of `$Register`
4773
+ */
4774
+ function $getGlobalScope() {
4775
+ return Function('return this')();
4776
+ }
4777
+
4778
+ /**
4779
+ * @@@
4780
+ *
4781
+ * @param text @@@
4782
+ * @returns @@@
4783
+ * @example 'HELLO_WORLD'
4784
+ * @example 'I_LOVE_PROMPTBOOK'
4785
+ * @public exported from `@promptbook/utils`
4786
+ */
4787
+ function normalizeTo_SCREAMING_CASE(text) {
4788
+ var e_1, _a;
4789
+ var charType;
4790
+ var lastCharType = 'OTHER';
4791
+ var normalizedName = '';
4792
+ try {
4793
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4794
+ var char = text_1_1.value;
4795
+ var normalizedChar = void 0;
4796
+ if (/^[a-z]$/.test(char)) {
4797
+ charType = 'LOWERCASE';
4798
+ normalizedChar = char.toUpperCase();
4799
+ }
4800
+ else if (/^[A-Z]$/.test(char)) {
4801
+ charType = 'UPPERCASE';
4802
+ normalizedChar = char;
4803
+ }
4804
+ else if (/^[0-9]$/.test(char)) {
4805
+ charType = 'NUMBER';
4806
+ normalizedChar = char;
4807
+ }
4808
+ else {
4809
+ charType = 'OTHER';
4810
+ normalizedChar = '_';
4811
+ }
4812
+ if (charType !== lastCharType &&
4813
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4814
+ !(lastCharType === 'NUMBER') &&
4815
+ !(charType === 'NUMBER')) {
4816
+ normalizedName += '_';
4817
+ }
4818
+ normalizedName += normalizedChar;
4819
+ lastCharType = charType;
4820
+ }
4821
+ }
4822
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4823
+ finally {
4824
+ try {
4825
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
4826
+ }
4827
+ finally { if (e_1) throw e_1.error; }
4828
+ }
4829
+ normalizedName = normalizedName.replace(/_+/g, '_');
4830
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
4831
+ normalizedName = normalizedName.replace(/^_/, '');
4832
+ normalizedName = normalizedName.replace(/_$/, '');
4833
+ return normalizedName;
4834
+ }
4835
+ /**
4836
+ * TODO: Tests
4837
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
4838
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
4839
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
4840
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
4841
+ * TODO: [🌺] Use some intermediate util splitWords
4842
+ */
4843
+
4844
+ /**
4845
+ * @@@
4846
+ *
4847
+ * @param text @@@
4848
+ * @returns @@@
4849
+ * @example 'hello_world'
4850
+ * @example 'i_love_promptbook'
4851
+ * @public exported from `@promptbook/utils`
4852
+ */
4853
+ function normalizeTo_snake_case(text) {
4854
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
4855
+ }
4856
+
4857
+ /**
4858
+ * Register is @@@
4859
+ *
4860
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
4861
+ *
4862
+ * @private internal utility, exported are only signleton instances of this class
4863
+ */
4864
+ var $Register = /** @class */ (function () {
4865
+ function $Register(registerName) {
4866
+ this.registerName = registerName;
4867
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
4868
+ var globalScope = $getGlobalScope();
4869
+ if (globalScope[storageName] === undefined) {
4870
+ globalScope[storageName] = [];
4871
+ }
4872
+ else if (!Array.isArray(globalScope[storageName])) {
4873
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
4874
+ }
4875
+ this.storage = globalScope[storageName];
4876
+ }
4877
+ $Register.prototype.list = function () {
4878
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
4879
+ return this.storage;
4880
+ };
4881
+ $Register.prototype.register = function (registered) {
4882
+ var packageName = registered.packageName, className = registered.className;
4883
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
4884
+ var existingRegistration = this.storage[existingRegistrationIndex];
4885
+ if (!existingRegistration) {
4886
+ this.storage.push(registered);
4887
+ }
4888
+ else {
4889
+ this.storage[existingRegistrationIndex] = registered;
4890
+ }
4891
+ return {
4892
+ registerName: this.registerName,
4893
+ packageName: packageName,
4894
+ className: className,
4895
+ get isDestroyed() {
4896
+ return false;
4897
+ },
4898
+ destroy: function () {
4899
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
4900
+ },
4901
+ };
4902
+ };
4903
+ return $Register;
4904
+ }());
4905
+
4906
+ /**
4907
+ * @@@
4995
4908
  *
4909
+ * Note: `$` is used to indicate that this interacts with the global scope
4910
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
4996
4911
  * @public exported from `@promptbook/core`
4997
4912
  */
4998
- var KnowledgeScrapeError = /** @class */ (function (_super) {
4999
- __extends(KnowledgeScrapeError, _super);
5000
- function KnowledgeScrapeError(message) {
5001
- var _this = _super.call(this, message) || this;
5002
- _this.name = 'KnowledgeScrapeError';
5003
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
5004
- return _this;
5005
- }
5006
- return KnowledgeScrapeError;
5007
- }(Error));
4913
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4914
+ /**
4915
+ * TODO: [®] DRY Register logic
4916
+ */
5008
4917
 
5009
4918
  /**
5010
4919
  * @@@
@@ -5013,12 +4922,12 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
5013
4922
  * @singleton Only one instance of each register is created per build, but thare can be more @@@
5014
4923
  * @public exported from `@promptbook/core`
5015
4924
  */
5016
- var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4925
+ var $scrapersRegister = new $Register('scraper_constructors');
5017
4926
  /**
5018
4927
  * TODO: [®] DRY Register logic
5019
4928
  */
5020
4929
 
5021
- // TODO: !!!!!! Maybe delete this function
4930
+ // TODO: !!!!!!last - Maybe delete this function
5022
4931
  /**
5023
4932
  * Creates a message with all registered scrapers
5024
4933
  *
@@ -5185,7 +5094,6 @@ function normalizeToKebabCase(text) {
5185
5094
  * @private within the repository
5186
5095
  */
5187
5096
  function sourceContentToName(sourceContent) {
5188
- // TODO: !!!!!! Better name for source than gibberish hash
5189
5097
  var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
5190
5098
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5191
5099
  .toString( /* hex */)
@@ -5298,7 +5206,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5298
5206
  case 0:
5299
5207
  sourceContent = knowledgeSource.sourceContent;
5300
5208
  name = knowledgeSource.name;
5301
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5209
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
5302
5210
  TODO_USE(isVerbose);
5303
5211
  if (!name) {
5304
5212
  name = sourceContentToName(sourceContent);
@@ -5388,7 +5296,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5388
5296
  content = _a.sent();
5389
5297
  return [2 /*return*/, new Blob([
5390
5298
  content,
5391
- // <- TODO: !!!!!! Maybe not working
5299
+ // <- TODO: !!!!!! Test that this is working
5392
5300
  ], { type: mimeType_1 })];
5393
5301
  }
5394
5302
  });
@@ -5451,7 +5359,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
5451
5359
  return __generator(this, function (_c) {
5452
5360
  switch (_c.label) {
5453
5361
  case 0:
5454
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5362
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5455
5363
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
5456
5364
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
5457
5365
  var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
@@ -5480,7 +5388,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
5480
5388
  case 4:
5481
5389
  partialPiecesUnchecked = _d.sent();
5482
5390
  if (partialPiecesUnchecked !== null) {
5483
- partialPieces = partialPiecesUnchecked;
5391
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
5392
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
5484
5393
  return [3 /*break*/, 6];
5485
5394
  }
5486
5395
  _d.label = 5;
@@ -5528,7 +5437,7 @@ TODO: [🧊] This is how it can look in future
5528
5437
  > /**
5529
5438
  > * Unprepared knowledge
5530
5439
  > * /
5531
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
5440
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
5532
5441
  > };
5533
5442
  >
5534
5443
  > export async function prepareKnowledgePieces(
@@ -5586,7 +5495,7 @@ function prepareTemplates(pipeline, tools, options) {
5586
5495
  return __generator(this, function (_b) {
5587
5496
  switch (_b.label) {
5588
5497
  case 0:
5589
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
5498
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
5590
5499
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
5591
5500
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
5592
5501
  TODO_USE(parameters);
@@ -5648,7 +5557,7 @@ function preparePipeline(pipeline, tools, options) {
5648
5557
  if (isPipelinePrepared(pipeline)) {
5649
5558
  return [2 /*return*/, pipeline];
5650
5559
  }
5651
- rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5560
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5652
5561
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
5653
5562
  if (tools === undefined || tools.llm === undefined) {
5654
5563
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
@@ -5706,7 +5615,9 @@ function preparePipeline(pipeline, tools, options) {
5706
5615
  // ----- /Templates preparation -----
5707
5616
  // Note: Count total usage
5708
5617
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
5709
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
5618
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
5619
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
5620
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
5710
5621
  }
5711
5622
  });
5712
5623
  });
@@ -8630,7 +8541,7 @@ function $registeredLlmToolsMessage() {
8630
8541
  */
8631
8542
  function createLlmToolsFromConfiguration(configuration, options) {
8632
8543
  if (options === void 0) { options = {}; }
8633
- var _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a, userId = options.userId;
8544
+ var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
8634
8545
  var llmTools = configuration.map(function (llmConfiguration) {
8635
8546
  var registeredItem = $llmToolsRegister
8636
8547
  .list()
@@ -8691,6 +8602,115 @@ function $provideLlmToolsFromEnv(options) {
8691
8602
  * TODO: [®] DRY Register logic
8692
8603
  */
8693
8604
 
8605
+ /**
8606
+ * @@@
8607
+ *
8608
+ * @public exported from `@promptbook/node`
8609
+ */
8610
+ function $provideExecutablesForNode(options) {
8611
+ return __awaiter(this, void 0, void 0, function () {
8612
+ var _a, _b, isAutoInstalled, _c, isVerbose;
8613
+ return __generator(this, function (_d) {
8614
+ if (!$isRunningInNode()) {
8615
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8616
+ }
8617
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8618
+ TODO_USE(isAutoInstalled);
8619
+ TODO_USE(isVerbose);
8620
+ return [2 /*return*/, {
8621
+ // TODO: !!!!!! use `locate-app` library here
8622
+ pandocPath: 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe',
8623
+ libreOfficePath: 'C:/Program Files/LibreOffice/program/swriter.exe',
8624
+ }];
8625
+ });
8626
+ });
8627
+ }
8628
+ /**
8629
+ * TODO: [🧠] THis should be maybe in different folder
8630
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8631
+ */
8632
+
8633
+ /**
8634
+ * @@@
8635
+ *
8636
+ * @public exported from `@promptbook/node`
8637
+ */
8638
+ function $provideFilesystemForNode(options) {
8639
+ if (!$isRunningInNode()) {
8640
+ throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
8641
+ }
8642
+ var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
8643
+ TODO_USE(isVerbose);
8644
+ return {
8645
+ stat: stat,
8646
+ access: access,
8647
+ constants: constants,
8648
+ readFile: readFile,
8649
+ readdir: readdir,
8650
+ };
8651
+ }
8652
+ /**
8653
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8654
+ */
8655
+
8656
+ /**
8657
+ * @@@
8658
+ *
8659
+ * 1) @@@
8660
+ * 2) @@@
8661
+ *
8662
+ * @public exported from `@promptbook/node`
8663
+ */
8664
+ function $provideScrapersForNode(tools, options) {
8665
+ return __awaiter(this, void 0, void 0, function () {
8666
+ var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
8667
+ var e_1, _f;
8668
+ return __generator(this, function (_g) {
8669
+ switch (_g.label) {
8670
+ case 0:
8671
+ if (!$isRunningInNode()) {
8672
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8673
+ }
8674
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8675
+ TODO_USE(isAutoInstalled);
8676
+ TODO_USE(isVerbose);
8677
+ scrapers = [];
8678
+ _g.label = 1;
8679
+ case 1:
8680
+ _g.trys.push([1, 6, 7, 8]);
8681
+ _d = __values($scrapersRegister.list()), _e = _d.next();
8682
+ _g.label = 2;
8683
+ case 2:
8684
+ if (!!_e.done) return [3 /*break*/, 5];
8685
+ scraperFactory = _e.value;
8686
+ return [4 /*yield*/, scraperFactory(tools, options || {})];
8687
+ case 3:
8688
+ scraper = _g.sent();
8689
+ scrapers.push(scraper);
8690
+ _g.label = 4;
8691
+ case 4:
8692
+ _e = _d.next();
8693
+ return [3 /*break*/, 2];
8694
+ case 5: return [3 /*break*/, 8];
8695
+ case 6:
8696
+ e_1_1 = _g.sent();
8697
+ e_1 = { error: e_1_1 };
8698
+ return [3 /*break*/, 8];
8699
+ case 7:
8700
+ try {
8701
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
8702
+ }
8703
+ finally { if (e_1) throw e_1.error; }
8704
+ return [7 /*endfinally*/];
8705
+ case 8: return [2 /*return*/, scrapers];
8706
+ }
8707
+ });
8708
+ });
8709
+ }
8710
+ /**
8711
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8712
+ */
8713
+
8694
8714
  /**
8695
8715
  * Extracts code block from markdown.
8696
8716
  *
@@ -9145,7 +9165,7 @@ var JavascriptExecutionTools = JavascriptEvalExecutionTools;
9145
9165
  */
9146
9166
  function $provideExecutionToolsForNode(options) {
9147
9167
  return __awaiter(this, void 0, void 0, function () {
9148
- var fs, llm, tools;
9168
+ var fs, llm, executables, tools;
9149
9169
  var _a;
9150
9170
  return __generator(this, function (_b) {
9151
9171
  switch (_b.label) {
@@ -9155,12 +9175,16 @@ function $provideExecutionToolsForNode(options) {
9155
9175
  }
9156
9176
  fs = $provideFilesystemForNode();
9157
9177
  llm = $provideLlmToolsFromEnv(options);
9178
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
9179
+ case 1:
9180
+ executables = _b.sent();
9158
9181
  _a = {
9159
9182
  llm: llm,
9160
- fs: fs
9183
+ fs: fs,
9184
+ executables: executables
9161
9185
  };
9162
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
9163
- case 1:
9186
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
9187
+ case 2:
9164
9188
  tools = (_a.scrapers = _b.sent(),
9165
9189
  _a.script = [new JavascriptExecutionTools(options)],
9166
9190
  _a);
@@ -9398,7 +9422,9 @@ function createCollectionFromDirectory(path, tools, options) {
9398
9422
  throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
9399
9423
  // <- TODO: [🧠] What is the best error type here`
9400
9424
  }
9401
- makedLibraryFilePath = join(path, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".json"));
9425
+ makedLibraryFilePath = join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
9426
+ // <- TODO: [🦒] Allow to override (pass different value into the function)
9427
+ , ".json"));
9402
9428
  return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
9403
9429
  case 3:
9404
9430
  if (!(_f.sent())) {
@@ -9409,7 +9435,7 @@ function createCollectionFromDirectory(path, tools, options) {
9409
9435
  // TODO: !! Implement;
9410
9436
  // TODO: [🌗]
9411
9437
  }
9412
- _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9438
+ _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9413
9439
  collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
9414
9440
  var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
9415
9441
  var e_1, _a;
@@ -9881,7 +9907,7 @@ function $currentDate() {
9881
9907
  function cacheLlmTools(llmTools, options) {
9882
9908
  var _this = this;
9883
9909
  if (options === void 0) { options = {}; }
9884
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isReloaded, isReloaded = _b === void 0 ? false : _b;
9910
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
9885
9911
  var proxyTools = __assign(__assign({}, llmTools), {
9886
9912
  // <- Note: [🥫]
9887
9913
  get title() {
@@ -9903,7 +9929,7 @@ function cacheLlmTools(llmTools, options) {
9903
9929
  key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
9904
9930
  '-' +
9905
9931
  sha256(hexEncoder.parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
9906
- if (!!isReloaded) return [3 /*break*/, 2];
9932
+ if (!!isCacheReloaded) return [3 /*break*/, 2];
9907
9933
  return [4 /*yield*/, storage.getItem(key)];
9908
9934
  case 1:
9909
9935
  _a = _c.sent();
@@ -9989,12 +10015,14 @@ function $provideLlmToolsForCli(options) {
9989
10015
  if (!$isRunningInNode()) {
9990
10016
  throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
9991
10017
  }
9992
- var _a = (options !== null && options !== void 0 ? options : {}).isCacheCleaned, isCacheCleaned = _a === void 0 ? false : _a;
10018
+ var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
9993
10019
  return cacheLlmTools(countTotalUsage(
9994
10020
  // <- Note: for example here we don`t want the [🌯]
9995
10021
  $provideLlmToolsFromEnv()), {
9996
- storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, { rootFolderPath: join(process.cwd(), EXECUTIONS_CACHE_DIRNAME) }),
9997
- isReloaded: isCacheCleaned,
10022
+ storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
10023
+ rootFolderPath: join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
10024
+ }),
10025
+ isCacheReloaded: isCacheReloaded,
9998
10026
  });
9999
10027
  }
10000
10028
  /**
@@ -10021,13 +10049,13 @@ function initializeMakeCommand(program) {
10021
10049
  makeCommand.option('-f, --format <format>', spaceTrim$1("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳‍🌈] */);
10022
10050
  makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
10023
10051
  makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
10024
- makeCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache", false);
10052
+ makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
10025
10053
  makeCommand.option('--verbose', "Is output verbose", false);
10026
- makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), PIPELINE_COLLECTION_BASE_FILENAME);
10054
+ makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
10027
10055
  makeCommand.action(function (path, _a) {
10028
- var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10056
+ var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10029
10057
  return __awaiter(_this, void 0, void 0, function () {
10030
- var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10058
+ var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10031
10059
  var _d, e_2, _e, e_1, _f;
10032
10060
  var _this = this;
10033
10061
  return __generator(this, function (_g) {
@@ -10041,22 +10069,25 @@ function initializeMakeCommand(program) {
10041
10069
  .split(',')
10042
10070
  .map(function (_) { return _.trim(); })
10043
10071
  .filter(function (_) { return _ !== ''; });
10044
- if (outFile !== PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10072
+ if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10045
10073
  console.error(colors.red("You can only use one format if you specify --out-file"));
10046
10074
  process.exit(1);
10047
10075
  }
10048
10076
  options = {
10049
10077
  isVerbose: isVerbose,
10050
- isCacheCleaned: isCacheCleaned,
10078
+ isCacheReloaded: isCacheReloaded,
10051
10079
  };
10052
10080
  fs = $provideFilesystemForNode(options);
10053
10081
  llm = $provideLlmToolsForCli(options);
10082
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10083
+ case 1:
10084
+ executables = _g.sent();
10054
10085
  _d = {
10055
10086
  llm: llm,
10056
10087
  fs: fs
10057
10088
  };
10058
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10059
- case 1:
10089
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10090
+ case 2:
10060
10091
  tools = (_d.scrapers = _g.sent(),
10061
10092
  _d.script = [
10062
10093
  /*new JavascriptExecutionTools(options)*/
@@ -10065,31 +10096,31 @@ function initializeMakeCommand(program) {
10065
10096
  return [4 /*yield*/, createCollectionFromDirectory(path, tools, {
10066
10097
  isVerbose: isVerbose,
10067
10098
  isRecursive: true,
10068
- // <- TODO: [🍖] isCacheReloaded
10099
+ // <- TODO: [🍖] Add `intermediateFilesStrategy`
10069
10100
  })];
10070
- case 2:
10071
- collection = _g.sent();
10072
- _g.label = 3;
10073
10101
  case 3:
10074
- _g.trys.push([3, 15, 16, 17]);
10075
- validations_1 = __values(validations), validations_1_1 = validations_1.next();
10102
+ collection = _g.sent();
10076
10103
  _g.label = 4;
10077
10104
  case 4:
10078
- if (!!validations_1_1.done) return [3 /*break*/, 14];
10079
- validation_1 = validations_1_1.value;
10105
+ _g.trys.push([4, 16, 17, 18]);
10106
+ validations_1 = __values(validations), validations_1_1 = validations_1.next();
10080
10107
  _g.label = 5;
10081
10108
  case 5:
10082
- _g.trys.push([5, 11, 12, 13]);
10109
+ if (!!validations_1_1.done) return [3 /*break*/, 15];
10110
+ validation_1 = validations_1_1.value;
10111
+ _g.label = 6;
10112
+ case 6:
10113
+ _g.trys.push([6, 12, 13, 14]);
10083
10114
  e_1 = void 0;
10084
10115
  return [4 /*yield*/, collection.listPipelines()];
10085
- case 6:
10086
- _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10087
- _g.label = 7;
10088
10116
  case 7:
10089
- if (!!_c.done) return [3 /*break*/, 10];
10117
+ _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10118
+ _g.label = 8;
10119
+ case 8:
10120
+ if (!!_c.done) return [3 /*break*/, 11];
10090
10121
  pipelineUrl = _c.value;
10091
10122
  return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
10092
- case 8:
10123
+ case 9:
10093
10124
  pipeline = _g.sent();
10094
10125
  if (validation_1 === 'logic') {
10095
10126
  validatePipeline(pipeline);
@@ -10097,37 +10128,37 @@ function initializeMakeCommand(program) {
10097
10128
  console.info(colors.cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
10098
10129
  }
10099
10130
  }
10100
- _g.label = 9;
10101
- case 9:
10131
+ _g.label = 10;
10132
+ case 10:
10102
10133
  _c = _b.next();
10103
- return [3 /*break*/, 7];
10104
- case 10: return [3 /*break*/, 13];
10105
- case 11:
10134
+ return [3 /*break*/, 8];
10135
+ case 11: return [3 /*break*/, 14];
10136
+ case 12:
10106
10137
  e_1_1 = _g.sent();
10107
10138
  e_1 = { error: e_1_1 };
10108
- return [3 /*break*/, 13];
10109
- case 12:
10139
+ return [3 /*break*/, 14];
10140
+ case 13:
10110
10141
  try {
10111
10142
  if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
10112
10143
  }
10113
10144
  finally { if (e_1) throw e_1.error; }
10114
10145
  return [7 /*endfinally*/];
10115
- case 13:
10146
+ case 14:
10116
10147
  validations_1_1 = validations_1.next();
10117
- return [3 /*break*/, 4];
10118
- case 14: return [3 /*break*/, 17];
10119
- case 15:
10148
+ return [3 /*break*/, 5];
10149
+ case 15: return [3 /*break*/, 18];
10150
+ case 16:
10120
10151
  e_2_1 = _g.sent();
10121
10152
  e_2 = { error: e_2_1 };
10122
- return [3 /*break*/, 17];
10123
- case 16:
10153
+ return [3 /*break*/, 18];
10154
+ case 17:
10124
10155
  try {
10125
10156
  if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
10126
10157
  }
10127
10158
  finally { if (e_2) throw e_2.error; }
10128
10159
  return [7 /*endfinally*/];
10129
- case 17: return [4 /*yield*/, collectionToJson(collection)];
10130
- case 18:
10160
+ case 18: return [4 /*yield*/, collectionToJson(collection)];
10161
+ case 19:
10131
10162
  collectionJson = _g.sent();
10132
10163
  collectionJsonString = stringifyPipelineJson(collectionJson).trim();
10133
10164
  collectionJsonItems = (function () {
@@ -10146,9 +10177,9 @@ function initializeMakeCommand(program) {
10146
10177
  return __generator(this, function (_a) {
10147
10178
  switch (_a.label) {
10148
10179
  case 0:
10149
- filename = outFile !== PIPELINE_COLLECTION_BASE_FILENAME
10180
+ filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
10150
10181
  ? outFile
10151
- : join(path, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10182
+ : join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10152
10183
  if (!outFile.endsWith(".".concat(extension))) {
10153
10184
  console.warn(colors.yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
10154
10185
  }
@@ -10164,27 +10195,27 @@ function initializeMakeCommand(program) {
10164
10195
  }
10165
10196
  });
10166
10197
  }); };
10167
- if (!formats.includes('json')) return [3 /*break*/, 20];
10198
+ if (!formats.includes('json')) return [3 /*break*/, 21];
10168
10199
  formats = formats.filter(function (format) { return format !== 'json'; });
10169
10200
  return [4 /*yield*/, saveFile('json', collectionJsonString)];
10170
- case 19:
10171
- _g.sent();
10172
- _g.label = 20;
10173
10201
  case 20:
10174
- if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 22];
10202
+ _g.sent();
10203
+ _g.label = 21;
10204
+ case 21:
10205
+ if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
10175
10206
  formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
10176
10207
  return [4 /*yield*/, saveFile('js', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
10177
- case 21:
10178
- (_g.sent()) + '\n';
10179
- _g.label = 22;
10180
10208
  case 22:
10181
- if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 24];
10209
+ (_g.sent()) + '\n';
10210
+ _g.label = 23;
10211
+ case 23:
10212
+ if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
10182
10213
  formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
10183
10214
  return [4 /*yield*/, saveFile('ts', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
10184
- case 23:
10185
- _g.sent();
10186
- _g.label = 24;
10187
10215
  case 24:
10216
+ _g.sent();
10217
+ _g.label = 25;
10218
+ case 25:
10188
10219
  if (formats.length > 0) {
10189
10220
  console.warn(colors.yellow("Format ".concat(formats.join(' and '), " is not supported")));
10190
10221
  }
@@ -10446,76 +10477,79 @@ function initializeTestCommand(program) {
10446
10477
  // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument
10447
10478
  'Pipelines to test as glob pattern');
10448
10479
  testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
10449
- testCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache ", false);
10480
+ testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
10450
10481
  testCommand.option('-v, --verbose', "Is output verbose", false);
10451
10482
  testCommand.action(function (filesGlob, _a) {
10452
- var ignore = _a.ignore, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose;
10483
+ var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
10453
10484
  return __awaiter(_this, void 0, void 0, function () {
10454
- var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10485
+ var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10455
10486
  var _d, e_1, _e;
10456
10487
  return __generator(this, function (_f) {
10457
10488
  switch (_f.label) {
10458
10489
  case 0:
10459
10490
  options = {
10460
10491
  isVerbose: isVerbose,
10461
- isCacheCleaned: isCacheCleaned,
10492
+ isCacheReloaded: isCacheReloaded,
10462
10493
  };
10463
10494
  fs = $provideFilesystemForNode(options);
10464
10495
  llm = $provideLlmToolsForCli(options);
10496
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10497
+ case 1:
10498
+ executables = _f.sent();
10465
10499
  _d = {
10466
10500
  llm: llm,
10467
10501
  fs: fs
10468
10502
  };
10469
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10470
- case 1:
10503
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10504
+ case 2:
10471
10505
  tools = (_d.scrapers = _f.sent(),
10472
10506
  _d.script = [
10473
10507
  /*new JavascriptExecutionTools(options)*/
10474
10508
  ],
10475
10509
  _d);
10476
10510
  return [4 /*yield*/, glob(filesGlob, { ignore: ignore })];
10477
- case 2:
10478
- filenames = _f.sent();
10479
- _f.label = 3;
10480
10511
  case 3:
10481
- _f.trys.push([3, 15, 16, 17]);
10482
- filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10512
+ filenames = _f.sent();
10483
10513
  _f.label = 4;
10484
10514
  case 4:
10485
- if (!!filenames_1_1.done) return [3 /*break*/, 14];
10486
- filename = filenames_1_1.value;
10515
+ _f.trys.push([4, 16, 17, 18]);
10516
+ filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10487
10517
  _f.label = 5;
10488
10518
  case 5:
10489
- _f.trys.push([5, 12, , 13]);
10519
+ if (!!filenames_1_1.done) return [3 /*break*/, 15];
10520
+ filename = filenames_1_1.value;
10521
+ _f.label = 6;
10522
+ case 6:
10523
+ _f.trys.push([6, 13, , 14]);
10490
10524
  pipeline = void 0;
10491
- if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 8];
10525
+ if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
10492
10526
  return [4 /*yield*/, readFile(filename, 'utf-8')];
10493
- case 6:
10527
+ case 7:
10494
10528
  pipelineMarkdown = (_f.sent());
10495
10529
  return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
10496
- case 7:
10530
+ case 8:
10497
10531
  pipeline = _f.sent();
10498
10532
  if (isVerbose) {
10499
10533
  console.info(colors.green("Parsed ".concat(filename)));
10500
10534
  }
10501
- _f.label = 8;
10502
- case 8:
10503
- if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 10];
10535
+ _f.label = 9;
10536
+ case 9:
10537
+ if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
10504
10538
  _c = (_b = JSON).parse;
10505
10539
  return [4 /*yield*/, readFile(filename, 'utf-8')];
10506
- case 9:
10507
- pipeline = _c.apply(_b, [_f.sent()]);
10508
- return [3 /*break*/, 11];
10509
10540
  case 10:
10541
+ pipeline = _c.apply(_b, [_f.sent()]);
10542
+ return [3 /*break*/, 12];
10543
+ case 11:
10510
10544
  if (isVerbose) {
10511
10545
  console.info(colors.gray("Skipping ".concat(filename)));
10512
10546
  }
10513
- return [3 /*break*/, 13];
10514
- case 11:
10547
+ return [3 /*break*/, 14];
10548
+ case 12:
10515
10549
  validatePipeline(pipeline);
10516
10550
  console.info(colors.green("Validated ".concat(filename)));
10517
- return [3 /*break*/, 13];
10518
- case 12:
10551
+ return [3 /*break*/, 14];
10552
+ case 13:
10519
10553
  error_1 = _f.sent();
10520
10554
  if (!(error_1 instanceof Error)) {
10521
10555
  throw error_1;
@@ -10524,22 +10558,22 @@ function initializeTestCommand(program) {
10524
10558
  console.error(colors.bgRed(error_1.name /* <- 11:11 */));
10525
10559
  console.error(colors.red(error_1.stack || error_1.message));
10526
10560
  process.exit(1);
10527
- return [3 /*break*/, 13];
10528
- case 13:
10561
+ return [3 /*break*/, 14];
10562
+ case 14:
10529
10563
  filenames_1_1 = filenames_1.next();
10530
- return [3 /*break*/, 4];
10531
- case 14: return [3 /*break*/, 17];
10532
- case 15:
10564
+ return [3 /*break*/, 5];
10565
+ case 15: return [3 /*break*/, 18];
10566
+ case 16:
10533
10567
  e_1_1 = _f.sent();
10534
10568
  e_1 = { error: e_1_1 };
10535
- return [3 /*break*/, 17];
10536
- case 16:
10569
+ return [3 /*break*/, 18];
10570
+ case 17:
10537
10571
  try {
10538
10572
  if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
10539
10573
  }
10540
10574
  finally { if (e_1) throw e_1.error; }
10541
10575
  return [7 /*endfinally*/];
10542
- case 17:
10576
+ case 18:
10543
10577
  console.info(colors.green("All pipelines are valid"));
10544
10578
  process.exit(0);
10545
10579
  return [2 /*return*/];
@@ -13010,12 +13044,12 @@ function $execCommand(options) {
13010
13044
  */
13011
13045
  function getScraperIntermediateSource(source, options) {
13012
13046
  return __awaiter(this, void 0, void 0, function () {
13013
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13047
+ var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13014
13048
  return __generator(this, function (_a) {
13015
13049
  switch (_a.label) {
13016
13050
  case 0:
13017
13051
  sourceFilename = source.filename, url = source.url;
13018
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
13052
+ rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
13019
13053
  hash = SHA256(
13020
13054
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
13021
13055
  hexEncoder.parse(sourceFilename || url || 'untitled'))
@@ -13025,7 +13059,7 @@ function getScraperIntermediateSource(source, options) {
13025
13059
  pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
13026
13060
  name = pieces.join('-').split('--').join('-');
13027
13061
  // <- TODO: Use MAX_FILENAME_LENGTH
13028
- TODO_USE(rootDirname); // <- TODO: !!!!!!
13062
+ TODO_USE(rootDirname); // <- TODO: [😡]
13029
13063
  cacheFilename = join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
13030
13064
  cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
13031
13065
  .join('/') +
@@ -13045,7 +13079,7 @@ function getScraperIntermediateSource(source, options) {
13045
13079
  return __generator(this, function (_a) {
13046
13080
  switch (_a.label) {
13047
13081
  case 0:
13048
- if (!isCacheCleaned) return [3 /*break*/, 2];
13082
+ if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
13049
13083
  if (isVerbose) {
13050
13084
  console.info('legacyDocumentScraper: Clening cache');
13051
13085
  }
@@ -13069,7 +13103,7 @@ function getScraperIntermediateSource(source, options) {
13069
13103
  /**
13070
13104
  * Note: Not using `FileCacheStorage` for two reasons:
13071
13105
  * 1) Need to store more than serialized JSONs
13072
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
13106
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
13073
13107
  * TODO: [🐱‍🐉][🧠] Make some smart crop
13074
13108
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
13075
13109
  */
@@ -13086,7 +13120,7 @@ var markdownScraperMetadata = $deepFreeze({
13086
13120
  mimeTypes: ['text/markdown', 'text/plain'],
13087
13121
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13088
13122
  isAvilableInBrowser: true,
13089
- requiredExecutables: ['!!!!!!'],
13123
+ requiredExecutables: [],
13090
13124
  }); /* <- TODO: [🤛] */
13091
13125
  /**
13092
13126
  * Registration of known scraper metadata
@@ -13130,7 +13164,7 @@ var MarkdownScraper = /** @class */ (function () {
13130
13164
  return __generator(this, function (_k) {
13131
13165
  switch (_k.label) {
13132
13166
  case 0:
13133
- _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
13167
+ _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
13134
13168
  llm = this.tools.llm;
13135
13169
  if (llm === undefined) {
13136
13170
  throw new MissingToolsError('LLM tools are required for scraping external files');
@@ -13229,7 +13263,8 @@ var MarkdownScraper = /** @class */ (function () {
13229
13263
  embeddingResult = _c.sent();
13230
13264
  index.push({
13231
13265
  modelName: embeddingResult.modelName,
13232
- position: embeddingResult.content,
13266
+ position: __spreadArray([], __read(embeddingResult.content), false),
13267
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
13233
13268
  });
13234
13269
  _c.label = 6;
13235
13270
  case 6: return [3 /*break*/, 8];
@@ -13280,7 +13315,7 @@ var documentScraperMetadata = $deepFreeze({
13280
13315
  mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
13281
13316
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13282
13317
  isAvilableInBrowser: false,
13283
- requiredExecutables: ['!!!!!!'],
13318
+ requiredExecutables: ['Pandoc'],
13284
13319
  }); /* <- TODO: [🤛] */
13285
13320
  /**
13286
13321
  * Registration of known scraper metadata
@@ -13320,12 +13355,13 @@ var DocumentScraper = /** @class */ (function () {
13320
13355
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13321
13356
  */
13322
13357
  DocumentScraper.prototype.$convert = function (source) {
13358
+ var _a;
13323
13359
  return __awaiter(this, void 0, void 0, function () {
13324
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, command_1;
13360
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
13325
13361
  return __generator(this, function (_g) {
13326
13362
  switch (_g.label) {
13327
13363
  case 0:
13328
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13364
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13329
13365
  if (!$isRunningInNode()) {
13330
13366
  throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
13331
13367
  }
@@ -13333,7 +13369,7 @@ var DocumentScraper = /** @class */ (function () {
13333
13369
  throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
13334
13370
  // <- TODO: [🧠] What is the best error type here`
13335
13371
  }
13336
- if (externalProgramsPaths.pandocPath === undefined) {
13372
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
13337
13373
  throw new MissingToolsError('Pandoc is required for scraping .docx files');
13338
13374
  }
13339
13375
  if (source.filename === null) {
@@ -13344,7 +13380,7 @@ var DocumentScraper = /** @class */ (function () {
13344
13380
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13345
13381
  rootDirname: rootDirname,
13346
13382
  cacheDirname: cacheDirname,
13347
- isCacheCleaned: isCacheCleaned,
13383
+ intermediateFilesStrategy: intermediateFilesStrategy,
13348
13384
  extension: 'md',
13349
13385
  isVerbose: isVerbose,
13350
13386
  })];
@@ -13353,11 +13389,9 @@ var DocumentScraper = /** @class */ (function () {
13353
13389
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13354
13390
  case 2:
13355
13391
  if (!!(_g.sent())) return [3 /*break*/, 5];
13356
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13357
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13392
+ command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13358
13393
  return [4 /*yield*/, $execCommand(command_1)];
13359
13394
  case 3:
13360
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13361
13395
  _g.sent();
13362
13396
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13363
13397
  case 4:
@@ -13436,7 +13470,11 @@ var legacyDocumentScraperMetadata = $deepFreeze({
13436
13470
  mimeTypes: ['application/msword', 'text/rtf'],
13437
13471
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13438
13472
  isAvilableInBrowser: false,
13439
- requiredExecutables: ['!!!!!!'],
13473
+ requiredExecutables: [
13474
+ 'Pandoc',
13475
+ 'LibreOffice',
13476
+ // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
13477
+ ],
13440
13478
  }); /* <- TODO: [🤛] */
13441
13479
  /**
13442
13480
  * Registration of known scraper metadata
@@ -13449,7 +13487,7 @@ var legacyDocumentScraperMetadata = $deepFreeze({
13449
13487
  var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
13450
13488
 
13451
13489
  /**
13452
- * Scraper for .docx files
13490
+ * Scraper for old document files (like .doc and .rtf)
13453
13491
  *
13454
13492
  * @see `documentationUrl` for more details
13455
13493
  * @public exported from `@promptbook/legacy-documents`
@@ -13476,12 +13514,13 @@ var LegacyDocumentScraper = /** @class */ (function () {
13476
13514
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13477
13515
  */
13478
13516
  LegacyDocumentScraper.prototype.$convert = function (source) {
13517
+ var _a;
13479
13518
  return __awaiter(this, void 0, void 0, function () {
13480
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13519
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13481
13520
  return __generator(this, function (_g) {
13482
13521
  switch (_g.label) {
13483
13522
  case 0:
13484
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13523
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13485
13524
  if (!$isRunningInNode()) {
13486
13525
  throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
13487
13526
  }
@@ -13489,7 +13528,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
13489
13528
  throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
13490
13529
  // <- TODO: [🧠] What is the best error type here`
13491
13530
  }
13492
- if (externalProgramsPaths.libreOfficePath === undefined) {
13531
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
13493
13532
  throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
13494
13533
  }
13495
13534
  if (source.filename === null) {
@@ -13500,7 +13539,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
13500
13539
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13501
13540
  rootDirname: rootDirname,
13502
13541
  cacheDirname: cacheDirname,
13503
- isCacheCleaned: isCacheCleaned,
13542
+ intermediateFilesStrategy: intermediateFilesStrategy,
13504
13543
  extension: 'docx',
13505
13544
  isVerbose: isVerbose,
13506
13545
  })];
@@ -13515,11 +13554,9 @@ var LegacyDocumentScraper = /** @class */ (function () {
13515
13554
  documentSourceOutdirPathForLibreOffice_1 = join(dirname(cacheFilehandler.filename), 'libreoffice')
13516
13555
  .split('\\')
13517
13556
  .join('/');
13518
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13519
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13557
+ command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13520
13558
  return [4 /*yield*/, $execCommand(command_1)];
13521
13559
  case 3:
13522
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13523
13560
  _g.sent();
13524
13561
  return [4 /*yield*/, readdir(documentSourceOutdirPathForLibreOffice_1)];
13525
13562
  case 4:
@@ -13676,7 +13713,7 @@ var pdfScraperMetadata = $deepFreeze({
13676
13713
  mimeTypes: ['application/pdf'],
13677
13714
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13678
13715
  isAvilableInBrowser: true,
13679
- requiredExecutables: ['!!!!!!'],
13716
+ requiredExecutables: [],
13680
13717
  }); /* <- TODO: [🤛] */
13681
13718
  /**
13682
13719
  * Registration of known scraper metadata
@@ -13689,7 +13726,7 @@ var pdfScraperMetadata = $deepFreeze({
13689
13726
  var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
13690
13727
 
13691
13728
  /**
13692
- * Scraper for .docx files
13729
+ * Scraper for .pdf files
13693
13730
  *
13694
13731
  * @see `documentationUrl` for more details
13695
13732
  * @public exported from `@promptbook/pdf`
@@ -13732,9 +13769,8 @@ var PdfScraper = /** @class */ (function () {
13732
13769
  TODO_USE(this.options);
13733
13770
  /*
13734
13771
  const {
13735
- externalProgramsPaths = {},
13736
13772
  cacheDirname = SCRAPE_CACHE_DIRNAME,
13737
- isCacheCleaned = false,
13773
+ intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
13738
13774
  isVerbose = IS_VERBOSE,
13739
13775
  } = options;
13740
13776
  */
@@ -13787,7 +13823,7 @@ var websiteScraperMetadata = $deepFreeze({
13787
13823
  mimeTypes: ['text/html'],
13788
13824
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13789
13825
  isAvilableInBrowser: false,
13790
- requiredExecutables: ['!!!!!!'],
13826
+ requiredExecutables: [],
13791
13827
  }); /* <- TODO: [🤛] */
13792
13828
  /**
13793
13829
  * Registration of known scraper metadata
@@ -13800,32 +13836,29 @@ var websiteScraperMetadata = $deepFreeze({
13800
13836
  var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
13801
13837
 
13802
13838
  /**
13803
- * A converter instance that uses showdown and highlight extensions
13839
+ * Create a new showdown converter instance
13804
13840
  *
13805
- * @type {Converter}
13806
- * @private for markdown and html knowledge scrapers
13807
- */
13808
- var markdownConverter = new Converter({
13809
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
13810
- /*
13811
- > import showdownHighlight from 'showdown-highlight';
13812
- > extensions: [
13813
- > showdownHighlight({
13814
- > // Whether to add the classes to the <pre> tag, default is false
13815
- > pre: true,
13816
- > // Whether to use hljs' auto language detection, default is true
13817
- > auto_detection: true,
13818
- > }),
13819
- > ],
13820
- */
13821
- });
13822
- /**
13823
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
13824
- * TODO: !!!!!! Lazy-make converter
13841
+ * @private utility of `WebsiteScraper`
13825
13842
  */
13843
+ function createShowdownConverter() {
13844
+ return new Converter({
13845
+ flavor: 'github',
13846
+ /*
13847
+ > import showdownHighlight from 'showdown-highlight';
13848
+ > extensions: [
13849
+ > showdownHighlight({
13850
+ > // Whether to add the classes to the <pre> tag, default is false
13851
+ > pre: true,
13852
+ > // Whether to use hljs' auto language detection, default is true
13853
+ > auto_detection: true,
13854
+ > }),
13855
+ > ],
13856
+ */
13857
+ });
13858
+ }
13826
13859
 
13827
13860
  /**
13828
- * Scraper for .docx files
13861
+ * Scraper for websites
13829
13862
  *
13830
13863
  * @see `documentationUrl` for more details
13831
13864
  * @public exported from `@promptbook/website-crawler`
@@ -13835,6 +13868,7 @@ var WebsiteScraper = /** @class */ (function () {
13835
13868
  this.tools = tools;
13836
13869
  this.options = options;
13837
13870
  this.markdownScraper = new MarkdownScraper(tools, options);
13871
+ this.showdownConverter = createShowdownConverter();
13838
13872
  }
13839
13873
  Object.defineProperty(WebsiteScraper.prototype, "metadata", {
13840
13874
  /**
@@ -13855,12 +13889,11 @@ var WebsiteScraper = /** @class */ (function () {
13855
13889
  return __awaiter(this, void 0, void 0, function () {
13856
13890
  var _a, _b,
13857
13891
  // TODO: [🧠] Maybe in node use headless browser not just JSDOM
13858
- // externalProgramsPaths = {},
13859
- rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
13892
+ rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
13860
13893
  return __generator(this, function (_g) {
13861
13894
  switch (_g.label) {
13862
13895
  case 0:
13863
- _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? SCRAPE_CACHE_DIRNAME : _c, _d = _a.isCacheCleaned, isCacheCleaned = _d === void 0 ? false : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e;
13896
+ _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
13864
13897
  // TODO: !!!!!! Does this work in browser? Make it work.
13865
13898
  if (source.url === null) {
13866
13899
  throw new KnowledgeScrapeError('Website scraper requires URL');
@@ -13873,10 +13906,6 @@ var WebsiteScraper = /** @class */ (function () {
13873
13906
  }]))();
13874
13907
  reader = new Readability(jsdom.window.document);
13875
13908
  article = reader.parse();
13876
- console.log(article);
13877
- return [4 /*yield*/, forTime(10000)];
13878
- case 2:
13879
- _g.sent();
13880
13909
  html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
13881
13910
  // Note: Unwrap html such as it is convertable by `markdownConverter`
13882
13911
  for (i = 0; i < 2; i++) {
@@ -13888,16 +13917,16 @@ var WebsiteScraper = /** @class */ (function () {
13888
13917
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13889
13918
  rootDirname: rootDirname,
13890
13919
  cacheDirname: cacheDirname,
13891
- isCacheCleaned: isCacheCleaned,
13920
+ intermediateFilesStrategy: intermediateFilesStrategy,
13892
13921
  extension: 'html',
13893
13922
  isVerbose: isVerbose,
13894
13923
  })];
13895
- case 3:
13924
+ case 2:
13896
13925
  cacheFilehandler = _g.sent();
13897
13926
  return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
13898
- case 4:
13927
+ case 3:
13899
13928
  _g.sent();
13900
- markdown = markdownConverter.makeMarkdown(html, jsdom.window.document);
13929
+ markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
13901
13930
  return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
13902
13931
  }
13903
13932
  });
@@ -13941,7 +13970,6 @@ var WebsiteScraper = /** @class */ (function () {
13941
13970
  return WebsiteScraper;
13942
13971
  }());
13943
13972
  /**
13944
- * TODO: !!!!!! Put into separate package
13945
13973
  * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
13946
13974
  * TODO: [🪂] Do it in parallel 11:11
13947
13975
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools