@promptbook/cli 0.72.0-14 → 0.72.0-15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +1 -1
  2. package/esm/index.es.js +544 -516
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +22 -18
  5. package/esm/typings/src/_packages/node.index.d.ts +2 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +10 -10
  7. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  8. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  10. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  11. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  12. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  14. package/esm/typings/src/config.d.ts +21 -14
  15. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  16. package/esm/typings/src/execution/Executables.d.ts +18 -0
  17. package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
  18. package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
  19. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  20. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
  21. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  22. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
  23. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
  24. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
  25. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  26. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  27. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  28. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  29. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  30. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
  31. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
  32. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  33. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  36. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
  38. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
  39. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
  40. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  41. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +3 -3
  45. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +2 -2
  46. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
  47. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  48. package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -5
  49. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
  50. package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
  51. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
  52. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
  53. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
  54. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
  55. package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
  56. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  57. package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
  58. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
  59. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
  60. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
  61. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
  62. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
  63. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
  64. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
  65. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
  66. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
  67. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
  68. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
  69. package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
  70. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  71. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  72. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  73. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  74. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  75. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  76. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  77. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  78. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  79. package/esm/typings/src/types/Prompt.d.ts +1 -1
  80. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  81. package/esm/typings/src/utils/$Register.d.ts +1 -1
  82. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  83. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  84. package/esm/typings/src/utils/emojis.d.ts +1 -1
  85. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
  86. package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
  87. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
  88. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  89. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  90. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  91. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  92. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  93. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  94. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  95. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  96. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  97. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  98. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  99. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  100. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  101. package/package.json +3 -2
  102. package/umd/index.umd.js +544 -516
  103. package/umd/index.umd.js.map +1 -1
  104. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  105. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  106. /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/umd/index.umd.js CHANGED
@@ -39,7 +39,7 @@
39
39
  /**
40
40
  * The version of the Promptbook library
41
41
  */
42
- var PROMPTBOOK_VERSION = '0.72.0-13';
42
+ var PROMPTBOOK_VERSION = '0.72.0-14';
43
43
  // TODO: [main] !!!! List here all the versions and annotate + put into script
44
44
 
45
45
  /*! *****************************************************************************
@@ -412,7 +412,8 @@
412
412
  *
413
413
  * @public exported from `@promptbook/core`
414
414
  */
415
- var CLAIM = "Supercharge LLM models with Promptbook";
415
+ var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
416
+ // <- TODO: [🐊] Pick the best claim
416
417
  /**
417
418
  * Warning message for the generated sections and files files
418
419
  *
@@ -445,23 +446,30 @@
445
446
  */
446
447
  var IMMEDIATE_TIME = 10;
447
448
  /**
448
- * The maximum number of (LLM) tasks running in parallel
449
+ * The maximum length of the (generated) filename
449
450
  *
450
451
  * @public exported from `@promptbook/core`
451
452
  */
452
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
453
+ var MAX_FILENAME_LENGTH = 30;
453
454
  /**
454
- * The maximum number of attempts to execute LLM task before giving up
455
+ * Strategy for caching the intermediate results for knowledge sources
455
456
  *
456
457
  * @public exported from `@promptbook/core`
457
458
  */
458
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
459
+ var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
460
+ // <- TODO: [😡] Change to 'VISIBLE'
459
461
  /**
460
- * The maximum length of the (generated) filename
462
+ * The maximum number of (LLM) tasks running in parallel
461
463
  *
462
464
  * @public exported from `@promptbook/core`
463
465
  */
464
- var MAX_FILENAME_LENGTH = 30;
466
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
467
+ /**
468
+ * The maximum number of attempts to execute LLM task before giving up
469
+ *
470
+ * @public exported from `@promptbook/core`
471
+ */
472
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
465
473
  /**
466
474
  * Where to store the cache of executions for promptbook CLI
467
475
  *
@@ -469,7 +477,7 @@
469
477
  *
470
478
  * @public exported from `@promptbook/core`
471
479
  */
472
- var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
480
+ var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
473
481
  /**
474
482
  * Where to store the scrape cache
475
483
  *
@@ -477,13 +485,13 @@
477
485
  *
478
486
  * @public exported from `@promptbook/core`
479
487
  */
480
- var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
488
+ var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
481
489
  /**
482
490
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
483
491
  *
484
492
  * @public exported from `@promptbook/core`
485
493
  */
486
- var PIPELINE_COLLECTION_BASE_FILENAME = "index";
494
+ var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
487
495
  /**
488
496
  * Nonce which is used for replacing things in strings
489
497
  *
@@ -547,13 +555,13 @@
547
555
  *
548
556
  * @public exported from `@promptbook/core`
549
557
  */
550
- var IS_VERBOSE = false;
558
+ var DEFAULT_IS_VERBOSE = false;
551
559
  /**
552
560
  * @@@
553
561
  *
554
562
  * @public exported from `@promptbook/core`
555
563
  */
556
- var IS_AUTO_INSTALLED = false;
564
+ var DEFAULT_IS_AUTO_INSTALLED = false;
557
565
  /**
558
566
  * @@@
559
567
  *
@@ -626,271 +634,6 @@
626
634
  * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
627
635
  */
628
636
 
629
- /**
630
- * Just marks a place of place where should be something implemented
631
- * No side effects.
632
- *
633
- * Note: It can be usefull suppressing eslint errors of unused variables
634
- *
635
- * @param value any values
636
- * @returns void
637
- * @private within the repository
638
- */
639
- function TODO_USE() {
640
- var value = [];
641
- for (var _i = 0; _i < arguments.length; _i++) {
642
- value[_i] = arguments[_i];
643
- }
644
- }
645
-
646
- /**
647
- * @@@
648
- *
649
- * @public exported from `@promptbook/node`
650
- */
651
- function $provideFilesystemForNode(options) {
652
- if (!$isRunningInNode()) {
653
- throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
654
- }
655
- var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
656
- TODO_USE(isVerbose);
657
- return {
658
- stat: promises.stat,
659
- access: promises.access,
660
- constants: promises.constants,
661
- readFile: promises.readFile,
662
- readdir: promises.readdir,
663
- };
664
- }
665
- /**
666
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
667
- */
668
-
669
- /**
670
- * This error type indicates that some part of the code is not implemented yet
671
- *
672
- * @public exported from `@promptbook/core`
673
- */
674
- var NotYetImplementedError = /** @class */ (function (_super) {
675
- __extends(NotYetImplementedError, _super);
676
- function NotYetImplementedError(message) {
677
- var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
678
- _this.name = 'NotYetImplementedError';
679
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
680
- return _this;
681
- }
682
- return NotYetImplementedError;
683
- }(Error));
684
-
685
- /**
686
- * @@@
687
- *
688
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
689
- *
690
- * @private internal function of `$Register`
691
- */
692
- function $getGlobalScope() {
693
- return Function('return this')();
694
- }
695
-
696
- /**
697
- * @@@
698
- *
699
- * @param text @@@
700
- * @returns @@@
701
- * @example 'HELLO_WORLD'
702
- * @example 'I_LOVE_PROMPTBOOK'
703
- * @public exported from `@promptbook/utils`
704
- */
705
- function normalizeTo_SCREAMING_CASE(text) {
706
- var e_1, _a;
707
- var charType;
708
- var lastCharType = 'OTHER';
709
- var normalizedName = '';
710
- try {
711
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
712
- var char = text_1_1.value;
713
- var normalizedChar = void 0;
714
- if (/^[a-z]$/.test(char)) {
715
- charType = 'LOWERCASE';
716
- normalizedChar = char.toUpperCase();
717
- }
718
- else if (/^[A-Z]$/.test(char)) {
719
- charType = 'UPPERCASE';
720
- normalizedChar = char;
721
- }
722
- else if (/^[0-9]$/.test(char)) {
723
- charType = 'NUMBER';
724
- normalizedChar = char;
725
- }
726
- else {
727
- charType = 'OTHER';
728
- normalizedChar = '_';
729
- }
730
- if (charType !== lastCharType &&
731
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
732
- !(lastCharType === 'NUMBER') &&
733
- !(charType === 'NUMBER')) {
734
- normalizedName += '_';
735
- }
736
- normalizedName += normalizedChar;
737
- lastCharType = charType;
738
- }
739
- }
740
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
741
- finally {
742
- try {
743
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
744
- }
745
- finally { if (e_1) throw e_1.error; }
746
- }
747
- normalizedName = normalizedName.replace(/_+/g, '_');
748
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
749
- normalizedName = normalizedName.replace(/^_/, '');
750
- normalizedName = normalizedName.replace(/_$/, '');
751
- return normalizedName;
752
- }
753
- /**
754
- * TODO: Tests
755
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
756
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
757
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
758
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
759
- * TODO: [🌺] Use some intermediate util splitWords
760
- */
761
-
762
- /**
763
- * @@@
764
- *
765
- * @param text @@@
766
- * @returns @@@
767
- * @example 'hello_world'
768
- * @example 'i_love_promptbook'
769
- * @public exported from `@promptbook/utils`
770
- */
771
- function normalizeTo_snake_case(text) {
772
- return normalizeTo_SCREAMING_CASE(text).toLowerCase();
773
- }
774
-
775
- /**
776
- * Register is @@@
777
- *
778
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
779
- *
780
- * @private internal utility, exported are only signleton instances of this class
781
- */
782
- var $Register = /** @class */ (function () {
783
- function $Register(registerName) {
784
- this.registerName = registerName;
785
- var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
786
- var globalScope = $getGlobalScope();
787
- if (globalScope[storageName] === undefined) {
788
- globalScope[storageName] = [];
789
- }
790
- else if (!Array.isArray(globalScope[storageName])) {
791
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
792
- }
793
- this.storage = globalScope[storageName];
794
- }
795
- $Register.prototype.list = function () {
796
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
797
- return this.storage;
798
- };
799
- $Register.prototype.register = function (registered) {
800
- var packageName = registered.packageName, className = registered.className;
801
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
802
- var existingRegistration = this.storage[existingRegistrationIndex];
803
- if (!existingRegistration) {
804
- this.storage.push(registered);
805
- }
806
- else {
807
- this.storage[existingRegistrationIndex] = registered;
808
- }
809
- return {
810
- registerName: this.registerName,
811
- packageName: packageName,
812
- className: className,
813
- get isDestroyed() {
814
- return false;
815
- },
816
- destroy: function () {
817
- throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
818
- },
819
- };
820
- };
821
- return $Register;
822
- }());
823
-
824
- /**
825
- * @@@
826
- *
827
- * Note: `$` is used to indicate that this interacts with the global scope
828
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
829
- * @public exported from `@promptbook/core`
830
- */
831
- var $scrapersRegister = new $Register('scraper_constructors');
832
- /**
833
- * TODO: [®] DRY Register logic
834
- */
835
-
836
- /**
837
- * !!!!!!
838
- *
839
- * 1) @@@
840
- * 2) @@@
841
- *
842
- * @public exported from `@promptbook/node`
843
- */
844
- function $provideScrapersForNode(tools, options) {
845
- return __awaiter(this, void 0, void 0, function () {
846
- var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
847
- var e_1, _f;
848
- return __generator(this, function (_g) {
849
- switch (_g.label) {
850
- case 0:
851
- if (!$isRunningInNode()) {
852
- throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
853
- }
854
- _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
855
- TODO_USE(isAutoInstalled);
856
- TODO_USE(isVerbose);
857
- scrapers = [];
858
- _g.label = 1;
859
- case 1:
860
- _g.trys.push([1, 6, 7, 8]);
861
- _d = __values($scrapersRegister.list()), _e = _d.next();
862
- _g.label = 2;
863
- case 2:
864
- if (!!_e.done) return [3 /*break*/, 5];
865
- scraperFactory = _e.value;
866
- return [4 /*yield*/, scraperFactory(tools, options || {})];
867
- case 3:
868
- scraper = _g.sent();
869
- scrapers.push(scraper);
870
- _g.label = 4;
871
- case 4:
872
- _e = _d.next();
873
- return [3 /*break*/, 2];
874
- case 5: return [3 /*break*/, 8];
875
- case 6:
876
- e_1_1 = _g.sent();
877
- e_1 = { error: e_1_1 };
878
- return [3 /*break*/, 8];
879
- case 7:
880
- try {
881
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
882
- }
883
- finally { if (e_1) throw e_1.error; }
884
- return [7 /*endfinally*/];
885
- case 8: return [2 /*return*/, scrapers];
886
- }
887
- });
888
- });
889
- }
890
- /**
891
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
892
- */
893
-
894
637
  /**
895
638
  * Converts PipelineCollection to serialized JSON
896
639
  *
@@ -976,7 +719,6 @@
976
719
  pipelineString += '\n\n';
977
720
  pipelineString += description;
978
721
  }
979
- // TODO:> const commands: Array<Command>
980
722
  var commands = [];
981
723
  if (pipelineUrl) {
982
724
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -1032,7 +774,6 @@
1032
774
  pipelineString += '\n\n';
1033
775
  pipelineString += description_1;
1034
776
  }
1035
- // TODO:> const commands: Array<Command>
1036
777
  var commands_1 = [];
1037
778
  var contentLanguage = 'text';
1038
779
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -1331,8 +1072,7 @@
1331
1072
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1332
1073
  *
1333
1074
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1334
- * @public exported from `@promptbook/types`
1335
- * TODO: !!!!!! Export as runtime class not just type
1075
+ * @public exported from `@promptbook/core`
1336
1076
  */
1337
1077
  var MultipleLlmExecutionTools = /** @class */ (function () {
1338
1078
  /**
@@ -2491,6 +2231,22 @@
2491
2231
  return LimitReachedError;
2492
2232
  }(Error));
2493
2233
 
2234
+ /**
2235
+ * This error type indicates that some part of the code is not implemented yet
2236
+ *
2237
+ * @public exported from `@promptbook/core`
2238
+ */
2239
+ var NotYetImplementedError = /** @class */ (function (_super) {
2240
+ __extends(NotYetImplementedError, _super);
2241
+ function NotYetImplementedError(message) {
2242
+ var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2243
+ _this.name = 'NotYetImplementedError';
2244
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2245
+ return _this;
2246
+ }
2247
+ return NotYetImplementedError;
2248
+ }(Error));
2249
+
2494
2250
  /**
2495
2251
  * Index of all custom errors
2496
2252
  *
@@ -2552,6 +2308,7 @@
2552
2308
  }
2553
2309
  }
2554
2310
  /**
2311
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2555
2312
  * TODO: [🧠] Can this return type be better typed than void
2556
2313
  */
2557
2314
 
@@ -2809,6 +2566,23 @@
2809
2566
  return union;
2810
2567
  }
2811
2568
 
2569
+ /**
2570
+ * Just marks a place of place where should be something implemented
2571
+ * No side effects.
2572
+ *
2573
+ * Note: It can be usefull suppressing eslint errors of unused variables
2574
+ *
2575
+ * @param value any values
2576
+ * @returns void
2577
+ * @private within the repository
2578
+ */
2579
+ function TODO_USE() {
2580
+ var value = [];
2581
+ for (var _i = 0; _i < arguments.length; _i++) {
2582
+ value[_i] = arguments[_i];
2583
+ }
2584
+ }
2585
+
2812
2586
  /**
2813
2587
  * This error indicates problems parsing the format value
2814
2588
  *
@@ -3841,12 +3615,11 @@
3841
3615
  */
3842
3616
  function executeAttempts(options) {
3843
3617
  return __awaiter(this, void 0, void 0, function () {
3844
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3618
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3845
3619
  return __generator(this, function (_a) {
3846
3620
  switch (_a.label) {
3847
3621
  case 0:
3848
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3849
- maxExecutionAttempts = settings.maxExecutionAttempts;
3622
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
3850
3623
  $ongoingTemplateResult = {
3851
3624
  $result: null,
3852
3625
  $resultString: null,
@@ -4212,12 +3985,12 @@
4212
3985
  */
4213
3986
  function executeFormatSubvalues(options) {
4214
3987
  return __awaiter(this, void 0, void 0, function () {
4215
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3988
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4216
3989
  var _this = this;
4217
3990
  return __generator(this, function (_a) {
4218
3991
  switch (_a.label) {
4219
3992
  case 0:
4220
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3993
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4221
3994
  if (template.foreach === undefined) {
4222
3995
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4223
3996
  }
@@ -4247,7 +4020,7 @@
4247
4020
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4248
4021
  }
4249
4022
  if (formatDefinition.formatName === 'CSV') {
4250
- formatSettings = settings.csvSettings;
4023
+ formatSettings = csvSettings;
4251
4024
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4252
4025
  }
4253
4026
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4400,13 +4173,12 @@
4400
4173
  */
4401
4174
  function executeTemplate(options) {
4402
4175
  return __awaiter(this, void 0, void 0, function () {
4403
- var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4404
- var e_1, _f, _g;
4405
- return __generator(this, function (_h) {
4406
- switch (_h.label) {
4176
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4177
+ var e_1, _g, _h;
4178
+ return __generator(this, function (_j) {
4179
+ switch (_j.label) {
4407
4180
  case 0:
4408
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4409
- maxExecutionAttempts = settings.maxExecutionAttempts;
4181
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4410
4182
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4411
4183
  title = currentTemplate.title;
4412
4184
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -4421,7 +4193,7 @@
4421
4193
  // <- [🍸]
4422
4194
  })];
4423
4195
  case 1:
4424
- _h.sent();
4196
+ _j.sent();
4425
4197
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
4426
4198
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
4427
4199
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4432,15 +4204,15 @@
4432
4204
  .map(function (name) { return "{".concat(name, "}"); })
4433
4205
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4434
4206
  }
4435
- _b = (_a = Object).freeze;
4436
- _c = [{}];
4207
+ _c = (_b = Object).freeze;
4208
+ _d = [{}];
4437
4209
  return [4 /*yield*/, getReservedParametersForTemplate({
4438
4210
  preparedPipeline: preparedPipeline,
4439
4211
  template: currentTemplate,
4440
4212
  pipelineIdentification: pipelineIdentification,
4441
4213
  })];
4442
4214
  case 2:
4443
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4215
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
4444
4216
  definedParameterNames = new Set(Object.keys(definedParameters));
4445
4217
  parameters = {};
4446
4218
  _loop_1 = function (parameterName) {
@@ -4460,15 +4232,15 @@
4460
4232
  try {
4461
4233
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4462
4234
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4463
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4464
- parameterName = _e.value;
4235
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
4236
+ parameterName = _f.value;
4465
4237
  _loop_1(parameterName);
4466
4238
  }
4467
4239
  }
4468
4240
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4469
4241
  finally {
4470
4242
  try {
4471
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4243
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
4472
4244
  }
4473
4245
  finally { if (e_1) throw e_1.error; }
4474
4246
  }
@@ -4488,12 +4260,11 @@
4488
4260
  template: currentTemplate,
4489
4261
  preparedPipeline: preparedPipeline,
4490
4262
  tools: tools,
4491
- settings: settings,
4492
4263
  $executionReport: $executionReport,
4493
4264
  pipelineIdentification: pipelineIdentification,
4494
4265
  })];
4495
4266
  case 3:
4496
- resultString = _h.sent();
4267
+ resultString = _j.sent();
4497
4268
  return [4 /*yield*/, onProgress({
4498
4269
  name: name,
4499
4270
  title: title,
@@ -4505,12 +4276,12 @@
4505
4276
  // <- [🍸]
4506
4277
  })];
4507
4278
  case 4:
4508
- _h.sent();
4509
- return [2 /*return*/, Object.freeze((_g = {},
4510
- _g[currentTemplate.resultingParameterName] =
4279
+ _j.sent();
4280
+ return [2 /*return*/, Object.freeze((_h = {},
4281
+ _h[currentTemplate.resultingParameterName] =
4511
4282
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4512
4283
  resultString,
4513
- _g))];
4284
+ _h))];
4514
4285
  }
4515
4286
  });
4516
4287
  });
@@ -4569,13 +4340,12 @@
4569
4340
  */
4570
4341
  function executePipeline(options) {
4571
4342
  return __awaiter(this, void 0, void 0, function () {
4572
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4343
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4573
4344
  var e_1, _f, e_2, _g;
4574
4345
  return __generator(this, function (_h) {
4575
4346
  switch (_h.label) {
4576
4347
  case 0:
4577
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4578
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4348
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4579
4349
  preparedPipeline = options.preparedPipeline;
4580
4350
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4581
4351
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -4760,12 +4530,7 @@
4760
4530
  return [3 /*break*/, 4];
4761
4531
  case 3:
4762
4532
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
4763
- work_1 = executeTemplate({
4764
- currentTemplate: currentTemplate,
4765
- preparedPipeline: preparedPipeline,
4766
- parametersToPass: parametersToPass,
4767
- tools: tools,
4768
- onProgress: function (progress) {
4533
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
4769
4534
  if (isReturned) {
4770
4535
  throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4771
4536
  .split('\n')
@@ -4775,11 +4540,7 @@
4775
4540
  if (onProgress) {
4776
4541
  onProgress(progress);
4777
4542
  }
4778
- },
4779
- settings: settings,
4780
- $executionReport: executionReport,
4781
- pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
4782
- })
4543
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
4783
4544
  .then(function (newParametersToPass) {
4784
4545
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
4785
4546
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -4883,8 +4644,7 @@
4883
4644
  */
4884
4645
  function createPipelineExecutor(options) {
4885
4646
  var _this = this;
4886
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
4887
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
4647
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
4888
4648
  validatePipeline(pipeline);
4889
4649
  var pipelineIdentification = (function () {
4890
4650
  // Note: This is a 😐 implementation of [🚞]
@@ -4918,14 +4678,12 @@
4918
4678
  tools: tools,
4919
4679
  onProgress: onProgress,
4920
4680
  pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
4921
- settings: {
4922
- maxExecutionAttempts: maxExecutionAttempts,
4923
- maxParallelCount: maxParallelCount,
4924
- csvSettings: csvSettings,
4925
- isVerbose: isVerbose,
4926
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4927
- rootDirname: rootDirname,
4928
- },
4681
+ maxExecutionAttempts: maxExecutionAttempts,
4682
+ maxParallelCount: maxParallelCount,
4683
+ csvSettings: csvSettings,
4684
+ isVerbose: isVerbose,
4685
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4686
+ rootDirname: rootDirname,
4929
4687
  })];
4930
4688
  });
4931
4689
  }); };
@@ -4948,7 +4706,7 @@
4948
4706
  return __generator(this, function (_d) {
4949
4707
  switch (_d.label) {
4950
4708
  case 0:
4951
- _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4709
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4952
4710
  if (tools === undefined || tools.llm === undefined) {
4953
4711
  throw new MissingToolsError('LLM tools are required for preparing persona');
4954
4712
  }
@@ -5004,20 +4762,171 @@
5004
4762
  */
5005
4763
 
5006
4764
  /**
5007
- * This error indicates that the promptbook can not retrieve knowledge from external sources
4765
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
4766
+ *
4767
+ * @public exported from `@promptbook/core`
4768
+ */
4769
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4770
+ __extends(KnowledgeScrapeError, _super);
4771
+ function KnowledgeScrapeError(message) {
4772
+ var _this = _super.call(this, message) || this;
4773
+ _this.name = 'KnowledgeScrapeError';
4774
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4775
+ return _this;
4776
+ }
4777
+ return KnowledgeScrapeError;
4778
+ }(Error));
4779
+
4780
+ /**
4781
+ * @@@
4782
+ *
4783
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4784
+ *
4785
+ * @private internal function of `$Register`
4786
+ */
4787
+ function $getGlobalScope() {
4788
+ return Function('return this')();
4789
+ }
4790
+
4791
+ /**
4792
+ * @@@
4793
+ *
4794
+ * @param text @@@
4795
+ * @returns @@@
4796
+ * @example 'HELLO_WORLD'
4797
+ * @example 'I_LOVE_PROMPTBOOK'
4798
+ * @public exported from `@promptbook/utils`
4799
+ */
4800
+ function normalizeTo_SCREAMING_CASE(text) {
4801
+ var e_1, _a;
4802
+ var charType;
4803
+ var lastCharType = 'OTHER';
4804
+ var normalizedName = '';
4805
+ try {
4806
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4807
+ var char = text_1_1.value;
4808
+ var normalizedChar = void 0;
4809
+ if (/^[a-z]$/.test(char)) {
4810
+ charType = 'LOWERCASE';
4811
+ normalizedChar = char.toUpperCase();
4812
+ }
4813
+ else if (/^[A-Z]$/.test(char)) {
4814
+ charType = 'UPPERCASE';
4815
+ normalizedChar = char;
4816
+ }
4817
+ else if (/^[0-9]$/.test(char)) {
4818
+ charType = 'NUMBER';
4819
+ normalizedChar = char;
4820
+ }
4821
+ else {
4822
+ charType = 'OTHER';
4823
+ normalizedChar = '_';
4824
+ }
4825
+ if (charType !== lastCharType &&
4826
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4827
+ !(lastCharType === 'NUMBER') &&
4828
+ !(charType === 'NUMBER')) {
4829
+ normalizedName += '_';
4830
+ }
4831
+ normalizedName += normalizedChar;
4832
+ lastCharType = charType;
4833
+ }
4834
+ }
4835
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4836
+ finally {
4837
+ try {
4838
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
4839
+ }
4840
+ finally { if (e_1) throw e_1.error; }
4841
+ }
4842
+ normalizedName = normalizedName.replace(/_+/g, '_');
4843
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
4844
+ normalizedName = normalizedName.replace(/^_/, '');
4845
+ normalizedName = normalizedName.replace(/_$/, '');
4846
+ return normalizedName;
4847
+ }
4848
+ /**
4849
+ * TODO: Tests
4850
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
4851
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
4852
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
4853
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
4854
+ * TODO: [🌺] Use some intermediate util splitWords
4855
+ */
4856
+
4857
+ /**
4858
+ * @@@
4859
+ *
4860
+ * @param text @@@
4861
+ * @returns @@@
4862
+ * @example 'hello_world'
4863
+ * @example 'i_love_promptbook'
4864
+ * @public exported from `@promptbook/utils`
4865
+ */
4866
+ function normalizeTo_snake_case(text) {
4867
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
4868
+ }
4869
+
4870
+ /**
4871
+ * Register is @@@
4872
+ *
4873
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
4874
+ *
4875
+ * @private internal utility, exported are only signleton instances of this class
4876
+ */
4877
+ var $Register = /** @class */ (function () {
4878
+ function $Register(registerName) {
4879
+ this.registerName = registerName;
4880
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
4881
+ var globalScope = $getGlobalScope();
4882
+ if (globalScope[storageName] === undefined) {
4883
+ globalScope[storageName] = [];
4884
+ }
4885
+ else if (!Array.isArray(globalScope[storageName])) {
4886
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
4887
+ }
4888
+ this.storage = globalScope[storageName];
4889
+ }
4890
+ $Register.prototype.list = function () {
4891
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
4892
+ return this.storage;
4893
+ };
4894
+ $Register.prototype.register = function (registered) {
4895
+ var packageName = registered.packageName, className = registered.className;
4896
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
4897
+ var existingRegistration = this.storage[existingRegistrationIndex];
4898
+ if (!existingRegistration) {
4899
+ this.storage.push(registered);
4900
+ }
4901
+ else {
4902
+ this.storage[existingRegistrationIndex] = registered;
4903
+ }
4904
+ return {
4905
+ registerName: this.registerName,
4906
+ packageName: packageName,
4907
+ className: className,
4908
+ get isDestroyed() {
4909
+ return false;
4910
+ },
4911
+ destroy: function () {
4912
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
4913
+ },
4914
+ };
4915
+ };
4916
+ return $Register;
4917
+ }());
4918
+
4919
+ /**
4920
+ * @@@
5008
4921
  *
4922
+ * Note: `$` is used to indicate that this interacts with the global scope
4923
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
5009
4924
  * @public exported from `@promptbook/core`
5010
4925
  */
5011
- var KnowledgeScrapeError = /** @class */ (function (_super) {
5012
- __extends(KnowledgeScrapeError, _super);
5013
- function KnowledgeScrapeError(message) {
5014
- var _this = _super.call(this, message) || this;
5015
- _this.name = 'KnowledgeScrapeError';
5016
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
5017
- return _this;
5018
- }
5019
- return KnowledgeScrapeError;
5020
- }(Error));
4926
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4927
+ /**
4928
+ * TODO: [®] DRY Register logic
4929
+ */
5021
4930
 
5022
4931
  /**
5023
4932
  * @@@
@@ -5026,12 +4935,12 @@
5026
4935
  * @singleton Only one instance of each register is created per build, but thare can be more @@@
5027
4936
  * @public exported from `@promptbook/core`
5028
4937
  */
5029
- var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4938
+ var $scrapersRegister = new $Register('scraper_constructors');
5030
4939
  /**
5031
4940
  * TODO: [®] DRY Register logic
5032
4941
  */
5033
4942
 
5034
- // TODO: !!!!!! Maybe delete this function
4943
+ // TODO: !!!!!!last - Maybe delete this function
5035
4944
  /**
5036
4945
  * Creates a message with all registered scrapers
5037
4946
  *
@@ -5198,7 +5107,6 @@
5198
5107
  * @private within the repository
5199
5108
  */
5200
5109
  function sourceContentToName(sourceContent) {
5201
- // TODO: !!!!!! Better name for source than gibberish hash
5202
5110
  var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
5203
5111
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5204
5112
  .toString( /* hex */)
@@ -5311,7 +5219,7 @@
5311
5219
  case 0:
5312
5220
  sourceContent = knowledgeSource.sourceContent;
5313
5221
  name = knowledgeSource.name;
5314
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5222
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
5315
5223
  TODO_USE(isVerbose);
5316
5224
  if (!name) {
5317
5225
  name = sourceContentToName(sourceContent);
@@ -5401,7 +5309,7 @@
5401
5309
  content = _a.sent();
5402
5310
  return [2 /*return*/, new Blob([
5403
5311
  content,
5404
- // <- TODO: !!!!!! Maybe not working
5312
+ // <- TODO: !!!!!! Test that this is working
5405
5313
  ], { type: mimeType_1 })];
5406
5314
  }
5407
5315
  });
@@ -5464,7 +5372,7 @@
5464
5372
  return __generator(this, function (_c) {
5465
5373
  switch (_c.label) {
5466
5374
  case 0:
5467
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5375
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5468
5376
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
5469
5377
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
5470
5378
  var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
@@ -5493,7 +5401,8 @@
5493
5401
  case 4:
5494
5402
  partialPiecesUnchecked = _d.sent();
5495
5403
  if (partialPiecesUnchecked !== null) {
5496
- partialPieces = partialPiecesUnchecked;
5404
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
5405
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
5497
5406
  return [3 /*break*/, 6];
5498
5407
  }
5499
5408
  _d.label = 5;
@@ -5541,7 +5450,7 @@
5541
5450
  > /**
5542
5451
  > * Unprepared knowledge
5543
5452
  > * /
5544
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
5453
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
5545
5454
  > };
5546
5455
  >
5547
5456
  > export async function prepareKnowledgePieces(
@@ -5599,7 +5508,7 @@
5599
5508
  return __generator(this, function (_b) {
5600
5509
  switch (_b.label) {
5601
5510
  case 0:
5602
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
5511
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
5603
5512
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
5604
5513
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
5605
5514
  TODO_USE(parameters);
@@ -5661,7 +5570,7 @@
5661
5570
  if (isPipelinePrepared(pipeline)) {
5662
5571
  return [2 /*return*/, pipeline];
5663
5572
  }
5664
- rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5573
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5665
5574
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
5666
5575
  if (tools === undefined || tools.llm === undefined) {
5667
5576
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
@@ -5719,7 +5628,9 @@
5719
5628
  // ----- /Templates preparation -----
5720
5629
  // Note: Count total usage
5721
5630
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
5722
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
5631
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
5632
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
5633
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
5723
5634
  }
5724
5635
  });
5725
5636
  });
@@ -8643,7 +8554,7 @@
8643
8554
  */
8644
8555
  function createLlmToolsFromConfiguration(configuration, options) {
8645
8556
  if (options === void 0) { options = {}; }
8646
- var _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a, userId = options.userId;
8557
+ var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
8647
8558
  var llmTools = configuration.map(function (llmConfiguration) {
8648
8559
  var registeredItem = $llmToolsRegister
8649
8560
  .list()
@@ -8704,6 +8615,115 @@
8704
8615
  * TODO: [®] DRY Register logic
8705
8616
  */
8706
8617
 
8618
+ /**
8619
+ * @@@
8620
+ *
8621
+ * @public exported from `@promptbook/node`
8622
+ */
8623
+ function $provideExecutablesForNode(options) {
8624
+ return __awaiter(this, void 0, void 0, function () {
8625
+ var _a, _b, isAutoInstalled, _c, isVerbose;
8626
+ return __generator(this, function (_d) {
8627
+ if (!$isRunningInNode()) {
8628
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8629
+ }
8630
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8631
+ TODO_USE(isAutoInstalled);
8632
+ TODO_USE(isVerbose);
8633
+ return [2 /*return*/, {
8634
+ // TODO: !!!!!! use `locate-app` library here
8635
+ pandocPath: 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe',
8636
+ libreOfficePath: 'C:/Program Files/LibreOffice/program/swriter.exe',
8637
+ }];
8638
+ });
8639
+ });
8640
+ }
8641
+ /**
8642
+ * TODO: [🧠] THis should be maybe in different folder
8643
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8644
+ */
8645
+
8646
+ /**
8647
+ * @@@
8648
+ *
8649
+ * @public exported from `@promptbook/node`
8650
+ */
8651
+ function $provideFilesystemForNode(options) {
8652
+ if (!$isRunningInNode()) {
8653
+ throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
8654
+ }
8655
+ var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
8656
+ TODO_USE(isVerbose);
8657
+ return {
8658
+ stat: promises.stat,
8659
+ access: promises.access,
8660
+ constants: promises.constants,
8661
+ readFile: promises.readFile,
8662
+ readdir: promises.readdir,
8663
+ };
8664
+ }
8665
+ /**
8666
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8667
+ */
8668
+
8669
+ /**
8670
+ * @@@
8671
+ *
8672
+ * 1) @@@
8673
+ * 2) @@@
8674
+ *
8675
+ * @public exported from `@promptbook/node`
8676
+ */
8677
+ function $provideScrapersForNode(tools, options) {
8678
+ return __awaiter(this, void 0, void 0, function () {
8679
+ var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
8680
+ var e_1, _f;
8681
+ return __generator(this, function (_g) {
8682
+ switch (_g.label) {
8683
+ case 0:
8684
+ if (!$isRunningInNode()) {
8685
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8686
+ }
8687
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8688
+ TODO_USE(isAutoInstalled);
8689
+ TODO_USE(isVerbose);
8690
+ scrapers = [];
8691
+ _g.label = 1;
8692
+ case 1:
8693
+ _g.trys.push([1, 6, 7, 8]);
8694
+ _d = __values($scrapersRegister.list()), _e = _d.next();
8695
+ _g.label = 2;
8696
+ case 2:
8697
+ if (!!_e.done) return [3 /*break*/, 5];
8698
+ scraperFactory = _e.value;
8699
+ return [4 /*yield*/, scraperFactory(tools, options || {})];
8700
+ case 3:
8701
+ scraper = _g.sent();
8702
+ scrapers.push(scraper);
8703
+ _g.label = 4;
8704
+ case 4:
8705
+ _e = _d.next();
8706
+ return [3 /*break*/, 2];
8707
+ case 5: return [3 /*break*/, 8];
8708
+ case 6:
8709
+ e_1_1 = _g.sent();
8710
+ e_1 = { error: e_1_1 };
8711
+ return [3 /*break*/, 8];
8712
+ case 7:
8713
+ try {
8714
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
8715
+ }
8716
+ finally { if (e_1) throw e_1.error; }
8717
+ return [7 /*endfinally*/];
8718
+ case 8: return [2 /*return*/, scrapers];
8719
+ }
8720
+ });
8721
+ });
8722
+ }
8723
+ /**
8724
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8725
+ */
8726
+
8707
8727
  /**
8708
8728
  * Extracts code block from markdown.
8709
8729
  *
@@ -9158,7 +9178,7 @@
9158
9178
  */
9159
9179
  function $provideExecutionToolsForNode(options) {
9160
9180
  return __awaiter(this, void 0, void 0, function () {
9161
- var fs, llm, tools;
9181
+ var fs, llm, executables, tools;
9162
9182
  var _a;
9163
9183
  return __generator(this, function (_b) {
9164
9184
  switch (_b.label) {
@@ -9168,12 +9188,16 @@
9168
9188
  }
9169
9189
  fs = $provideFilesystemForNode();
9170
9190
  llm = $provideLlmToolsFromEnv(options);
9191
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
9192
+ case 1:
9193
+ executables = _b.sent();
9171
9194
  _a = {
9172
9195
  llm: llm,
9173
- fs: fs
9196
+ fs: fs,
9197
+ executables: executables
9174
9198
  };
9175
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
9176
- case 1:
9199
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
9200
+ case 2:
9177
9201
  tools = (_a.scrapers = _b.sent(),
9178
9202
  _a.script = [new JavascriptExecutionTools(options)],
9179
9203
  _a);
@@ -9411,7 +9435,9 @@
9411
9435
  throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
9412
9436
  // <- TODO: [🧠] What is the best error type here`
9413
9437
  }
9414
- makedLibraryFilePath = path.join(path$1, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".json"));
9438
+ makedLibraryFilePath = path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
9439
+ // <- TODO: [🦒] Allow to override (pass different value into the function)
9440
+ , ".json"));
9415
9441
  return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
9416
9442
  case 3:
9417
9443
  if (!(_f.sent())) {
@@ -9422,7 +9448,7 @@
9422
9448
  // TODO: !! Implement;
9423
9449
  // TODO: [🌗]
9424
9450
  }
9425
- _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9451
+ _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9426
9452
  collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
9427
9453
  var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
9428
9454
  var e_1, _a;
@@ -9894,7 +9920,7 @@
9894
9920
  function cacheLlmTools(llmTools, options) {
9895
9921
  var _this = this;
9896
9922
  if (options === void 0) { options = {}; }
9897
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isReloaded, isReloaded = _b === void 0 ? false : _b;
9923
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
9898
9924
  var proxyTools = __assign(__assign({}, llmTools), {
9899
9925
  // <- Note: [🥫]
9900
9926
  get title() {
@@ -9916,7 +9942,7 @@
9916
9942
  key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
9917
9943
  '-' +
9918
9944
  sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
9919
- if (!!isReloaded) return [3 /*break*/, 2];
9945
+ if (!!isCacheReloaded) return [3 /*break*/, 2];
9920
9946
  return [4 /*yield*/, storage.getItem(key)];
9921
9947
  case 1:
9922
9948
  _a = _c.sent();
@@ -10002,12 +10028,14 @@
10002
10028
  if (!$isRunningInNode()) {
10003
10029
  throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
10004
10030
  }
10005
- var _a = (options !== null && options !== void 0 ? options : {}).isCacheCleaned, isCacheCleaned = _a === void 0 ? false : _a;
10031
+ var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
10006
10032
  return cacheLlmTools(countTotalUsage(
10007
10033
  // <- Note: for example here we don`t want the [🌯]
10008
10034
  $provideLlmToolsFromEnv()), {
10009
- storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, { rootFolderPath: path.join(process.cwd(), EXECUTIONS_CACHE_DIRNAME) }),
10010
- isReloaded: isCacheCleaned,
10035
+ storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
10036
+ rootFolderPath: path.join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
10037
+ }),
10038
+ isCacheReloaded: isCacheReloaded,
10011
10039
  });
10012
10040
  }
10013
10041
  /**
@@ -10034,13 +10062,13 @@
10034
10062
  makeCommand.option('-f, --format <format>', spaceTrim__default["default"]("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳‍🌈] */);
10035
10063
  makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
10036
10064
  makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
10037
- makeCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache", false);
10065
+ makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
10038
10066
  makeCommand.option('--verbose', "Is output verbose", false);
10039
- makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), PIPELINE_COLLECTION_BASE_FILENAME);
10067
+ makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
10040
10068
  makeCommand.action(function (path$1, _a) {
10041
- var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10069
+ var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10042
10070
  return __awaiter(_this, void 0, void 0, function () {
10043
- var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10071
+ var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10044
10072
  var _d, e_2, _e, e_1, _f;
10045
10073
  var _this = this;
10046
10074
  return __generator(this, function (_g) {
@@ -10054,22 +10082,25 @@
10054
10082
  .split(',')
10055
10083
  .map(function (_) { return _.trim(); })
10056
10084
  .filter(function (_) { return _ !== ''; });
10057
- if (outFile !== PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10085
+ if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10058
10086
  console.error(colors__default["default"].red("You can only use one format if you specify --out-file"));
10059
10087
  process.exit(1);
10060
10088
  }
10061
10089
  options = {
10062
10090
  isVerbose: isVerbose,
10063
- isCacheCleaned: isCacheCleaned,
10091
+ isCacheReloaded: isCacheReloaded,
10064
10092
  };
10065
10093
  fs = $provideFilesystemForNode(options);
10066
10094
  llm = $provideLlmToolsForCli(options);
10095
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10096
+ case 1:
10097
+ executables = _g.sent();
10067
10098
  _d = {
10068
10099
  llm: llm,
10069
10100
  fs: fs
10070
10101
  };
10071
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10072
- case 1:
10102
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10103
+ case 2:
10073
10104
  tools = (_d.scrapers = _g.sent(),
10074
10105
  _d.script = [
10075
10106
  /*new JavascriptExecutionTools(options)*/
@@ -10078,31 +10109,31 @@
10078
10109
  return [4 /*yield*/, createCollectionFromDirectory(path$1, tools, {
10079
10110
  isVerbose: isVerbose,
10080
10111
  isRecursive: true,
10081
- // <- TODO: [🍖] isCacheReloaded
10112
+ // <- TODO: [🍖] Add `intermediateFilesStrategy`
10082
10113
  })];
10083
- case 2:
10084
- collection = _g.sent();
10085
- _g.label = 3;
10086
10114
  case 3:
10087
- _g.trys.push([3, 15, 16, 17]);
10088
- validations_1 = __values(validations), validations_1_1 = validations_1.next();
10115
+ collection = _g.sent();
10089
10116
  _g.label = 4;
10090
10117
  case 4:
10091
- if (!!validations_1_1.done) return [3 /*break*/, 14];
10092
- validation_1 = validations_1_1.value;
10118
+ _g.trys.push([4, 16, 17, 18]);
10119
+ validations_1 = __values(validations), validations_1_1 = validations_1.next();
10093
10120
  _g.label = 5;
10094
10121
  case 5:
10095
- _g.trys.push([5, 11, 12, 13]);
10122
+ if (!!validations_1_1.done) return [3 /*break*/, 15];
10123
+ validation_1 = validations_1_1.value;
10124
+ _g.label = 6;
10125
+ case 6:
10126
+ _g.trys.push([6, 12, 13, 14]);
10096
10127
  e_1 = void 0;
10097
10128
  return [4 /*yield*/, collection.listPipelines()];
10098
- case 6:
10099
- _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10100
- _g.label = 7;
10101
10129
  case 7:
10102
- if (!!_c.done) return [3 /*break*/, 10];
10130
+ _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10131
+ _g.label = 8;
10132
+ case 8:
10133
+ if (!!_c.done) return [3 /*break*/, 11];
10103
10134
  pipelineUrl = _c.value;
10104
10135
  return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
10105
- case 8:
10136
+ case 9:
10106
10137
  pipeline = _g.sent();
10107
10138
  if (validation_1 === 'logic') {
10108
10139
  validatePipeline(pipeline);
@@ -10110,37 +10141,37 @@
10110
10141
  console.info(colors__default["default"].cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
10111
10142
  }
10112
10143
  }
10113
- _g.label = 9;
10114
- case 9:
10144
+ _g.label = 10;
10145
+ case 10:
10115
10146
  _c = _b.next();
10116
- return [3 /*break*/, 7];
10117
- case 10: return [3 /*break*/, 13];
10118
- case 11:
10147
+ return [3 /*break*/, 8];
10148
+ case 11: return [3 /*break*/, 14];
10149
+ case 12:
10119
10150
  e_1_1 = _g.sent();
10120
10151
  e_1 = { error: e_1_1 };
10121
- return [3 /*break*/, 13];
10122
- case 12:
10152
+ return [3 /*break*/, 14];
10153
+ case 13:
10123
10154
  try {
10124
10155
  if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
10125
10156
  }
10126
10157
  finally { if (e_1) throw e_1.error; }
10127
10158
  return [7 /*endfinally*/];
10128
- case 13:
10159
+ case 14:
10129
10160
  validations_1_1 = validations_1.next();
10130
- return [3 /*break*/, 4];
10131
- case 14: return [3 /*break*/, 17];
10132
- case 15:
10161
+ return [3 /*break*/, 5];
10162
+ case 15: return [3 /*break*/, 18];
10163
+ case 16:
10133
10164
  e_2_1 = _g.sent();
10134
10165
  e_2 = { error: e_2_1 };
10135
- return [3 /*break*/, 17];
10136
- case 16:
10166
+ return [3 /*break*/, 18];
10167
+ case 17:
10137
10168
  try {
10138
10169
  if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
10139
10170
  }
10140
10171
  finally { if (e_2) throw e_2.error; }
10141
10172
  return [7 /*endfinally*/];
10142
- case 17: return [4 /*yield*/, collectionToJson(collection)];
10143
- case 18:
10173
+ case 18: return [4 /*yield*/, collectionToJson(collection)];
10174
+ case 19:
10144
10175
  collectionJson = _g.sent();
10145
10176
  collectionJsonString = stringifyPipelineJson(collectionJson).trim();
10146
10177
  collectionJsonItems = (function () {
@@ -10159,9 +10190,9 @@
10159
10190
  return __generator(this, function (_a) {
10160
10191
  switch (_a.label) {
10161
10192
  case 0:
10162
- filename = outFile !== PIPELINE_COLLECTION_BASE_FILENAME
10193
+ filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
10163
10194
  ? outFile
10164
- : path.join(path$1, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10195
+ : path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10165
10196
  if (!outFile.endsWith(".".concat(extension))) {
10166
10197
  console.warn(colors__default["default"].yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
10167
10198
  }
@@ -10177,27 +10208,27 @@
10177
10208
  }
10178
10209
  });
10179
10210
  }); };
10180
- if (!formats.includes('json')) return [3 /*break*/, 20];
10211
+ if (!formats.includes('json')) return [3 /*break*/, 21];
10181
10212
  formats = formats.filter(function (format) { return format !== 'json'; });
10182
10213
  return [4 /*yield*/, saveFile('json', collectionJsonString)];
10183
- case 19:
10184
- _g.sent();
10185
- _g.label = 20;
10186
10214
  case 20:
10187
- if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 22];
10215
+ _g.sent();
10216
+ _g.label = 21;
10217
+ case 21:
10218
+ if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
10188
10219
  formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
10189
10220
  return [4 /*yield*/, saveFile('js', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
10190
- case 21:
10191
- (_g.sent()) + '\n';
10192
- _g.label = 22;
10193
10221
  case 22:
10194
- if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 24];
10222
+ (_g.sent()) + '\n';
10223
+ _g.label = 23;
10224
+ case 23:
10225
+ if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
10195
10226
  formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
10196
10227
  return [4 /*yield*/, saveFile('ts', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
10197
- case 23:
10198
- _g.sent();
10199
- _g.label = 24;
10200
10228
  case 24:
10229
+ _g.sent();
10230
+ _g.label = 25;
10231
+ case 25:
10201
10232
  if (formats.length > 0) {
10202
10233
  console.warn(colors__default["default"].yellow("Format ".concat(formats.join(' and '), " is not supported")));
10203
10234
  }
@@ -10459,76 +10490,79 @@
10459
10490
  // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument
10460
10491
  'Pipelines to test as glob pattern');
10461
10492
  testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
10462
- testCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache ", false);
10493
+ testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
10463
10494
  testCommand.option('-v, --verbose', "Is output verbose", false);
10464
10495
  testCommand.action(function (filesGlob, _a) {
10465
- var ignore = _a.ignore, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose;
10496
+ var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
10466
10497
  return __awaiter(_this, void 0, void 0, function () {
10467
- var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10498
+ var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10468
10499
  var _d, e_1, _e;
10469
10500
  return __generator(this, function (_f) {
10470
10501
  switch (_f.label) {
10471
10502
  case 0:
10472
10503
  options = {
10473
10504
  isVerbose: isVerbose,
10474
- isCacheCleaned: isCacheCleaned,
10505
+ isCacheReloaded: isCacheReloaded,
10475
10506
  };
10476
10507
  fs = $provideFilesystemForNode(options);
10477
10508
  llm = $provideLlmToolsForCli(options);
10509
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10510
+ case 1:
10511
+ executables = _f.sent();
10478
10512
  _d = {
10479
10513
  llm: llm,
10480
10514
  fs: fs
10481
10515
  };
10482
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10483
- case 1:
10516
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10517
+ case 2:
10484
10518
  tools = (_d.scrapers = _f.sent(),
10485
10519
  _d.script = [
10486
10520
  /*new JavascriptExecutionTools(options)*/
10487
10521
  ],
10488
10522
  _d);
10489
10523
  return [4 /*yield*/, glob__default["default"](filesGlob, { ignore: ignore })];
10490
- case 2:
10491
- filenames = _f.sent();
10492
- _f.label = 3;
10493
10524
  case 3:
10494
- _f.trys.push([3, 15, 16, 17]);
10495
- filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10525
+ filenames = _f.sent();
10496
10526
  _f.label = 4;
10497
10527
  case 4:
10498
- if (!!filenames_1_1.done) return [3 /*break*/, 14];
10499
- filename = filenames_1_1.value;
10528
+ _f.trys.push([4, 16, 17, 18]);
10529
+ filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10500
10530
  _f.label = 5;
10501
10531
  case 5:
10502
- _f.trys.push([5, 12, , 13]);
10532
+ if (!!filenames_1_1.done) return [3 /*break*/, 15];
10533
+ filename = filenames_1_1.value;
10534
+ _f.label = 6;
10535
+ case 6:
10536
+ _f.trys.push([6, 13, , 14]);
10503
10537
  pipeline = void 0;
10504
- if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 8];
10538
+ if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
10505
10539
  return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
10506
- case 6:
10540
+ case 7:
10507
10541
  pipelineMarkdown = (_f.sent());
10508
10542
  return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
10509
- case 7:
10543
+ case 8:
10510
10544
  pipeline = _f.sent();
10511
10545
  if (isVerbose) {
10512
10546
  console.info(colors__default["default"].green("Parsed ".concat(filename)));
10513
10547
  }
10514
- _f.label = 8;
10515
- case 8:
10516
- if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 10];
10548
+ _f.label = 9;
10549
+ case 9:
10550
+ if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
10517
10551
  _c = (_b = JSON).parse;
10518
10552
  return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
10519
- case 9:
10520
- pipeline = _c.apply(_b, [_f.sent()]);
10521
- return [3 /*break*/, 11];
10522
10553
  case 10:
10554
+ pipeline = _c.apply(_b, [_f.sent()]);
10555
+ return [3 /*break*/, 12];
10556
+ case 11:
10523
10557
  if (isVerbose) {
10524
10558
  console.info(colors__default["default"].gray("Skipping ".concat(filename)));
10525
10559
  }
10526
- return [3 /*break*/, 13];
10527
- case 11:
10560
+ return [3 /*break*/, 14];
10561
+ case 12:
10528
10562
  validatePipeline(pipeline);
10529
10563
  console.info(colors__default["default"].green("Validated ".concat(filename)));
10530
- return [3 /*break*/, 13];
10531
- case 12:
10564
+ return [3 /*break*/, 14];
10565
+ case 13:
10532
10566
  error_1 = _f.sent();
10533
10567
  if (!(error_1 instanceof Error)) {
10534
10568
  throw error_1;
@@ -10537,22 +10571,22 @@
10537
10571
  console.error(colors__default["default"].bgRed(error_1.name /* <- 11:11 */));
10538
10572
  console.error(colors__default["default"].red(error_1.stack || error_1.message));
10539
10573
  process.exit(1);
10540
- return [3 /*break*/, 13];
10541
- case 13:
10574
+ return [3 /*break*/, 14];
10575
+ case 14:
10542
10576
  filenames_1_1 = filenames_1.next();
10543
- return [3 /*break*/, 4];
10544
- case 14: return [3 /*break*/, 17];
10545
- case 15:
10577
+ return [3 /*break*/, 5];
10578
+ case 15: return [3 /*break*/, 18];
10579
+ case 16:
10546
10580
  e_1_1 = _f.sent();
10547
10581
  e_1 = { error: e_1_1 };
10548
- return [3 /*break*/, 17];
10549
- case 16:
10582
+ return [3 /*break*/, 18];
10583
+ case 17:
10550
10584
  try {
10551
10585
  if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
10552
10586
  }
10553
10587
  finally { if (e_1) throw e_1.error; }
10554
10588
  return [7 /*endfinally*/];
10555
- case 17:
10589
+ case 18:
10556
10590
  console.info(colors__default["default"].green("All pipelines are valid"));
10557
10591
  process.exit(0);
10558
10592
  return [2 /*return*/];
@@ -13023,12 +13057,12 @@
13023
13057
  */
13024
13058
  function getScraperIntermediateSource(source, options) {
13025
13059
  return __awaiter(this, void 0, void 0, function () {
13026
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13060
+ var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13027
13061
  return __generator(this, function (_a) {
13028
13062
  switch (_a.label) {
13029
13063
  case 0:
13030
13064
  sourceFilename = source.filename, url = source.url;
13031
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
13065
+ rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
13032
13066
  hash = cryptoJs.SHA256(
13033
13067
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
13034
13068
  hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
@@ -13038,7 +13072,7 @@
13038
13072
  pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
13039
13073
  name = pieces.join('-').split('--').join('-');
13040
13074
  // <- TODO: Use MAX_FILENAME_LENGTH
13041
- TODO_USE(rootDirname); // <- TODO: !!!!!!
13075
+ TODO_USE(rootDirname); // <- TODO: [😡]
13042
13076
  cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
13043
13077
  cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
13044
13078
  .join('/') +
@@ -13058,7 +13092,7 @@
13058
13092
  return __generator(this, function (_a) {
13059
13093
  switch (_a.label) {
13060
13094
  case 0:
13061
- if (!isCacheCleaned) return [3 /*break*/, 2];
13095
+ if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
13062
13096
  if (isVerbose) {
13063
13097
  console.info('legacyDocumentScraper: Clening cache');
13064
13098
  }
@@ -13082,7 +13116,7 @@
13082
13116
  /**
13083
13117
  * Note: Not using `FileCacheStorage` for two reasons:
13084
13118
  * 1) Need to store more than serialized JSONs
13085
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
13119
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
13086
13120
  * TODO: [🐱‍🐉][🧠] Make some smart crop
13087
13121
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
13088
13122
  */
@@ -13099,7 +13133,7 @@
13099
13133
  mimeTypes: ['text/markdown', 'text/plain'],
13100
13134
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13101
13135
  isAvilableInBrowser: true,
13102
- requiredExecutables: ['!!!!!!'],
13136
+ requiredExecutables: [],
13103
13137
  }); /* <- TODO: [🤛] */
13104
13138
  /**
13105
13139
  * Registration of known scraper metadata
@@ -13143,7 +13177,7 @@
13143
13177
  return __generator(this, function (_k) {
13144
13178
  switch (_k.label) {
13145
13179
  case 0:
13146
- _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
13180
+ _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
13147
13181
  llm = this.tools.llm;
13148
13182
  if (llm === undefined) {
13149
13183
  throw new MissingToolsError('LLM tools are required for scraping external files');
@@ -13242,7 +13276,8 @@
13242
13276
  embeddingResult = _c.sent();
13243
13277
  index.push({
13244
13278
  modelName: embeddingResult.modelName,
13245
- position: embeddingResult.content,
13279
+ position: __spreadArray([], __read(embeddingResult.content), false),
13280
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
13246
13281
  });
13247
13282
  _c.label = 6;
13248
13283
  case 6: return [3 /*break*/, 8];
@@ -13293,7 +13328,7 @@
13293
13328
  mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
13294
13329
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13295
13330
  isAvilableInBrowser: false,
13296
- requiredExecutables: ['!!!!!!'],
13331
+ requiredExecutables: ['Pandoc'],
13297
13332
  }); /* <- TODO: [🤛] */
13298
13333
  /**
13299
13334
  * Registration of known scraper metadata
@@ -13333,12 +13368,13 @@
13333
13368
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13334
13369
  */
13335
13370
  DocumentScraper.prototype.$convert = function (source) {
13371
+ var _a;
13336
13372
  return __awaiter(this, void 0, void 0, function () {
13337
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, command_1;
13373
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
13338
13374
  return __generator(this, function (_g) {
13339
13375
  switch (_g.label) {
13340
13376
  case 0:
13341
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13377
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13342
13378
  if (!$isRunningInNode()) {
13343
13379
  throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
13344
13380
  }
@@ -13346,7 +13382,7 @@
13346
13382
  throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
13347
13383
  // <- TODO: [🧠] What is the best error type here`
13348
13384
  }
13349
- if (externalProgramsPaths.pandocPath === undefined) {
13385
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
13350
13386
  throw new MissingToolsError('Pandoc is required for scraping .docx files');
13351
13387
  }
13352
13388
  if (source.filename === null) {
@@ -13357,7 +13393,7 @@
13357
13393
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13358
13394
  rootDirname: rootDirname,
13359
13395
  cacheDirname: cacheDirname,
13360
- isCacheCleaned: isCacheCleaned,
13396
+ intermediateFilesStrategy: intermediateFilesStrategy,
13361
13397
  extension: 'md',
13362
13398
  isVerbose: isVerbose,
13363
13399
  })];
@@ -13366,11 +13402,9 @@
13366
13402
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13367
13403
  case 2:
13368
13404
  if (!!(_g.sent())) return [3 /*break*/, 5];
13369
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13370
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13405
+ command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13371
13406
  return [4 /*yield*/, $execCommand(command_1)];
13372
13407
  case 3:
13373
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13374
13408
  _g.sent();
13375
13409
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13376
13410
  case 4:
@@ -13449,7 +13483,11 @@
13449
13483
  mimeTypes: ['application/msword', 'text/rtf'],
13450
13484
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13451
13485
  isAvilableInBrowser: false,
13452
- requiredExecutables: ['!!!!!!'],
13486
+ requiredExecutables: [
13487
+ 'Pandoc',
13488
+ 'LibreOffice',
13489
+ // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
13490
+ ],
13453
13491
  }); /* <- TODO: [🤛] */
13454
13492
  /**
13455
13493
  * Registration of known scraper metadata
@@ -13462,7 +13500,7 @@
13462
13500
  var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
13463
13501
 
13464
13502
  /**
13465
- * Scraper for .docx files
13503
+ * Scraper for old document files (like .doc and .rtf)
13466
13504
  *
13467
13505
  * @see `documentationUrl` for more details
13468
13506
  * @public exported from `@promptbook/legacy-documents`
@@ -13489,12 +13527,13 @@
13489
13527
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13490
13528
  */
13491
13529
  LegacyDocumentScraper.prototype.$convert = function (source) {
13530
+ var _a;
13492
13531
  return __awaiter(this, void 0, void 0, function () {
13493
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13532
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13494
13533
  return __generator(this, function (_g) {
13495
13534
  switch (_g.label) {
13496
13535
  case 0:
13497
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13536
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13498
13537
  if (!$isRunningInNode()) {
13499
13538
  throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
13500
13539
  }
@@ -13502,7 +13541,7 @@
13502
13541
  throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
13503
13542
  // <- TODO: [🧠] What is the best error type here`
13504
13543
  }
13505
- if (externalProgramsPaths.libreOfficePath === undefined) {
13544
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
13506
13545
  throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
13507
13546
  }
13508
13547
  if (source.filename === null) {
@@ -13513,7 +13552,7 @@
13513
13552
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13514
13553
  rootDirname: rootDirname,
13515
13554
  cacheDirname: cacheDirname,
13516
- isCacheCleaned: isCacheCleaned,
13555
+ intermediateFilesStrategy: intermediateFilesStrategy,
13517
13556
  extension: 'docx',
13518
13557
  isVerbose: isVerbose,
13519
13558
  })];
@@ -13528,11 +13567,9 @@
13528
13567
  documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
13529
13568
  .split('\\')
13530
13569
  .join('/');
13531
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13532
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13570
+ command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13533
13571
  return [4 /*yield*/, $execCommand(command_1)];
13534
13572
  case 3:
13535
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13536
13573
  _g.sent();
13537
13574
  return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
13538
13575
  case 4:
@@ -13689,7 +13726,7 @@
13689
13726
  mimeTypes: ['application/pdf'],
13690
13727
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13691
13728
  isAvilableInBrowser: true,
13692
- requiredExecutables: ['!!!!!!'],
13729
+ requiredExecutables: [],
13693
13730
  }); /* <- TODO: [🤛] */
13694
13731
  /**
13695
13732
  * Registration of known scraper metadata
@@ -13702,7 +13739,7 @@
13702
13739
  var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
13703
13740
 
13704
13741
  /**
13705
- * Scraper for .docx files
13742
+ * Scraper for .pdf files
13706
13743
  *
13707
13744
  * @see `documentationUrl` for more details
13708
13745
  * @public exported from `@promptbook/pdf`
@@ -13745,9 +13782,8 @@
13745
13782
  TODO_USE(this.options);
13746
13783
  /*
13747
13784
  const {
13748
- externalProgramsPaths = {},
13749
13785
  cacheDirname = SCRAPE_CACHE_DIRNAME,
13750
- isCacheCleaned = false,
13786
+ intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
13751
13787
  isVerbose = IS_VERBOSE,
13752
13788
  } = options;
13753
13789
  */
@@ -13800,7 +13836,7 @@
13800
13836
  mimeTypes: ['text/html'],
13801
13837
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13802
13838
  isAvilableInBrowser: false,
13803
- requiredExecutables: ['!!!!!!'],
13839
+ requiredExecutables: [],
13804
13840
  }); /* <- TODO: [🤛] */
13805
13841
  /**
13806
13842
  * Registration of known scraper metadata
@@ -13813,32 +13849,29 @@
13813
13849
  var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
13814
13850
 
13815
13851
  /**
13816
- * A converter instance that uses showdown and highlight extensions
13852
+ * Create a new showdown converter instance
13817
13853
  *
13818
- * @type {Converter}
13819
- * @private for markdown and html knowledge scrapers
13820
- */
13821
- var markdownConverter = new showdown.Converter({
13822
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
13823
- /*
13824
- > import showdownHighlight from 'showdown-highlight';
13825
- > extensions: [
13826
- > showdownHighlight({
13827
- > // Whether to add the classes to the <pre> tag, default is false
13828
- > pre: true,
13829
- > // Whether to use hljs' auto language detection, default is true
13830
- > auto_detection: true,
13831
- > }),
13832
- > ],
13833
- */
13834
- });
13835
- /**
13836
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
13837
- * TODO: !!!!!! Lazy-make converter
13854
+ * @private utility of `WebsiteScraper`
13838
13855
  */
13856
+ function createShowdownConverter() {
13857
+ return new showdown.Converter({
13858
+ flavor: 'github',
13859
+ /*
13860
+ > import showdownHighlight from 'showdown-highlight';
13861
+ > extensions: [
13862
+ > showdownHighlight({
13863
+ > // Whether to add the classes to the <pre> tag, default is false
13864
+ > pre: true,
13865
+ > // Whether to use hljs' auto language detection, default is true
13866
+ > auto_detection: true,
13867
+ > }),
13868
+ > ],
13869
+ */
13870
+ });
13871
+ }
13839
13872
 
13840
13873
  /**
13841
- * Scraper for .docx files
13874
+ * Scraper for websites
13842
13875
  *
13843
13876
  * @see `documentationUrl` for more details
13844
13877
  * @public exported from `@promptbook/website-crawler`
@@ -13848,6 +13881,7 @@
13848
13881
  this.tools = tools;
13849
13882
  this.options = options;
13850
13883
  this.markdownScraper = new MarkdownScraper(tools, options);
13884
+ this.showdownConverter = createShowdownConverter();
13851
13885
  }
13852
13886
  Object.defineProperty(WebsiteScraper.prototype, "metadata", {
13853
13887
  /**
@@ -13868,12 +13902,11 @@
13868
13902
  return __awaiter(this, void 0, void 0, function () {
13869
13903
  var _a, _b,
13870
13904
  // TODO: [🧠] Maybe in node use headless browser not just JSDOM
13871
- // externalProgramsPaths = {},
13872
- rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
13905
+ rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
13873
13906
  return __generator(this, function (_g) {
13874
13907
  switch (_g.label) {
13875
13908
  case 0:
13876
- _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? SCRAPE_CACHE_DIRNAME : _c, _d = _a.isCacheCleaned, isCacheCleaned = _d === void 0 ? false : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e;
13909
+ _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
13877
13910
  // TODO: !!!!!! Does this work in browser? Make it work.
13878
13911
  if (source.url === null) {
13879
13912
  throw new KnowledgeScrapeError('Website scraper requires URL');
@@ -13886,10 +13919,6 @@
13886
13919
  }]))();
13887
13920
  reader = new readability.Readability(jsdom$1.window.document);
13888
13921
  article = reader.parse();
13889
- console.log(article);
13890
- return [4 /*yield*/, waitasecond.forTime(10000)];
13891
- case 2:
13892
- _g.sent();
13893
13922
  html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
13894
13923
  // Note: Unwrap html such as it is convertable by `markdownConverter`
13895
13924
  for (i = 0; i < 2; i++) {
@@ -13901,16 +13930,16 @@
13901
13930
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13902
13931
  rootDirname: rootDirname,
13903
13932
  cacheDirname: cacheDirname,
13904
- isCacheCleaned: isCacheCleaned,
13933
+ intermediateFilesStrategy: intermediateFilesStrategy,
13905
13934
  extension: 'html',
13906
13935
  isVerbose: isVerbose,
13907
13936
  })];
13908
- case 3:
13937
+ case 2:
13909
13938
  cacheFilehandler = _g.sent();
13910
13939
  return [4 /*yield*/, promises.writeFile(cacheFilehandler.filename, html, 'utf-8')];
13911
- case 4:
13940
+ case 3:
13912
13941
  _g.sent();
13913
- markdown = markdownConverter.makeMarkdown(html, jsdom$1.window.document);
13942
+ markdown = this.showdownConverter.makeMarkdown(html, jsdom$1.window.document);
13914
13943
  return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
13915
13944
  }
13916
13945
  });
@@ -13954,7 +13983,6 @@
13954
13983
  return WebsiteScraper;
13955
13984
  }());
13956
13985
  /**
13957
- * TODO: !!!!!! Put into separate package
13958
13986
  * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
13959
13987
  * TODO: [🪂] Do it in parallel 11:11
13960
13988
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools