@promptbook/cli 0.72.0-9 → 0.72.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +4 -8
  2. package/esm/index.es.js +987 -649
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +24 -18
  5. package/esm/typings/src/_packages/node.index.d.ts +2 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +18 -12
  7. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  8. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  10. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  11. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  12. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  14. package/esm/typings/src/config.d.ts +21 -14
  15. package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
  16. package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
  17. package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
  18. package/esm/typings/src/executables/locateApp.d.ts +33 -0
  19. package/esm/typings/src/executables/locateApp.test.d.ts +1 -0
  20. package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
  21. package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
  22. package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
  23. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  24. package/esm/typings/src/execution/Executables.d.ts +18 -0
  25. package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
  26. package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
  27. package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
  28. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  29. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
  30. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  31. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
  32. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
  33. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
  34. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  35. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  36. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  37. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  38. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  39. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
  40. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
  41. package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +7 -0
  42. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  43. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +2 -2
  45. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  46. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  47. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -1
  48. package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
  49. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  50. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
  51. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
  52. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
  53. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  54. package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
  55. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
  56. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
  57. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  58. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
  59. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +15 -6
  60. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
  61. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +6 -12
  62. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
  63. package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +9 -14
  64. package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +23 -8
  65. package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
  66. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
  67. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  68. package/esm/typings/src/scrapers/_common/Scraper.d.ts +2 -9
  69. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
  70. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
  71. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
  72. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
  73. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
  74. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
  75. package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
  76. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  77. package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
  78. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
  79. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
  80. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
  81. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
  82. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
  83. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
  84. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
  85. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
  86. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
  87. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
  88. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
  89. package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
  90. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  91. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.test.d.ts +1 -0
  92. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  93. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  94. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  95. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  96. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  97. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  98. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  99. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  100. package/esm/typings/src/types/Prompt.d.ts +2 -1
  101. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  102. package/esm/typings/src/types/typeAliases.d.ts +11 -8
  103. package/esm/typings/src/utils/$Register.d.ts +1 -1
  104. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  105. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  106. package/esm/typings/src/utils/emojis.d.ts +1 -1
  107. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +4 -2
  108. package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
  109. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -0
  110. package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
  111. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  112. package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
  113. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  114. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  115. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  116. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  117. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  118. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  119. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  120. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  121. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  122. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  123. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  124. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  125. package/package.json +3 -2
  126. package/umd/index.umd.js +988 -651
  127. package/umd/index.umd.js.map +1 -1
  128. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  129. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  130. package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
  131. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
  132. /package/esm/typings/src/{scrapers/website/WebsiteScraper.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
  133. /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
package/esm/index.es.js CHANGED
@@ -2,7 +2,7 @@ import commander from 'commander';
2
2
  import spaceTrim$1, { spaceTrim } from 'spacetrim';
3
3
  import colors from 'colors';
4
4
  import { forTime } from 'waitasecond';
5
- import { stat, access, constants, readFile, readdir, writeFile, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
5
+ import { stat, access, constants, readFile, writeFile, readdir, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
6
6
  import { join, basename, dirname } from 'path';
7
7
  import { format } from 'prettier';
8
8
  import parserHtml from 'prettier/parser-html';
@@ -10,6 +10,8 @@ import { unparse, parse } from 'papaparse';
10
10
  import { SHA256 } from 'crypto-js';
11
11
  import hexEncoder from 'crypto-js/enc-hex';
12
12
  import { lookup } from 'mime-types';
13
+ import { exec as exec$2, spawn } from 'child_process';
14
+ import { promisify } from 'util';
13
15
  import * as dotenv from 'dotenv';
14
16
  import sha256 from 'crypto-js/sha256';
15
17
  import glob from 'glob-promise';
@@ -17,7 +19,6 @@ import { io } from 'socket.io-client';
17
19
  import Anthropic from '@anthropic-ai/sdk';
18
20
  import { OpenAIClient, AzureKeyCredential } from '@azure/openai';
19
21
  import OpenAI from 'openai';
20
- import { spawn } from 'child_process';
21
22
  import { Readability } from '@mozilla/readability';
22
23
  import { JSDOM } from 'jsdom';
23
24
  import { Converter } from 'showdown';
@@ -26,7 +27,7 @@ import { Converter } from 'showdown';
26
27
  /**
27
28
  * The version of the Promptbook library
28
29
  */
29
- var PROMPTBOOK_VERSION = '0.72.0-8';
30
+ var PROMPTBOOK_VERSION = '0.72.0-34';
30
31
  // TODO: [main] !!!! List here all the versions and annotate + put into script
31
32
 
32
33
  /*! *****************************************************************************
@@ -399,7 +400,8 @@ var GENERATOR_WARNING = "\u26A0\uFE0F WARNING: This code has been generated so t
399
400
  *
400
401
  * @public exported from `@promptbook/core`
401
402
  */
402
- var CLAIM = "Supercharge LLM models with Promptbook";
403
+ var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
404
+ // <- TODO: [🐊] Pick the best claim
403
405
  /**
404
406
  * Warning message for the generated sections and files files
405
407
  *
@@ -419,6 +421,7 @@ var LOOP_LIMIT = 1000;
419
421
  * @private within the repository - too low-level in comparison with other `MAX_...`
420
422
  */
421
423
  var CONNECTION_TIMEOUT_MS = 7 * 1000;
424
+ // <- TODO: [⏳] Standartize timeouts, Make DEFAULT_TIMEOUT_MS as global constant
422
425
  /**
423
426
  * How many times to retry the connections
424
427
  *
@@ -432,23 +435,30 @@ var CONNECTION_RETRIES_LIMIT = 5;
432
435
  */
433
436
  var IMMEDIATE_TIME = 10;
434
437
  /**
435
- * The maximum number of (LLM) tasks running in parallel
438
+ * The maximum length of the (generated) filename
436
439
  *
437
440
  * @public exported from `@promptbook/core`
438
441
  */
439
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
442
+ var MAX_FILENAME_LENGTH = 30;
440
443
  /**
441
- * The maximum number of attempts to execute LLM task before giving up
444
+ * Strategy for caching the intermediate results for knowledge sources
442
445
  *
443
446
  * @public exported from `@promptbook/core`
444
447
  */
445
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
448
+ var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
449
+ // <- TODO: [😡] Change to 'VISIBLE'
446
450
  /**
447
- * The maximum length of the (generated) filename
451
+ * The maximum number of (LLM) tasks running in parallel
448
452
  *
449
453
  * @public exported from `@promptbook/core`
450
454
  */
451
- var MAX_FILENAME_LENGTH = 30;
455
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
456
+ /**
457
+ * The maximum number of attempts to execute LLM task before giving up
458
+ *
459
+ * @public exported from `@promptbook/core`
460
+ */
461
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
452
462
  /**
453
463
  * Where to store the cache of executions for promptbook CLI
454
464
  *
@@ -456,7 +466,7 @@ var MAX_FILENAME_LENGTH = 30;
456
466
  *
457
467
  * @public exported from `@promptbook/core`
458
468
  */
459
- var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
469
+ var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
460
470
  /**
461
471
  * Where to store the scrape cache
462
472
  *
@@ -464,13 +474,13 @@ var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
464
474
  *
465
475
  * @public exported from `@promptbook/core`
466
476
  */
467
- var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
477
+ var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
468
478
  /**
469
479
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
470
480
  *
471
481
  * @public exported from `@promptbook/core`
472
482
  */
473
- var PIPELINE_COLLECTION_BASE_FILENAME = "index";
483
+ var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
474
484
  /**
475
485
  * Nonce which is used for replacing things in strings
476
486
  *
@@ -534,13 +544,13 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
534
544
  *
535
545
  * @public exported from `@promptbook/core`
536
546
  */
537
- var IS_VERBOSE = false;
547
+ var DEFAULT_IS_VERBOSE = false;
538
548
  /**
539
549
  * @@@
540
550
  *
541
551
  * @public exported from `@promptbook/core`
542
552
  */
543
- var IS_AUTO_INSTALLED = false;
553
+ var DEFAULT_IS_AUTO_INSTALLED = false;
544
554
  /**
545
555
  * @@@
546
556
  *
@@ -613,271 +623,6 @@ function initializeHelloCommand(program) {
613
623
  * Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
614
624
  */
615
625
 
616
- /**
617
- * Just marks a place of place where should be something implemented
618
- * No side effects.
619
- *
620
- * Note: It can be usefull suppressing eslint errors of unused variables
621
- *
622
- * @param value any values
623
- * @returns void
624
- * @private within the repository
625
- */
626
- function TODO_USE() {
627
- var value = [];
628
- for (var _i = 0; _i < arguments.length; _i++) {
629
- value[_i] = arguments[_i];
630
- }
631
- }
632
-
633
- /**
634
- * @@@
635
- *
636
- * @public exported from `@promptbook/node`
637
- */
638
- function $provideFilesystemForNode(options) {
639
- if (!$isRunningInNode()) {
640
- throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
641
- }
642
- var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
643
- TODO_USE(isVerbose);
644
- return {
645
- stat: stat,
646
- access: access,
647
- constants: constants,
648
- readFile: readFile,
649
- readdir: readdir,
650
- };
651
- }
652
- /**
653
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
654
- */
655
-
656
- /**
657
- * This error type indicates that some part of the code is not implemented yet
658
- *
659
- * @public exported from `@promptbook/core`
660
- */
661
- var NotYetImplementedError = /** @class */ (function (_super) {
662
- __extends(NotYetImplementedError, _super);
663
- function NotYetImplementedError(message) {
664
- var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
665
- _this.name = 'NotYetImplementedError';
666
- Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
667
- return _this;
668
- }
669
- return NotYetImplementedError;
670
- }(Error));
671
-
672
- /**
673
- * @@@
674
- *
675
- * Note: `$` is used to indicate that this function is not a pure function - it access global scope
676
- *
677
- * @private internal function of `$Register`
678
- */
679
- function $getGlobalScope() {
680
- return Function('return this')();
681
- }
682
-
683
- /**
684
- * @@@
685
- *
686
- * @param text @@@
687
- * @returns @@@
688
- * @example 'HELLO_WORLD'
689
- * @example 'I_LOVE_PROMPTBOOK'
690
- * @public exported from `@promptbook/utils`
691
- */
692
- function normalizeTo_SCREAMING_CASE(text) {
693
- var e_1, _a;
694
- var charType;
695
- var lastCharType = 'OTHER';
696
- var normalizedName = '';
697
- try {
698
- for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
699
- var char = text_1_1.value;
700
- var normalizedChar = void 0;
701
- if (/^[a-z]$/.test(char)) {
702
- charType = 'LOWERCASE';
703
- normalizedChar = char.toUpperCase();
704
- }
705
- else if (/^[A-Z]$/.test(char)) {
706
- charType = 'UPPERCASE';
707
- normalizedChar = char;
708
- }
709
- else if (/^[0-9]$/.test(char)) {
710
- charType = 'NUMBER';
711
- normalizedChar = char;
712
- }
713
- else {
714
- charType = 'OTHER';
715
- normalizedChar = '_';
716
- }
717
- if (charType !== lastCharType &&
718
- !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
719
- !(lastCharType === 'NUMBER') &&
720
- !(charType === 'NUMBER')) {
721
- normalizedName += '_';
722
- }
723
- normalizedName += normalizedChar;
724
- lastCharType = charType;
725
- }
726
- }
727
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
728
- finally {
729
- try {
730
- if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
731
- }
732
- finally { if (e_1) throw e_1.error; }
733
- }
734
- normalizedName = normalizedName.replace(/_+/g, '_');
735
- normalizedName = normalizedName.replace(/_?\/_?/g, '/');
736
- normalizedName = normalizedName.replace(/^_/, '');
737
- normalizedName = normalizedName.replace(/_$/, '');
738
- return normalizedName;
739
- }
740
- /**
741
- * TODO: Tests
742
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
743
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
744
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
745
- * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
746
- * TODO: [🌺] Use some intermediate util splitWords
747
- */
748
-
749
- /**
750
- * @@@
751
- *
752
- * @param text @@@
753
- * @returns @@@
754
- * @example 'hello_world'
755
- * @example 'i_love_promptbook'
756
- * @public exported from `@promptbook/utils`
757
- */
758
- function normalizeTo_snake_case(text) {
759
- return normalizeTo_SCREAMING_CASE(text).toLowerCase();
760
- }
761
-
762
- /**
763
- * Register is @@@
764
- *
765
- * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
766
- *
767
- * @private internal utility, exported are only signleton instances of this class
768
- */
769
- var $Register = /** @class */ (function () {
770
- function $Register(registerName) {
771
- this.registerName = registerName;
772
- var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
773
- var globalScope = $getGlobalScope();
774
- if (globalScope[storageName] === undefined) {
775
- globalScope[storageName] = [];
776
- }
777
- else if (!Array.isArray(globalScope[storageName])) {
778
- throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
779
- }
780
- this.storage = globalScope[storageName];
781
- }
782
- $Register.prototype.list = function () {
783
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
784
- return this.storage;
785
- };
786
- $Register.prototype.register = function (registered) {
787
- var packageName = registered.packageName, className = registered.className;
788
- var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
789
- var existingRegistration = this.storage[existingRegistrationIndex];
790
- if (!existingRegistration) {
791
- this.storage.push(registered);
792
- }
793
- else {
794
- this.storage[existingRegistrationIndex] = registered;
795
- }
796
- return {
797
- registerName: this.registerName,
798
- packageName: packageName,
799
- className: className,
800
- get isDestroyed() {
801
- return false;
802
- },
803
- destroy: function () {
804
- throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
805
- },
806
- };
807
- };
808
- return $Register;
809
- }());
810
-
811
- /**
812
- * @@@
813
- *
814
- * Note: `$` is used to indicate that this interacts with the global scope
815
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
816
- * @public exported from `@promptbook/core`
817
- */
818
- var $scrapersRegister = new $Register('scraper_constructors');
819
- /**
820
- * TODO: [®] DRY Register logic
821
- */
822
-
823
- /**
824
- * !!!!!!
825
- *
826
- * 1) @@@
827
- * 2) @@@
828
- *
829
- * @public exported from `@promptbook/node`
830
- */
831
- function $provideScrapersForNode(tools, options) {
832
- return __awaiter(this, void 0, void 0, function () {
833
- var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
834
- var e_1, _f;
835
- return __generator(this, function (_g) {
836
- switch (_g.label) {
837
- case 0:
838
- if (!$isRunningInNode()) {
839
- throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
840
- }
841
- _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
842
- TODO_USE(isAutoInstalled);
843
- TODO_USE(isVerbose);
844
- scrapers = [];
845
- _g.label = 1;
846
- case 1:
847
- _g.trys.push([1, 6, 7, 8]);
848
- _d = __values($scrapersRegister.list()), _e = _d.next();
849
- _g.label = 2;
850
- case 2:
851
- if (!!_e.done) return [3 /*break*/, 5];
852
- scraperFactory = _e.value;
853
- return [4 /*yield*/, scraperFactory(tools, options || {})];
854
- case 3:
855
- scraper = _g.sent();
856
- scrapers.push(scraper);
857
- _g.label = 4;
858
- case 4:
859
- _e = _d.next();
860
- return [3 /*break*/, 2];
861
- case 5: return [3 /*break*/, 8];
862
- case 6:
863
- e_1_1 = _g.sent();
864
- e_1 = { error: e_1_1 };
865
- return [3 /*break*/, 8];
866
- case 7:
867
- try {
868
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
869
- }
870
- finally { if (e_1) throw e_1.error; }
871
- return [7 /*endfinally*/];
872
- case 8: return [2 /*return*/, scrapers];
873
- }
874
- });
875
- });
876
- }
877
- /**
878
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
879
- */
880
-
881
626
  /**
882
627
  * Converts PipelineCollection to serialized JSON
883
628
  *
@@ -963,7 +708,6 @@ function pipelineJsonToString(pipelineJson) {
963
708
  pipelineString += '\n\n';
964
709
  pipelineString += description;
965
710
  }
966
- // TODO:> const commands: Array<Command>
967
711
  var commands = [];
968
712
  if (pipelineUrl) {
969
713
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -1019,7 +763,6 @@ function pipelineJsonToString(pipelineJson) {
1019
763
  pipelineString += '\n\n';
1020
764
  pipelineString += description_1;
1021
765
  }
1022
- // TODO:> const commands: Array<Command>
1023
766
  var commands_1 = [];
1024
767
  var contentLanguage = 'text';
1025
768
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -1318,8 +1061,7 @@ var PipelineExecutionError = /** @class */ (function (_super) {
1318
1061
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
1319
1062
  *
1320
1063
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
1321
- * @public exported from `@promptbook/types`
1322
- * TODO: !!!!!! Export as runtime class not just type
1064
+ * @public exported from `@promptbook/core`
1323
1065
  */
1324
1066
  var MultipleLlmExecutionTools = /** @class */ (function () {
1325
1067
  /**
@@ -2478,6 +2220,22 @@ var LimitReachedError = /** @class */ (function (_super) {
2478
2220
  return LimitReachedError;
2479
2221
  }(Error));
2480
2222
 
2223
+ /**
2224
+ * This error type indicates that some part of the code is not implemented yet
2225
+ *
2226
+ * @public exported from `@promptbook/core`
2227
+ */
2228
+ var NotYetImplementedError = /** @class */ (function (_super) {
2229
+ __extends(NotYetImplementedError, _super);
2230
+ function NotYetImplementedError(message) {
2231
+ var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
2232
+ _this.name = 'NotYetImplementedError';
2233
+ Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
2234
+ return _this;
2235
+ }
2236
+ return NotYetImplementedError;
2237
+ }(Error));
2238
+
2481
2239
  /**
2482
2240
  * Index of all custom errors
2483
2241
  *
@@ -2539,6 +2297,7 @@ function assertsExecutionSuccessful(executionResult) {
2539
2297
  }
2540
2298
  }
2541
2299
  /**
2300
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2542
2301
  * TODO: [🧠] Can this return type be better typed than void
2543
2302
  */
2544
2303
 
@@ -2796,6 +2555,23 @@ function union() {
2796
2555
  return union;
2797
2556
  }
2798
2557
 
2558
+ /**
2559
+ * Just marks a place of place where should be something implemented
2560
+ * No side effects.
2561
+ *
2562
+ * Note: It can be usefull suppressing eslint errors of unused variables
2563
+ *
2564
+ * @param value any values
2565
+ * @returns void
2566
+ * @private within the repository
2567
+ */
2568
+ function TODO_USE() {
2569
+ var value = [];
2570
+ for (var _i = 0; _i < arguments.length; _i++) {
2571
+ value[_i] = arguments[_i];
2572
+ }
2573
+ }
2574
+
2799
2575
  /**
2800
2576
  * This error indicates problems parsing the format value
2801
2577
  *
@@ -3828,12 +3604,11 @@ function checkExpectations(expectations, value) {
3828
3604
  */
3829
3605
  function executeAttempts(options) {
3830
3606
  return __awaiter(this, void 0, void 0, function () {
3831
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3607
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
3832
3608
  return __generator(this, function (_a) {
3833
3609
  switch (_a.label) {
3834
3610
  case 0:
3835
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
3836
- maxExecutionAttempts = settings.maxExecutionAttempts;
3611
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
3837
3612
  $ongoingTemplateResult = {
3838
3613
  $result: null,
3839
3614
  $resultString: null,
@@ -4199,12 +3974,12 @@ function executeAttempts(options) {
4199
3974
  */
4200
3975
  function executeFormatSubvalues(options) {
4201
3976
  return __awaiter(this, void 0, void 0, function () {
4202
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
3977
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4203
3978
  var _this = this;
4204
3979
  return __generator(this, function (_a) {
4205
3980
  switch (_a.label) {
4206
3981
  case 0:
4207
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
3982
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4208
3983
  if (template.foreach === undefined) {
4209
3984
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4210
3985
  }
@@ -4234,7 +4009,7 @@ function executeFormatSubvalues(options) {
4234
4009
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4235
4010
  }
4236
4011
  if (formatDefinition.formatName === 'CSV') {
4237
- formatSettings = settings.csvSettings;
4012
+ formatSettings = csvSettings;
4238
4013
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4239
4014
  }
4240
4015
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4387,13 +4162,12 @@ function getReservedParametersForTemplate(options) {
4387
4162
  */
4388
4163
  function executeTemplate(options) {
4389
4164
  return __awaiter(this, void 0, void 0, function () {
4390
- var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4391
- var e_1, _f, _g;
4392
- return __generator(this, function (_h) {
4393
- switch (_h.label) {
4165
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4166
+ var e_1, _g, _h;
4167
+ return __generator(this, function (_j) {
4168
+ switch (_j.label) {
4394
4169
  case 0:
4395
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4396
- maxExecutionAttempts = settings.maxExecutionAttempts;
4170
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4397
4171
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4398
4172
  title = currentTemplate.title;
4399
4173
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -4408,7 +4182,7 @@ function executeTemplate(options) {
4408
4182
  // <- [🍸]
4409
4183
  })];
4410
4184
  case 1:
4411
- _h.sent();
4185
+ _j.sent();
4412
4186
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
4413
4187
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
4414
4188
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4419,15 +4193,15 @@ function executeTemplate(options) {
4419
4193
  .map(function (name) { return "{".concat(name, "}"); })
4420
4194
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
4421
4195
  }
4422
- _b = (_a = Object).freeze;
4423
- _c = [{}];
4196
+ _c = (_b = Object).freeze;
4197
+ _d = [{}];
4424
4198
  return [4 /*yield*/, getReservedParametersForTemplate({
4425
4199
  preparedPipeline: preparedPipeline,
4426
4200
  template: currentTemplate,
4427
4201
  pipelineIdentification: pipelineIdentification,
4428
4202
  })];
4429
4203
  case 2:
4430
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4204
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
4431
4205
  definedParameterNames = new Set(Object.keys(definedParameters));
4432
4206
  parameters = {};
4433
4207
  _loop_1 = function (parameterName) {
@@ -4447,15 +4221,15 @@ function executeTemplate(options) {
4447
4221
  try {
4448
4222
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
4449
4223
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4450
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
4451
- parameterName = _e.value;
4224
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
4225
+ parameterName = _f.value;
4452
4226
  _loop_1(parameterName);
4453
4227
  }
4454
4228
  }
4455
4229
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4456
4230
  finally {
4457
4231
  try {
4458
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4232
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
4459
4233
  }
4460
4234
  finally { if (e_1) throw e_1.error; }
4461
4235
  }
@@ -4475,12 +4249,11 @@ function executeTemplate(options) {
4475
4249
  template: currentTemplate,
4476
4250
  preparedPipeline: preparedPipeline,
4477
4251
  tools: tools,
4478
- settings: settings,
4479
4252
  $executionReport: $executionReport,
4480
4253
  pipelineIdentification: pipelineIdentification,
4481
4254
  })];
4482
4255
  case 3:
4483
- resultString = _h.sent();
4256
+ resultString = _j.sent();
4484
4257
  return [4 /*yield*/, onProgress({
4485
4258
  name: name,
4486
4259
  title: title,
@@ -4492,12 +4265,12 @@ function executeTemplate(options) {
4492
4265
  // <- [🍸]
4493
4266
  })];
4494
4267
  case 4:
4495
- _h.sent();
4496
- return [2 /*return*/, Object.freeze((_g = {},
4497
- _g[currentTemplate.resultingParameterName] =
4268
+ _j.sent();
4269
+ return [2 /*return*/, Object.freeze((_h = {},
4270
+ _h[currentTemplate.resultingParameterName] =
4498
4271
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4499
4272
  resultString,
4500
- _g))];
4273
+ _h))];
4501
4274
  }
4502
4275
  });
4503
4276
  });
@@ -4556,13 +4329,12 @@ function filterJustOutputParameters(options) {
4556
4329
  */
4557
4330
  function executePipeline(options) {
4558
4331
  return __awaiter(this, void 0, void 0, function () {
4559
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4332
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
4560
4333
  var e_1, _f, e_2, _g;
4561
4334
  return __generator(this, function (_h) {
4562
4335
  switch (_h.label) {
4563
4336
  case 0:
4564
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4565
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4337
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4566
4338
  preparedPipeline = options.preparedPipeline;
4567
4339
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
4568
4340
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -4747,12 +4519,7 @@ function executePipeline(options) {
4747
4519
  return [3 /*break*/, 4];
4748
4520
  case 3:
4749
4521
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
4750
- work_1 = executeTemplate({
4751
- currentTemplate: currentTemplate,
4752
- preparedPipeline: preparedPipeline,
4753
- parametersToPass: parametersToPass,
4754
- tools: tools,
4755
- onProgress: function (progress) {
4522
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
4756
4523
  if (isReturned) {
4757
4524
  throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
4758
4525
  .split('\n')
@@ -4762,11 +4529,7 @@ function executePipeline(options) {
4762
4529
  if (onProgress) {
4763
4530
  onProgress(progress);
4764
4531
  }
4765
- },
4766
- settings: settings,
4767
- $executionReport: executionReport,
4768
- pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
4769
- })
4532
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
4770
4533
  .then(function (newParametersToPass) {
4771
4534
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
4772
4535
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -4870,8 +4633,7 @@ function executePipeline(options) {
4870
4633
  */
4871
4634
  function createPipelineExecutor(options) {
4872
4635
  var _this = this;
4873
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
4874
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
4636
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
4875
4637
  validatePipeline(pipeline);
4876
4638
  var pipelineIdentification = (function () {
4877
4639
  // Note: This is a 😐 implementation of [🚞]
@@ -4905,14 +4667,12 @@ function createPipelineExecutor(options) {
4905
4667
  tools: tools,
4906
4668
  onProgress: onProgress,
4907
4669
  pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
4908
- settings: {
4909
- maxExecutionAttempts: maxExecutionAttempts,
4910
- maxParallelCount: maxParallelCount,
4911
- csvSettings: csvSettings,
4912
- isVerbose: isVerbose,
4913
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4914
- rootDirname: rootDirname,
4915
- },
4670
+ maxExecutionAttempts: maxExecutionAttempts,
4671
+ maxParallelCount: maxParallelCount,
4672
+ csvSettings: csvSettings,
4673
+ isVerbose: isVerbose,
4674
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4675
+ rootDirname: rootDirname,
4916
4676
  })];
4917
4677
  });
4918
4678
  }); };
@@ -4935,7 +4695,7 @@ function preparePersona(personaDescription, tools, options) {
4935
4695
  return __generator(this, function (_d) {
4936
4696
  switch (_d.label) {
4937
4697
  case 0:
4938
- _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
4698
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
4939
4699
  if (tools === undefined || tools.llm === undefined) {
4940
4700
  throw new MissingToolsError('LLM tools are required for preparing persona');
4941
4701
  }
@@ -4991,20 +4751,171 @@ function preparePersona(personaDescription, tools, options) {
4991
4751
  */
4992
4752
 
4993
4753
  /**
4994
- * This error indicates that the promptbook can not retrieve knowledge from external sources
4754
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
4755
+ *
4756
+ * @public exported from `@promptbook/core`
4757
+ */
4758
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
4759
+ __extends(KnowledgeScrapeError, _super);
4760
+ function KnowledgeScrapeError(message) {
4761
+ var _this = _super.call(this, message) || this;
4762
+ _this.name = 'KnowledgeScrapeError';
4763
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
4764
+ return _this;
4765
+ }
4766
+ return KnowledgeScrapeError;
4767
+ }(Error));
4768
+
4769
+ /**
4770
+ * @@@
4771
+ *
4772
+ * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4773
+ *
4774
+ * @private internal function of `$Register`
4775
+ */
4776
+ function $getGlobalScope() {
4777
+ return Function('return this')();
4778
+ }
4779
+
4780
+ /**
4781
+ * @@@
4782
+ *
4783
+ * @param text @@@
4784
+ * @returns @@@
4785
+ * @example 'HELLO_WORLD'
4786
+ * @example 'I_LOVE_PROMPTBOOK'
4787
+ * @public exported from `@promptbook/utils`
4788
+ */
4789
+ function normalizeTo_SCREAMING_CASE(text) {
4790
+ var e_1, _a;
4791
+ var charType;
4792
+ var lastCharType = 'OTHER';
4793
+ var normalizedName = '';
4794
+ try {
4795
+ for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
4796
+ var char = text_1_1.value;
4797
+ var normalizedChar = void 0;
4798
+ if (/^[a-z]$/.test(char)) {
4799
+ charType = 'LOWERCASE';
4800
+ normalizedChar = char.toUpperCase();
4801
+ }
4802
+ else if (/^[A-Z]$/.test(char)) {
4803
+ charType = 'UPPERCASE';
4804
+ normalizedChar = char;
4805
+ }
4806
+ else if (/^[0-9]$/.test(char)) {
4807
+ charType = 'NUMBER';
4808
+ normalizedChar = char;
4809
+ }
4810
+ else {
4811
+ charType = 'OTHER';
4812
+ normalizedChar = '_';
4813
+ }
4814
+ if (charType !== lastCharType &&
4815
+ !(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
4816
+ !(lastCharType === 'NUMBER') &&
4817
+ !(charType === 'NUMBER')) {
4818
+ normalizedName += '_';
4819
+ }
4820
+ normalizedName += normalizedChar;
4821
+ lastCharType = charType;
4822
+ }
4823
+ }
4824
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
4825
+ finally {
4826
+ try {
4827
+ if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
4828
+ }
4829
+ finally { if (e_1) throw e_1.error; }
4830
+ }
4831
+ normalizedName = normalizedName.replace(/_+/g, '_');
4832
+ normalizedName = normalizedName.replace(/_?\/_?/g, '/');
4833
+ normalizedName = normalizedName.replace(/^_/, '');
4834
+ normalizedName = normalizedName.replace(/_$/, '');
4835
+ return normalizedName;
4836
+ }
4837
+ /**
4838
+ * TODO: Tests
4839
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
4840
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
4841
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
4842
+ * > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
4843
+ * TODO: [🌺] Use some intermediate util splitWords
4844
+ */
4845
+
4846
+ /**
4847
+ * @@@
4848
+ *
4849
+ * @param text @@@
4850
+ * @returns @@@
4851
+ * @example 'hello_world'
4852
+ * @example 'i_love_promptbook'
4853
+ * @public exported from `@promptbook/utils`
4854
+ */
4855
+ function normalizeTo_snake_case(text) {
4856
+ return normalizeTo_SCREAMING_CASE(text).toLowerCase();
4857
+ }
4858
+
4859
+ /**
4860
+ * Register is @@@
4861
+ *
4862
+ * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
4863
+ *
4864
+ * @private internal utility, exported are only signleton instances of this class
4865
+ */
4866
+ var $Register = /** @class */ (function () {
4867
+ function $Register(registerName) {
4868
+ this.registerName = registerName;
4869
+ var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
4870
+ var globalScope = $getGlobalScope();
4871
+ if (globalScope[storageName] === undefined) {
4872
+ globalScope[storageName] = [];
4873
+ }
4874
+ else if (!Array.isArray(globalScope[storageName])) {
4875
+ throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
4876
+ }
4877
+ this.storage = globalScope[storageName];
4878
+ }
4879
+ $Register.prototype.list = function () {
4880
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
4881
+ return this.storage;
4882
+ };
4883
+ $Register.prototype.register = function (registered) {
4884
+ var packageName = registered.packageName, className = registered.className;
4885
+ var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
4886
+ var existingRegistration = this.storage[existingRegistrationIndex];
4887
+ if (!existingRegistration) {
4888
+ this.storage.push(registered);
4889
+ }
4890
+ else {
4891
+ this.storage[existingRegistrationIndex] = registered;
4892
+ }
4893
+ return {
4894
+ registerName: this.registerName,
4895
+ packageName: packageName,
4896
+ className: className,
4897
+ get isDestroyed() {
4898
+ return false;
4899
+ },
4900
+ destroy: function () {
4901
+ throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
4902
+ },
4903
+ };
4904
+ };
4905
+ return $Register;
4906
+ }());
4907
+
4908
+ /**
4909
+ * @@@
4995
4910
  *
4911
+ * Note: `$` is used to indicate that this interacts with the global scope
4912
+ * @singleton Only one instance of each register is created per build, but thare can be more @@@
4996
4913
  * @public exported from `@promptbook/core`
4997
4914
  */
4998
- var KnowledgeScrapeError = /** @class */ (function (_super) {
4999
- __extends(KnowledgeScrapeError, _super);
5000
- function KnowledgeScrapeError(message) {
5001
- var _this = _super.call(this, message) || this;
5002
- _this.name = 'KnowledgeScrapeError';
5003
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
5004
- return _this;
5005
- }
5006
- return KnowledgeScrapeError;
5007
- }(Error));
4915
+ var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4916
+ /**
4917
+ * TODO: [®] DRY Register logic
4918
+ */
5008
4919
 
5009
4920
  /**
5010
4921
  * @@@
@@ -5013,12 +4924,11 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
5013
4924
  * @singleton Only one instance of each register is created per build, but thare can be more @@@
5014
4925
  * @public exported from `@promptbook/core`
5015
4926
  */
5016
- var $scrapersMetadataRegister = new $Register('scrapers_metadata');
4927
+ var $scrapersRegister = new $Register('scraper_constructors');
5017
4928
  /**
5018
4929
  * TODO: [®] DRY Register logic
5019
4930
  */
5020
4931
 
5021
- // TODO: !!!!!! Maybe delete this function
5022
4932
  /**
5023
4933
  * Creates a message with all registered scrapers
5024
4934
  *
@@ -5185,7 +5095,6 @@ function normalizeToKebabCase(text) {
5185
5095
  * @private within the repository
5186
5096
  */
5187
5097
  function sourceContentToName(sourceContent) {
5188
- // TODO: !!!!!! Better name for source than gibberish hash
5189
5098
  var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
5190
5099
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
5191
5100
  .toString( /* hex */)
@@ -5285,20 +5194,20 @@ function isValidFilePath(filename) {
5285
5194
  /**
5286
5195
  * @@@
5287
5196
  *
5288
- * @private for scraper utilities
5197
+ * @public exported from `@promptbook/core`
5289
5198
  */
5290
5199
  function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5291
5200
  var _a;
5292
5201
  return __awaiter(this, void 0, void 0, function () {
5293
5202
  var sourceContent, name, _b, _c, rootDirname, _d,
5294
5203
  // <- TODO: process.cwd() if running in Node.js
5295
- isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
5204
+ isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
5296
5205
  return __generator(this, function (_e) {
5297
5206
  switch (_e.label) {
5298
5207
  case 0:
5299
5208
  sourceContent = knowledgeSource.sourceContent;
5300
5209
  name = knowledgeSource.name;
5301
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
5210
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
5302
5211
  TODO_USE(isVerbose);
5303
5212
  if (!name) {
5304
5213
  name = sourceContentToName(sourceContent);
@@ -5314,19 +5223,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5314
5223
  filename: null,
5315
5224
  url: url,
5316
5225
  mimeType: mimeType,
5317
- asBlob: function () {
5318
- return __awaiter(this, void 0, void 0, function () {
5319
- var content;
5320
- return __generator(this, function (_a) {
5321
- switch (_a.label) {
5322
- case 0: return [4 /*yield*/, response_1.blob()];
5323
- case 1:
5324
- content = _a.sent();
5325
- return [2 /*return*/, content];
5326
- }
5327
- });
5328
- });
5329
- },
5226
+ /*
5227
+ TODO: [🥽]
5228
+ > async asBlob() {
5229
+ > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
5230
+ > const content = await response.blob();
5231
+ > return content;
5232
+ > },
5233
+ */
5330
5234
  asJson: function () {
5331
5235
  return __awaiter(this, void 0, void 0, function () {
5332
5236
  var content;
@@ -5366,34 +5270,31 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5366
5270
  }
5367
5271
  filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
5368
5272
  fileExtension = getFileExtension(filename_1);
5369
- mimeType_1 = extensionToMimeType(fileExtension || '');
5273
+ mimeType = extensionToMimeType(fileExtension || '');
5370
5274
  return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
5371
5275
  case 3:
5372
5276
  if (!(_e.sent())) {
5373
5277
  throw new NotFoundError(spaceTrim$1(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
5374
5278
  }
5375
- // TODO: !!!!!! Test security file - file is scoped to the project (maybe do this in `filesystemTools`)
5279
+ // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
5376
5280
  return [2 /*return*/, {
5377
5281
  source: name,
5378
5282
  filename: filename_1,
5379
5283
  url: null,
5380
- mimeType: mimeType_1,
5381
- asBlob: function () {
5382
- return __awaiter(this, void 0, void 0, function () {
5383
- var content;
5384
- return __generator(this, function (_a) {
5385
- switch (_a.label) {
5386
- case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
5387
- case 1:
5388
- content = _a.sent();
5389
- return [2 /*return*/, new Blob([
5390
- content,
5391
- // <- TODO: !!!!!! Maybe not working
5392
- ], { type: mimeType_1 })];
5393
- }
5394
- });
5395
- });
5396
- },
5284
+ mimeType: mimeType,
5285
+ /*
5286
+ TODO: [🥽]
5287
+ > async asBlob() {
5288
+ > const content = await tools.fs!.readFile(filename);
5289
+ > return new Blob(
5290
+ > [
5291
+ > content,
5292
+ > // <- TODO: [🥽] This is NOT tested, test it
5293
+ > ],
5294
+ > { type: mimeType },
5295
+ > );
5296
+ > },
5297
+ */
5397
5298
  asJson: function () {
5398
5299
  return __awaiter(this, void 0, void 0, function () {
5399
5300
  var _a, _b;
@@ -5429,9 +5330,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5429
5330
  asJson: function () {
5430
5331
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
5431
5332
  },
5432
- asBlob: function () {
5433
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
5434
- },
5333
+ /*
5334
+ TODO: [🥽]
5335
+ > asBlob() {
5336
+ > throw new UnexpectedError(
5337
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
5338
+ > );
5339
+ > },
5340
+ */
5435
5341
  }];
5436
5342
  }
5437
5343
  });
@@ -5451,7 +5357,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
5451
5357
  return __generator(this, function (_c) {
5452
5358
  switch (_c.label) {
5453
5359
  case 0:
5454
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5360
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5455
5361
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
5456
5362
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
5457
5363
  var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
@@ -5480,7 +5386,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
5480
5386
  case 4:
5481
5387
  partialPiecesUnchecked = _d.sent();
5482
5388
  if (partialPiecesUnchecked !== null) {
5483
- partialPieces = partialPiecesUnchecked;
5389
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
5390
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
5484
5391
  return [3 /*break*/, 6];
5485
5392
  }
5486
5393
  _d.label = 5;
@@ -5528,7 +5435,7 @@ TODO: [🧊] This is how it can look in future
5528
5435
  > /**
5529
5436
  > * Unprepared knowledge
5530
5437
  > * /
5531
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
5438
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
5532
5439
  > };
5533
5440
  >
5534
5441
  > export async function prepareKnowledgePieces(
@@ -5586,7 +5493,7 @@ function prepareTemplates(pipeline, tools, options) {
5586
5493
  return __generator(this, function (_b) {
5587
5494
  switch (_b.label) {
5588
5495
  case 0:
5589
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
5496
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
5590
5497
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
5591
5498
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
5592
5499
  TODO_USE(parameters);
@@ -5648,7 +5555,7 @@ function preparePipeline(pipeline, tools, options) {
5648
5555
  if (isPipelinePrepared(pipeline)) {
5649
5556
  return [2 /*return*/, pipeline];
5650
5557
  }
5651
- rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
5558
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
5652
5559
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
5653
5560
  if (tools === undefined || tools.llm === undefined) {
5654
5561
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
@@ -5706,7 +5613,9 @@ function preparePipeline(pipeline, tools, options) {
5706
5613
  // ----- /Templates preparation -----
5707
5614
  // Note: Count total usage
5708
5615
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
5709
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
5616
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
5617
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
5618
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
5710
5619
  }
5711
5620
  });
5712
5621
  });
@@ -5753,6 +5662,7 @@ var knowledgeCommandParser = {
5753
5662
  'KNOWLEDGE ./hejny-cv.md',
5754
5663
  'KNOWLEDGE ./hejny-cv.pdf',
5755
5664
  'KNOWLEDGE ./hejny-cv.docx',
5665
+ // <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
5756
5666
  ],
5757
5667
  /**
5758
5668
  * Parses the KNOWLEDGE command
@@ -8427,38 +8337,359 @@ function pipelineStringToJsonSync(pipelineString) {
8427
8337
  * Note: This function does not validate logic of the pipeline only the parsing
8428
8338
  * Note: This function acts as compilation process
8429
8339
  *
8430
- * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8431
- * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
8432
- * @param options - Options and tools for the compilation
8433
- * @returns {Promptbook} compiled in JSON format (.ptbk.json)
8434
- * @throws {ParseError} if the promptbook string is not valid
8435
- * @public exported from `@promptbook/core`
8340
+ * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
8341
+ * @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
8342
+ * @param options - Options and tools for the compilation
8343
+ * @returns {Promptbook} compiled in JSON format (.ptbk.json)
8344
+ * @throws {ParseError} if the promptbook string is not valid
8345
+ * @public exported from `@promptbook/core`
8346
+ */
8347
+ function pipelineStringToJson(pipelineString, tools, options) {
8348
+ return __awaiter(this, void 0, void 0, function () {
8349
+ var pipelineJson;
8350
+ return __generator(this, function (_a) {
8351
+ switch (_a.label) {
8352
+ case 0:
8353
+ pipelineJson = pipelineStringToJsonSync(pipelineString);
8354
+ if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8355
+ return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
8356
+ rootDirname: null,
8357
+ })];
8358
+ case 1:
8359
+ pipelineJson = _a.sent();
8360
+ _a.label = 2;
8361
+ case 2:
8362
+ // Note: No need to use `$asDeeplyFrozenSerializableJson` because `pipelineStringToJsonSync` and `preparePipeline` already do that
8363
+ return [2 /*return*/, pipelineJson];
8364
+ }
8365
+ });
8366
+ });
8367
+ }
8368
+ /**
8369
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
8370
+ * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
8371
+ * TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
8372
+ */
8373
+
8374
+ // Note: We want to use the `exec` as async function
8375
+ var exec$1 = promisify(exec$2);
8376
+ /**
8377
+ * @@@
8378
+ *
8379
+ * @private within the repository
8380
+ */
8381
+ function locateAppOnLinux(_a) {
8382
+ var appName = _a.appName, linuxWhich = _a.linuxWhich;
8383
+ return __awaiter(this, void 0, void 0, function () {
8384
+ var _b, stderr, stdout, error_1;
8385
+ return __generator(this, function (_c) {
8386
+ switch (_c.label) {
8387
+ case 0:
8388
+ _c.trys.push([0, 2, , 3]);
8389
+ return [4 /*yield*/, exec$1("which ".concat(linuxWhich))];
8390
+ case 1:
8391
+ _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
8392
+ if (!stderr && stdout) {
8393
+ return [2 /*return*/, stdout.trim()];
8394
+ }
8395
+ throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr));
8396
+ case 2:
8397
+ error_1 = _c.sent();
8398
+ if (!(error_1 instanceof Error)) {
8399
+ throw error_1;
8400
+ }
8401
+ return [2 /*return*/, null];
8402
+ case 3: return [2 /*return*/];
8403
+ }
8404
+ });
8405
+ });
8406
+ }
8407
+ /**
8408
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8409
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8410
+ */
8411
+
8412
+ /**
8413
+ * @@@
8414
+ *
8415
+ * @public exported from `@promptbook/node`
8416
+ */
8417
+ function $provideFilesystemForNode(options) {
8418
+ if (!$isRunningInNode()) {
8419
+ throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
8420
+ }
8421
+ var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
8422
+ TODO_USE(isVerbose);
8423
+ return {
8424
+ stat: stat,
8425
+ access: access,
8426
+ constants: constants,
8427
+ readFile: readFile,
8428
+ writeFile: writeFile,
8429
+ readdir: readdir,
8430
+ };
8431
+ }
8432
+ /**
8433
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8434
+ */
8435
+
8436
+ /**
8437
+ * Checks if the file is executable
8438
+ *
8439
+ * @private within the repository
8440
+ */
8441
+ function isExecutable(path, fs) {
8442
+ return __awaiter(this, void 0, void 0, function () {
8443
+ return __generator(this, function (_a) {
8444
+ switch (_a.label) {
8445
+ case 0:
8446
+ _a.trys.push([0, 2, , 3]);
8447
+ return [4 /*yield*/, fs.access(path, fs.constants.X_OK)];
8448
+ case 1:
8449
+ _a.sent();
8450
+ return [2 /*return*/, true];
8451
+ case 2:
8452
+ _a.sent();
8453
+ return [2 /*return*/, false];
8454
+ case 3: return [2 /*return*/];
8455
+ }
8456
+ });
8457
+ });
8458
+ }
8459
+ /**
8460
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
8461
+ * TODO: [🖇] What about symlinks?
8462
+ */
8463
+
8464
+ // Note: Module `userhome` has no types available, so it is imported using `require`
8465
+ // @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions
8466
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
8467
+ var userhome = require('userhome');
8468
+ // Note: We want to use the `exec` as async function
8469
+ var exec = promisify(exec$2);
8470
+ /**
8471
+ * @@@
8472
+ *
8473
+ * @private within the repository
8474
+ */
8475
+ function locateAppOnMacOs(_a) {
8476
+ var appName = _a.appName, macOsName = _a.macOsName;
8477
+ return __awaiter(this, void 0, void 0, function () {
8478
+ var toExec, regPath, altPath, _b, stderr, stdout, error_1;
8479
+ return __generator(this, function (_c) {
8480
+ switch (_c.label) {
8481
+ case 0:
8482
+ _c.trys.push([0, 6, , 7]);
8483
+ toExec = "/Contents/MacOS/".concat(macOsName);
8484
+ regPath = "/Applications/".concat(macOsName, ".app") + toExec;
8485
+ altPath = userhome(regPath.slice(1));
8486
+ return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())];
8487
+ case 1:
8488
+ if (!_c.sent()) return [3 /*break*/, 2];
8489
+ return [2 /*return*/, regPath];
8490
+ case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())];
8491
+ case 3:
8492
+ if (_c.sent()) {
8493
+ return [2 /*return*/, altPath];
8494
+ }
8495
+ _c.label = 4;
8496
+ case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))];
8497
+ case 5:
8498
+ _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
8499
+ if (!stderr && stdout) {
8500
+ return [2 /*return*/, stdout.trim() + toExec];
8501
+ }
8502
+ throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr));
8503
+ case 6:
8504
+ error_1 = _c.sent();
8505
+ if (!(error_1 instanceof Error)) {
8506
+ throw error_1;
8507
+ }
8508
+ return [2 /*return*/, null];
8509
+ case 7: return [2 /*return*/];
8510
+ }
8511
+ });
8512
+ });
8513
+ }
8514
+ /**
8515
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8516
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8517
+ */
8518
+
8519
+ /**
8520
+ * @@@
8521
+ *
8522
+ * @private within the repository
8523
+ */
8524
+ function locateAppOnWindows(_a) {
8525
+ var appName = _a.appName, windowsSuffix = _a.windowsSuffix;
8526
+ return __awaiter(this, void 0, void 0, function () {
8527
+ var prefixes, prefixes_1, prefixes_1_1, prefix, path, e_1_1, error_1;
8528
+ var e_1, _b;
8529
+ return __generator(this, function (_c) {
8530
+ switch (_c.label) {
8531
+ case 0:
8532
+ _c.trys.push([0, 9, , 10]);
8533
+ prefixes = [
8534
+ process.env.LOCALAPPDATA,
8535
+ join(process.env.LOCALAPPDATA || '', 'Programs'),
8536
+ process.env.PROGRAMFILES,
8537
+ process.env['PROGRAMFILES(X86)'],
8538
+ ];
8539
+ _c.label = 1;
8540
+ case 1:
8541
+ _c.trys.push([1, 6, 7, 8]);
8542
+ prefixes_1 = __values(prefixes), prefixes_1_1 = prefixes_1.next();
8543
+ _c.label = 2;
8544
+ case 2:
8545
+ if (!!prefixes_1_1.done) return [3 /*break*/, 5];
8546
+ prefix = prefixes_1_1.value;
8547
+ path = prefix + windowsSuffix;
8548
+ return [4 /*yield*/, isExecutable(path, $provideFilesystemForNode())];
8549
+ case 3:
8550
+ if (_c.sent()) {
8551
+ return [2 /*return*/, path];
8552
+ }
8553
+ _c.label = 4;
8554
+ case 4:
8555
+ prefixes_1_1 = prefixes_1.next();
8556
+ return [3 /*break*/, 2];
8557
+ case 5: return [3 /*break*/, 8];
8558
+ case 6:
8559
+ e_1_1 = _c.sent();
8560
+ e_1 = { error: e_1_1 };
8561
+ return [3 /*break*/, 8];
8562
+ case 7:
8563
+ try {
8564
+ if (prefixes_1_1 && !prefixes_1_1.done && (_b = prefixes_1.return)) _b.call(prefixes_1);
8565
+ }
8566
+ finally { if (e_1) throw e_1.error; }
8567
+ return [7 /*endfinally*/];
8568
+ case 8: throw new Error("Can not locate app ".concat(appName, " on Windows."));
8569
+ case 9:
8570
+ error_1 = _c.sent();
8571
+ if (!(error_1 instanceof Error)) {
8572
+ throw error_1;
8573
+ }
8574
+ return [2 /*return*/, null];
8575
+ case 10: return [2 /*return*/];
8576
+ }
8577
+ });
8578
+ });
8579
+ }
8580
+ /**
8581
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8582
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8583
+ */
8584
+
8585
+ /**
8586
+ * Locates an application on the system
8587
+ *
8588
+ * @private within the repository
8589
+ */
8590
+ function locateApp(options) {
8591
+ if (!$isRunningInNode()) {
8592
+ throw new EnvironmentMismatchError('Locating apps works only in Node.js environment');
8593
+ }
8594
+ var appName = options.appName, linuxWhich = options.linuxWhich, windowsSuffix = options.windowsSuffix, macOsName = options.macOsName;
8595
+ if (process.platform === 'win32') {
8596
+ if (windowsSuffix) {
8597
+ return locateAppOnWindows({ appName: appName, windowsSuffix: windowsSuffix });
8598
+ }
8599
+ else {
8600
+ throw new Error("".concat(appName, " is not available on Windows."));
8601
+ }
8602
+ }
8603
+ else if (process.platform === 'darwin') {
8604
+ if (macOsName) {
8605
+ return locateAppOnMacOs({ appName: appName, macOsName: macOsName });
8606
+ }
8607
+ else {
8608
+ throw new Error("".concat(appName, " is not available on macOS."));
8609
+ }
8610
+ }
8611
+ else {
8612
+ if (linuxWhich) {
8613
+ return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich });
8614
+ }
8615
+ else {
8616
+ throw new Error("".concat(appName, " is not available on Linux."));
8617
+ }
8618
+ }
8619
+ }
8620
+ /**
8621
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
8622
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8623
+ */
8624
+
8625
+ /**
8626
+ * @@@
8627
+ *
8628
+ * @private within the repository
8629
+ */
8630
+ function locateLibreoffice() {
8631
+ return locateApp({
8632
+ appName: 'Libreoffice',
8633
+ linuxWhich: 'libreoffice',
8634
+ windowsSuffix: '\\LibreOffice\\program\\soffice.exe',
8635
+ macOsName: 'LibreOffice',
8636
+ });
8637
+ }
8638
+ /**
8639
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents`
8640
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8641
+ */
8642
+
8643
+ /**
8644
+ * @@@
8645
+ *
8646
+ * @private within the repository
8647
+ */
8648
+ function locatePandoc() {
8649
+ return locateApp({
8650
+ appName: 'Pandoc',
8651
+ linuxWhich: 'pandoc',
8652
+ windowsSuffix: '\\Pandoc\\pandoc.exe',
8653
+ macOsName: 'Pandoc',
8654
+ });
8655
+ }
8656
+ /**
8657
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents`
8658
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8659
+ */
8660
+
8661
+ /**
8662
+ * @@@
8663
+ *
8664
+ * @public exported from `@promptbook/node`
8436
8665
  */
8437
- function pipelineStringToJson(pipelineString, tools, options) {
8666
+ function $provideExecutablesForNode(options) {
8438
8667
  return __awaiter(this, void 0, void 0, function () {
8439
- var pipelineJson;
8440
- return __generator(this, function (_a) {
8441
- switch (_a.label) {
8668
+ var _a, _b, isAutoInstalled, _c, isVerbose;
8669
+ var _d;
8670
+ return __generator(this, function (_e) {
8671
+ switch (_e.label) {
8442
8672
  case 0:
8443
- pipelineJson = pipelineStringToJsonSync(pipelineString);
8444
- if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
8445
- return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
8446
- rootDirname: null,
8447
- })];
8673
+ if (!$isRunningInNode()) {
8674
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8675
+ }
8676
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8677
+ TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables
8678
+ TODO_USE(isVerbose);
8679
+ _d = {};
8680
+ return [4 /*yield*/, locatePandoc()];
8448
8681
  case 1:
8449
- pipelineJson = _a.sent();
8450
- _a.label = 2;
8451
- case 2:
8452
- // Note: No need to use `$asDeeplyFrozenSerializableJson` because `pipelineStringToJsonSync` and `preparePipeline` already do that
8453
- return [2 /*return*/, pipelineJson];
8682
+ _d.pandocPath = (_e.sent()) || undefined;
8683
+ return [4 /*yield*/, locateLibreoffice()];
8684
+ case 2: return [2 /*return*/, (_d.libreOfficePath = (_e.sent()) || undefined,
8685
+ _d)];
8454
8686
  }
8455
8687
  });
8456
8688
  });
8457
8689
  }
8458
8690
  /**
8459
- * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
8460
- * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
8461
- * TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
8691
+ * TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided
8692
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8462
8693
  */
8463
8694
 
8464
8695
  /**
@@ -8630,7 +8861,7 @@ function $registeredLlmToolsMessage() {
8630
8861
  */
8631
8862
  function createLlmToolsFromConfiguration(configuration, options) {
8632
8863
  if (options === void 0) { options = {}; }
8633
- var _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
8864
+ var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
8634
8865
  var llmTools = configuration.map(function (llmConfiguration) {
8635
8866
  var registeredItem = $llmToolsRegister
8636
8867
  .list()
@@ -8641,7 +8872,7 @@ function createLlmToolsFromConfiguration(configuration, options) {
8641
8872
  if (registeredItem === undefined) {
8642
8873
  throw new Error(spaceTrim$1(function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
8643
8874
  }
8644
- return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
8875
+ return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
8645
8876
  });
8646
8877
  return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
8647
8878
  }
@@ -8691,6 +8922,64 @@ function $provideLlmToolsFromEnv(options) {
8691
8922
  * TODO: [®] DRY Register logic
8692
8923
  */
8693
8924
 
8925
+ /**
8926
+ * @@@
8927
+ *
8928
+ * 1) @@@
8929
+ * 2) @@@
8930
+ *
8931
+ * @public exported from `@promptbook/node`
8932
+ */
8933
+ function $provideScrapersForNode(tools, options) {
8934
+ return __awaiter(this, void 0, void 0, function () {
8935
+ var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
8936
+ var e_1, _f;
8937
+ return __generator(this, function (_g) {
8938
+ switch (_g.label) {
8939
+ case 0:
8940
+ if (!$isRunningInNode()) {
8941
+ throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
8942
+ }
8943
+ _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
8944
+ TODO_USE(isAutoInstalled);
8945
+ TODO_USE(isVerbose);
8946
+ scrapers = [];
8947
+ _g.label = 1;
8948
+ case 1:
8949
+ _g.trys.push([1, 6, 7, 8]);
8950
+ _d = __values($scrapersRegister.list()), _e = _d.next();
8951
+ _g.label = 2;
8952
+ case 2:
8953
+ if (!!_e.done) return [3 /*break*/, 5];
8954
+ scraperFactory = _e.value;
8955
+ return [4 /*yield*/, scraperFactory(tools, options || {})];
8956
+ case 3:
8957
+ scraper = _g.sent();
8958
+ scrapers.push(scraper);
8959
+ _g.label = 4;
8960
+ case 4:
8961
+ _e = _d.next();
8962
+ return [3 /*break*/, 2];
8963
+ case 5: return [3 /*break*/, 8];
8964
+ case 6:
8965
+ e_1_1 = _g.sent();
8966
+ e_1 = { error: e_1_1 };
8967
+ return [3 /*break*/, 8];
8968
+ case 7:
8969
+ try {
8970
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
8971
+ }
8972
+ finally { if (e_1) throw e_1.error; }
8973
+ return [7 /*endfinally*/];
8974
+ case 8: return [2 /*return*/, scrapers];
8975
+ }
8976
+ });
8977
+ });
8978
+ }
8979
+ /**
8980
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
8981
+ */
8982
+
8694
8983
  /**
8695
8984
  * Extracts code block from markdown.
8696
8985
  *
@@ -9145,7 +9434,7 @@ var JavascriptExecutionTools = JavascriptEvalExecutionTools;
9145
9434
  */
9146
9435
  function $provideExecutionToolsForNode(options) {
9147
9436
  return __awaiter(this, void 0, void 0, function () {
9148
- var fs, llm, tools;
9437
+ var fs, llm, executables, tools;
9149
9438
  var _a;
9150
9439
  return __generator(this, function (_b) {
9151
9440
  switch (_b.label) {
@@ -9155,12 +9444,16 @@ function $provideExecutionToolsForNode(options) {
9155
9444
  }
9156
9445
  fs = $provideFilesystemForNode();
9157
9446
  llm = $provideLlmToolsFromEnv(options);
9447
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
9448
+ case 1:
9449
+ executables = _b.sent();
9158
9450
  _a = {
9159
9451
  llm: llm,
9160
- fs: fs
9452
+ fs: fs,
9453
+ executables: executables
9161
9454
  };
9162
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
9163
- case 1:
9455
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
9456
+ case 2:
9164
9457
  tools = (_a.scrapers = _b.sent(),
9165
9458
  _a.script = [new JavascriptExecutionTools(options)],
9166
9459
  _a);
@@ -9398,7 +9691,9 @@ function createCollectionFromDirectory(path, tools, options) {
9398
9691
  throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
9399
9692
  // <- TODO: [🧠] What is the best error type here`
9400
9693
  }
9401
- makedLibraryFilePath = join(path, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".json"));
9694
+ makedLibraryFilePath = join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
9695
+ // <- TODO: [🦒] Allow to override (pass different value into the function)
9696
+ , ".json"));
9402
9697
  return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
9403
9698
  case 3:
9404
9699
  if (!(_f.sent())) {
@@ -9409,7 +9704,7 @@ function createCollectionFromDirectory(path, tools, options) {
9409
9704
  // TODO: !! Implement;
9410
9705
  // TODO: [🌗]
9411
9706
  }
9412
- _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9707
+ _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
9413
9708
  collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
9414
9709
  var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
9415
9710
  var e_1, _a;
@@ -9881,7 +10176,7 @@ function $currentDate() {
9881
10176
  function cacheLlmTools(llmTools, options) {
9882
10177
  var _this = this;
9883
10178
  if (options === void 0) { options = {}; }
9884
- var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isReloaded, isReloaded = _b === void 0 ? false : _b;
10179
+ var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
9885
10180
  var proxyTools = __assign(__assign({}, llmTools), {
9886
10181
  // <- Note: [🥫]
9887
10182
  get title() {
@@ -9903,7 +10198,7 @@ function cacheLlmTools(llmTools, options) {
9903
10198
  key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
9904
10199
  '-' +
9905
10200
  sha256(hexEncoder.parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
9906
- if (!!isReloaded) return [3 /*break*/, 2];
10201
+ if (!!isCacheReloaded) return [3 /*break*/, 2];
9907
10202
  return [4 /*yield*/, storage.getItem(key)];
9908
10203
  case 1:
9909
10204
  _a = _c.sent();
@@ -9989,12 +10284,14 @@ function $provideLlmToolsForCli(options) {
9989
10284
  if (!$isRunningInNode()) {
9990
10285
  throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
9991
10286
  }
9992
- var _a = (options !== null && options !== void 0 ? options : {}).isCacheCleaned, isCacheCleaned = _a === void 0 ? false : _a;
10287
+ var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
9993
10288
  return cacheLlmTools(countTotalUsage(
9994
10289
  // <- Note: for example here we don`t want the [🌯]
9995
10290
  $provideLlmToolsFromEnv()), {
9996
- storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, { rootFolderPath: join(process.cwd(), EXECUTIONS_CACHE_DIRNAME) }),
9997
- isReloaded: isCacheCleaned,
10291
+ storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
10292
+ rootFolderPath: join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
10293
+ }),
10294
+ isCacheReloaded: isCacheReloaded,
9998
10295
  });
9999
10296
  }
10000
10297
  /**
@@ -10021,13 +10318,13 @@ function initializeMakeCommand(program) {
10021
10318
  makeCommand.option('-f, --format <format>', spaceTrim$1("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳‍🌈] */);
10022
10319
  makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
10023
10320
  makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
10024
- makeCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache", false);
10321
+ makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
10025
10322
  makeCommand.option('--verbose', "Is output verbose", false);
10026
- makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), PIPELINE_COLLECTION_BASE_FILENAME);
10323
+ makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
10027
10324
  makeCommand.action(function (path, _a) {
10028
- var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10325
+ var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
10029
10326
  return __awaiter(_this, void 0, void 0, function () {
10030
- var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10327
+ var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
10031
10328
  var _d, e_2, _e, e_1, _f;
10032
10329
  var _this = this;
10033
10330
  return __generator(this, function (_g) {
@@ -10041,22 +10338,25 @@ function initializeMakeCommand(program) {
10041
10338
  .split(',')
10042
10339
  .map(function (_) { return _.trim(); })
10043
10340
  .filter(function (_) { return _ !== ''; });
10044
- if (outFile !== PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10341
+ if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
10045
10342
  console.error(colors.red("You can only use one format if you specify --out-file"));
10046
10343
  process.exit(1);
10047
10344
  }
10048
10345
  options = {
10049
10346
  isVerbose: isVerbose,
10050
- isCacheCleaned: isCacheCleaned,
10347
+ isCacheReloaded: isCacheReloaded,
10051
10348
  };
10052
10349
  fs = $provideFilesystemForNode(options);
10053
10350
  llm = $provideLlmToolsForCli(options);
10351
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10352
+ case 1:
10353
+ executables = _g.sent();
10054
10354
  _d = {
10055
10355
  llm: llm,
10056
10356
  fs: fs
10057
10357
  };
10058
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10059
- case 1:
10358
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10359
+ case 2:
10060
10360
  tools = (_d.scrapers = _g.sent(),
10061
10361
  _d.script = [
10062
10362
  /*new JavascriptExecutionTools(options)*/
@@ -10065,31 +10365,31 @@ function initializeMakeCommand(program) {
10065
10365
  return [4 /*yield*/, createCollectionFromDirectory(path, tools, {
10066
10366
  isVerbose: isVerbose,
10067
10367
  isRecursive: true,
10068
- // <- TODO: [🍖] isCacheReloaded
10368
+ // <- TODO: [🍖] Add `intermediateFilesStrategy`
10069
10369
  })];
10070
- case 2:
10071
- collection = _g.sent();
10072
- _g.label = 3;
10073
10370
  case 3:
10074
- _g.trys.push([3, 15, 16, 17]);
10075
- validations_1 = __values(validations), validations_1_1 = validations_1.next();
10371
+ collection = _g.sent();
10076
10372
  _g.label = 4;
10077
10373
  case 4:
10078
- if (!!validations_1_1.done) return [3 /*break*/, 14];
10079
- validation_1 = validations_1_1.value;
10374
+ _g.trys.push([4, 16, 17, 18]);
10375
+ validations_1 = __values(validations), validations_1_1 = validations_1.next();
10080
10376
  _g.label = 5;
10081
10377
  case 5:
10082
- _g.trys.push([5, 11, 12, 13]);
10378
+ if (!!validations_1_1.done) return [3 /*break*/, 15];
10379
+ validation_1 = validations_1_1.value;
10380
+ _g.label = 6;
10381
+ case 6:
10382
+ _g.trys.push([6, 12, 13, 14]);
10083
10383
  e_1 = void 0;
10084
10384
  return [4 /*yield*/, collection.listPipelines()];
10085
- case 6:
10086
- _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10087
- _g.label = 7;
10088
10385
  case 7:
10089
- if (!!_c.done) return [3 /*break*/, 10];
10386
+ _b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
10387
+ _g.label = 8;
10388
+ case 8:
10389
+ if (!!_c.done) return [3 /*break*/, 11];
10090
10390
  pipelineUrl = _c.value;
10091
10391
  return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
10092
- case 8:
10392
+ case 9:
10093
10393
  pipeline = _g.sent();
10094
10394
  if (validation_1 === 'logic') {
10095
10395
  validatePipeline(pipeline);
@@ -10097,37 +10397,37 @@ function initializeMakeCommand(program) {
10097
10397
  console.info(colors.cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
10098
10398
  }
10099
10399
  }
10100
- _g.label = 9;
10101
- case 9:
10400
+ _g.label = 10;
10401
+ case 10:
10102
10402
  _c = _b.next();
10103
- return [3 /*break*/, 7];
10104
- case 10: return [3 /*break*/, 13];
10105
- case 11:
10403
+ return [3 /*break*/, 8];
10404
+ case 11: return [3 /*break*/, 14];
10405
+ case 12:
10106
10406
  e_1_1 = _g.sent();
10107
10407
  e_1 = { error: e_1_1 };
10108
- return [3 /*break*/, 13];
10109
- case 12:
10408
+ return [3 /*break*/, 14];
10409
+ case 13:
10110
10410
  try {
10111
10411
  if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
10112
10412
  }
10113
10413
  finally { if (e_1) throw e_1.error; }
10114
10414
  return [7 /*endfinally*/];
10115
- case 13:
10415
+ case 14:
10116
10416
  validations_1_1 = validations_1.next();
10117
- return [3 /*break*/, 4];
10118
- case 14: return [3 /*break*/, 17];
10119
- case 15:
10417
+ return [3 /*break*/, 5];
10418
+ case 15: return [3 /*break*/, 18];
10419
+ case 16:
10120
10420
  e_2_1 = _g.sent();
10121
10421
  e_2 = { error: e_2_1 };
10122
- return [3 /*break*/, 17];
10123
- case 16:
10422
+ return [3 /*break*/, 18];
10423
+ case 17:
10124
10424
  try {
10125
10425
  if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
10126
10426
  }
10127
10427
  finally { if (e_2) throw e_2.error; }
10128
10428
  return [7 /*endfinally*/];
10129
- case 17: return [4 /*yield*/, collectionToJson(collection)];
10130
- case 18:
10429
+ case 18: return [4 /*yield*/, collectionToJson(collection)];
10430
+ case 19:
10131
10431
  collectionJson = _g.sent();
10132
10432
  collectionJsonString = stringifyPipelineJson(collectionJson).trim();
10133
10433
  collectionJsonItems = (function () {
@@ -10146,9 +10446,9 @@ function initializeMakeCommand(program) {
10146
10446
  return __generator(this, function (_a) {
10147
10447
  switch (_a.label) {
10148
10448
  case 0:
10149
- filename = outFile !== PIPELINE_COLLECTION_BASE_FILENAME
10449
+ filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
10150
10450
  ? outFile
10151
- : join(path, "".concat(PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10451
+ : join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
10152
10452
  if (!outFile.endsWith(".".concat(extension))) {
10153
10453
  console.warn(colors.yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
10154
10454
  }
@@ -10164,27 +10464,27 @@ function initializeMakeCommand(program) {
10164
10464
  }
10165
10465
  });
10166
10466
  }); };
10167
- if (!formats.includes('json')) return [3 /*break*/, 20];
10467
+ if (!formats.includes('json')) return [3 /*break*/, 21];
10168
10468
  formats = formats.filter(function (format) { return format !== 'json'; });
10169
10469
  return [4 /*yield*/, saveFile('json', collectionJsonString)];
10170
- case 19:
10171
- _g.sent();
10172
- _g.label = 20;
10173
10470
  case 20:
10174
- if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 22];
10471
+ _g.sent();
10472
+ _g.label = 21;
10473
+ case 21:
10474
+ if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
10175
10475
  formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
10176
10476
  return [4 /*yield*/, saveFile('js', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
10177
- case 21:
10178
- (_g.sent()) + '\n';
10179
- _g.label = 22;
10180
10477
  case 22:
10181
- if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 24];
10478
+ (_g.sent()) + '\n';
10479
+ _g.label = 23;
10480
+ case 23:
10481
+ if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
10182
10482
  formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
10183
10483
  return [4 /*yield*/, saveFile('ts', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
10184
- case 23:
10185
- _g.sent();
10186
- _g.label = 24;
10187
10484
  case 24:
10485
+ _g.sent();
10486
+ _g.label = 25;
10487
+ case 25:
10188
10488
  if (formats.length > 0) {
10189
10489
  console.warn(colors.yellow("Format ".concat(formats.join(' and '), " is not supported")));
10190
10490
  }
@@ -10446,76 +10746,79 @@ function initializeTestCommand(program) {
10446
10746
  // <- TODO: [🧟‍♂️] Unite path to promptbook collection argument
10447
10747
  'Pipelines to test as glob pattern');
10448
10748
  testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
10449
- testCommand.option('--reload-cache', "Call LLM models even if same prompt with result is in the cache ", false);
10749
+ testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
10450
10750
  testCommand.option('-v, --verbose', "Is output verbose", false);
10451
10751
  testCommand.action(function (filesGlob, _a) {
10452
- var ignore = _a.ignore, isCacheCleaned = _a.reloadCache, isVerbose = _a.verbose;
10752
+ var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
10453
10753
  return __awaiter(_this, void 0, void 0, function () {
10454
- var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10754
+ var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
10455
10755
  var _d, e_1, _e;
10456
10756
  return __generator(this, function (_f) {
10457
10757
  switch (_f.label) {
10458
10758
  case 0:
10459
10759
  options = {
10460
10760
  isVerbose: isVerbose,
10461
- isCacheCleaned: isCacheCleaned,
10761
+ isCacheReloaded: isCacheReloaded,
10462
10762
  };
10463
10763
  fs = $provideFilesystemForNode(options);
10464
10764
  llm = $provideLlmToolsForCli(options);
10765
+ return [4 /*yield*/, $provideExecutablesForNode(options)];
10766
+ case 1:
10767
+ executables = _f.sent();
10465
10768
  _d = {
10466
10769
  llm: llm,
10467
10770
  fs: fs
10468
10771
  };
10469
- return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
10470
- case 1:
10772
+ return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
10773
+ case 2:
10471
10774
  tools = (_d.scrapers = _f.sent(),
10472
10775
  _d.script = [
10473
10776
  /*new JavascriptExecutionTools(options)*/
10474
10777
  ],
10475
10778
  _d);
10476
10779
  return [4 /*yield*/, glob(filesGlob, { ignore: ignore })];
10477
- case 2:
10478
- filenames = _f.sent();
10479
- _f.label = 3;
10480
10780
  case 3:
10481
- _f.trys.push([3, 15, 16, 17]);
10482
- filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10781
+ filenames = _f.sent();
10483
10782
  _f.label = 4;
10484
10783
  case 4:
10485
- if (!!filenames_1_1.done) return [3 /*break*/, 14];
10486
- filename = filenames_1_1.value;
10784
+ _f.trys.push([4, 16, 17, 18]);
10785
+ filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
10487
10786
  _f.label = 5;
10488
10787
  case 5:
10489
- _f.trys.push([5, 12, , 13]);
10788
+ if (!!filenames_1_1.done) return [3 /*break*/, 15];
10789
+ filename = filenames_1_1.value;
10790
+ _f.label = 6;
10791
+ case 6:
10792
+ _f.trys.push([6, 13, , 14]);
10490
10793
  pipeline = void 0;
10491
- if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 8];
10794
+ if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
10492
10795
  return [4 /*yield*/, readFile(filename, 'utf-8')];
10493
- case 6:
10796
+ case 7:
10494
10797
  pipelineMarkdown = (_f.sent());
10495
10798
  return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
10496
- case 7:
10799
+ case 8:
10497
10800
  pipeline = _f.sent();
10498
10801
  if (isVerbose) {
10499
10802
  console.info(colors.green("Parsed ".concat(filename)));
10500
10803
  }
10501
- _f.label = 8;
10502
- case 8:
10503
- if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 10];
10804
+ _f.label = 9;
10805
+ case 9:
10806
+ if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
10504
10807
  _c = (_b = JSON).parse;
10505
10808
  return [4 /*yield*/, readFile(filename, 'utf-8')];
10506
- case 9:
10507
- pipeline = _c.apply(_b, [_f.sent()]);
10508
- return [3 /*break*/, 11];
10509
10809
  case 10:
10810
+ pipeline = _c.apply(_b, [_f.sent()]);
10811
+ return [3 /*break*/, 12];
10812
+ case 11:
10510
10813
  if (isVerbose) {
10511
10814
  console.info(colors.gray("Skipping ".concat(filename)));
10512
10815
  }
10513
- return [3 /*break*/, 13];
10514
- case 11:
10816
+ return [3 /*break*/, 14];
10817
+ case 12:
10515
10818
  validatePipeline(pipeline);
10516
10819
  console.info(colors.green("Validated ".concat(filename)));
10517
- return [3 /*break*/, 13];
10518
- case 12:
10820
+ return [3 /*break*/, 14];
10821
+ case 13:
10519
10822
  error_1 = _f.sent();
10520
10823
  if (!(error_1 instanceof Error)) {
10521
10824
  throw error_1;
@@ -10524,22 +10827,22 @@ function initializeTestCommand(program) {
10524
10827
  console.error(colors.bgRed(error_1.name /* <- 11:11 */));
10525
10828
  console.error(colors.red(error_1.stack || error_1.message));
10526
10829
  process.exit(1);
10527
- return [3 /*break*/, 13];
10528
- case 13:
10830
+ return [3 /*break*/, 14];
10831
+ case 14:
10529
10832
  filenames_1_1 = filenames_1.next();
10530
- return [3 /*break*/, 4];
10531
- case 14: return [3 /*break*/, 17];
10532
- case 15:
10833
+ return [3 /*break*/, 5];
10834
+ case 15: return [3 /*break*/, 18];
10835
+ case 16:
10533
10836
  e_1_1 = _f.sent();
10534
10837
  e_1 = { error: e_1_1 };
10535
- return [3 /*break*/, 17];
10536
- case 16:
10838
+ return [3 /*break*/, 18];
10839
+ case 17:
10537
10840
  try {
10538
10841
  if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
10539
10842
  }
10540
10843
  finally { if (e_1) throw e_1.error; }
10541
10844
  return [7 /*endfinally*/];
10542
- case 17:
10845
+ case 18:
10543
10846
  console.info(colors.green("All pipelines are valid"));
10544
10847
  process.exit(0);
10545
10848
  return [2 /*return*/];
@@ -10653,6 +10956,7 @@ var _AnthropicClaudeMetadataRegistration = $llmToolsMetadataRegister.register({
10653
10956
  * @public exported from `@promptbook/remote-client`
10654
10957
  */
10655
10958
  var RemoteLlmExecutionTools = /** @class */ (function () {
10959
+ /* <- TODO: [🍚] `, Destroyable` */
10656
10960
  function RemoteLlmExecutionTools(options) {
10657
10961
  this.options = options;
10658
10962
  }
@@ -10702,12 +11006,16 @@ var RemoteLlmExecutionTools = /** @class */ (function () {
10702
11006
  if (this.options.isAnonymous) {
10703
11007
  socket.emit('listModels-request', {
10704
11008
  isAnonymous: true,
11009
+ userId: this.options.userId,
10705
11010
  llmToolsConfiguration: this.options.llmToolsConfiguration,
10706
11011
  } /* <- TODO: [🤛] */);
10707
11012
  }
10708
11013
  else {
10709
11014
  socket.emit('listModels-request', {
10710
11015
  isAnonymous: false,
11016
+ appId: this.options.appId,
11017
+ userId: this.options.userId,
11018
+ customOptions: this.options.customOptions,
10711
11019
  } /* <- TODO: [🤛] */);
10712
11020
  }
10713
11021
  return [4 /*yield*/, new Promise(function (resolve, reject) {
@@ -10801,7 +11109,9 @@ var RemoteLlmExecutionTools = /** @class */ (function () {
10801
11109
  else {
10802
11110
  socket.emit('prompt-request', {
10803
11111
  isAnonymous: false,
11112
+ appId: this.options.appId,
10804
11113
  userId: this.options.userId,
11114
+ customOptions: this.options.customOptions,
10805
11115
  prompt: prompt,
10806
11116
  } /* <- TODO: [🤛] */);
10807
11117
  }
@@ -11283,7 +11593,7 @@ var AnthropicClaudeExecutionTools = /** @class */ (function () {
11283
11593
  */
11284
11594
  var createAnthropicClaudeExecutionTools = Object.assign(function (options) {
11285
11595
  if (options.isProxied) {
11286
- return new RemoteLlmExecutionTools(__assign(__assign({}, options), { isAnonymous: true, llmToolsConfiguration: [
11596
+ return new RemoteLlmExecutionTools(__assign(__assign({}, options), { userId: null, isAnonymous: true, llmToolsConfiguration: [
11287
11597
  {
11288
11598
  title: 'Anthropic Claude (proxied)',
11289
11599
  packageName: '@promptbook/anthropic-claude',
@@ -11843,12 +12153,12 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11843
12153
  * Calls OpenAI API to use a chat model.
11844
12154
  */
11845
12155
  AzureOpenAiExecutionTools.prototype.callChatModel = function (prompt) {
11846
- var _a, _b;
12156
+ var _a, _b, _c;
11847
12157
  return __awaiter(this, void 0, void 0, function () {
11848
12158
  var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, messages, start, complete, rawRequest, rawResponse, resultContent, usage, error_1;
11849
12159
  var _this = this;
11850
- return __generator(this, function (_c) {
11851
- switch (_c.label) {
12160
+ return __generator(this, function (_d) {
12161
+ switch (_d.label) {
11852
12162
  case 0:
11853
12163
  if (this.options.isVerbose) {
11854
12164
  console.info('💬 OpenAI callChatModel call');
@@ -11856,20 +12166,20 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11856
12166
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
11857
12167
  return [4 /*yield*/, this.getClient()];
11858
12168
  case 1:
11859
- client = _c.sent();
12169
+ client = _d.sent();
11860
12170
  // TODO: [☂] Use here more modelRequirements
11861
12171
  if (modelRequirements.modelVariant !== 'CHAT') {
11862
12172
  throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
11863
12173
  }
11864
- _c.label = 2;
12174
+ _d.label = 2;
11865
12175
  case 2:
11866
- _c.trys.push([2, 4, , 5]);
12176
+ _d.trys.push([2, 4, , 5]);
11867
12177
  modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
11868
12178
  modelSettings = {
11869
12179
  maxTokens: modelRequirements.maxTokens,
11870
12180
  // <- TODO: [🌾] Make some global max cap for maxTokens
11871
12181
  temperature: modelRequirements.temperature,
11872
- user: this.options.user,
12182
+ user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
11873
12183
  // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
11874
12184
  // <- Note: [🧆]
11875
12185
  };
@@ -11900,7 +12210,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11900
12210
  throw error;
11901
12211
  })];
11902
12212
  case 3:
11903
- rawResponse = _c.sent();
12213
+ rawResponse = _d.sent();
11904
12214
  if (this.options.isVerbose) {
11905
12215
  console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
11906
12216
  }
@@ -11919,8 +12229,8 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11919
12229
  complete = getCurrentIsoDate();
11920
12230
  usage = {
11921
12231
  price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
11922
- input: __assign({ tokensCount: uncertainNumber((_a = rawResponse.usage) === null || _a === void 0 ? void 0 : _a.promptTokens) }, computeUsageCounts(prompt.content)),
11923
- output: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completionTokens) }, computeUsageCounts(prompt.content)),
12232
+ input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
12233
+ output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
11924
12234
  };
11925
12235
  return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools ChatPromptResult', {
11926
12236
  content: resultContent,
@@ -11936,7 +12246,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11936
12246
  // <- [🗯]
11937
12247
  })];
11938
12248
  case 4:
11939
- error_1 = _c.sent();
12249
+ error_1 = _d.sent();
11940
12250
  throw this.transformAzureError(error_1);
11941
12251
  case 5: return [2 /*return*/];
11942
12252
  }
@@ -11947,12 +12257,12 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11947
12257
  * Calls Azure OpenAI API to use a complete model.
11948
12258
  */
11949
12259
  AzureOpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
11950
- var _a, _b;
12260
+ var _a, _b, _c;
11951
12261
  return __awaiter(this, void 0, void 0, function () {
11952
12262
  var content, parameters, modelRequirements, client, modelName, modelSettings, start, complete, rawPromptContent, rawRequest, rawResponse, resultContent, usage, error_2;
11953
12263
  var _this = this;
11954
- return __generator(this, function (_c) {
11955
- switch (_c.label) {
12264
+ return __generator(this, function (_d) {
12265
+ switch (_d.label) {
11956
12266
  case 0:
11957
12267
  if (this.options.isVerbose) {
11958
12268
  console.info('🖋 OpenAI callCompletionModel call');
@@ -11960,20 +12270,20 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11960
12270
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
11961
12271
  return [4 /*yield*/, this.getClient()];
11962
12272
  case 1:
11963
- client = _c.sent();
12273
+ client = _d.sent();
11964
12274
  // TODO: [☂] Use here more modelRequirements
11965
12275
  if (modelRequirements.modelVariant !== 'COMPLETION') {
11966
12276
  throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
11967
12277
  }
11968
- _c.label = 2;
12278
+ _d.label = 2;
11969
12279
  case 2:
11970
- _c.trys.push([2, 4, , 5]);
12280
+ _d.trys.push([2, 4, , 5]);
11971
12281
  modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
11972
12282
  modelSettings = {
11973
12283
  maxTokens: modelRequirements.maxTokens || 2000,
11974
12284
  // <- TODO: [🌾] Make some global max cap for maxTokens
11975
12285
  temperature: modelRequirements.temperature,
11976
- user: this.options.user,
12286
+ user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
11977
12287
  // <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
11978
12288
  // <- Note: [🧆]
11979
12289
  };
@@ -11996,7 +12306,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
11996
12306
  throw error;
11997
12307
  })];
11998
12308
  case 3:
11999
- rawResponse = _c.sent();
12309
+ rawResponse = _d.sent();
12000
12310
  if (this.options.isVerbose) {
12001
12311
  console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
12002
12312
  }
@@ -12012,8 +12322,8 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
12012
12322
  complete = getCurrentIsoDate();
12013
12323
  usage = {
12014
12324
  price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
12015
- input: __assign({ tokensCount: uncertainNumber((_a = rawResponse.usage) === null || _a === void 0 ? void 0 : _a.promptTokens) }, computeUsageCounts(prompt.content)),
12016
- output: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completionTokens) }, computeUsageCounts(prompt.content)),
12325
+ input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
12326
+ output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
12017
12327
  };
12018
12328
  return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools CompletionPromptResult', {
12019
12329
  content: resultContent,
@@ -12029,7 +12339,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
12029
12339
  // <- [🗯]
12030
12340
  })];
12031
12341
  case 4:
12032
- error_2 = _c.sent();
12342
+ error_2 = _d.sent();
12033
12343
  throw this.transformAzureError(error_2);
12034
12344
  case 5: return [2 /*return*/];
12035
12345
  }
@@ -12254,7 +12564,6 @@ var OpenAiExecutionTools = /** @class */ (function () {
12254
12564
  * @param options which are relevant are directly passed to the OpenAI client
12255
12565
  */
12256
12566
  function OpenAiExecutionTools(options) {
12257
- if (options === void 0) { options = {}; }
12258
12567
  this.options = options;
12259
12568
  /**
12260
12569
  * OpenAI API client.
@@ -12282,8 +12591,8 @@ var OpenAiExecutionTools = /** @class */ (function () {
12282
12591
  if (this.client === null) {
12283
12592
  openAiOptions = __assign({}, this.options);
12284
12593
  delete openAiOptions.isVerbose;
12285
- delete openAiOptions.user;
12286
- this.client = new OpenAI(__assign({}, openAiOptions));
12594
+ delete openAiOptions.userId;
12595
+ this.client = new OpenAI(openAiOptions);
12287
12596
  }
12288
12597
  return [2 /*return*/, this.client];
12289
12598
  });
@@ -12330,11 +12639,12 @@ var OpenAiExecutionTools = /** @class */ (function () {
12330
12639
  * Calls OpenAI API to use a chat model.
12331
12640
  */
12332
12641
  OpenAiExecutionTools.prototype.callChatModel = function (prompt) {
12642
+ var _a;
12333
12643
  return __awaiter(this, void 0, void 0, function () {
12334
12644
  var content, parameters, modelRequirements, format, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
12335
12645
  var _this = this;
12336
- return __generator(this, function (_a) {
12337
- switch (_a.label) {
12646
+ return __generator(this, function (_b) {
12647
+ switch (_b.label) {
12338
12648
  case 0:
12339
12649
  if (this.options.isVerbose) {
12340
12650
  console.info('💬 OpenAI callChatModel call', { prompt: prompt });
@@ -12342,7 +12652,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
12342
12652
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements, format = prompt.format;
12343
12653
  return [4 /*yield*/, this.getClient()];
12344
12654
  case 1:
12345
- client = _a.sent();
12655
+ client = _b.sent();
12346
12656
  // TODO: [☂] Use here more modelRequirements
12347
12657
  if (modelRequirements.modelVariant !== 'CHAT') {
12348
12658
  throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
@@ -12374,7 +12684,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
12374
12684
  role: 'user',
12375
12685
  content: rawPromptContent,
12376
12686
  },
12377
- ], false), user: this.options.user });
12687
+ ], false), user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
12378
12688
  start = getCurrentIsoDate();
12379
12689
  if (this.options.isVerbose) {
12380
12690
  console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
@@ -12386,7 +12696,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
12386
12696
  throw error;
12387
12697
  })];
12388
12698
  case 2:
12389
- rawResponse = _a.sent();
12699
+ rawResponse = _b.sent();
12390
12700
  if (this.options.isVerbose) {
12391
12701
  console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
12392
12702
  }
@@ -12425,11 +12735,12 @@ var OpenAiExecutionTools = /** @class */ (function () {
12425
12735
  * Calls OpenAI API to use a complete model.
12426
12736
  */
12427
12737
  OpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
12738
+ var _a;
12428
12739
  return __awaiter(this, void 0, void 0, function () {
12429
12740
  var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
12430
12741
  var _this = this;
12431
- return __generator(this, function (_a) {
12432
- switch (_a.label) {
12742
+ return __generator(this, function (_b) {
12743
+ switch (_b.label) {
12433
12744
  case 0:
12434
12745
  if (this.options.isVerbose) {
12435
12746
  console.info('🖋 OpenAI callCompletionModel call', { prompt: prompt });
@@ -12437,7 +12748,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
12437
12748
  content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
12438
12749
  return [4 /*yield*/, this.getClient()];
12439
12750
  case 1:
12440
- client = _a.sent();
12751
+ client = _b.sent();
12441
12752
  // TODO: [☂] Use here more modelRequirements
12442
12753
  if (modelRequirements.modelVariant !== 'COMPLETION') {
12443
12754
  throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
@@ -12452,7 +12763,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
12452
12763
  // <- Note: [🧆]
12453
12764
  };
12454
12765
  rawPromptContent = replaceParameters(content, __assign(__assign({}, parameters), { modelName: modelName }));
12455
- rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: this.options.user });
12766
+ rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
12456
12767
  start = getCurrentIsoDate();
12457
12768
  if (this.options.isVerbose) {
12458
12769
  console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
@@ -12464,7 +12775,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
12464
12775
  throw error;
12465
12776
  })];
12466
12777
  case 2:
12467
- rawResponse = _a.sent();
12778
+ rawResponse = _b.sent();
12468
12779
  if (this.options.isVerbose) {
12469
12780
  console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
12470
12781
  }
@@ -12690,17 +13001,16 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
12690
13001
  assistant_id: this.assistantId,
12691
13002
  thread: {
12692
13003
  messages: [
12693
- // TODO: !!!!!! Allow threads to be passed
13004
+ // TODO: [🗯] !! Allow threads to be passed
12694
13005
  { role: 'user', content: rawPromptContent },
12695
13006
  ],
12696
13007
  },
12697
- // !!!!!! user: this.options.user,
13008
+ // <- TODO: Add user identification here> user: this.options.user,
12698
13009
  };
12699
13010
  start = getCurrentIsoDate();
12700
13011
  if (this.options.isVerbose) {
12701
13012
  console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
12702
13013
  }
12703
- console.log('!!!!!! OpenAI client', client);
12704
13014
  return [4 /*yield*/, client.beta.threads.createAndRunStream(rawRequest)];
12705
13015
  case 2:
12706
13016
  stream = _g.sent();
@@ -12718,7 +13028,7 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
12718
13028
  messageDelta.content[0].type === 'text') {
12719
13029
  console.info('messageDelta', (_a = messageDelta.content[0].text) === null || _a === void 0 ? void 0 : _a.value);
12720
13030
  }
12721
- // TODO: !!!!!! report progress
13031
+ // <- TODO: [🐚] Make streaming and running tasks working
12722
13032
  });
12723
13033
  stream.on('messageCreated', function (message) {
12724
13034
  if (_this.options.isVerbose) {
@@ -12746,18 +13056,20 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
12746
13056
  throw new PipelineExecutionError("There is NOT 'text' BUT ".concat((_b = rawResponse[0].content[0]) === null || _b === void 0 ? void 0 : _b.type, " finalMessages content type from OpenAI"));
12747
13057
  }
12748
13058
  resultContent = (_c = rawResponse[0].content[0]) === null || _c === void 0 ? void 0 : _c.text.value;
12749
- // <- TODO: !!!!!! There are also annotations, maybe use them
13059
+ // <- TODO: [🧠] There are also annotations, maybe use them
12750
13060
  // eslint-disable-next-line prefer-const
12751
13061
  complete = getCurrentIsoDate();
12752
13062
  usage = UNCERTAIN_USAGE;
12753
- // TODO: !!!!!!> = computeOpenAiUsage(content, resultContent || '', rawResponse);
13063
+ // <- TODO: [🥘] Compute real usage for assistant
13064
+ // ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse);
12754
13065
  if (resultContent === null) {
12755
13066
  throw new PipelineExecutionError('No response message from OpenAI');
12756
13067
  }
12757
13068
  return [2 /*return*/, $asDeeplyFrozenSerializableJson('OpenAiAssistantExecutionTools ChatPromptResult', {
12758
13069
  content: resultContent,
12759
13070
  modelName: 'assistant',
12760
- // <- TODO: !!!!!! Can we detect really used model: rawResponse.model || modelName,
13071
+ // <- TODO: [🥘] Detect used model in assistant
13072
+ // ?> model: rawResponse.model || modelName,
12761
13073
  timing: {
12762
13074
  start: start,
12763
13075
  complete: complete,
@@ -12775,7 +13087,6 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
12775
13087
  return OpenAiAssistantExecutionTools;
12776
13088
  }(OpenAiExecutionTools));
12777
13089
  /**
12778
- * TODO: !!!!!! DO not use colors - can be used in browser
12779
13090
  * TODO: [🧠][🧙‍♂️] Maybe there can be some wizzard for thoose who want to use just OpenAI
12780
13091
  * TODO: Maybe make custom OpenAiError
12781
13092
  * TODO: [🧠][🈁] Maybe use `isDeterministic` from options
@@ -12847,22 +13158,26 @@ var _OpenAiAssistantRegistration = $llmToolsRegister.register(createOpenAiAssist
12847
13158
  /**
12848
13159
  * Normalize options for `execCommand` and `execCommands`
12849
13160
  *
13161
+ * Note: `$` is used to indicate that this function behaves differently according to `process.platform`
13162
+ *
12850
13163
  * @private internal utility of `execCommand` and `execCommands`
12851
13164
  */
12852
- function execCommandNormalizeOptions(options) {
13165
+ function $execCommandNormalizeOptions(options) {
12853
13166
  var _a;
12854
- var _b, _c, _d;
13167
+ var _b, _c, _d, _e;
12855
13168
  var command;
12856
13169
  var cwd;
12857
13170
  var crashOnError;
12858
13171
  var args = [];
12859
13172
  var timeout;
13173
+ var isVerbose;
12860
13174
  if (typeof options === 'string') {
12861
13175
  // TODO: [1] DRY default values
12862
13176
  command = options;
12863
13177
  cwd = process.cwd();
12864
13178
  crashOnError = true;
12865
- timeout = Infinity;
13179
+ timeout = Infinity; // <- TODO: [⏳]
13180
+ isVerbose = DEFAULT_IS_VERBOSE;
12866
13181
  }
12867
13182
  else {
12868
13183
  /*
@@ -12878,6 +13193,7 @@ function execCommandNormalizeOptions(options) {
12878
13193
  cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
12879
13194
  crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
12880
13195
  timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
13196
+ isVerbose = (_e = options.isVerbose) !== null && _e !== void 0 ? _e : DEFAULT_IS_VERBOSE;
12881
13197
  }
12882
13198
  // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
12883
13199
  var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
@@ -12899,16 +13215,21 @@ function execCommandNormalizeOptions(options) {
12899
13215
  if (['ts-node'].includes(humanReadableCommand)) {
12900
13216
  humanReadableCommand += " ".concat(args[1]);
12901
13217
  }
12902
- return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
13218
+ if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
13219
+ command = "".concat(command, ".cmd");
13220
+ }
13221
+ return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout, isVerbose: isVerbose };
12903
13222
  }
12904
13223
  // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
12905
13224
 
12906
13225
  /**
12907
13226
  * Run one command in a shell
12908
13227
  *
13228
+ *
12909
13229
  * Note: There are 2 similar functions in the codebase:
12910
13230
  * - `$execCommand` which runs a single command
12911
13231
  * - `$execCommands` which runs multiple commands
13232
+ * Note: `$` is used to indicate that this function is not a pure function - it runs a command in a shell
12912
13233
  *
12913
13234
  * @public exported from `@promptbook/node`
12914
13235
  */
@@ -12918,7 +13239,7 @@ function $execCommand(options) {
12918
13239
  }
12919
13240
  return new Promise(function (resolve, reject) {
12920
13241
  // eslint-disable-next-line prefer-const
12921
- var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
13242
+ var _a = $execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
12922
13243
  if (timeout !== Infinity) {
12923
13244
  // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
12924
13245
  forTime(timeout).then(function () {
@@ -12931,24 +13252,26 @@ function $execCommand(options) {
12931
13252
  }
12932
13253
  });
12933
13254
  }
12934
- if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
12935
- command = "".concat(command, ".cmd");
13255
+ if (isVerbose) {
13256
+ console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
12936
13257
  }
12937
- // !!!!!! Verbose mode - to all consoles
12938
- console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
12939
13258
  try {
12940
13259
  var commandProcess = spawn(command, args, { cwd: cwd, shell: true });
12941
- commandProcess.on('message', function (message) {
12942
- console.info({ message: message });
12943
- });
13260
+ if (isVerbose) {
13261
+ commandProcess.on('message', function (message) {
13262
+ console.info({ message: message });
13263
+ });
13264
+ }
12944
13265
  var output_1 = [];
12945
13266
  commandProcess.stdout.on('data', function (stdout) {
12946
13267
  output_1.push(stdout.toString());
12947
- console.info(stdout.toString());
13268
+ if (isVerbose) {
13269
+ console.info(stdout.toString());
13270
+ }
12948
13271
  });
12949
13272
  commandProcess.stderr.on('data', function (stderr) {
12950
13273
  output_1.push(stderr.toString());
12951
- if (stderr.toString().trim()) {
13274
+ if (isVerbose && stderr.toString().trim()) {
12952
13275
  console.warn(stderr.toString());
12953
13276
  }
12954
13277
  });
@@ -12959,7 +13282,9 @@ function $execCommand(options) {
12959
13282
  "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
12960
13283
  }
12961
13284
  else {
12962
- console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
13285
+ if (isVerbose) {
13286
+ console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
13287
+ }
12963
13288
  resolve(spaceTrim(output_1.join('\n')));
12964
13289
  }
12965
13290
  }
@@ -12978,7 +13303,9 @@ function $execCommand(options) {
12978
13303
  reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
12979
13304
  }
12980
13305
  else {
12981
- console.warn(error);
13306
+ if (isVerbose) {
13307
+ console.warn(error);
13308
+ }
12982
13309
  resolve(spaceTrim(output_1.join('\n')));
12983
13310
  }
12984
13311
  });
@@ -13002,12 +13329,12 @@ function $execCommand(options) {
13002
13329
  */
13003
13330
  function getScraperIntermediateSource(source, options) {
13004
13331
  return __awaiter(this, void 0, void 0, function () {
13005
- var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13332
+ var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
13006
13333
  return __generator(this, function (_a) {
13007
13334
  switch (_a.label) {
13008
13335
  case 0:
13009
13336
  sourceFilename = source.filename, url = source.url;
13010
- rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
13337
+ rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
13011
13338
  hash = SHA256(
13012
13339
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
13013
13340
  hexEncoder.parse(sourceFilename || url || 'untitled'))
@@ -13017,7 +13344,7 @@ function getScraperIntermediateSource(source, options) {
13017
13344
  pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
13018
13345
  name = pieces.join('-').split('--').join('-');
13019
13346
  // <- TODO: Use MAX_FILENAME_LENGTH
13020
- TODO_USE(rootDirname); // <- TODO: !!!!!!
13347
+ TODO_USE(rootDirname); // <- TODO: [😡]
13021
13348
  cacheFilename = join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
13022
13349
  cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
13023
13350
  .join('/') +
@@ -13037,7 +13364,7 @@ function getScraperIntermediateSource(source, options) {
13037
13364
  return __generator(this, function (_a) {
13038
13365
  switch (_a.label) {
13039
13366
  case 0:
13040
- if (!isCacheCleaned) return [3 /*break*/, 2];
13367
+ if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
13041
13368
  if (isVerbose) {
13042
13369
  console.info('legacyDocumentScraper: Clening cache');
13043
13370
  }
@@ -13061,7 +13388,7 @@ function getScraperIntermediateSource(source, options) {
13061
13388
  /**
13062
13389
  * Note: Not using `FileCacheStorage` for two reasons:
13063
13390
  * 1) Need to store more than serialized JSONs
13064
- * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
13391
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
13065
13392
  * TODO: [🐱‍🐉][🧠] Make some smart crop
13066
13393
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
13067
13394
  */
@@ -13078,7 +13405,7 @@ var markdownScraperMetadata = $deepFreeze({
13078
13405
  mimeTypes: ['text/markdown', 'text/plain'],
13079
13406
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13080
13407
  isAvilableInBrowser: true,
13081
- requiredExecutables: ['!!!!!!'],
13408
+ requiredExecutables: [],
13082
13409
  }); /* <- TODO: [🤛] */
13083
13410
  /**
13084
13411
  * Registration of known scraper metadata
@@ -13122,7 +13449,7 @@ var MarkdownScraper = /** @class */ (function () {
13122
13449
  return __generator(this, function (_k) {
13123
13450
  switch (_k.label) {
13124
13451
  case 0:
13125
- _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
13452
+ _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
13126
13453
  llm = this.tools.llm;
13127
13454
  if (llm === undefined) {
13128
13455
  throw new MissingToolsError('LLM tools are required for scraping external files');
@@ -13221,7 +13548,8 @@ var MarkdownScraper = /** @class */ (function () {
13221
13548
  embeddingResult = _c.sent();
13222
13549
  index.push({
13223
13550
  modelName: embeddingResult.modelName,
13224
- position: embeddingResult.content,
13551
+ position: __spreadArray([], __read(embeddingResult.content), false),
13552
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
13225
13553
  });
13226
13554
  _c.label = 6;
13227
13555
  case 6: return [3 /*break*/, 8];
@@ -13272,7 +13600,7 @@ var documentScraperMetadata = $deepFreeze({
13272
13600
  mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
13273
13601
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13274
13602
  isAvilableInBrowser: false,
13275
- requiredExecutables: ['!!!!!!'],
13603
+ requiredExecutables: ['Pandoc'],
13276
13604
  }); /* <- TODO: [🤛] */
13277
13605
  /**
13278
13606
  * Registration of known scraper metadata
@@ -13312,12 +13640,13 @@ var DocumentScraper = /** @class */ (function () {
13312
13640
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13313
13641
  */
13314
13642
  DocumentScraper.prototype.$convert = function (source) {
13643
+ var _a;
13315
13644
  return __awaiter(this, void 0, void 0, function () {
13316
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, command_1;
13645
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
13317
13646
  return __generator(this, function (_g) {
13318
13647
  switch (_g.label) {
13319
13648
  case 0:
13320
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13649
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13321
13650
  if (!$isRunningInNode()) {
13322
13651
  throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
13323
13652
  }
@@ -13325,7 +13654,7 @@ var DocumentScraper = /** @class */ (function () {
13325
13654
  throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
13326
13655
  // <- TODO: [🧠] What is the best error type here`
13327
13656
  }
13328
- if (externalProgramsPaths.pandocPath === undefined) {
13657
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
13329
13658
  throw new MissingToolsError('Pandoc is required for scraping .docx files');
13330
13659
  }
13331
13660
  if (source.filename === null) {
@@ -13336,7 +13665,7 @@ var DocumentScraper = /** @class */ (function () {
13336
13665
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13337
13666
  rootDirname: rootDirname,
13338
13667
  cacheDirname: cacheDirname,
13339
- isCacheCleaned: isCacheCleaned,
13668
+ intermediateFilesStrategy: intermediateFilesStrategy,
13340
13669
  extension: 'md',
13341
13670
  isVerbose: isVerbose,
13342
13671
  })];
@@ -13345,11 +13674,9 @@ var DocumentScraper = /** @class */ (function () {
13345
13674
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13346
13675
  case 2:
13347
13676
  if (!!(_g.sent())) return [3 /*break*/, 5];
13348
- command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13349
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13677
+ command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
13350
13678
  return [4 /*yield*/, $execCommand(command_1)];
13351
13679
  case 3:
13352
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
13353
13680
  _g.sent();
13354
13681
  return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
13355
13682
  case 4:
@@ -13394,9 +13721,14 @@ var DocumentScraper = /** @class */ (function () {
13394
13721
  asJson: function () {
13395
13722
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
13396
13723
  },
13397
- asBlob: function () {
13398
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
13399
- },
13724
+ /*
13725
+ TODO: [🥽]
13726
+ > asBlob() {
13727
+ > throw new UnexpectedError(
13728
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
13729
+ > );
13730
+ > },
13731
+ */
13400
13732
  };
13401
13733
  knowledge = this.markdownScraper.scrape(markdownSource);
13402
13734
  return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13428,7 +13760,11 @@ var legacyDocumentScraperMetadata = $deepFreeze({
13428
13760
  mimeTypes: ['application/msword', 'text/rtf'],
13429
13761
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13430
13762
  isAvilableInBrowser: false,
13431
- requiredExecutables: ['!!!!!!'],
13763
+ requiredExecutables: [
13764
+ 'Pandoc',
13765
+ 'LibreOffice',
13766
+ // <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
13767
+ ],
13432
13768
  }); /* <- TODO: [🤛] */
13433
13769
  /**
13434
13770
  * Registration of known scraper metadata
@@ -13441,7 +13777,7 @@ var legacyDocumentScraperMetadata = $deepFreeze({
13441
13777
  var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
13442
13778
 
13443
13779
  /**
13444
- * Scraper for .docx files
13780
+ * Scraper for old document files (like .doc and .rtf)
13445
13781
  *
13446
13782
  * @see `documentationUrl` for more details
13447
13783
  * @public exported from `@promptbook/legacy-documents`
@@ -13468,12 +13804,13 @@ var LegacyDocumentScraper = /** @class */ (function () {
13468
13804
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
13469
13805
  */
13470
13806
  LegacyDocumentScraper.prototype.$convert = function (source) {
13807
+ var _a;
13471
13808
  return __awaiter(this, void 0, void 0, function () {
13472
- var _a, _b, externalProgramsPaths, _c, rootDirname, _d, cacheDirname, _e, isCacheCleaned, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13809
+ var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
13473
13810
  return __generator(this, function (_g) {
13474
13811
  switch (_g.label) {
13475
13812
  case 0:
13476
- _a = this.options, _b = _a.externalProgramsPaths, externalProgramsPaths = _b === void 0 ? {} : _b, _c = _a.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _a.cacheDirname, cacheDirname = _d === void 0 ? SCRAPE_CACHE_DIRNAME : _d, _e = _a.isCacheCleaned, isCacheCleaned = _e === void 0 ? false : _e, _f = _a.isVerbose, isVerbose = _f === void 0 ? IS_VERBOSE : _f;
13813
+ _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
13477
13814
  if (!$isRunningInNode()) {
13478
13815
  throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
13479
13816
  }
@@ -13481,7 +13818,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
13481
13818
  throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
13482
13819
  // <- TODO: [🧠] What is the best error type here`
13483
13820
  }
13484
- if (externalProgramsPaths.libreOfficePath === undefined) {
13821
+ if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
13485
13822
  throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
13486
13823
  }
13487
13824
  if (source.filename === null) {
@@ -13492,7 +13829,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
13492
13829
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13493
13830
  rootDirname: rootDirname,
13494
13831
  cacheDirname: cacheDirname,
13495
- isCacheCleaned: isCacheCleaned,
13832
+ intermediateFilesStrategy: intermediateFilesStrategy,
13496
13833
  extension: 'docx',
13497
13834
  isVerbose: isVerbose,
13498
13835
  })];
@@ -13507,11 +13844,9 @@ var LegacyDocumentScraper = /** @class */ (function () {
13507
13844
  documentSourceOutdirPathForLibreOffice_1 = join(dirname(cacheFilehandler.filename), 'libreoffice')
13508
13845
  .split('\\')
13509
13846
  .join('/');
13510
- command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13511
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13847
+ command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
13512
13848
  return [4 /*yield*/, $execCommand(command_1)];
13513
13849
  case 3:
13514
- // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
13515
13850
  _g.sent();
13516
13851
  return [4 /*yield*/, readdir(documentSourceOutdirPathForLibreOffice_1)];
13517
13852
  case 4:
@@ -13559,9 +13894,14 @@ var LegacyDocumentScraper = /** @class */ (function () {
13559
13894
  asJson: function () {
13560
13895
  throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
13561
13896
  },
13562
- asBlob: function () {
13563
- throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
13564
- },
13897
+ /*
13898
+ TODO: [🥽]
13899
+ > asBlob() {
13900
+ > throw new UnexpectedError(
13901
+ > 'Did not expect that `documentScraper` would need to get the content `asBlob`',
13902
+ > );
13903
+ > },
13904
+ */
13565
13905
  };
13566
13906
  knowledge = this.documentScraper.scrape(markdownSource);
13567
13907
  return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13668,7 +14008,7 @@ var pdfScraperMetadata = $deepFreeze({
13668
14008
  mimeTypes: ['application/pdf'],
13669
14009
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13670
14010
  isAvilableInBrowser: true,
13671
- requiredExecutables: ['!!!!!!'],
14011
+ requiredExecutables: [],
13672
14012
  }); /* <- TODO: [🤛] */
13673
14013
  /**
13674
14014
  * Registration of known scraper metadata
@@ -13681,7 +14021,7 @@ var pdfScraperMetadata = $deepFreeze({
13681
14021
  var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
13682
14022
 
13683
14023
  /**
13684
- * Scraper for .docx files
14024
+ * Scraper for .pdf files
13685
14025
  *
13686
14026
  * @see `documentationUrl` for more details
13687
14027
  * @public exported from `@promptbook/pdf`
@@ -13724,9 +14064,8 @@ var PdfScraper = /** @class */ (function () {
13724
14064
  TODO_USE(this.options);
13725
14065
  /*
13726
14066
  const {
13727
- externalProgramsPaths = {},
13728
14067
  cacheDirname = SCRAPE_CACHE_DIRNAME,
13729
- isCacheCleaned = false,
14068
+ intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
13730
14069
  isVerbose = IS_VERBOSE,
13731
14070
  } = options;
13732
14071
  */
@@ -13779,7 +14118,7 @@ var websiteScraperMetadata = $deepFreeze({
13779
14118
  mimeTypes: ['text/html'],
13780
14119
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
13781
14120
  isAvilableInBrowser: false,
13782
- requiredExecutables: ['!!!!!!'],
14121
+ requiredExecutables: [],
13783
14122
  }); /* <- TODO: [🤛] */
13784
14123
  /**
13785
14124
  * Registration of known scraper metadata
@@ -13792,32 +14131,29 @@ var websiteScraperMetadata = $deepFreeze({
13792
14131
  var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
13793
14132
 
13794
14133
  /**
13795
- * A converter instance that uses showdown and highlight extensions
14134
+ * Create a new showdown converter instance
13796
14135
  *
13797
- * @type {Converter}
13798
- * @private for markdown and html knowledge scrapers
13799
- */
13800
- var markdownConverter = new Converter({
13801
- flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
13802
- /*
13803
- > import showdownHighlight from 'showdown-highlight';
13804
- > extensions: [
13805
- > showdownHighlight({
13806
- > // Whether to add the classes to the <pre> tag, default is false
13807
- > pre: true,
13808
- > // Whether to use hljs' auto language detection, default is true
13809
- > auto_detection: true,
13810
- > }),
13811
- > ],
13812
- */
13813
- });
13814
- /**
13815
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
13816
- * TODO: !!!!!! Lazy-make converter
14136
+ * @private utility of `WebsiteScraper`
13817
14137
  */
14138
+ function createShowdownConverter() {
14139
+ return new Converter({
14140
+ flavor: 'github',
14141
+ /*
14142
+ > import showdownHighlight from 'showdown-highlight';
14143
+ > extensions: [
14144
+ > showdownHighlight({
14145
+ > // Whether to add the classes to the <pre> tag, default is false
14146
+ > pre: true,
14147
+ > // Whether to use hljs' auto language detection, default is true
14148
+ > auto_detection: true,
14149
+ > }),
14150
+ > ],
14151
+ */
14152
+ });
14153
+ }
13818
14154
 
13819
14155
  /**
13820
- * Scraper for .docx files
14156
+ * Scraper for websites
13821
14157
  *
13822
14158
  * @see `documentationUrl` for more details
13823
14159
  * @public exported from `@promptbook/website-crawler`
@@ -13827,6 +14163,7 @@ var WebsiteScraper = /** @class */ (function () {
13827
14163
  this.tools = tools;
13828
14164
  this.options = options;
13829
14165
  this.markdownScraper = new MarkdownScraper(tools, options);
14166
+ this.showdownConverter = createShowdownConverter();
13830
14167
  }
13831
14168
  Object.defineProperty(WebsiteScraper.prototype, "metadata", {
13832
14169
  /**
@@ -13847,16 +14184,17 @@ var WebsiteScraper = /** @class */ (function () {
13847
14184
  return __awaiter(this, void 0, void 0, function () {
13848
14185
  var _a, _b,
13849
14186
  // TODO: [🧠] Maybe in node use headless browser not just JSDOM
13850
- // externalProgramsPaths = {},
13851
- rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
14187
+ rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
13852
14188
  return __generator(this, function (_g) {
13853
14189
  switch (_g.label) {
13854
14190
  case 0:
13855
- _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? SCRAPE_CACHE_DIRNAME : _c, _d = _a.isCacheCleaned, isCacheCleaned = _d === void 0 ? false : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e;
13856
- // TODO: !!!!!! Does this work in browser? Make it work.
14191
+ _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
13857
14192
  if (source.url === null) {
13858
14193
  throw new KnowledgeScrapeError('Website scraper requires URL');
13859
14194
  }
14195
+ if (this.tools.fs === undefined) {
14196
+ throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
14197
+ }
13860
14198
  _f = JSDOM.bind;
13861
14199
  return [4 /*yield*/, source.asText()];
13862
14200
  case 1:
@@ -13865,10 +14203,6 @@ var WebsiteScraper = /** @class */ (function () {
13865
14203
  }]))();
13866
14204
  reader = new Readability(jsdom.window.document);
13867
14205
  article = reader.parse();
13868
- console.log(article);
13869
- return [4 /*yield*/, forTime(10000)];
13870
- case 2:
13871
- _g.sent();
13872
14206
  html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
13873
14207
  // Note: Unwrap html such as it is convertable by `markdownConverter`
13874
14208
  for (i = 0; i < 2; i++) {
@@ -13880,16 +14214,16 @@ var WebsiteScraper = /** @class */ (function () {
13880
14214
  return [4 /*yield*/, getScraperIntermediateSource(source, {
13881
14215
  rootDirname: rootDirname,
13882
14216
  cacheDirname: cacheDirname,
13883
- isCacheCleaned: isCacheCleaned,
14217
+ intermediateFilesStrategy: intermediateFilesStrategy,
13884
14218
  extension: 'html',
13885
14219
  isVerbose: isVerbose,
13886
14220
  })];
13887
- case 3:
14221
+ case 2:
13888
14222
  cacheFilehandler = _g.sent();
13889
- return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
13890
- case 4:
14223
+ return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8')];
14224
+ case 3:
13891
14225
  _g.sent();
13892
- markdown = markdownConverter.makeMarkdown(html, jsdom.window.document);
14226
+ markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
13893
14227
  return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
13894
14228
  }
13895
14229
  });
@@ -13917,9 +14251,14 @@ var WebsiteScraper = /** @class */ (function () {
13917
14251
  asJson: function () {
13918
14252
  throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
13919
14253
  },
13920
- asBlob: function () {
13921
- throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
13922
- },
14254
+ /*
14255
+ TODO: [🥽]
14256
+ > asBlob() {
14257
+ > throw new UnexpectedError(
14258
+ > 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
14259
+ > );
14260
+ > },
14261
+ */
13923
14262
  };
13924
14263
  knowledge = this.markdownScraper.scrape(markdownSource);
13925
14264
  return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13933,7 +14272,6 @@ var WebsiteScraper = /** @class */ (function () {
13933
14272
  return WebsiteScraper;
13934
14273
  }());
13935
14274
  /**
13936
- * TODO: !!!!!! Put into separate package
13937
14275
  * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
13938
14276
  * TODO: [🪂] Do it in parallel 11:11
13939
14277
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools