@promptbook/markdown-utils 0.71.0-17 → 0.71.0-19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +2 -4
  2. package/esm/index.es.js +59 -73
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +24 -18
  5. package/esm/typings/src/_packages/node.index.d.ts +2 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +10 -10
  7. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  8. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  10. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  11. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  12. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  14. package/esm/typings/src/config.d.ts +21 -14
  15. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  16. package/esm/typings/src/execution/Executables.d.ts +18 -0
  17. package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
  18. package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
  19. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  20. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
  21. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  22. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
  23. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
  24. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
  25. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  26. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  27. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  28. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  29. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  30. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
  31. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
  32. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  33. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  36. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
  38. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
  39. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
  40. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  41. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
  45. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
  46. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
  47. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  48. package/esm/typings/src/scrapers/_common/Scraper.d.ts +2 -2
  49. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
  50. package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
  51. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
  52. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
  53. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
  54. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
  55. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
  56. package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
  57. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  58. package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
  59. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
  60. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
  61. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
  62. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
  63. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
  64. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
  65. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
  66. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
  67. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
  68. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
  69. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
  70. package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
  71. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  72. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  73. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  74. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  75. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  76. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  77. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  78. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  79. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  80. package/esm/typings/src/types/Prompt.d.ts +1 -1
  81. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  82. package/esm/typings/src/utils/$Register.d.ts +1 -1
  83. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  84. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  85. package/esm/typings/src/utils/emojis.d.ts +1 -1
  86. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
  87. package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
  88. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
  89. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  90. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  91. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  92. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  93. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  94. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  95. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  96. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  97. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  98. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  99. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  100. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  101. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  102. package/package.json +3 -2
  103. package/umd/index.umd.js +59 -73
  104. package/umd/index.umd.js.map +1 -1
  105. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  106. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  107. /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  # ![Promptbook logo - cube with letters P and B](./other/design/logo-h1.png) Promptbook
4
4
 
5
- Supercharge your use of large language models
5
+ Build responsible, controlled and transparent applications on top of LLM models!
6
6
 
7
7
 
8
8
 
@@ -21,11 +21,9 @@ Supercharge your use of large language models
21
21
  - ✨ **Support of [OpenAI o1 model](https://openai.com/o1/)**
22
22
 
23
23
  <blockquote style="color: #ff8811">
24
- <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
24
+ <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
25
25
  </blockquote>
26
26
 
27
-
28
-
29
27
  ## 📦 Package `@promptbook/markdown-utils`
30
28
 
31
29
  - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
package/esm/index.es.js CHANGED
@@ -12,7 +12,7 @@ import { unparse, parse } from 'papaparse';
12
12
  /**
13
13
  * The version of the Promptbook library
14
14
  */
15
- var PROMPTBOOK_VERSION = '0.71.0-16';
15
+ var PROMPTBOOK_VERSION = '0.71.0-18';
16
16
  // TODO: [main] !!!! List here all the versions and annotate + put into script
17
17
 
18
18
  /*! *****************************************************************************
@@ -413,7 +413,6 @@ function pipelineJsonToString(pipelineJson) {
413
413
  pipelineString += '\n\n';
414
414
  pipelineString += description;
415
415
  }
416
- // TODO:> const commands: Array<Command>
417
416
  var commands = [];
418
417
  if (pipelineUrl) {
419
418
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -469,7 +468,6 @@ function pipelineJsonToString(pipelineJson) {
469
468
  pipelineString += '\n\n';
470
469
  pipelineString += description_1;
471
470
  }
472
- // TODO:> const commands: Array<Command>
473
471
  var commands_1 = [];
474
472
  var contentLanguage = 'text';
475
473
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -835,18 +833,19 @@ var LOOP_LIMIT = 1000;
835
833
  * @private within the repository - too low-level in comparison with other `MAX_...`
836
834
  */
837
835
  var IMMEDIATE_TIME = 10;
836
+ // <- TODO: [😡] Change to 'VISIBLE'
838
837
  /**
839
838
  * The maximum number of (LLM) tasks running in parallel
840
839
  *
841
840
  * @public exported from `@promptbook/core`
842
841
  */
843
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
842
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
844
843
  /**
845
844
  * The maximum number of attempts to execute LLM task before giving up
846
845
  *
847
846
  * @public exported from `@promptbook/core`
848
847
  */
849
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
848
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
850
849
  /**
851
850
  * Nonce which is used for replacing things in strings
852
851
  *
@@ -898,7 +897,7 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
898
897
  *
899
898
  * @public exported from `@promptbook/core`
900
899
  */
901
- var IS_VERBOSE = false;
900
+ var DEFAULT_IS_VERBOSE = false;
902
901
  /**
903
902
  * @@@
904
903
  *
@@ -2091,6 +2090,7 @@ function assertsExecutionSuccessful(executionResult) {
2091
2090
  }
2092
2091
  }
2093
2092
  /**
2093
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2094
2094
  * TODO: [🧠] Can this return type be better typed than void
2095
2095
  */
2096
2096
 
@@ -2431,8 +2431,7 @@ function countTotalUsage(llmTools) {
2431
2431
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2432
2432
  *
2433
2433
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
2434
- * @public exported from `@promptbook/types`
2435
- * TODO: !!!!!! Export as runtime class not just type
2434
+ * @public exported from `@promptbook/core`
2436
2435
  */
2437
2436
  var MultipleLlmExecutionTools = /** @class */ (function () {
2438
2437
  /**
@@ -2752,7 +2751,7 @@ function preparePersona(personaDescription, tools, options) {
2752
2751
  return __generator(this, function (_d) {
2753
2752
  switch (_d.label) {
2754
2753
  case 0:
2755
- _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
2754
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
2756
2755
  if (tools === undefined || tools.llm === undefined) {
2757
2756
  throw new MissingToolsError('LLM tools are required for preparing persona');
2758
2757
  }
@@ -2934,7 +2933,7 @@ var $Register = /** @class */ (function () {
2934
2933
  this.storage = globalScope[storageName];
2935
2934
  }
2936
2935
  $Register.prototype.list = function () {
2937
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
2936
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
2938
2937
  return this.storage;
2939
2938
  };
2940
2939
  $Register.prototype.register = function (registered) {
@@ -2986,7 +2985,7 @@ var $scrapersRegister = new $Register('scraper_constructors');
2986
2985
  * TODO: [®] DRY Register logic
2987
2986
  */
2988
2987
 
2989
- // TODO: !!!!!! Maybe delete this function
2988
+ // TODO: !!!!!!last - Maybe delete this function
2990
2989
  /**
2991
2990
  * Creates a message with all registered scrapers
2992
2991
  *
@@ -3094,7 +3093,6 @@ function $registeredScrapersMessage() {
3094
3093
  * @private within the repository
3095
3094
  */
3096
3095
  function sourceContentToName(sourceContent) {
3097
- // TODO: !!!!!! Better name for source than gibberish hash
3098
3096
  var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
3099
3097
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
3100
3098
  .toString( /* hex */)
@@ -3168,7 +3166,7 @@ function isFileExisting(filename, fs) {
3168
3166
  /**
3169
3167
  * @@@
3170
3168
  *
3171
- * @private for scraper utilities
3169
+ * @public exported from `@promptbook/core`
3172
3170
  */
3173
3171
  function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3174
3172
  var _a;
@@ -3181,7 +3179,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3181
3179
  case 0:
3182
3180
  sourceContent = knowledgeSource.sourceContent;
3183
3181
  name = knowledgeSource.name;
3184
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
3182
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
3185
3183
  TODO_USE(isVerbose);
3186
3184
  if (!name) {
3187
3185
  name = sourceContentToName(sourceContent);
@@ -3271,7 +3269,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3271
3269
  content = _a.sent();
3272
3270
  return [2 /*return*/, new Blob([
3273
3271
  content,
3274
- // <- TODO: !!!!!! Maybe not working
3272
+ // <- TODO: !!!!!! Test that this is working
3275
3273
  ], { type: mimeType_1 })];
3276
3274
  }
3277
3275
  });
@@ -3334,7 +3332,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
3334
3332
  return __generator(this, function (_c) {
3335
3333
  switch (_c.label) {
3336
3334
  case 0:
3337
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
3335
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
3338
3336
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
3339
3337
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
3340
3338
  var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
@@ -3363,7 +3361,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
3363
3361
  case 4:
3364
3362
  partialPiecesUnchecked = _d.sent();
3365
3363
  if (partialPiecesUnchecked !== null) {
3366
- partialPieces = partialPiecesUnchecked;
3364
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
3365
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3367
3366
  return [3 /*break*/, 6];
3368
3367
  }
3369
3368
  _d.label = 5;
@@ -3411,7 +3410,7 @@ TODO: [🧊] This is how it can look in future
3411
3410
  > /**
3412
3411
  > * Unprepared knowledge
3413
3412
  > * /
3414
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
3413
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
3415
3414
  > };
3416
3415
  >
3417
3416
  > export async function prepareKnowledgePieces(
@@ -3469,7 +3468,7 @@ function prepareTemplates(pipeline, tools, options) {
3469
3468
  return __generator(this, function (_b) {
3470
3469
  switch (_b.label) {
3471
3470
  case 0:
3472
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
3471
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
3473
3472
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
3474
3473
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
3475
3474
  TODO_USE(parameters);
@@ -3531,7 +3530,7 @@ function preparePipeline(pipeline, tools, options) {
3531
3530
  if (isPipelinePrepared(pipeline)) {
3532
3531
  return [2 /*return*/, pipeline];
3533
3532
  }
3534
- rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
3533
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
3535
3534
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
3536
3535
  if (tools === undefined || tools.llm === undefined) {
3537
3536
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
@@ -3589,7 +3588,9 @@ function preparePipeline(pipeline, tools, options) {
3589
3588
  // ----- /Templates preparation -----
3590
3589
  // Note: Count total usage
3591
3590
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
3592
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
3591
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
3592
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
3593
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
3593
3594
  }
3594
3595
  });
3595
3596
  });
@@ -4423,12 +4424,11 @@ function checkExpectations(expectations, value) {
4423
4424
  */
4424
4425
  function executeAttempts(options) {
4425
4426
  return __awaiter(this, void 0, void 0, function () {
4426
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4427
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4427
4428
  return __generator(this, function (_a) {
4428
4429
  switch (_a.label) {
4429
4430
  case 0:
4430
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4431
- maxExecutionAttempts = settings.maxExecutionAttempts;
4431
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
4432
4432
  $ongoingTemplateResult = {
4433
4433
  $result: null,
4434
4434
  $resultString: null,
@@ -4794,12 +4794,12 @@ function executeAttempts(options) {
4794
4794
  */
4795
4795
  function executeFormatSubvalues(options) {
4796
4796
  return __awaiter(this, void 0, void 0, function () {
4797
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4797
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4798
4798
  var _this = this;
4799
4799
  return __generator(this, function (_a) {
4800
4800
  switch (_a.label) {
4801
4801
  case 0:
4802
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4802
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4803
4803
  if (template.foreach === undefined) {
4804
4804
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4805
4805
  }
@@ -4829,7 +4829,7 @@ function executeFormatSubvalues(options) {
4829
4829
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4830
4830
  }
4831
4831
  if (formatDefinition.formatName === 'CSV') {
4832
- formatSettings = settings.csvSettings;
4832
+ formatSettings = csvSettings;
4833
4833
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4834
4834
  }
4835
4835
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -4982,13 +4982,12 @@ function getReservedParametersForTemplate(options) {
4982
4982
  */
4983
4983
  function executeTemplate(options) {
4984
4984
  return __awaiter(this, void 0, void 0, function () {
4985
- var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4986
- var e_1, _f, _g;
4987
- return __generator(this, function (_h) {
4988
- switch (_h.label) {
4985
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4986
+ var e_1, _g, _h;
4987
+ return __generator(this, function (_j) {
4988
+ switch (_j.label) {
4989
4989
  case 0:
4990
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4991
- maxExecutionAttempts = settings.maxExecutionAttempts;
4990
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
4992
4991
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
4993
4992
  title = currentTemplate.title;
4994
4993
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -5003,7 +5002,7 @@ function executeTemplate(options) {
5003
5002
  // <- [🍸]
5004
5003
  })];
5005
5004
  case 1:
5006
- _h.sent();
5005
+ _j.sent();
5007
5006
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
5008
5007
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
5009
5008
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -5014,15 +5013,15 @@ function executeTemplate(options) {
5014
5013
  .map(function (name) { return "{".concat(name, "}"); })
5015
5014
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
5016
5015
  }
5017
- _b = (_a = Object).freeze;
5018
- _c = [{}];
5016
+ _c = (_b = Object).freeze;
5017
+ _d = [{}];
5019
5018
  return [4 /*yield*/, getReservedParametersForTemplate({
5020
5019
  preparedPipeline: preparedPipeline,
5021
5020
  template: currentTemplate,
5022
5021
  pipelineIdentification: pipelineIdentification,
5023
5022
  })];
5024
5023
  case 2:
5025
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
5024
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
5026
5025
  definedParameterNames = new Set(Object.keys(definedParameters));
5027
5026
  parameters = {};
5028
5027
  _loop_1 = function (parameterName) {
@@ -5042,15 +5041,15 @@ function executeTemplate(options) {
5042
5041
  try {
5043
5042
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
5044
5043
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
5045
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
5046
- parameterName = _e.value;
5044
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
5045
+ parameterName = _f.value;
5047
5046
  _loop_1(parameterName);
5048
5047
  }
5049
5048
  }
5050
5049
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
5051
5050
  finally {
5052
5051
  try {
5053
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
5052
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
5054
5053
  }
5055
5054
  finally { if (e_1) throw e_1.error; }
5056
5055
  }
@@ -5070,12 +5069,11 @@ function executeTemplate(options) {
5070
5069
  template: currentTemplate,
5071
5070
  preparedPipeline: preparedPipeline,
5072
5071
  tools: tools,
5073
- settings: settings,
5074
5072
  $executionReport: $executionReport,
5075
5073
  pipelineIdentification: pipelineIdentification,
5076
5074
  })];
5077
5075
  case 3:
5078
- resultString = _h.sent();
5076
+ resultString = _j.sent();
5079
5077
  return [4 /*yield*/, onProgress({
5080
5078
  name: name,
5081
5079
  title: title,
@@ -5087,12 +5085,12 @@ function executeTemplate(options) {
5087
5085
  // <- [🍸]
5088
5086
  })];
5089
5087
  case 4:
5090
- _h.sent();
5091
- return [2 /*return*/, Object.freeze((_g = {},
5092
- _g[currentTemplate.resultingParameterName] =
5088
+ _j.sent();
5089
+ return [2 /*return*/, Object.freeze((_h = {},
5090
+ _h[currentTemplate.resultingParameterName] =
5093
5091
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
5094
5092
  resultString,
5095
- _g))];
5093
+ _h))];
5096
5094
  }
5097
5095
  });
5098
5096
  });
@@ -5151,13 +5149,12 @@ function filterJustOutputParameters(options) {
5151
5149
  */
5152
5150
  function executePipeline(options) {
5153
5151
  return __awaiter(this, void 0, void 0, function () {
5154
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
5152
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
5155
5153
  var e_1, _f, e_2, _g;
5156
5154
  return __generator(this, function (_h) {
5157
5155
  switch (_h.label) {
5158
5156
  case 0:
5159
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
5160
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
5157
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
5161
5158
  preparedPipeline = options.preparedPipeline;
5162
5159
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
5163
5160
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -5342,12 +5339,7 @@ function executePipeline(options) {
5342
5339
  return [3 /*break*/, 4];
5343
5340
  case 3:
5344
5341
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
5345
- work_1 = executeTemplate({
5346
- currentTemplate: currentTemplate,
5347
- preparedPipeline: preparedPipeline,
5348
- parametersToPass: parametersToPass,
5349
- tools: tools,
5350
- onProgress: function (progress) {
5342
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
5351
5343
  if (isReturned) {
5352
5344
  throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
5353
5345
  .split('\n')
@@ -5357,11 +5349,7 @@ function executePipeline(options) {
5357
5349
  if (onProgress) {
5358
5350
  onProgress(progress);
5359
5351
  }
5360
- },
5361
- settings: settings,
5362
- $executionReport: executionReport,
5363
- pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
5364
- })
5352
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
5365
5353
  .then(function (newParametersToPass) {
5366
5354
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
5367
5355
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -5465,8 +5453,7 @@ function executePipeline(options) {
5465
5453
  */
5466
5454
  function createPipelineExecutor(options) {
5467
5455
  var _this = this;
5468
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
5469
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
5456
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
5470
5457
  validatePipeline(pipeline);
5471
5458
  var pipelineIdentification = (function () {
5472
5459
  // Note: This is a 😐 implementation of [🚞]
@@ -5500,14 +5487,12 @@ function createPipelineExecutor(options) {
5500
5487
  tools: tools,
5501
5488
  onProgress: onProgress,
5502
5489
  pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
5503
- settings: {
5504
- maxExecutionAttempts: maxExecutionAttempts,
5505
- maxParallelCount: maxParallelCount,
5506
- csvSettings: csvSettings,
5507
- isVerbose: isVerbose,
5508
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5509
- rootDirname: rootDirname,
5510
- },
5490
+ maxExecutionAttempts: maxExecutionAttempts,
5491
+ maxParallelCount: maxParallelCount,
5492
+ csvSettings: csvSettings,
5493
+ isVerbose: isVerbose,
5494
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5495
+ rootDirname: rootDirname,
5511
5496
  })];
5512
5497
  });
5513
5498
  }); };
@@ -5529,7 +5514,7 @@ var markdownScraperMetadata = $deepFreeze({
5529
5514
  mimeTypes: ['text/markdown', 'text/plain'],
5530
5515
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5531
5516
  isAvilableInBrowser: true,
5532
- requiredExecutables: ['!!!!!!'],
5517
+ requiredExecutables: [],
5533
5518
  }); /* <- TODO: [🤛] */
5534
5519
  /**
5535
5520
  * Registration of known scraper metadata
@@ -5573,7 +5558,7 @@ var MarkdownScraper = /** @class */ (function () {
5573
5558
  return __generator(this, function (_k) {
5574
5559
  switch (_k.label) {
5575
5560
  case 0:
5576
- _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
5561
+ _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
5577
5562
  llm = this.tools.llm;
5578
5563
  if (llm === undefined) {
5579
5564
  throw new MissingToolsError('LLM tools are required for scraping external files');
@@ -5672,7 +5657,8 @@ var MarkdownScraper = /** @class */ (function () {
5672
5657
  embeddingResult = _c.sent();
5673
5658
  index.push({
5674
5659
  modelName: embeddingResult.modelName,
5675
- position: embeddingResult.content,
5660
+ position: __spreadArray([], __read(embeddingResult.content), false),
5661
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
5676
5662
  });
5677
5663
  _c.label = 6;
5678
5664
  case 6: return [3 /*break*/, 8];