@promptbook/markdown-utils 0.72.0-13 → 0.72.0-15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +1 -1
  2. package/esm/index.es.js +58 -72
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +22 -18
  5. package/esm/typings/src/_packages/node.index.d.ts +2 -0
  6. package/esm/typings/src/_packages/types.index.d.ts +10 -10
  7. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  8. package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
  10. package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
  11. package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
  12. package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
  14. package/esm/typings/src/config.d.ts +21 -14
  15. package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
  16. package/esm/typings/src/execution/Executables.d.ts +18 -0
  17. package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
  18. package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
  19. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  20. package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
  21. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
  22. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
  23. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
  24. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
  25. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
  26. package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
  27. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
  28. package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
  29. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
  30. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
  31. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
  32. package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +7 -0
  33. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
  34. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  36. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
  38. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
  39. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
  40. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
  41. package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
  42. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
  43. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  44. package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
  45. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +3 -3
  46. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +2 -2
  47. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
  48. package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
  49. package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -5
  50. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
  51. package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
  52. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
  53. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
  54. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
  55. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
  56. package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
  57. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  58. package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
  59. package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
  60. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
  61. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
  62. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
  63. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
  64. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
  65. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
  66. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
  67. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
  68. package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
  69. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
  70. package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
  71. package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
  72. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  73. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
  74. package/esm/typings/src/types/Arrayable.d.ts +1 -1
  75. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
  76. package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
  77. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
  78. package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
  79. package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
  80. package/esm/typings/src/types/Prompt.d.ts +1 -1
  81. package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
  82. package/esm/typings/src/utils/$Register.d.ts +1 -1
  83. package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
  84. package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
  85. package/esm/typings/src/utils/emojis.d.ts +1 -1
  86. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
  87. package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
  88. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
  89. package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
  90. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
  91. package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
  92. package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
  93. package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
  94. package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
  95. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
  96. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  97. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  98. package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
  99. package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
  100. package/esm/typings/src/utils/sets/union.d.ts +1 -1
  101. package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
  102. package/package.json +3 -2
  103. package/umd/index.umd.js +58 -72
  104. package/umd/index.umd.js.map +1 -1
  105. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
  106. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
  107. /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  # ![Promptbook logo - cube with letters P and B](./other/design/logo-h1.png) Promptbook
4
4
 
5
- Supercharge your use of large language models
5
+ Build responsible, controlled and transparent applications on top of LLM models!
6
6
 
7
7
 
8
8
 
package/esm/index.es.js CHANGED
@@ -12,7 +12,7 @@ import { unparse, parse } from 'papaparse';
12
12
  /**
13
13
  * The version of the Promptbook library
14
14
  */
15
- var PROMPTBOOK_VERSION = '0.72.0-12';
15
+ var PROMPTBOOK_VERSION = '0.72.0-14';
16
16
  // TODO: [main] !!!! List here all the versions and annotate + put into script
17
17
 
18
18
  /*! *****************************************************************************
@@ -413,7 +413,6 @@ function pipelineJsonToString(pipelineJson) {
413
413
  pipelineString += '\n\n';
414
414
  pipelineString += description;
415
415
  }
416
- // TODO:> const commands: Array<Command>
417
416
  var commands = [];
418
417
  if (pipelineUrl) {
419
418
  commands.push("PIPELINE URL ".concat(pipelineUrl));
@@ -469,7 +468,6 @@ function pipelineJsonToString(pipelineJson) {
469
468
  pipelineString += '\n\n';
470
469
  pipelineString += description_1;
471
470
  }
472
- // TODO:> const commands: Array<Command>
473
471
  var commands_1 = [];
474
472
  var contentLanguage = 'text';
475
473
  if (templateType === 'PROMPT_TEMPLATE') {
@@ -835,18 +833,19 @@ var LOOP_LIMIT = 1000;
835
833
  * @private within the repository - too low-level in comparison with other `MAX_...`
836
834
  */
837
835
  var IMMEDIATE_TIME = 10;
836
+ // <- TODO: [😡] Change to 'VISIBLE'
838
837
  /**
839
838
  * The maximum number of (LLM) tasks running in parallel
840
839
  *
841
840
  * @public exported from `@promptbook/core`
842
841
  */
843
- var MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
842
+ var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
844
843
  /**
845
844
  * The maximum number of attempts to execute LLM task before giving up
846
845
  *
847
846
  * @public exported from `@promptbook/core`
848
847
  */
849
- var MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
848
+ var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
850
849
  /**
851
850
  * Nonce which is used for replacing things in strings
852
851
  *
@@ -898,7 +897,7 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
898
897
  *
899
898
  * @public exported from `@promptbook/core`
900
899
  */
901
- var IS_VERBOSE = false;
900
+ var DEFAULT_IS_VERBOSE = false;
902
901
  /**
903
902
  * @@@
904
903
  *
@@ -2091,6 +2090,7 @@ function assertsExecutionSuccessful(executionResult) {
2091
2090
  }
2092
2091
  }
2093
2092
  /**
2093
+ * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2094
2094
  * TODO: [🧠] Can this return type be better typed than void
2095
2095
  */
2096
2096
 
@@ -2276,8 +2276,7 @@ $deepFreeze({
2276
2276
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
2277
2277
  *
2278
2278
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
2279
- * @public exported from `@promptbook/types`
2280
- * TODO: !!!!!! Export as runtime class not just type
2279
+ * @public exported from `@promptbook/core`
2281
2280
  */
2282
2281
  var MultipleLlmExecutionTools = /** @class */ (function () {
2283
2282
  /**
@@ -2779,7 +2778,7 @@ function preparePersona(personaDescription, tools, options) {
2779
2778
  return __generator(this, function (_d) {
2780
2779
  switch (_d.label) {
2781
2780
  case 0:
2782
- _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
2781
+ _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
2783
2782
  if (tools === undefined || tools.llm === undefined) {
2784
2783
  throw new MissingToolsError('LLM tools are required for preparing persona');
2785
2784
  }
@@ -2961,7 +2960,7 @@ var $Register = /** @class */ (function () {
2961
2960
  this.storage = globalScope[storageName];
2962
2961
  }
2963
2962
  $Register.prototype.list = function () {
2964
- // <- TODO: ReadonlyDeep<Array<TRegistered>>
2963
+ // <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
2965
2964
  return this.storage;
2966
2965
  };
2967
2966
  $Register.prototype.register = function (registered) {
@@ -3013,7 +3012,7 @@ var $scrapersRegister = new $Register('scraper_constructors');
3013
3012
  * TODO: [®] DRY Register logic
3014
3013
  */
3015
3014
 
3016
- // TODO: !!!!!! Maybe delete this function
3015
+ // TODO: !!!!!!last - Maybe delete this function
3017
3016
  /**
3018
3017
  * Creates a message with all registered scrapers
3019
3018
  *
@@ -3121,7 +3120,6 @@ function $registeredScrapersMessage() {
3121
3120
  * @private within the repository
3122
3121
  */
3123
3122
  function sourceContentToName(sourceContent) {
3124
- // TODO: !!!!!! Better name for source than gibberish hash
3125
3123
  var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
3126
3124
  // <- TODO: [🥬] Encapsulate sha256 to some private utility function
3127
3125
  .toString( /* hex */)
@@ -3208,7 +3206,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3208
3206
  case 0:
3209
3207
  sourceContent = knowledgeSource.sourceContent;
3210
3208
  name = knowledgeSource.name;
3211
- _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
3209
+ _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
3212
3210
  TODO_USE(isVerbose);
3213
3211
  if (!name) {
3214
3212
  name = sourceContentToName(sourceContent);
@@ -3298,7 +3296,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3298
3296
  content = _a.sent();
3299
3297
  return [2 /*return*/, new Blob([
3300
3298
  content,
3301
- // <- TODO: !!!!!! Maybe not working
3299
+ // <- TODO: !!!!!! Test that this is working
3302
3300
  ], { type: mimeType_1 })];
3303
3301
  }
3304
3302
  });
@@ -3361,7 +3359,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
3361
3359
  return __generator(this, function (_c) {
3362
3360
  switch (_c.label) {
3363
3361
  case 0:
3364
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
3362
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
3365
3363
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
3366
3364
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
3367
3365
  var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
@@ -3390,7 +3388,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
3390
3388
  case 4:
3391
3389
  partialPiecesUnchecked = _d.sent();
3392
3390
  if (partialPiecesUnchecked !== null) {
3393
- partialPieces = partialPiecesUnchecked;
3391
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
3392
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3394
3393
  return [3 /*break*/, 6];
3395
3394
  }
3396
3395
  _d.label = 5;
@@ -3438,7 +3437,7 @@ TODO: [🧊] This is how it can look in future
3438
3437
  > /**
3439
3438
  > * Unprepared knowledge
3440
3439
  > * /
3441
- > readonly knowledgeSources: Array<KnowledgeSourceJson>;
3440
+ > readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
3442
3441
  > };
3443
3442
  >
3444
3443
  > export async function prepareKnowledgePieces(
@@ -3496,7 +3495,7 @@ function prepareTemplates(pipeline, tools, options) {
3496
3495
  return __generator(this, function (_b) {
3497
3496
  switch (_b.label) {
3498
3497
  case 0:
3499
- _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
3498
+ _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
3500
3499
  templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
3501
3500
  // TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
3502
3501
  TODO_USE(parameters);
@@ -3558,7 +3557,7 @@ function preparePipeline(pipeline, tools, options) {
3558
3557
  if (isPipelinePrepared(pipeline)) {
3559
3558
  return [2 /*return*/, pipeline];
3560
3559
  }
3561
- rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
3560
+ rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
3562
3561
  parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
3563
3562
  if (tools === undefined || tools.llm === undefined) {
3564
3563
  throw new MissingToolsError('LLM tools are required for preparing the pipeline');
@@ -3616,7 +3615,9 @@ function preparePipeline(pipeline, tools, options) {
3616
3615
  // ----- /Templates preparation -----
3617
3616
  // Note: Count total usage
3618
3617
  currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
3619
- return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: templatesPrepared, knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations }))];
3618
+ return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
3619
+ // <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
3620
+ knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
3620
3621
  }
3621
3622
  });
3622
3623
  });
@@ -4450,12 +4451,11 @@ function checkExpectations(expectations, value) {
4450
4451
  */
4451
4452
  function executeAttempts(options) {
4452
4453
  return __awaiter(this, void 0, void 0, function () {
4453
- var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4454
+ var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
4454
4455
  return __generator(this, function (_a) {
4455
4456
  switch (_a.label) {
4456
4457
  case 0:
4457
- jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
4458
- maxExecutionAttempts = settings.maxExecutionAttempts;
4458
+ jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
4459
4459
  $ongoingTemplateResult = {
4460
4460
  $result: null,
4461
4461
  $resultString: null,
@@ -4821,12 +4821,12 @@ function executeAttempts(options) {
4821
4821
  */
4822
4822
  function executeFormatSubvalues(options) {
4823
4823
  return __awaiter(this, void 0, void 0, function () {
4824
- var template, jokerParameterNames, parameters, priority, pipelineIdentification, settings, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4824
+ var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
4825
4825
  var _this = this;
4826
4826
  return __generator(this, function (_a) {
4827
4827
  switch (_a.label) {
4828
4828
  case 0:
4829
- template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
4829
+ template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
4830
4830
  if (template.foreach === undefined) {
4831
4831
  return [2 /*return*/, /* not await */ executeAttempts(options)];
4832
4832
  }
@@ -4856,7 +4856,7 @@ function executeFormatSubvalues(options) {
4856
4856
  .join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
4857
4857
  }
4858
4858
  if (formatDefinition.formatName === 'CSV') {
4859
- formatSettings = settings.csvSettings;
4859
+ formatSettings = csvSettings;
4860
4860
  // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4861
4861
  }
4862
4862
  return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
@@ -5009,13 +5009,12 @@ function getReservedParametersForTemplate(options) {
5009
5009
  */
5010
5010
  function executeTemplate(options) {
5011
5011
  return __awaiter(this, void 0, void 0, function () {
5012
- var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, settings, $executionReport, pipelineIdentification, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
5013
- var e_1, _f, _g;
5014
- return __generator(this, function (_h) {
5015
- switch (_h.label) {
5012
+ var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
5013
+ var e_1, _g, _h;
5014
+ return __generator(this, function (_j) {
5015
+ switch (_j.label) {
5016
5016
  case 0:
5017
- currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, settings = options.settings, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification;
5018
- maxExecutionAttempts = settings.maxExecutionAttempts;
5017
+ currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
5019
5018
  name = "pipeline-executor-frame-".concat(currentTemplate.name);
5020
5019
  title = currentTemplate.title;
5021
5020
  priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
@@ -5030,7 +5029,7 @@ function executeTemplate(options) {
5030
5029
  // <- [🍸]
5031
5030
  })];
5032
5031
  case 1:
5033
- _h.sent();
5032
+ _j.sent();
5034
5033
  usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
5035
5034
  dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
5036
5035
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -5041,15 +5040,15 @@ function executeTemplate(options) {
5041
5040
  .map(function (name) { return "{".concat(name, "}"); })
5042
5041
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
5043
5042
  }
5044
- _b = (_a = Object).freeze;
5045
- _c = [{}];
5043
+ _c = (_b = Object).freeze;
5044
+ _d = [{}];
5046
5045
  return [4 /*yield*/, getReservedParametersForTemplate({
5047
5046
  preparedPipeline: preparedPipeline,
5048
5047
  template: currentTemplate,
5049
5048
  pipelineIdentification: pipelineIdentification,
5050
5049
  })];
5051
5050
  case 2:
5052
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
5051
+ definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
5053
5052
  definedParameterNames = new Set(Object.keys(definedParameters));
5054
5053
  parameters = {};
5055
5054
  _loop_1 = function (parameterName) {
@@ -5069,15 +5068,15 @@ function executeTemplate(options) {
5069
5068
  try {
5070
5069
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this template
5071
5070
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
5072
- for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
5073
- parameterName = _e.value;
5071
+ for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
5072
+ parameterName = _f.value;
5074
5073
  _loop_1(parameterName);
5075
5074
  }
5076
5075
  }
5077
5076
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
5078
5077
  finally {
5079
5078
  try {
5080
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
5079
+ if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
5081
5080
  }
5082
5081
  finally { if (e_1) throw e_1.error; }
5083
5082
  }
@@ -5097,12 +5096,11 @@ function executeTemplate(options) {
5097
5096
  template: currentTemplate,
5098
5097
  preparedPipeline: preparedPipeline,
5099
5098
  tools: tools,
5100
- settings: settings,
5101
5099
  $executionReport: $executionReport,
5102
5100
  pipelineIdentification: pipelineIdentification,
5103
5101
  })];
5104
5102
  case 3:
5105
- resultString = _h.sent();
5103
+ resultString = _j.sent();
5106
5104
  return [4 /*yield*/, onProgress({
5107
5105
  name: name,
5108
5106
  title: title,
@@ -5114,12 +5112,12 @@ function executeTemplate(options) {
5114
5112
  // <- [🍸]
5115
5113
  })];
5116
5114
  case 4:
5117
- _h.sent();
5118
- return [2 /*return*/, Object.freeze((_g = {},
5119
- _g[currentTemplate.resultingParameterName] =
5115
+ _j.sent();
5116
+ return [2 /*return*/, Object.freeze((_h = {},
5117
+ _h[currentTemplate.resultingParameterName] =
5120
5118
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
5121
5119
  resultString,
5122
- _g))];
5120
+ _h))];
5123
5121
  }
5124
5122
  });
5125
5123
  });
@@ -5178,13 +5176,12 @@ function filterJustOutputParameters(options) {
5178
5176
  */
5179
5177
  function executePipeline(options) {
5180
5178
  return __awaiter(this, void 0, void 0, function () {
5181
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
5179
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
5182
5180
  var e_1, _f, e_2, _g;
5183
5181
  return __generator(this, function (_h) {
5184
5182
  switch (_h.label) {
5185
5183
  case 0:
5186
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
5187
- maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
5184
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
5188
5185
  preparedPipeline = options.preparedPipeline;
5189
5186
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
5190
5187
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -5369,12 +5366,7 @@ function executePipeline(options) {
5369
5366
  return [3 /*break*/, 4];
5370
5367
  case 3:
5371
5368
  unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
5372
- work_1 = executeTemplate({
5373
- currentTemplate: currentTemplate,
5374
- preparedPipeline: preparedPipeline,
5375
- parametersToPass: parametersToPass,
5376
- tools: tools,
5377
- onProgress: function (progress) {
5369
+ work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
5378
5370
  if (isReturned) {
5379
5371
  throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
5380
5372
  .split('\n')
@@ -5384,11 +5376,7 @@ function executePipeline(options) {
5384
5376
  if (onProgress) {
5385
5377
  onProgress(progress);
5386
5378
  }
5387
- },
5388
- settings: settings,
5389
- $executionReport: executionReport,
5390
- pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
5391
- })
5379
+ }, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
5392
5380
  .then(function (newParametersToPass) {
5393
5381
  parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
5394
5382
  resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
@@ -5492,8 +5480,7 @@ function executePipeline(options) {
5492
5480
  */
5493
5481
  function createPipelineExecutor(options) {
5494
5482
  var _this = this;
5495
- var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
5496
- var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
5483
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
5497
5484
  validatePipeline(pipeline);
5498
5485
  var pipelineIdentification = (function () {
5499
5486
  // Note: This is a 😐 implementation of [🚞]
@@ -5527,14 +5514,12 @@ function createPipelineExecutor(options) {
5527
5514
  tools: tools,
5528
5515
  onProgress: onProgress,
5529
5516
  pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
5530
- settings: {
5531
- maxExecutionAttempts: maxExecutionAttempts,
5532
- maxParallelCount: maxParallelCount,
5533
- csvSettings: csvSettings,
5534
- isVerbose: isVerbose,
5535
- isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5536
- rootDirname: rootDirname,
5537
- },
5517
+ maxExecutionAttempts: maxExecutionAttempts,
5518
+ maxParallelCount: maxParallelCount,
5519
+ csvSettings: csvSettings,
5520
+ isVerbose: isVerbose,
5521
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5522
+ rootDirname: rootDirname,
5538
5523
  })];
5539
5524
  });
5540
5525
  }); };
@@ -5556,7 +5541,7 @@ var markdownScraperMetadata = $deepFreeze({
5556
5541
  mimeTypes: ['text/markdown', 'text/plain'],
5557
5542
  documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
5558
5543
  isAvilableInBrowser: true,
5559
- requiredExecutables: ['!!!!!!'],
5544
+ requiredExecutables: [],
5560
5545
  }); /* <- TODO: [🤛] */
5561
5546
  /**
5562
5547
  * Registration of known scraper metadata
@@ -5600,7 +5585,7 @@ var MarkdownScraper = /** @class */ (function () {
5600
5585
  return __generator(this, function (_k) {
5601
5586
  switch (_k.label) {
5602
5587
  case 0:
5603
- _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
5588
+ _a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
5604
5589
  llm = this.tools.llm;
5605
5590
  if (llm === undefined) {
5606
5591
  throw new MissingToolsError('LLM tools are required for scraping external files');
@@ -5699,7 +5684,8 @@ var MarkdownScraper = /** @class */ (function () {
5699
5684
  embeddingResult = _c.sent();
5700
5685
  index.push({
5701
5686
  modelName: embeddingResult.modelName,
5702
- position: embeddingResult.content,
5687
+ position: __spreadArray([], __read(embeddingResult.content), false),
5688
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
5703
5689
  });
5704
5690
  _c.label = 6;
5705
5691
  case 6: return [3 /*break*/, 8];