@promptbook/markdown-utils 0.72.0-13 → 0.72.0-15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/esm/index.es.js +58 -72
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +22 -18
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +10 -10
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +7 -0
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +2 -2
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -5
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +1 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
- package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -2
- package/umd/index.umd.js +58 -72
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/README.md
CHANGED
package/esm/index.es.js
CHANGED
|
@@ -12,7 +12,7 @@ import { unparse, parse } from 'papaparse';
|
|
|
12
12
|
/**
|
|
13
13
|
* The version of the Promptbook library
|
|
14
14
|
*/
|
|
15
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
15
|
+
var PROMPTBOOK_VERSION = '0.72.0-14';
|
|
16
16
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
17
17
|
|
|
18
18
|
/*! *****************************************************************************
|
|
@@ -413,7 +413,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
413
413
|
pipelineString += '\n\n';
|
|
414
414
|
pipelineString += description;
|
|
415
415
|
}
|
|
416
|
-
// TODO:> const commands: Array<Command>
|
|
417
416
|
var commands = [];
|
|
418
417
|
if (pipelineUrl) {
|
|
419
418
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -469,7 +468,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
469
468
|
pipelineString += '\n\n';
|
|
470
469
|
pipelineString += description_1;
|
|
471
470
|
}
|
|
472
|
-
// TODO:> const commands: Array<Command>
|
|
473
471
|
var commands_1 = [];
|
|
474
472
|
var contentLanguage = 'text';
|
|
475
473
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -835,18 +833,19 @@ var LOOP_LIMIT = 1000;
|
|
|
835
833
|
* @private within the repository - too low-level in comparison with other `MAX_...`
|
|
836
834
|
*/
|
|
837
835
|
var IMMEDIATE_TIME = 10;
|
|
836
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
838
837
|
/**
|
|
839
838
|
* The maximum number of (LLM) tasks running in parallel
|
|
840
839
|
*
|
|
841
840
|
* @public exported from `@promptbook/core`
|
|
842
841
|
*/
|
|
843
|
-
var
|
|
842
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
844
843
|
/**
|
|
845
844
|
* The maximum number of attempts to execute LLM task before giving up
|
|
846
845
|
*
|
|
847
846
|
* @public exported from `@promptbook/core`
|
|
848
847
|
*/
|
|
849
|
-
var
|
|
848
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
850
849
|
/**
|
|
851
850
|
* Nonce which is used for replacing things in strings
|
|
852
851
|
*
|
|
@@ -898,7 +897,7 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
898
897
|
*
|
|
899
898
|
* @public exported from `@promptbook/core`
|
|
900
899
|
*/
|
|
901
|
-
var
|
|
900
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
902
901
|
/**
|
|
903
902
|
* @@@
|
|
904
903
|
*
|
|
@@ -2091,6 +2090,7 @@ function assertsExecutionSuccessful(executionResult) {
|
|
|
2091
2090
|
}
|
|
2092
2091
|
}
|
|
2093
2092
|
/**
|
|
2093
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2094
2094
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2095
2095
|
*/
|
|
2096
2096
|
|
|
@@ -2276,8 +2276,7 @@ $deepFreeze({
|
|
|
2276
2276
|
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
2277
2277
|
*
|
|
2278
2278
|
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
2279
|
-
* @public exported from `@promptbook/
|
|
2280
|
-
* TODO: !!!!!! Export as runtime class not just type
|
|
2279
|
+
* @public exported from `@promptbook/core`
|
|
2281
2280
|
*/
|
|
2282
2281
|
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
2283
2282
|
/**
|
|
@@ -2779,7 +2778,7 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
2779
2778
|
return __generator(this, function (_d) {
|
|
2780
2779
|
switch (_d.label) {
|
|
2781
2780
|
case 0:
|
|
2782
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
2781
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
2783
2782
|
if (tools === undefined || tools.llm === undefined) {
|
|
2784
2783
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
2785
2784
|
}
|
|
@@ -2961,7 +2960,7 @@ var $Register = /** @class */ (function () {
|
|
|
2961
2960
|
this.storage = globalScope[storageName];
|
|
2962
2961
|
}
|
|
2963
2962
|
$Register.prototype.list = function () {
|
|
2964
|
-
// <- TODO: ReadonlyDeep<
|
|
2963
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
2965
2964
|
return this.storage;
|
|
2966
2965
|
};
|
|
2967
2966
|
$Register.prototype.register = function (registered) {
|
|
@@ -3013,7 +3012,7 @@ var $scrapersRegister = new $Register('scraper_constructors');
|
|
|
3013
3012
|
* TODO: [®] DRY Register logic
|
|
3014
3013
|
*/
|
|
3015
3014
|
|
|
3016
|
-
// TODO: !!!!!! Maybe delete this function
|
|
3015
|
+
// TODO: !!!!!!last - Maybe delete this function
|
|
3017
3016
|
/**
|
|
3018
3017
|
* Creates a message with all registered scrapers
|
|
3019
3018
|
*
|
|
@@ -3121,7 +3120,6 @@ function $registeredScrapersMessage() {
|
|
|
3121
3120
|
* @private within the repository
|
|
3122
3121
|
*/
|
|
3123
3122
|
function sourceContentToName(sourceContent) {
|
|
3124
|
-
// TODO: !!!!!! Better name for source than gibberish hash
|
|
3125
3123
|
var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
|
|
3126
3124
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
3127
3125
|
.toString( /* hex */)
|
|
@@ -3208,7 +3206,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
3208
3206
|
case 0:
|
|
3209
3207
|
sourceContent = knowledgeSource.sourceContent;
|
|
3210
3208
|
name = knowledgeSource.name;
|
|
3211
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
3209
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
3212
3210
|
TODO_USE(isVerbose);
|
|
3213
3211
|
if (!name) {
|
|
3214
3212
|
name = sourceContentToName(sourceContent);
|
|
@@ -3298,7 +3296,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
3298
3296
|
content = _a.sent();
|
|
3299
3297
|
return [2 /*return*/, new Blob([
|
|
3300
3298
|
content,
|
|
3301
|
-
// <- TODO: !!!!!!
|
|
3299
|
+
// <- TODO: !!!!!! Test that this is working
|
|
3302
3300
|
], { type: mimeType_1 })];
|
|
3303
3301
|
}
|
|
3304
3302
|
});
|
|
@@ -3361,7 +3359,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3361
3359
|
return __generator(this, function (_c) {
|
|
3362
3360
|
switch (_c.label) {
|
|
3363
3361
|
case 0:
|
|
3364
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3362
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3365
3363
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3366
3364
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3367
3365
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -3390,7 +3388,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3390
3388
|
case 4:
|
|
3391
3389
|
partialPiecesUnchecked = _d.sent();
|
|
3392
3390
|
if (partialPiecesUnchecked !== null) {
|
|
3393
|
-
partialPieces = partialPiecesUnchecked;
|
|
3391
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3392
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3394
3393
|
return [3 /*break*/, 6];
|
|
3395
3394
|
}
|
|
3396
3395
|
_d.label = 5;
|
|
@@ -3438,7 +3437,7 @@ TODO: [🧊] This is how it can look in future
|
|
|
3438
3437
|
> /**
|
|
3439
3438
|
> * Unprepared knowledge
|
|
3440
3439
|
> * /
|
|
3441
|
-
> readonly knowledgeSources:
|
|
3440
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
3442
3441
|
> };
|
|
3443
3442
|
>
|
|
3444
3443
|
> export async function prepareKnowledgePieces(
|
|
@@ -3496,7 +3495,7 @@ function prepareTemplates(pipeline, tools, options) {
|
|
|
3496
3495
|
return __generator(this, function (_b) {
|
|
3497
3496
|
switch (_b.label) {
|
|
3498
3497
|
case 0:
|
|
3499
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3498
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
3500
3499
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
3501
3500
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
3502
3501
|
TODO_USE(parameters);
|
|
@@ -3558,7 +3557,7 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
3558
3557
|
if (isPipelinePrepared(pipeline)) {
|
|
3559
3558
|
return [2 /*return*/, pipeline];
|
|
3560
3559
|
}
|
|
3561
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3560
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3562
3561
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
3563
3562
|
if (tools === undefined || tools.llm === undefined) {
|
|
3564
3563
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -3616,7 +3615,9 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
3616
3615
|
// ----- /Templates preparation -----
|
|
3617
3616
|
// Note: Count total usage
|
|
3618
3617
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
3619
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
3618
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
3619
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
3620
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
3620
3621
|
}
|
|
3621
3622
|
});
|
|
3622
3623
|
});
|
|
@@ -4450,12 +4451,11 @@ function checkExpectations(expectations, value) {
|
|
|
4450
4451
|
*/
|
|
4451
4452
|
function executeAttempts(options) {
|
|
4452
4453
|
return __awaiter(this, void 0, void 0, function () {
|
|
4453
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
4454
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
4454
4455
|
return __generator(this, function (_a) {
|
|
4455
4456
|
switch (_a.label) {
|
|
4456
4457
|
case 0:
|
|
4457
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
4458
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4458
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
4459
4459
|
$ongoingTemplateResult = {
|
|
4460
4460
|
$result: null,
|
|
4461
4461
|
$resultString: null,
|
|
@@ -4821,12 +4821,12 @@ function executeAttempts(options) {
|
|
|
4821
4821
|
*/
|
|
4822
4822
|
function executeFormatSubvalues(options) {
|
|
4823
4823
|
return __awaiter(this, void 0, void 0, function () {
|
|
4824
|
-
var template, jokerParameterNames, parameters, priority,
|
|
4824
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4825
4825
|
var _this = this;
|
|
4826
4826
|
return __generator(this, function (_a) {
|
|
4827
4827
|
switch (_a.label) {
|
|
4828
4828
|
case 0:
|
|
4829
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
4829
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4830
4830
|
if (template.foreach === undefined) {
|
|
4831
4831
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4832
4832
|
}
|
|
@@ -4856,7 +4856,7 @@ function executeFormatSubvalues(options) {
|
|
|
4856
4856
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4857
4857
|
}
|
|
4858
4858
|
if (formatDefinition.formatName === 'CSV') {
|
|
4859
|
-
formatSettings =
|
|
4859
|
+
formatSettings = csvSettings;
|
|
4860
4860
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4861
4861
|
}
|
|
4862
4862
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -5009,13 +5009,12 @@ function getReservedParametersForTemplate(options) {
|
|
|
5009
5009
|
*/
|
|
5010
5010
|
function executeTemplate(options) {
|
|
5011
5011
|
return __awaiter(this, void 0, void 0, function () {
|
|
5012
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
5013
|
-
var e_1,
|
|
5014
|
-
return __generator(this, function (
|
|
5015
|
-
switch (
|
|
5012
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
5013
|
+
var e_1, _g, _h;
|
|
5014
|
+
return __generator(this, function (_j) {
|
|
5015
|
+
switch (_j.label) {
|
|
5016
5016
|
case 0:
|
|
5017
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
5018
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
5017
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
5019
5018
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
5020
5019
|
title = currentTemplate.title;
|
|
5021
5020
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -5030,7 +5029,7 @@ function executeTemplate(options) {
|
|
|
5030
5029
|
// <- [🍸]
|
|
5031
5030
|
})];
|
|
5032
5031
|
case 1:
|
|
5033
|
-
|
|
5032
|
+
_j.sent();
|
|
5034
5033
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
5035
5034
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
5036
5035
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -5041,15 +5040,15 @@ function executeTemplate(options) {
|
|
|
5041
5040
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
5042
5041
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
5043
5042
|
}
|
|
5044
|
-
|
|
5045
|
-
|
|
5043
|
+
_c = (_b = Object).freeze;
|
|
5044
|
+
_d = [{}];
|
|
5046
5045
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
5047
5046
|
preparedPipeline: preparedPipeline,
|
|
5048
5047
|
template: currentTemplate,
|
|
5049
5048
|
pipelineIdentification: pipelineIdentification,
|
|
5050
5049
|
})];
|
|
5051
5050
|
case 2:
|
|
5052
|
-
definedParameters =
|
|
5051
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
5053
5052
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
5054
5053
|
parameters = {};
|
|
5055
5054
|
_loop_1 = function (parameterName) {
|
|
@@ -5069,15 +5068,15 @@ function executeTemplate(options) {
|
|
|
5069
5068
|
try {
|
|
5070
5069
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
5071
5070
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
5072
|
-
for (
|
|
5073
|
-
parameterName =
|
|
5071
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
5072
|
+
parameterName = _f.value;
|
|
5074
5073
|
_loop_1(parameterName);
|
|
5075
5074
|
}
|
|
5076
5075
|
}
|
|
5077
5076
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5078
5077
|
finally {
|
|
5079
5078
|
try {
|
|
5080
|
-
if (
|
|
5079
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
5081
5080
|
}
|
|
5082
5081
|
finally { if (e_1) throw e_1.error; }
|
|
5083
5082
|
}
|
|
@@ -5097,12 +5096,11 @@ function executeTemplate(options) {
|
|
|
5097
5096
|
template: currentTemplate,
|
|
5098
5097
|
preparedPipeline: preparedPipeline,
|
|
5099
5098
|
tools: tools,
|
|
5100
|
-
settings: settings,
|
|
5101
5099
|
$executionReport: $executionReport,
|
|
5102
5100
|
pipelineIdentification: pipelineIdentification,
|
|
5103
5101
|
})];
|
|
5104
5102
|
case 3:
|
|
5105
|
-
resultString =
|
|
5103
|
+
resultString = _j.sent();
|
|
5106
5104
|
return [4 /*yield*/, onProgress({
|
|
5107
5105
|
name: name,
|
|
5108
5106
|
title: title,
|
|
@@ -5114,12 +5112,12 @@ function executeTemplate(options) {
|
|
|
5114
5112
|
// <- [🍸]
|
|
5115
5113
|
})];
|
|
5116
5114
|
case 4:
|
|
5117
|
-
|
|
5118
|
-
return [2 /*return*/, Object.freeze((
|
|
5119
|
-
|
|
5115
|
+
_j.sent();
|
|
5116
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
5117
|
+
_h[currentTemplate.resultingParameterName] =
|
|
5120
5118
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
5121
5119
|
resultString,
|
|
5122
|
-
|
|
5120
|
+
_h))];
|
|
5123
5121
|
}
|
|
5124
5122
|
});
|
|
5125
5123
|
});
|
|
@@ -5178,13 +5176,12 @@ function filterJustOutputParameters(options) {
|
|
|
5178
5176
|
*/
|
|
5179
5177
|
function executePipeline(options) {
|
|
5180
5178
|
return __awaiter(this, void 0, void 0, function () {
|
|
5181
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
5179
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
5182
5180
|
var e_1, _f, e_2, _g;
|
|
5183
5181
|
return __generator(this, function (_h) {
|
|
5184
5182
|
switch (_h.label) {
|
|
5185
5183
|
case 0:
|
|
5186
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
5187
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
5184
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
5188
5185
|
preparedPipeline = options.preparedPipeline;
|
|
5189
5186
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
5190
5187
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -5369,12 +5366,7 @@ function executePipeline(options) {
|
|
|
5369
5366
|
return [3 /*break*/, 4];
|
|
5370
5367
|
case 3:
|
|
5371
5368
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
5372
|
-
work_1 = executeTemplate({
|
|
5373
|
-
currentTemplate: currentTemplate,
|
|
5374
|
-
preparedPipeline: preparedPipeline,
|
|
5375
|
-
parametersToPass: parametersToPass,
|
|
5376
|
-
tools: tools,
|
|
5377
|
-
onProgress: function (progress) {
|
|
5369
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
5378
5370
|
if (isReturned) {
|
|
5379
5371
|
throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
5380
5372
|
.split('\n')
|
|
@@ -5384,11 +5376,7 @@ function executePipeline(options) {
|
|
|
5384
5376
|
if (onProgress) {
|
|
5385
5377
|
onProgress(progress);
|
|
5386
5378
|
}
|
|
5387
|
-
},
|
|
5388
|
-
settings: settings,
|
|
5389
|
-
$executionReport: executionReport,
|
|
5390
|
-
pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
5391
|
-
})
|
|
5379
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
5392
5380
|
.then(function (newParametersToPass) {
|
|
5393
5381
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
5394
5382
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -5492,8 +5480,7 @@ function executePipeline(options) {
|
|
|
5492
5480
|
*/
|
|
5493
5481
|
function createPipelineExecutor(options) {
|
|
5494
5482
|
var _this = this;
|
|
5495
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
5496
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
5483
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
5497
5484
|
validatePipeline(pipeline);
|
|
5498
5485
|
var pipelineIdentification = (function () {
|
|
5499
5486
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -5527,14 +5514,12 @@ function createPipelineExecutor(options) {
|
|
|
5527
5514
|
tools: tools,
|
|
5528
5515
|
onProgress: onProgress,
|
|
5529
5516
|
pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
5530
|
-
|
|
5531
|
-
|
|
5532
|
-
|
|
5533
|
-
|
|
5534
|
-
|
|
5535
|
-
|
|
5536
|
-
rootDirname: rootDirname,
|
|
5537
|
-
},
|
|
5517
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
5518
|
+
maxParallelCount: maxParallelCount,
|
|
5519
|
+
csvSettings: csvSettings,
|
|
5520
|
+
isVerbose: isVerbose,
|
|
5521
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
5522
|
+
rootDirname: rootDirname,
|
|
5538
5523
|
})];
|
|
5539
5524
|
});
|
|
5540
5525
|
}); };
|
|
@@ -5556,7 +5541,7 @@ var markdownScraperMetadata = $deepFreeze({
|
|
|
5556
5541
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
5557
5542
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5558
5543
|
isAvilableInBrowser: true,
|
|
5559
|
-
requiredExecutables: [
|
|
5544
|
+
requiredExecutables: [],
|
|
5560
5545
|
}); /* <- TODO: [🤛] */
|
|
5561
5546
|
/**
|
|
5562
5547
|
* Registration of known scraper metadata
|
|
@@ -5600,7 +5585,7 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
5600
5585
|
return __generator(this, function (_k) {
|
|
5601
5586
|
switch (_k.label) {
|
|
5602
5587
|
case 0:
|
|
5603
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
5588
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
5604
5589
|
llm = this.tools.llm;
|
|
5605
5590
|
if (llm === undefined) {
|
|
5606
5591
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -5699,7 +5684,8 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
5699
5684
|
embeddingResult = _c.sent();
|
|
5700
5685
|
index.push({
|
|
5701
5686
|
modelName: embeddingResult.modelName,
|
|
5702
|
-
position: embeddingResult.content,
|
|
5687
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
5688
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
5703
5689
|
});
|
|
5704
5690
|
_c.label = 6;
|
|
5705
5691
|
case 6: return [3 /*break*/, 8];
|