@promptbook/markitdown 0.84.0 → 0.85.0-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/esm/index.es.js +213 -160
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +0 -2
- package/esm/typings/src/_packages/types.index.d.ts +14 -2
- package/esm/typings/src/_packages/utils.index.d.ts +0 -2
- package/esm/typings/src/cli/cli-commands/_boilerplate.d.ts +13 -0
- package/esm/typings/src/cli/cli-commands/start-server.d.ts +13 -0
- package/esm/typings/src/conversion/compilePipelineOnRemoteServer.d.ts +1 -1
- package/esm/typings/src/execution/AbstractTaskResult.d.ts +25 -0
- package/esm/typings/src/execution/ExecutionTask.d.ts +71 -0
- package/esm/typings/src/execution/PipelineExecutor.d.ts +2 -5
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -15
- package/esm/typings/src/execution/PromptbookFetch.d.ts +8 -1
- package/esm/typings/src/execution/{assertsExecutionSuccessful.d.ts → assertsTaskSuccessful.d.ts} +2 -3
- package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +0 -3
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -6
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +3 -6
- package/esm/typings/src/prepare/preparePipelineOnRemoteServer.d.ts +1 -1
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -0
- package/esm/typings/src/types/typeAliases.d.ts +2 -0
- package/esm/typings/src/utils/environment/$isRunningInBrowser.d.ts +2 -2
- package/esm/typings/src/utils/environment/$isRunningInJest.d.ts +2 -2
- package/esm/typings/src/utils/environment/$isRunningInNode.d.ts +2 -2
- package/esm/typings/src/utils/environment/$isRunningInWebWorker.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +2 -1
- package/esm/typings/src/utils/random/$randomToken.d.ts +13 -0
- package/esm/typings/src/wizzard/wizzard.d.ts +2 -3
- package/package.json +4 -2
- package/umd/index.umd.js +215 -164
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/remote-server/socket-types/_common/PromptbookServer_Progress.d.ts +0 -10
- package/esm/typings/src/types/TaskProgress.d.ts +0 -43
package/README.md
CHANGED
|
@@ -24,6 +24,10 @@
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
<blockquote style="color: #ff8811">
|
|
28
|
+
<b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
|
|
29
|
+
</blockquote>
|
|
30
|
+
|
|
27
31
|
## 📦 Package `@promptbook/markitdown`
|
|
28
32
|
|
|
29
33
|
- Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
|
package/esm/index.es.js
CHANGED
|
@@ -5,6 +5,8 @@ import hexEncoder from 'crypto-js/enc-hex';
|
|
|
5
5
|
import { basename, join, dirname } from 'path';
|
|
6
6
|
import { format } from 'prettier';
|
|
7
7
|
import parserHtml from 'prettier/parser-html';
|
|
8
|
+
import { BehaviorSubject, concat, from } from 'rxjs';
|
|
9
|
+
import { randomBytes } from 'crypto';
|
|
8
10
|
import { forTime } from 'waitasecond';
|
|
9
11
|
import sha256 from 'crypto-js/sha256';
|
|
10
12
|
import { lookup, extension } from 'mime-types';
|
|
@@ -24,7 +26,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
24
26
|
* @generated
|
|
25
27
|
* @see https://github.com/webgptorg/promptbook
|
|
26
28
|
*/
|
|
27
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.84.0
|
|
29
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.84.0';
|
|
28
30
|
/**
|
|
29
31
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
30
32
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -389,7 +391,7 @@ var UnexpectedError = /** @class */ (function (_super) {
|
|
|
389
391
|
*/
|
|
390
392
|
var $isRunningInNode = new Function("\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n");
|
|
391
393
|
/**
|
|
392
|
-
* TODO: [
|
|
394
|
+
* TODO: [🎺]
|
|
393
395
|
*/
|
|
394
396
|
|
|
395
397
|
/**
|
|
@@ -2228,6 +2230,58 @@ var PipelineExecutionError = /** @class */ (function (_super) {
|
|
|
2228
2230
|
return PipelineExecutionError;
|
|
2229
2231
|
}(Error));
|
|
2230
2232
|
|
|
2233
|
+
/**
|
|
2234
|
+
* Determine if the pipeline is fully prepared
|
|
2235
|
+
*
|
|
2236
|
+
* @see https://github.com/webgptorg/promptbook/discussions/196
|
|
2237
|
+
*
|
|
2238
|
+
* @public exported from `@promptbook/core`
|
|
2239
|
+
*/
|
|
2240
|
+
function isPipelinePrepared(pipeline) {
|
|
2241
|
+
// Note: Ignoring `pipeline.preparations` @@@
|
|
2242
|
+
// Note: Ignoring `pipeline.knowledgePieces` @@@
|
|
2243
|
+
if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
|
|
2244
|
+
return false;
|
|
2245
|
+
}
|
|
2246
|
+
if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
|
|
2247
|
+
return false;
|
|
2248
|
+
}
|
|
2249
|
+
if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
|
|
2250
|
+
return false;
|
|
2251
|
+
}
|
|
2252
|
+
/*
|
|
2253
|
+
TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them
|
|
2254
|
+
> if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) {
|
|
2255
|
+
> return false;
|
|
2256
|
+
> }
|
|
2257
|
+
*/
|
|
2258
|
+
return true;
|
|
2259
|
+
}
|
|
2260
|
+
/**
|
|
2261
|
+
* TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again
|
|
2262
|
+
* TODO: [🐠] Maybe base this on `makeValidator`
|
|
2263
|
+
* TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
|
|
2264
|
+
* TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
|
|
2265
|
+
* - [🏍] ? Is context in each task
|
|
2266
|
+
* - [♨] Are examples prepared
|
|
2267
|
+
* - [♨] Are tasks prepared
|
|
2268
|
+
*/
|
|
2269
|
+
|
|
2270
|
+
/**
|
|
2271
|
+
* Generates random token
|
|
2272
|
+
*
|
|
2273
|
+
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
2274
|
+
*
|
|
2275
|
+
* @private internal helper function
|
|
2276
|
+
* @returns secure random token
|
|
2277
|
+
*/
|
|
2278
|
+
function $randomToken(randomness) {
|
|
2279
|
+
return randomBytes(randomness).toString('hex');
|
|
2280
|
+
}
|
|
2281
|
+
/**
|
|
2282
|
+
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
2283
|
+
*/
|
|
2284
|
+
|
|
2231
2285
|
/**
|
|
2232
2286
|
* This error indicates problems parsing the format value
|
|
2233
2287
|
*
|
|
@@ -2413,9 +2467,9 @@ function deserializeError(error) {
|
|
|
2413
2467
|
*
|
|
2414
2468
|
* @param executionResult - The partial result of the Promptbook execution
|
|
2415
2469
|
* @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
|
|
2416
|
-
* @
|
|
2470
|
+
* @private internal helper function of `asPromise` method of `ExecutionTask`
|
|
2417
2471
|
*/
|
|
2418
|
-
function
|
|
2472
|
+
function assertsTaskSuccessful(executionResult) {
|
|
2419
2473
|
var e_1, _a;
|
|
2420
2474
|
var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors, warnings = executionResult.warnings;
|
|
2421
2475
|
try {
|
|
@@ -2450,121 +2504,54 @@ function assertsExecutionSuccessful(executionResult) {
|
|
|
2450
2504
|
}
|
|
2451
2505
|
}
|
|
2452
2506
|
/**
|
|
2453
|
-
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2454
2507
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2455
2508
|
*/
|
|
2456
2509
|
|
|
2457
2510
|
/**
|
|
2458
|
-
*
|
|
2459
|
-
*
|
|
2460
|
-
* @see https://github.com/webgptorg/promptbook/discussions/196
|
|
2461
|
-
*
|
|
2462
|
-
* @public exported from `@promptbook/core`
|
|
2463
|
-
*/
|
|
2464
|
-
function isPipelinePrepared(pipeline) {
|
|
2465
|
-
// Note: Ignoring `pipeline.preparations` @@@
|
|
2466
|
-
// Note: Ignoring `pipeline.knowledgePieces` @@@
|
|
2467
|
-
if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
|
|
2468
|
-
return false;
|
|
2469
|
-
}
|
|
2470
|
-
if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
|
|
2471
|
-
return false;
|
|
2472
|
-
}
|
|
2473
|
-
if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
|
|
2474
|
-
return false;
|
|
2475
|
-
}
|
|
2476
|
-
/*
|
|
2477
|
-
TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them
|
|
2478
|
-
> if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) {
|
|
2479
|
-
> return false;
|
|
2480
|
-
> }
|
|
2481
|
-
*/
|
|
2482
|
-
return true;
|
|
2483
|
-
}
|
|
2484
|
-
/**
|
|
2485
|
-
* TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again
|
|
2486
|
-
* TODO: [🐠] Maybe base this on `makeValidator`
|
|
2487
|
-
* TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
|
|
2488
|
-
* TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
|
|
2489
|
-
* - [🏍] ? Is context in each task
|
|
2490
|
-
* - [♨] Are examples prepared
|
|
2491
|
-
* - [♨] Are tasks prepared
|
|
2492
|
-
*/
|
|
2493
|
-
|
|
2494
|
-
/**
|
|
2495
|
-
* Format either small or big number
|
|
2511
|
+
* Helper to create a new task
|
|
2496
2512
|
*
|
|
2497
|
-
* @
|
|
2513
|
+
* @private internal helper function
|
|
2498
2514
|
*/
|
|
2499
|
-
function
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
}
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
}
|
|
2506
|
-
|
|
2507
|
-
return
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2515
|
+
function createTask(options) {
|
|
2516
|
+
var taskType = options.taskType, taskProcessCallback = options.taskProcessCallback;
|
|
2517
|
+
var taskId = "".concat(taskType.toLowerCase(), "-").concat($randomToken(256 /* <- TODO: !!! To global config */));
|
|
2518
|
+
var resultSubject = new BehaviorSubject({});
|
|
2519
|
+
var finalResultPromise = /* not await */ taskProcessCallback(function (newOngoingResult) {
|
|
2520
|
+
resultSubject.next(newOngoingResult);
|
|
2521
|
+
});
|
|
2522
|
+
function asPromise(options) {
|
|
2523
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
2524
|
+
var _a, isCrashedOnError, finalResult;
|
|
2525
|
+
return __generator(this, function (_b) {
|
|
2526
|
+
switch (_b.label) {
|
|
2527
|
+
case 0:
|
|
2528
|
+
_a = (options || {}).isCrashedOnError, isCrashedOnError = _a === void 0 ? true : _a;
|
|
2529
|
+
return [4 /*yield*/, finalResultPromise];
|
|
2530
|
+
case 1:
|
|
2531
|
+
finalResult = _b.sent();
|
|
2532
|
+
if (isCrashedOnError) {
|
|
2533
|
+
assertsTaskSuccessful(finalResult);
|
|
2534
|
+
}
|
|
2535
|
+
return [2 /*return*/, finalResult];
|
|
2536
|
+
}
|
|
2537
|
+
});
|
|
2538
|
+
});
|
|
2518
2539
|
}
|
|
2519
|
-
return
|
|
2540
|
+
return {
|
|
2541
|
+
taskType: taskType,
|
|
2542
|
+
taskId: taskId,
|
|
2543
|
+
asPromise: asPromise,
|
|
2544
|
+
asObservable: function () {
|
|
2545
|
+
return concat(resultSubject.asObservable(), from(asPromise({
|
|
2546
|
+
isCrashedOnError: true,
|
|
2547
|
+
})));
|
|
2548
|
+
},
|
|
2549
|
+
};
|
|
2520
2550
|
}
|
|
2521
|
-
|
|
2522
2551
|
/**
|
|
2523
|
-
*
|
|
2524
|
-
*
|
|
2525
|
-
*
|
|
2526
|
-
* Note: This function is not just calling `toString` method
|
|
2527
|
-
* It's more complex and can handle this conversion specifically for LLM models
|
|
2528
|
-
* See `VALUE_STRINGS`
|
|
2529
|
-
*
|
|
2530
|
-
* Note: There are 2 similar functions
|
|
2531
|
-
* - `valueToString` converts value to string for LLM models as human-readable string
|
|
2532
|
-
* - `asSerializable` converts value to string to preserve full information to be able to convert it back
|
|
2533
|
-
*
|
|
2534
|
-
* @public exported from `@promptbook/utils`
|
|
2552
|
+
* TODO: Maybe allow to terminate the task and add getter `isFinished` or `status`
|
|
2553
|
+
* TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
|
|
2535
2554
|
*/
|
|
2536
|
-
function valueToString(value) {
|
|
2537
|
-
try {
|
|
2538
|
-
if (value === '') {
|
|
2539
|
-
return VALUE_STRINGS.empty;
|
|
2540
|
-
}
|
|
2541
|
-
else if (value === null) {
|
|
2542
|
-
return VALUE_STRINGS.null;
|
|
2543
|
-
}
|
|
2544
|
-
else if (value === undefined) {
|
|
2545
|
-
return VALUE_STRINGS.undefined;
|
|
2546
|
-
}
|
|
2547
|
-
else if (typeof value === 'string') {
|
|
2548
|
-
return value;
|
|
2549
|
-
}
|
|
2550
|
-
else if (typeof value === 'number') {
|
|
2551
|
-
return numberToString(value);
|
|
2552
|
-
}
|
|
2553
|
-
else if (value instanceof Date) {
|
|
2554
|
-
return value.toISOString();
|
|
2555
|
-
}
|
|
2556
|
-
else {
|
|
2557
|
-
return JSON.stringify(value);
|
|
2558
|
-
}
|
|
2559
|
-
}
|
|
2560
|
-
catch (error) {
|
|
2561
|
-
if (!(error instanceof Error)) {
|
|
2562
|
-
throw error;
|
|
2563
|
-
}
|
|
2564
|
-
console.error(error);
|
|
2565
|
-
return VALUE_STRINGS.unserializable;
|
|
2566
|
-
}
|
|
2567
|
-
}
|
|
2568
2555
|
|
|
2569
2556
|
/**
|
|
2570
2557
|
* Serializes an error into a [🚉] JSON-serializable object
|
|
@@ -3228,10 +3215,9 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
3228
3215
|
return modelName;
|
|
3229
3216
|
})
|
|
3230
3217
|
.join(',');
|
|
3231
|
-
return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
|
|
3218
|
+
return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription }).asPromise()];
|
|
3232
3219
|
case 3:
|
|
3233
3220
|
result = _d.sent();
|
|
3234
|
-
assertsExecutionSuccessful(result);
|
|
3235
3221
|
outputParameters = result.outputParameters;
|
|
3236
3222
|
modelRequirementsRaw = outputParameters.modelRequirements;
|
|
3237
3223
|
modelRequirements = JSON.parse(modelRequirementsRaw);
|
|
@@ -4029,10 +4015,9 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
4029
4015
|
var content = _a.content;
|
|
4030
4016
|
return content;
|
|
4031
4017
|
}).join('\n\n'),
|
|
4032
|
-
})];
|
|
4018
|
+
}).asPromise()];
|
|
4033
4019
|
case 2:
|
|
4034
4020
|
result = _e.sent();
|
|
4035
|
-
assertsExecutionSuccessful(result);
|
|
4036
4021
|
outputParameters = result.outputParameters;
|
|
4037
4022
|
titleRaw = outputParameters.title;
|
|
4038
4023
|
if (isVerbose) {
|
|
@@ -4104,6 +4089,81 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
4104
4089
|
* @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format
|
|
4105
4090
|
*/
|
|
4106
4091
|
|
|
4092
|
+
/**
|
|
4093
|
+
* Format either small or big number
|
|
4094
|
+
*
|
|
4095
|
+
* @public exported from `@promptbook/utils`
|
|
4096
|
+
*/
|
|
4097
|
+
function numberToString(value) {
|
|
4098
|
+
if (value === 0) {
|
|
4099
|
+
return '0';
|
|
4100
|
+
}
|
|
4101
|
+
else if (Number.isNaN(value)) {
|
|
4102
|
+
return VALUE_STRINGS.nan;
|
|
4103
|
+
}
|
|
4104
|
+
else if (value === Infinity) {
|
|
4105
|
+
return VALUE_STRINGS.infinity;
|
|
4106
|
+
}
|
|
4107
|
+
else if (value === -Infinity) {
|
|
4108
|
+
return VALUE_STRINGS.negativeInfinity;
|
|
4109
|
+
}
|
|
4110
|
+
for (var exponent = 0; exponent < 15; exponent++) {
|
|
4111
|
+
var factor = Math.pow(10, exponent);
|
|
4112
|
+
var valueRounded = Math.round(value * factor) / factor;
|
|
4113
|
+
if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) {
|
|
4114
|
+
return valueRounded.toFixed(exponent);
|
|
4115
|
+
}
|
|
4116
|
+
}
|
|
4117
|
+
return value.toString();
|
|
4118
|
+
}
|
|
4119
|
+
|
|
4120
|
+
/**
|
|
4121
|
+
* Function `valueToString` will convert the given value to string
|
|
4122
|
+
* This is useful and used in the `templateParameters` function
|
|
4123
|
+
*
|
|
4124
|
+
* Note: This function is not just calling `toString` method
|
|
4125
|
+
* It's more complex and can handle this conversion specifically for LLM models
|
|
4126
|
+
* See `VALUE_STRINGS`
|
|
4127
|
+
*
|
|
4128
|
+
* Note: There are 2 similar functions
|
|
4129
|
+
* - `valueToString` converts value to string for LLM models as human-readable string
|
|
4130
|
+
* - `asSerializable` converts value to string to preserve full information to be able to convert it back
|
|
4131
|
+
*
|
|
4132
|
+
* @public exported from `@promptbook/utils`
|
|
4133
|
+
*/
|
|
4134
|
+
function valueToString(value) {
|
|
4135
|
+
try {
|
|
4136
|
+
if (value === '') {
|
|
4137
|
+
return VALUE_STRINGS.empty;
|
|
4138
|
+
}
|
|
4139
|
+
else if (value === null) {
|
|
4140
|
+
return VALUE_STRINGS.null;
|
|
4141
|
+
}
|
|
4142
|
+
else if (value === undefined) {
|
|
4143
|
+
return VALUE_STRINGS.undefined;
|
|
4144
|
+
}
|
|
4145
|
+
else if (typeof value === 'string') {
|
|
4146
|
+
return value;
|
|
4147
|
+
}
|
|
4148
|
+
else if (typeof value === 'number') {
|
|
4149
|
+
return numberToString(value);
|
|
4150
|
+
}
|
|
4151
|
+
else if (value instanceof Date) {
|
|
4152
|
+
return value.toISOString();
|
|
4153
|
+
}
|
|
4154
|
+
else {
|
|
4155
|
+
return JSON.stringify(value);
|
|
4156
|
+
}
|
|
4157
|
+
}
|
|
4158
|
+
catch (error) {
|
|
4159
|
+
if (!(error instanceof Error)) {
|
|
4160
|
+
throw error;
|
|
4161
|
+
}
|
|
4162
|
+
console.error(error);
|
|
4163
|
+
return VALUE_STRINGS.unserializable;
|
|
4164
|
+
}
|
|
4165
|
+
}
|
|
4166
|
+
|
|
4107
4167
|
/**
|
|
4108
4168
|
* Parses the given script and returns the list of all used variables that are not defined in the script
|
|
4109
4169
|
*
|
|
@@ -5548,27 +5608,20 @@ function getReservedParametersForTask(options) {
|
|
|
5548
5608
|
*/
|
|
5549
5609
|
function executeTask(options) {
|
|
5550
5610
|
return __awaiter(this, void 0, void 0, function () {
|
|
5551
|
-
var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed,
|
|
5552
|
-
var e_1,
|
|
5553
|
-
return __generator(this, function (
|
|
5554
|
-
switch (
|
|
5611
|
+
var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
5612
|
+
var _f, e_1, _g, _h, _j;
|
|
5613
|
+
return __generator(this, function (_k) {
|
|
5614
|
+
switch (_k.label) {
|
|
5555
5615
|
case 0:
|
|
5556
5616
|
currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts, maxParallelCount = options.maxParallelCount, csvSettings = options.csvSettings, isVerbose = options.isVerbose, rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, isAutoInstalled = options.isAutoInstalled, isNotPreparedWarningSupressed = options.isNotPreparedWarningSupressed;
|
|
5557
|
-
name = "pipeline-executor-frame-".concat(currentTask.name);
|
|
5558
|
-
title = currentTask.title;
|
|
5559
5617
|
priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
5560
5618
|
return [4 /*yield*/, onProgress({
|
|
5561
|
-
|
|
5562
|
-
|
|
5563
|
-
|
|
5564
|
-
isDone: false,
|
|
5565
|
-
taskType: currentTask.taskType,
|
|
5566
|
-
parameterName: currentTask.resultingParameterName,
|
|
5567
|
-
parameterValue: null,
|
|
5568
|
-
// <- [🍸]
|
|
5619
|
+
outputParameters: (_f = {},
|
|
5620
|
+
_f[currentTask.resultingParameterName] = '',
|
|
5621
|
+
_f),
|
|
5569
5622
|
})];
|
|
5570
5623
|
case 1:
|
|
5571
|
-
|
|
5624
|
+
_k.sent();
|
|
5572
5625
|
usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
5573
5626
|
dependentParameterNames = new Set(currentTask.dependentParameterNames);
|
|
5574
5627
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -5587,7 +5640,7 @@ function executeTask(options) {
|
|
|
5587
5640
|
pipelineIdentification: pipelineIdentification,
|
|
5588
5641
|
})];
|
|
5589
5642
|
case 2:
|
|
5590
|
-
definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(
|
|
5643
|
+
definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_k.sent())])), parametersToPass])]);
|
|
5591
5644
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
5592
5645
|
parameters = {};
|
|
5593
5646
|
_loop_1 = function (parameterName) {
|
|
@@ -5615,7 +5668,7 @@ function executeTask(options) {
|
|
|
5615
5668
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5616
5669
|
finally {
|
|
5617
5670
|
try {
|
|
5618
|
-
if (_e && !_e.done && (
|
|
5671
|
+
if (_e && !_e.done && (_g = _d.return)) _g.call(_d);
|
|
5619
5672
|
}
|
|
5620
5673
|
finally { if (e_1) throw e_1.error; }
|
|
5621
5674
|
}
|
|
@@ -5646,24 +5699,19 @@ function executeTask(options) {
|
|
|
5646
5699
|
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
5647
5700
|
})];
|
|
5648
5701
|
case 3:
|
|
5649
|
-
resultString =
|
|
5702
|
+
resultString = _k.sent();
|
|
5650
5703
|
return [4 /*yield*/, onProgress({
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
isDone: true,
|
|
5655
|
-
taskType: currentTask.taskType,
|
|
5656
|
-
parameterName: currentTask.resultingParameterName,
|
|
5657
|
-
parameterValue: resultString,
|
|
5658
|
-
// <- [🍸]
|
|
5704
|
+
outputParameters: (_h = {},
|
|
5705
|
+
_h[currentTask.resultingParameterName] = resultString,
|
|
5706
|
+
_h),
|
|
5659
5707
|
})];
|
|
5660
5708
|
case 4:
|
|
5661
|
-
|
|
5662
|
-
return [2 /*return*/, Object.freeze((
|
|
5663
|
-
|
|
5709
|
+
_k.sent();
|
|
5710
|
+
return [2 /*return*/, Object.freeze((_j = {},
|
|
5711
|
+
_j[currentTask.resultingParameterName] =
|
|
5664
5712
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
5665
5713
|
resultString,
|
|
5666
|
-
|
|
5714
|
+
_j))];
|
|
5667
5715
|
}
|
|
5668
5716
|
});
|
|
5669
5717
|
});
|
|
@@ -5671,9 +5719,6 @@ function executeTask(options) {
|
|
|
5671
5719
|
/**
|
|
5672
5720
|
* TODO: [🤹♂️]
|
|
5673
5721
|
*/
|
|
5674
|
-
/**
|
|
5675
|
-
* TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
|
|
5676
|
-
*/
|
|
5677
5722
|
|
|
5678
5723
|
/**
|
|
5679
5724
|
* @@@
|
|
@@ -5935,15 +5980,15 @@ function executePipeline(options) {
|
|
|
5935
5980
|
return [3 /*break*/, 4];
|
|
5936
5981
|
case 3:
|
|
5937
5982
|
unresovedTasks_1 = unresovedTasks_1.filter(function (task) { return task !== currentTask; });
|
|
5938
|
-
work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (
|
|
5983
|
+
work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (newOngoingResult) {
|
|
5939
5984
|
if (isReturned) {
|
|
5940
|
-
throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(
|
|
5985
|
+
throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(newOngoingResult, null, 4)
|
|
5941
5986
|
.split('\n')
|
|
5942
5987
|
.map(function (line) { return "> ".concat(line); })
|
|
5943
5988
|
.join('\n')), "\n "); }));
|
|
5944
5989
|
}
|
|
5945
5990
|
if (onProgress) {
|
|
5946
|
-
onProgress(
|
|
5991
|
+
onProgress(newOngoingResult);
|
|
5947
5992
|
}
|
|
5948
5993
|
}, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Task name: ").concat(currentTask.name, "\n Task title: ").concat(currentTask.title, "\n "); }) }))
|
|
5949
5994
|
.then(function (newParametersToPass) {
|
|
@@ -6046,9 +6091,6 @@ function executePipeline(options) {
|
|
|
6046
6091
|
});
|
|
6047
6092
|
});
|
|
6048
6093
|
}
|
|
6049
|
-
/**
|
|
6050
|
-
* TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
|
|
6051
|
-
*/
|
|
6052
6094
|
|
|
6053
6095
|
/**
|
|
6054
6096
|
* Creates executor function from pipeline and execution tools.
|
|
@@ -6080,7 +6122,7 @@ function createPipelineExecutor(options) {
|
|
|
6080
6122
|
console.warn(spaceTrim$1(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
|
|
6081
6123
|
}
|
|
6082
6124
|
var runCount = 0;
|
|
6083
|
-
var
|
|
6125
|
+
var pipelineExecutorWithCallback = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
|
|
6084
6126
|
return __generator(this, function (_a) {
|
|
6085
6127
|
runCount++;
|
|
6086
6128
|
return [2 /*return*/, /* not await */ executePipeline({
|
|
@@ -6105,11 +6147,23 @@ function createPipelineExecutor(options) {
|
|
|
6105
6147
|
})];
|
|
6106
6148
|
});
|
|
6107
6149
|
}); };
|
|
6150
|
+
var pipelineExecutor = function (inputParameters) {
|
|
6151
|
+
return createTask({
|
|
6152
|
+
taskType: 'EXECUTION',
|
|
6153
|
+
taskProcessCallback: function (updateOngoingResult) {
|
|
6154
|
+
var _this = this;
|
|
6155
|
+
return pipelineExecutorWithCallback(inputParameters, function (newOngoingResult) { return __awaiter(_this, void 0, void 0, function () {
|
|
6156
|
+
return __generator(this, function (_a) {
|
|
6157
|
+
updateOngoingResult(newOngoingResult);
|
|
6158
|
+
return [2 /*return*/];
|
|
6159
|
+
});
|
|
6160
|
+
}); });
|
|
6161
|
+
},
|
|
6162
|
+
});
|
|
6163
|
+
};
|
|
6164
|
+
// <- TODO: Make types such as there is no need to do `as` for `createTask`
|
|
6108
6165
|
return pipelineExecutor;
|
|
6109
6166
|
}
|
|
6110
|
-
/**
|
|
6111
|
-
* TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
|
|
6112
|
-
*/
|
|
6113
6167
|
|
|
6114
6168
|
/**
|
|
6115
6169
|
* Metadata of the scraper
|
|
@@ -6211,10 +6265,9 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
6211
6265
|
return [4 /*yield*/, source.asText()];
|
|
6212
6266
|
case 4:
|
|
6213
6267
|
knowledgeContent = _k.sent();
|
|
6214
|
-
return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
|
|
6268
|
+
return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent }).asPromise()];
|
|
6215
6269
|
case 5:
|
|
6216
6270
|
result = _k.sent();
|
|
6217
|
-
assertsExecutionSuccessful(result);
|
|
6218
6271
|
outputParameters = result.outputParameters;
|
|
6219
6272
|
knowledgePiecesRaw = outputParameters.knowledgePieces;
|
|
6220
6273
|
knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
|
|
@@ -6237,13 +6290,13 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
6237
6290
|
_c.label = 1;
|
|
6238
6291
|
case 1:
|
|
6239
6292
|
_c.trys.push([1, 7, , 8]);
|
|
6240
|
-
return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
|
|
6293
|
+
return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent }).asPromise()];
|
|
6241
6294
|
case 2:
|
|
6242
6295
|
titleResult = _c.sent();
|
|
6243
6296
|
_a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
|
|
6244
6297
|
title = spaceTrim(titleRaw) /* <- TODO: Maybe do in pipeline */;
|
|
6245
6298
|
name = titleToName(title);
|
|
6246
|
-
return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
|
|
6299
|
+
return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent }).asPromise()];
|
|
6247
6300
|
case 3:
|
|
6248
6301
|
keywordsResult = _c.sent();
|
|
6249
6302
|
_b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
|