@promptbook/website-crawler 0.75.6 โ 0.75.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +85 -67
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/types.index.d.ts +0 -2
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +1 -1
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +6 -4
- package/package.json +2 -2
- package/umd/index.umd.js +85 -67
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/_packages/langtail.index.d.ts +0 -6
- package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +0 -17
- package/esm/typings/src/llm-providers/langtail/LangtailExecutionToolsOptions.d.ts +0 -13
- package/esm/typings/src/llm-providers/langtail/playground/playground.d.ts +0 -5
|
@@ -55,7 +55,6 @@ import type { AnthropicClaudeExecutionToolsOptions } from '../llm-providers/anth
|
|
|
55
55
|
import type { AnthropicClaudeExecutionToolsDirectOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions';
|
|
56
56
|
import type { AnthropicClaudeExecutionToolsProxiedOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions';
|
|
57
57
|
import type { AzureOpenAiExecutionToolsOptions } from '../llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions';
|
|
58
|
-
import type { LangtailExecutionToolsOptions } from '../llm-providers/langtail/LangtailExecutionToolsOptions';
|
|
59
58
|
import type { OpenAiAssistantExecutionToolsOptions } from '../llm-providers/openai/OpenAiAssistantExecutionToolsOptions';
|
|
60
59
|
import type { OpenAiExecutionToolsOptions } from '../llm-providers/openai/OpenAiExecutionToolsOptions';
|
|
61
60
|
import type { PromptbookServer_Error } from '../llm-providers/remote/interfaces/PromptbookServer_Error';
|
|
@@ -309,7 +308,6 @@ export type { AnthropicClaudeExecutionToolsOptions };
|
|
|
309
308
|
export type { AnthropicClaudeExecutionToolsDirectOptions };
|
|
310
309
|
export type { AnthropicClaudeExecutionToolsProxiedOptions };
|
|
311
310
|
export type { AzureOpenAiExecutionToolsOptions };
|
|
312
|
-
export type { LangtailExecutionToolsOptions };
|
|
313
311
|
export type { OpenAiAssistantExecutionToolsOptions };
|
|
314
312
|
export type { OpenAiExecutionToolsOptions };
|
|
315
313
|
export type { PromptbookServer_Error };
|
|
@@ -9,7 +9,7 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
|
|
|
9
9
|
*
|
|
10
10
|
* @private internal type of `executePipeline`
|
|
11
11
|
*/
|
|
12
|
-
type ExecutePipelineOptions = CreatePipelineExecutorOptions & {
|
|
12
|
+
type ExecutePipelineOptions = Required<CreatePipelineExecutorOptions> & {
|
|
13
13
|
/**
|
|
14
14
|
* @@@
|
|
15
15
|
*/
|
|
@@ -10,7 +10,7 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
|
|
|
10
10
|
*
|
|
11
11
|
* @private internal type of `executeTask`
|
|
12
12
|
*/
|
|
13
|
-
type executeSingleTaskOptions = CreatePipelineExecutorOptions & {
|
|
13
|
+
type executeSingleTaskOptions = Required<CreatePipelineExecutorOptions> & {
|
|
14
14
|
/**
|
|
15
15
|
* @@@
|
|
16
16
|
*/
|
|
@@ -11,7 +11,7 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
|
|
|
11
11
|
*
|
|
12
12
|
* @private internal type of `executeAttempts`
|
|
13
13
|
*/
|
|
14
|
-
export type ExecuteAttemptsOptions = Omit<CreatePipelineExecutorOptions, 'pipeline'
|
|
14
|
+
export type ExecuteAttemptsOptions = Required<Omit<CreatePipelineExecutorOptions, 'pipeline'>> & {
|
|
15
15
|
/**
|
|
16
16
|
* @@@
|
|
17
17
|
*/
|
|
@@ -10,7 +10,7 @@ export type PrepareAndScrapeOptions = {
|
|
|
10
10
|
* Note: When the pipeline is not created from files, it is `null`
|
|
11
11
|
* Note: This folder must exist (=it is not created recursively)
|
|
12
12
|
*
|
|
13
|
-
* @default process.cwd()
|
|
13
|
+
* @default null or `process.cwd()` when created via `$provide...` function in node
|
|
14
14
|
*/
|
|
15
15
|
readonly rootDirname?: string_dirname | null;
|
|
16
16
|
/**
|
|
@@ -18,7 +18,7 @@ export type PrepareAndScrapeOptions = {
|
|
|
18
18
|
*
|
|
19
19
|
* Note: When the folder does not exist, it is created recursively
|
|
20
20
|
*
|
|
21
|
-
* @default
|
|
21
|
+
* @default DEFAULT_SCRAPE_CACHE_DIRNAME
|
|
22
22
|
*/
|
|
23
23
|
readonly cacheDirname?: string_dirname;
|
|
24
24
|
/**
|
|
@@ -30,17 +30,19 @@ export type PrepareAndScrapeOptions = {
|
|
|
30
30
|
/**
|
|
31
31
|
* Maximum number of tasks running in parallel
|
|
32
32
|
*
|
|
33
|
-
* @default
|
|
33
|
+
* @default DEFAULT_MAX_PARALLEL_COUNT
|
|
34
34
|
*/
|
|
35
35
|
readonly maxParallelCount?: number;
|
|
36
36
|
/**
|
|
37
37
|
* If true, the missing software is automatically installed
|
|
38
|
+
*
|
|
39
|
+
* @default DEFAULT_IS_AUTO_INSTALLED
|
|
38
40
|
*/
|
|
39
41
|
readonly isAutoInstalled?: boolean;
|
|
40
42
|
/**
|
|
41
43
|
* If true, the preparation logs additional information
|
|
42
44
|
*
|
|
43
|
-
* @default
|
|
45
|
+
* @default DEFAULT_IS_VERBOSE
|
|
44
46
|
*/
|
|
45
47
|
readonly isVerbose?: boolean;
|
|
46
48
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@promptbook/website-crawler",
|
|
3
|
-
"version": "0.75.
|
|
3
|
+
"version": "0.75.9",
|
|
4
4
|
"description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
|
|
5
5
|
"--note-0": " <- [๐]",
|
|
6
6
|
"private": false,
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"module": "./esm/index.es.js",
|
|
55
55
|
"typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
|
|
56
56
|
"peerDependencies": {
|
|
57
|
-
"@promptbook/core": "0.75.
|
|
57
|
+
"@promptbook/core": "0.75.9"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
60
|
"@mozilla/readability": "0.5.0",
|
package/umd/index.umd.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
*
|
|
23
23
|
* @see https://github.com/webgptorg/promptbook
|
|
24
24
|
*/
|
|
25
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
25
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.8';
|
|
26
26
|
/**
|
|
27
27
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
28
28
|
* Note: [๐] Ignore a discrepancy between file name and entity name
|
|
@@ -452,6 +452,12 @@
|
|
|
452
452
|
* @public exported from `@promptbook/core`
|
|
453
453
|
*/
|
|
454
454
|
var DEFAULT_IS_VERBOSE = false;
|
|
455
|
+
/**
|
|
456
|
+
* @@@
|
|
457
|
+
*
|
|
458
|
+
* @public exported from `@promptbook/core`
|
|
459
|
+
*/
|
|
460
|
+
var DEFAULT_IS_AUTO_INSTALLED = false;
|
|
455
461
|
/**
|
|
456
462
|
* @@@
|
|
457
463
|
*
|
|
@@ -5099,12 +5105,12 @@
|
|
|
5099
5105
|
*/
|
|
5100
5106
|
function executeTask(options) {
|
|
5101
5107
|
return __awaiter(this, void 0, void 0, function () {
|
|
5102
|
-
var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification,
|
|
5103
|
-
var e_1,
|
|
5104
|
-
return __generator(this, function (
|
|
5105
|
-
switch (
|
|
5108
|
+
var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
5109
|
+
var e_1, _f, _g;
|
|
5110
|
+
return __generator(this, function (_h) {
|
|
5111
|
+
switch (_h.label) {
|
|
5106
5112
|
case 0:
|
|
5107
|
-
currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification,
|
|
5113
|
+
currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts, maxParallelCount = options.maxParallelCount, csvSettings = options.csvSettings, isVerbose = options.isVerbose, rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, isAutoInstalled = options.isAutoInstalled, isNotPreparedWarningSupressed = options.isNotPreparedWarningSupressed;
|
|
5108
5114
|
name = "pipeline-executor-frame-".concat(currentTask.name);
|
|
5109
5115
|
title = currentTask.title;
|
|
5110
5116
|
priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
@@ -5119,7 +5125,7 @@
|
|
|
5119
5125
|
// <- [๐ธ]
|
|
5120
5126
|
})];
|
|
5121
5127
|
case 1:
|
|
5122
|
-
|
|
5128
|
+
_h.sent();
|
|
5123
5129
|
usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
5124
5130
|
dependentParameterNames = new Set(currentTask.dependentParameterNames);
|
|
5125
5131
|
// TODO: [๐ฉ๐พโ๐คโ๐ฉ๐ป] Use here `mapAvailableToExpectedParameters`
|
|
@@ -5130,15 +5136,15 @@
|
|
|
5130
5136
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
5131
5137
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
5132
5138
|
}
|
|
5133
|
-
|
|
5134
|
-
|
|
5139
|
+
_b = (_a = Object).freeze;
|
|
5140
|
+
_c = [{}];
|
|
5135
5141
|
return [4 /*yield*/, getReservedParametersForTask({
|
|
5136
5142
|
preparedPipeline: preparedPipeline,
|
|
5137
5143
|
task: currentTask,
|
|
5138
5144
|
pipelineIdentification: pipelineIdentification,
|
|
5139
5145
|
})];
|
|
5140
5146
|
case 2:
|
|
5141
|
-
definedParameters =
|
|
5147
|
+
definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
|
|
5142
5148
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
5143
5149
|
parameters = {};
|
|
5144
5150
|
_loop_1 = function (parameterName) {
|
|
@@ -5158,15 +5164,15 @@
|
|
|
5158
5164
|
try {
|
|
5159
5165
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this task
|
|
5160
5166
|
// TODO: [๐ฉ๐พโ๐คโ๐ฉ๐ป] Use here `mapAvailableToExpectedParameters`
|
|
5161
|
-
for (
|
|
5162
|
-
parameterName =
|
|
5167
|
+
for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
5168
|
+
parameterName = _e.value;
|
|
5163
5169
|
_loop_1(parameterName);
|
|
5164
5170
|
}
|
|
5165
5171
|
}
|
|
5166
5172
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5167
5173
|
finally {
|
|
5168
5174
|
try {
|
|
5169
|
-
if (
|
|
5175
|
+
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
5170
5176
|
}
|
|
5171
5177
|
finally { if (e_1) throw e_1.error; }
|
|
5172
5178
|
}
|
|
@@ -5186,9 +5192,18 @@
|
|
|
5186
5192
|
tools: tools,
|
|
5187
5193
|
$executionReport: $executionReport,
|
|
5188
5194
|
pipelineIdentification: pipelineIdentification,
|
|
5195
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
5196
|
+
maxParallelCount: maxParallelCount,
|
|
5197
|
+
csvSettings: csvSettings,
|
|
5198
|
+
isVerbose: isVerbose,
|
|
5199
|
+
rootDirname: rootDirname,
|
|
5200
|
+
cacheDirname: cacheDirname,
|
|
5201
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
5202
|
+
isAutoInstalled: isAutoInstalled,
|
|
5203
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
5189
5204
|
})];
|
|
5190
5205
|
case 3:
|
|
5191
|
-
resultString =
|
|
5206
|
+
resultString = _h.sent();
|
|
5192
5207
|
return [4 /*yield*/, onProgress({
|
|
5193
5208
|
name: name,
|
|
5194
5209
|
title: title,
|
|
@@ -5200,12 +5215,12 @@
|
|
|
5200
5215
|
// <- [๐ธ]
|
|
5201
5216
|
})];
|
|
5202
5217
|
case 4:
|
|
5203
|
-
|
|
5204
|
-
return [2 /*return*/, Object.freeze((
|
|
5205
|
-
|
|
5218
|
+
_h.sent();
|
|
5219
|
+
return [2 /*return*/, Object.freeze((_g = {},
|
|
5220
|
+
_g[currentTask.resultingParameterName] =
|
|
5206
5221
|
// <- Note: [๐ฉโ๐ฉโ๐ง] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
5207
5222
|
resultString,
|
|
5208
|
-
|
|
5223
|
+
_g))];
|
|
5209
5224
|
}
|
|
5210
5225
|
});
|
|
5211
5226
|
});
|
|
@@ -5264,12 +5279,12 @@
|
|
|
5264
5279
|
*/
|
|
5265
5280
|
function executePipeline(options) {
|
|
5266
5281
|
return __awaiter(this, void 0, void 0, function () {
|
|
5267
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname,
|
|
5268
|
-
var e_1,
|
|
5269
|
-
return __generator(this, function (
|
|
5270
|
-
switch (
|
|
5282
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _a, _b, parameter, e_1_1, _loop_1, _c, _d, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTasks_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
5283
|
+
var e_1, _e, e_2, _f;
|
|
5284
|
+
return __generator(this, function (_g) {
|
|
5285
|
+
switch (_g.label) {
|
|
5271
5286
|
case 0:
|
|
5272
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname,
|
|
5287
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, isVerbose = options.isVerbose;
|
|
5273
5288
|
preparedPipeline = options.preparedPipeline;
|
|
5274
5289
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
5275
5290
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -5278,9 +5293,9 @@
|
|
|
5278
5293
|
maxParallelCount: maxParallelCount,
|
|
5279
5294
|
})];
|
|
5280
5295
|
case 1:
|
|
5281
|
-
preparedPipeline =
|
|
5296
|
+
preparedPipeline = _g.sent();
|
|
5282
5297
|
setPreparedPipeline(preparedPipeline);
|
|
5283
|
-
|
|
5298
|
+
_g.label = 2;
|
|
5284
5299
|
case 2:
|
|
5285
5300
|
errors = [];
|
|
5286
5301
|
warnings = [];
|
|
@@ -5293,17 +5308,17 @@
|
|
|
5293
5308
|
promptExecutions: [],
|
|
5294
5309
|
};
|
|
5295
5310
|
isReturned = false;
|
|
5296
|
-
|
|
5311
|
+
_g.label = 3;
|
|
5297
5312
|
case 3:
|
|
5298
|
-
|
|
5299
|
-
|
|
5313
|
+
_g.trys.push([3, 9, 10, 11]);
|
|
5314
|
+
_a = __values(preparedPipeline.parameters.filter(function (_a) {
|
|
5300
5315
|
var isInput = _a.isInput;
|
|
5301
5316
|
return isInput;
|
|
5302
|
-
})),
|
|
5303
|
-
|
|
5317
|
+
})), _b = _a.next();
|
|
5318
|
+
_g.label = 4;
|
|
5304
5319
|
case 4:
|
|
5305
|
-
if (!!
|
|
5306
|
-
parameter =
|
|
5320
|
+
if (!!_b.done) return [3 /*break*/, 8];
|
|
5321
|
+
parameter = _b.value;
|
|
5307
5322
|
if (!(inputParameters[parameter.name] === undefined)) return [3 /*break*/, 7];
|
|
5308
5323
|
isReturned = true;
|
|
5309
5324
|
if (!(onProgress !== undefined)) return [3 /*break*/, 6];
|
|
@@ -5311,8 +5326,8 @@
|
|
|
5311
5326
|
return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
|
|
5312
5327
|
case 5:
|
|
5313
5328
|
// Note: Wait a short time to prevent race conditions
|
|
5314
|
-
|
|
5315
|
-
|
|
5329
|
+
_g.sent();
|
|
5330
|
+
_g.label = 6;
|
|
5316
5331
|
case 6: return [2 /*return*/, $asDeeplyFrozenSerializableJson("Unuccessful PipelineExecutorResult (with missing parameter {".concat(parameter.name, "}) PipelineExecutorResult"), {
|
|
5317
5332
|
isSuccessful: false,
|
|
5318
5333
|
errors: __spreadArray([
|
|
@@ -5325,24 +5340,24 @@
|
|
|
5325
5340
|
preparedPipeline: preparedPipeline,
|
|
5326
5341
|
})];
|
|
5327
5342
|
case 7:
|
|
5328
|
-
|
|
5343
|
+
_b = _a.next();
|
|
5329
5344
|
return [3 /*break*/, 4];
|
|
5330
5345
|
case 8: return [3 /*break*/, 11];
|
|
5331
5346
|
case 9:
|
|
5332
|
-
e_1_1 =
|
|
5347
|
+
e_1_1 = _g.sent();
|
|
5333
5348
|
e_1 = { error: e_1_1 };
|
|
5334
5349
|
return [3 /*break*/, 11];
|
|
5335
5350
|
case 10:
|
|
5336
5351
|
try {
|
|
5337
|
-
if (
|
|
5352
|
+
if (_b && !_b.done && (_e = _a.return)) _e.call(_a);
|
|
5338
5353
|
}
|
|
5339
5354
|
finally { if (e_1) throw e_1.error; }
|
|
5340
5355
|
return [7 /*endfinally*/];
|
|
5341
5356
|
case 11:
|
|
5342
5357
|
_loop_1 = function (parameterName) {
|
|
5343
5358
|
var parameter;
|
|
5344
|
-
return __generator(this, function (
|
|
5345
|
-
switch (
|
|
5359
|
+
return __generator(this, function (_h) {
|
|
5360
|
+
switch (_h.label) {
|
|
5346
5361
|
case 0:
|
|
5347
5362
|
parameter = preparedPipeline.parameters.find(function (_a) {
|
|
5348
5363
|
var name = _a.name;
|
|
@@ -5359,8 +5374,8 @@
|
|
|
5359
5374
|
return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
|
|
5360
5375
|
case 2:
|
|
5361
5376
|
// Note: Wait a short time to prevent race conditions
|
|
5362
|
-
|
|
5363
|
-
|
|
5377
|
+
_h.sent();
|
|
5378
|
+
_h.label = 3;
|
|
5364
5379
|
case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim.spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
|
|
5365
5380
|
isSuccessful: false,
|
|
5366
5381
|
errors: __spreadArray([
|
|
@@ -5376,39 +5391,39 @@
|
|
|
5376
5391
|
}
|
|
5377
5392
|
});
|
|
5378
5393
|
};
|
|
5379
|
-
|
|
5394
|
+
_g.label = 12;
|
|
5380
5395
|
case 12:
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5396
|
+
_g.trys.push([12, 17, 18, 19]);
|
|
5397
|
+
_c = __values(Object.keys(inputParameters)), _d = _c.next();
|
|
5398
|
+
_g.label = 13;
|
|
5384
5399
|
case 13:
|
|
5385
|
-
if (!!
|
|
5386
|
-
parameterName =
|
|
5400
|
+
if (!!_d.done) return [3 /*break*/, 16];
|
|
5401
|
+
parameterName = _d.value;
|
|
5387
5402
|
return [5 /*yield**/, _loop_1(parameterName)];
|
|
5388
5403
|
case 14:
|
|
5389
|
-
state_1 =
|
|
5404
|
+
state_1 = _g.sent();
|
|
5390
5405
|
if (typeof state_1 === "object")
|
|
5391
5406
|
return [2 /*return*/, state_1.value];
|
|
5392
|
-
|
|
5407
|
+
_g.label = 15;
|
|
5393
5408
|
case 15:
|
|
5394
|
-
|
|
5409
|
+
_d = _c.next();
|
|
5395
5410
|
return [3 /*break*/, 13];
|
|
5396
5411
|
case 16: return [3 /*break*/, 19];
|
|
5397
5412
|
case 17:
|
|
5398
|
-
e_2_1 =
|
|
5413
|
+
e_2_1 = _g.sent();
|
|
5399
5414
|
e_2 = { error: e_2_1 };
|
|
5400
5415
|
return [3 /*break*/, 19];
|
|
5401
5416
|
case 18:
|
|
5402
5417
|
try {
|
|
5403
|
-
if (
|
|
5418
|
+
if (_d && !_d.done && (_f = _c.return)) _f.call(_c);
|
|
5404
5419
|
}
|
|
5405
5420
|
finally { if (e_2) throw e_2.error; }
|
|
5406
5421
|
return [7 /*endfinally*/];
|
|
5407
5422
|
case 19:
|
|
5408
5423
|
parametersToPass = inputParameters;
|
|
5409
|
-
|
|
5424
|
+
_g.label = 20;
|
|
5410
5425
|
case 20:
|
|
5411
|
-
|
|
5426
|
+
_g.trys.push([20, 25, , 28]);
|
|
5412
5427
|
resovedParameterNames_1 = preparedPipeline.parameters
|
|
5413
5428
|
.filter(function (_a) {
|
|
5414
5429
|
var isInput = _a.isInput;
|
|
@@ -5423,8 +5438,8 @@
|
|
|
5423
5438
|
loopLimit = LOOP_LIMIT;
|
|
5424
5439
|
_loop_2 = function () {
|
|
5425
5440
|
var currentTask, work_1;
|
|
5426
|
-
return __generator(this, function (
|
|
5427
|
-
switch (
|
|
5441
|
+
return __generator(this, function (_j) {
|
|
5442
|
+
switch (_j.label) {
|
|
5428
5443
|
case 0:
|
|
5429
5444
|
if (loopLimit-- < 0) {
|
|
5430
5445
|
// Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
|
|
@@ -5460,7 +5475,7 @@
|
|
|
5460
5475
|
if (!!currentTask) return [3 /*break*/, 3];
|
|
5461
5476
|
/* [๐คนโโ๏ธ] */ return [4 /*yield*/, Promise.race(resolving_1)];
|
|
5462
5477
|
case 2:
|
|
5463
|
-
/* [๐คนโโ๏ธ] */
|
|
5478
|
+
/* [๐คนโโ๏ธ] */ _j.sent();
|
|
5464
5479
|
return [3 /*break*/, 4];
|
|
5465
5480
|
case 3:
|
|
5466
5481
|
unresovedTasks_1 = unresovedTasks_1.filter(function (task) { return task !== currentTask; });
|
|
@@ -5485,24 +5500,24 @@
|
|
|
5485
5500
|
// <- Note: Errors are catched here [3]
|
|
5486
5501
|
// TODO: BUT if in multiple tasks are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here
|
|
5487
5502
|
resolving_1.push(work_1);
|
|
5488
|
-
|
|
5503
|
+
_j.label = 4;
|
|
5489
5504
|
case 4: return [2 /*return*/];
|
|
5490
5505
|
}
|
|
5491
5506
|
});
|
|
5492
5507
|
};
|
|
5493
|
-
|
|
5508
|
+
_g.label = 21;
|
|
5494
5509
|
case 21:
|
|
5495
5510
|
if (!(unresovedTasks_1.length > 0)) return [3 /*break*/, 23];
|
|
5496
5511
|
return [5 /*yield**/, _loop_2()];
|
|
5497
5512
|
case 22:
|
|
5498
|
-
|
|
5513
|
+
_g.sent();
|
|
5499
5514
|
return [3 /*break*/, 21];
|
|
5500
5515
|
case 23: return [4 /*yield*/, Promise.all(resolving_1)];
|
|
5501
5516
|
case 24:
|
|
5502
|
-
|
|
5517
|
+
_g.sent();
|
|
5503
5518
|
return [3 /*break*/, 28];
|
|
5504
5519
|
case 25:
|
|
5505
|
-
error_1 =
|
|
5520
|
+
error_1 = _g.sent();
|
|
5506
5521
|
if (!(error_1 instanceof Error)) {
|
|
5507
5522
|
throw error_1;
|
|
5508
5523
|
}
|
|
@@ -5522,8 +5537,8 @@
|
|
|
5522
5537
|
return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
|
|
5523
5538
|
case 26:
|
|
5524
5539
|
// Note: Wait a short time to prevent race conditions
|
|
5525
|
-
|
|
5526
|
-
|
|
5540
|
+
_g.sent();
|
|
5541
|
+
_g.label = 27;
|
|
5527
5542
|
case 27: return [2 /*return*/, $asDeeplyFrozenSerializableJson('Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult', {
|
|
5528
5543
|
isSuccessful: false,
|
|
5529
5544
|
errors: __spreadArray([error_1], __read(errors), false).map(serializeError),
|
|
@@ -5550,8 +5565,8 @@
|
|
|
5550
5565
|
return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
|
|
5551
5566
|
case 29:
|
|
5552
5567
|
// Note: Wait a short time to prevent race conditions
|
|
5553
|
-
|
|
5554
|
-
|
|
5568
|
+
_g.sent();
|
|
5569
|
+
_g.label = 30;
|
|
5555
5570
|
case 30: return [2 /*return*/, $asDeeplyFrozenSerializableJson('Successful PipelineExecutorResult', {
|
|
5556
5571
|
isSuccessful: true,
|
|
5557
5572
|
errors: errors.map(serializeError),
|
|
@@ -5578,7 +5593,7 @@
|
|
|
5578
5593
|
*/
|
|
5579
5594
|
function createPipelineExecutor(options) {
|
|
5580
5595
|
var _this = this;
|
|
5581
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname =
|
|
5596
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.cacheDirname, cacheDirname = _f === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _f, _g = options.intermediateFilesStrategy, intermediateFilesStrategy = _g === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _g, _h = options.isAutoInstalled, isAutoInstalled = _h === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _h, _j = options.rootDirname, rootDirname = _j === void 0 ? null : _j;
|
|
5582
5597
|
validatePipeline(pipeline);
|
|
5583
5598
|
var pipelineIdentification = (function () {
|
|
5584
5599
|
// Note: This is a ๐ implementation of [๐]
|
|
@@ -5618,6 +5633,9 @@
|
|
|
5618
5633
|
isVerbose: isVerbose,
|
|
5619
5634
|
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
5620
5635
|
rootDirname: rootDirname,
|
|
5636
|
+
cacheDirname: cacheDirname,
|
|
5637
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
5638
|
+
isAutoInstalled: isAutoInstalled,
|
|
5621
5639
|
})];
|
|
5622
5640
|
});
|
|
5623
5641
|
}); };
|