@promptbook/website-crawler 0.75.6 โ†’ 0.75.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -55,7 +55,6 @@ import type { AnthropicClaudeExecutionToolsOptions } from '../llm-providers/anth
55
55
  import type { AnthropicClaudeExecutionToolsDirectOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions';
56
56
  import type { AnthropicClaudeExecutionToolsProxiedOptions } from '../llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions';
57
57
  import type { AzureOpenAiExecutionToolsOptions } from '../llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions';
58
- import type { LangtailExecutionToolsOptions } from '../llm-providers/langtail/LangtailExecutionToolsOptions';
59
58
  import type { OpenAiAssistantExecutionToolsOptions } from '../llm-providers/openai/OpenAiAssistantExecutionToolsOptions';
60
59
  import type { OpenAiExecutionToolsOptions } from '../llm-providers/openai/OpenAiExecutionToolsOptions';
61
60
  import type { PromptbookServer_Error } from '../llm-providers/remote/interfaces/PromptbookServer_Error';
@@ -309,7 +308,6 @@ export type { AnthropicClaudeExecutionToolsOptions };
309
308
  export type { AnthropicClaudeExecutionToolsDirectOptions };
310
309
  export type { AnthropicClaudeExecutionToolsProxiedOptions };
311
310
  export type { AzureOpenAiExecutionToolsOptions };
312
- export type { LangtailExecutionToolsOptions };
313
311
  export type { OpenAiAssistantExecutionToolsOptions };
314
312
  export type { OpenAiExecutionToolsOptions };
315
313
  export type { PromptbookServer_Error };
@@ -9,7 +9,7 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
9
9
  *
10
10
  * @private internal type of `executePipeline`
11
11
  */
12
- type ExecutePipelineOptions = CreatePipelineExecutorOptions & {
12
+ type ExecutePipelineOptions = Required<CreatePipelineExecutorOptions> & {
13
13
  /**
14
14
  * @@@
15
15
  */
@@ -10,7 +10,7 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
10
10
  *
11
11
  * @private internal type of `executeTask`
12
12
  */
13
- type executeSingleTaskOptions = CreatePipelineExecutorOptions & {
13
+ type executeSingleTaskOptions = Required<CreatePipelineExecutorOptions> & {
14
14
  /**
15
15
  * @@@
16
16
  */
@@ -11,7 +11,7 @@ import type { CreatePipelineExecutorOptions } from './00-CreatePipelineExecutorO
11
11
  *
12
12
  * @private internal type of `executeAttempts`
13
13
  */
14
- export type ExecuteAttemptsOptions = Omit<CreatePipelineExecutorOptions, 'pipeline'> & {
14
+ export type ExecuteAttemptsOptions = Required<Omit<CreatePipelineExecutorOptions, 'pipeline'>> & {
15
15
  /**
16
16
  * @@@
17
17
  */
@@ -10,7 +10,7 @@ export type PrepareAndScrapeOptions = {
10
10
  * Note: When the pipeline is not created from files, it is `null`
11
11
  * Note: This folder must exist (=it is not created recursively)
12
12
  *
13
- * @default process.cwd()
13
+ * @default null or `process.cwd()` when created via `$provide...` function in node
14
14
  */
15
15
  readonly rootDirname?: string_dirname | null;
16
16
  /**
@@ -18,7 +18,7 @@ export type PrepareAndScrapeOptions = {
18
18
  *
19
19
  * Note: When the folder does not exist, it is created recursively
20
20
  *
21
- * @default SCRAPE_CACHE_DIRNAME
21
+ * @default DEFAULT_SCRAPE_CACHE_DIRNAME
22
22
  */
23
23
  readonly cacheDirname?: string_dirname;
24
24
  /**
@@ -30,17 +30,19 @@ export type PrepareAndScrapeOptions = {
30
30
  /**
31
31
  * Maximum number of tasks running in parallel
32
32
  *
33
- * @default MAX_PARALLEL_COUNT
33
+ * @default DEFAULT_MAX_PARALLEL_COUNT
34
34
  */
35
35
  readonly maxParallelCount?: number;
36
36
  /**
37
37
  * If true, the missing software is automatically installed
38
+ *
39
+ * @default DEFAULT_IS_AUTO_INSTALLED
38
40
  */
39
41
  readonly isAutoInstalled?: boolean;
40
42
  /**
41
43
  * If true, the preparation logs additional information
42
44
  *
43
- * @default false
45
+ * @default DEFAULT_IS_VERBOSE
44
46
  */
45
47
  readonly isVerbose?: boolean;
46
48
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@promptbook/website-crawler",
3
- "version": "0.75.6",
3
+ "version": "0.75.9",
4
4
  "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
5
5
  "--note-0": " <- [๐ŸŠ]",
6
6
  "private": false,
@@ -54,7 +54,7 @@
54
54
  "module": "./esm/index.es.js",
55
55
  "typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
56
56
  "peerDependencies": {
57
- "@promptbook/core": "0.75.6"
57
+ "@promptbook/core": "0.75.9"
58
58
  },
59
59
  "dependencies": {
60
60
  "@mozilla/readability": "0.5.0",
package/umd/index.umd.js CHANGED
@@ -22,7 +22,7 @@
22
22
  *
23
23
  * @see https://github.com/webgptorg/promptbook
24
24
  */
25
- var PROMPTBOOK_ENGINE_VERSION = '0.75.5';
25
+ var PROMPTBOOK_ENGINE_VERSION = '0.75.8';
26
26
  /**
27
27
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
28
28
  * Note: [๐Ÿ’ž] Ignore a discrepancy between file name and entity name
@@ -452,6 +452,12 @@
452
452
  * @public exported from `@promptbook/core`
453
453
  */
454
454
  var DEFAULT_IS_VERBOSE = false;
455
+ /**
456
+ * @@@
457
+ *
458
+ * @public exported from `@promptbook/core`
459
+ */
460
+ var DEFAULT_IS_AUTO_INSTALLED = false;
455
461
  /**
456
462
  * @@@
457
463
  *
@@ -5099,12 +5105,12 @@
5099
5105
  */
5100
5106
  function executeTask(options) {
5101
5107
  return __awaiter(this, void 0, void 0, function () {
5102
- var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
5103
- var e_1, _g, _h;
5104
- return __generator(this, function (_j) {
5105
- switch (_j.label) {
5108
+ var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
5109
+ var e_1, _f, _g;
5110
+ return __generator(this, function (_h) {
5111
+ switch (_h.label) {
5106
5112
  case 0:
5107
- currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
5113
+ currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts, maxParallelCount = options.maxParallelCount, csvSettings = options.csvSettings, isVerbose = options.isVerbose, rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, isAutoInstalled = options.isAutoInstalled, isNotPreparedWarningSupressed = options.isNotPreparedWarningSupressed;
5108
5114
  name = "pipeline-executor-frame-".concat(currentTask.name);
5109
5115
  title = currentTask.title;
5110
5116
  priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
@@ -5119,7 +5125,7 @@
5119
5125
  // <- [๐Ÿธ]
5120
5126
  })];
5121
5127
  case 1:
5122
- _j.sent();
5128
+ _h.sent();
5123
5129
  usedParameterNames = extractParameterNamesFromTask(currentTask);
5124
5130
  dependentParameterNames = new Set(currentTask.dependentParameterNames);
5125
5131
  // TODO: [๐Ÿ‘ฉ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘ฉ๐Ÿป] Use here `mapAvailableToExpectedParameters`
@@ -5130,15 +5136,15 @@
5130
5136
  .map(function (name) { return "{".concat(name, "}"); })
5131
5137
  .join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
5132
5138
  }
5133
- _c = (_b = Object).freeze;
5134
- _d = [{}];
5139
+ _b = (_a = Object).freeze;
5140
+ _c = [{}];
5135
5141
  return [4 /*yield*/, getReservedParametersForTask({
5136
5142
  preparedPipeline: preparedPipeline,
5137
5143
  task: currentTask,
5138
5144
  pipelineIdentification: pipelineIdentification,
5139
5145
  })];
5140
5146
  case 2:
5141
- definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
5147
+ definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
5142
5148
  definedParameterNames = new Set(Object.keys(definedParameters));
5143
5149
  parameters = {};
5144
5150
  _loop_1 = function (parameterName) {
@@ -5158,15 +5164,15 @@
5158
5164
  try {
5159
5165
  // Note: [2] Check that all used parameters are defined and removing unused parameters for this task
5160
5166
  // TODO: [๐Ÿ‘ฉ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘ฉ๐Ÿป] Use here `mapAvailableToExpectedParameters`
5161
- for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
5162
- parameterName = _f.value;
5167
+ for (_d = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _e = _d.next(); !_e.done; _e = _d.next()) {
5168
+ parameterName = _e.value;
5163
5169
  _loop_1(parameterName);
5164
5170
  }
5165
5171
  }
5166
5172
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
5167
5173
  finally {
5168
5174
  try {
5169
- if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
5175
+ if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
5170
5176
  }
5171
5177
  finally { if (e_1) throw e_1.error; }
5172
5178
  }
@@ -5186,9 +5192,18 @@
5186
5192
  tools: tools,
5187
5193
  $executionReport: $executionReport,
5188
5194
  pipelineIdentification: pipelineIdentification,
5195
+ maxExecutionAttempts: maxExecutionAttempts,
5196
+ maxParallelCount: maxParallelCount,
5197
+ csvSettings: csvSettings,
5198
+ isVerbose: isVerbose,
5199
+ rootDirname: rootDirname,
5200
+ cacheDirname: cacheDirname,
5201
+ intermediateFilesStrategy: intermediateFilesStrategy,
5202
+ isAutoInstalled: isAutoInstalled,
5203
+ isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5189
5204
  })];
5190
5205
  case 3:
5191
- resultString = _j.sent();
5206
+ resultString = _h.sent();
5192
5207
  return [4 /*yield*/, onProgress({
5193
5208
  name: name,
5194
5209
  title: title,
@@ -5200,12 +5215,12 @@
5200
5215
  // <- [๐Ÿธ]
5201
5216
  })];
5202
5217
  case 4:
5203
- _j.sent();
5204
- return [2 /*return*/, Object.freeze((_h = {},
5205
- _h[currentTask.resultingParameterName] =
5218
+ _h.sent();
5219
+ return [2 /*return*/, Object.freeze((_g = {},
5220
+ _g[currentTask.resultingParameterName] =
5206
5221
  // <- Note: [๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘ง] No need to detect parameter collision here because pipeline checks logic consistency during construction
5207
5222
  resultString,
5208
- _h))];
5223
+ _g))];
5209
5224
  }
5210
5225
  });
5211
5226
  });
@@ -5264,12 +5279,12 @@
5264
5279
  */
5265
5280
  function executePipeline(options) {
5266
5281
  return __awaiter(this, void 0, void 0, function () {
5267
- var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTasks_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
5268
- var e_1, _f, e_2, _g;
5269
- return __generator(this, function (_h) {
5270
- switch (_h.label) {
5282
+ var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _a, _b, parameter, e_1_1, _loop_1, _c, _d, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTasks_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
5283
+ var e_1, _e, e_2, _f;
5284
+ return __generator(this, function (_g) {
5285
+ switch (_g.label) {
5271
5286
  case 0:
5272
- inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
5287
+ inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, isVerbose = options.isVerbose;
5273
5288
  preparedPipeline = options.preparedPipeline;
5274
5289
  if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
5275
5290
  return [4 /*yield*/, preparePipeline(pipeline, tools, {
@@ -5278,9 +5293,9 @@
5278
5293
  maxParallelCount: maxParallelCount,
5279
5294
  })];
5280
5295
  case 1:
5281
- preparedPipeline = _h.sent();
5296
+ preparedPipeline = _g.sent();
5282
5297
  setPreparedPipeline(preparedPipeline);
5283
- _h.label = 2;
5298
+ _g.label = 2;
5284
5299
  case 2:
5285
5300
  errors = [];
5286
5301
  warnings = [];
@@ -5293,17 +5308,17 @@
5293
5308
  promptExecutions: [],
5294
5309
  };
5295
5310
  isReturned = false;
5296
- _h.label = 3;
5311
+ _g.label = 3;
5297
5312
  case 3:
5298
- _h.trys.push([3, 9, 10, 11]);
5299
- _b = __values(preparedPipeline.parameters.filter(function (_a) {
5313
+ _g.trys.push([3, 9, 10, 11]);
5314
+ _a = __values(preparedPipeline.parameters.filter(function (_a) {
5300
5315
  var isInput = _a.isInput;
5301
5316
  return isInput;
5302
- })), _c = _b.next();
5303
- _h.label = 4;
5317
+ })), _b = _a.next();
5318
+ _g.label = 4;
5304
5319
  case 4:
5305
- if (!!_c.done) return [3 /*break*/, 8];
5306
- parameter = _c.value;
5320
+ if (!!_b.done) return [3 /*break*/, 8];
5321
+ parameter = _b.value;
5307
5322
  if (!(inputParameters[parameter.name] === undefined)) return [3 /*break*/, 7];
5308
5323
  isReturned = true;
5309
5324
  if (!(onProgress !== undefined)) return [3 /*break*/, 6];
@@ -5311,8 +5326,8 @@
5311
5326
  return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
5312
5327
  case 5:
5313
5328
  // Note: Wait a short time to prevent race conditions
5314
- _h.sent();
5315
- _h.label = 6;
5329
+ _g.sent();
5330
+ _g.label = 6;
5316
5331
  case 6: return [2 /*return*/, $asDeeplyFrozenSerializableJson("Unuccessful PipelineExecutorResult (with missing parameter {".concat(parameter.name, "}) PipelineExecutorResult"), {
5317
5332
  isSuccessful: false,
5318
5333
  errors: __spreadArray([
@@ -5325,24 +5340,24 @@
5325
5340
  preparedPipeline: preparedPipeline,
5326
5341
  })];
5327
5342
  case 7:
5328
- _c = _b.next();
5343
+ _b = _a.next();
5329
5344
  return [3 /*break*/, 4];
5330
5345
  case 8: return [3 /*break*/, 11];
5331
5346
  case 9:
5332
- e_1_1 = _h.sent();
5347
+ e_1_1 = _g.sent();
5333
5348
  e_1 = { error: e_1_1 };
5334
5349
  return [3 /*break*/, 11];
5335
5350
  case 10:
5336
5351
  try {
5337
- if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
5352
+ if (_b && !_b.done && (_e = _a.return)) _e.call(_a);
5338
5353
  }
5339
5354
  finally { if (e_1) throw e_1.error; }
5340
5355
  return [7 /*endfinally*/];
5341
5356
  case 11:
5342
5357
  _loop_1 = function (parameterName) {
5343
5358
  var parameter;
5344
- return __generator(this, function (_j) {
5345
- switch (_j.label) {
5359
+ return __generator(this, function (_h) {
5360
+ switch (_h.label) {
5346
5361
  case 0:
5347
5362
  parameter = preparedPipeline.parameters.find(function (_a) {
5348
5363
  var name = _a.name;
@@ -5359,8 +5374,8 @@
5359
5374
  return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
5360
5375
  case 2:
5361
5376
  // Note: Wait a short time to prevent race conditions
5362
- _j.sent();
5363
- _j.label = 3;
5377
+ _h.sent();
5378
+ _h.label = 3;
5364
5379
  case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim.spaceTrim(function (block) { return "\n Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n ").concat(block(pipelineIdentification), "\n "); }), {
5365
5380
  isSuccessful: false,
5366
5381
  errors: __spreadArray([
@@ -5376,39 +5391,39 @@
5376
5391
  }
5377
5392
  });
5378
5393
  };
5379
- _h.label = 12;
5394
+ _g.label = 12;
5380
5395
  case 12:
5381
- _h.trys.push([12, 17, 18, 19]);
5382
- _d = __values(Object.keys(inputParameters)), _e = _d.next();
5383
- _h.label = 13;
5396
+ _g.trys.push([12, 17, 18, 19]);
5397
+ _c = __values(Object.keys(inputParameters)), _d = _c.next();
5398
+ _g.label = 13;
5384
5399
  case 13:
5385
- if (!!_e.done) return [3 /*break*/, 16];
5386
- parameterName = _e.value;
5400
+ if (!!_d.done) return [3 /*break*/, 16];
5401
+ parameterName = _d.value;
5387
5402
  return [5 /*yield**/, _loop_1(parameterName)];
5388
5403
  case 14:
5389
- state_1 = _h.sent();
5404
+ state_1 = _g.sent();
5390
5405
  if (typeof state_1 === "object")
5391
5406
  return [2 /*return*/, state_1.value];
5392
- _h.label = 15;
5407
+ _g.label = 15;
5393
5408
  case 15:
5394
- _e = _d.next();
5409
+ _d = _c.next();
5395
5410
  return [3 /*break*/, 13];
5396
5411
  case 16: return [3 /*break*/, 19];
5397
5412
  case 17:
5398
- e_2_1 = _h.sent();
5413
+ e_2_1 = _g.sent();
5399
5414
  e_2 = { error: e_2_1 };
5400
5415
  return [3 /*break*/, 19];
5401
5416
  case 18:
5402
5417
  try {
5403
- if (_e && !_e.done && (_g = _d.return)) _g.call(_d);
5418
+ if (_d && !_d.done && (_f = _c.return)) _f.call(_c);
5404
5419
  }
5405
5420
  finally { if (e_2) throw e_2.error; }
5406
5421
  return [7 /*endfinally*/];
5407
5422
  case 19:
5408
5423
  parametersToPass = inputParameters;
5409
- _h.label = 20;
5424
+ _g.label = 20;
5410
5425
  case 20:
5411
- _h.trys.push([20, 25, , 28]);
5426
+ _g.trys.push([20, 25, , 28]);
5412
5427
  resovedParameterNames_1 = preparedPipeline.parameters
5413
5428
  .filter(function (_a) {
5414
5429
  var isInput = _a.isInput;
@@ -5423,8 +5438,8 @@
5423
5438
  loopLimit = LOOP_LIMIT;
5424
5439
  _loop_2 = function () {
5425
5440
  var currentTask, work_1;
5426
- return __generator(this, function (_k) {
5427
- switch (_k.label) {
5441
+ return __generator(this, function (_j) {
5442
+ switch (_j.label) {
5428
5443
  case 0:
5429
5444
  if (loopLimit-- < 0) {
5430
5445
  // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
@@ -5460,7 +5475,7 @@
5460
5475
  if (!!currentTask) return [3 /*break*/, 3];
5461
5476
  /* [๐Ÿคนโ€โ™‚๏ธ] */ return [4 /*yield*/, Promise.race(resolving_1)];
5462
5477
  case 2:
5463
- /* [๐Ÿคนโ€โ™‚๏ธ] */ _k.sent();
5478
+ /* [๐Ÿคนโ€โ™‚๏ธ] */ _j.sent();
5464
5479
  return [3 /*break*/, 4];
5465
5480
  case 3:
5466
5481
  unresovedTasks_1 = unresovedTasks_1.filter(function (task) { return task !== currentTask; });
@@ -5485,24 +5500,24 @@
5485
5500
  // <- Note: Errors are catched here [3]
5486
5501
  // TODO: BUT if in multiple tasks are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here
5487
5502
  resolving_1.push(work_1);
5488
- _k.label = 4;
5503
+ _j.label = 4;
5489
5504
  case 4: return [2 /*return*/];
5490
5505
  }
5491
5506
  });
5492
5507
  };
5493
- _h.label = 21;
5508
+ _g.label = 21;
5494
5509
  case 21:
5495
5510
  if (!(unresovedTasks_1.length > 0)) return [3 /*break*/, 23];
5496
5511
  return [5 /*yield**/, _loop_2()];
5497
5512
  case 22:
5498
- _h.sent();
5513
+ _g.sent();
5499
5514
  return [3 /*break*/, 21];
5500
5515
  case 23: return [4 /*yield*/, Promise.all(resolving_1)];
5501
5516
  case 24:
5502
- _h.sent();
5517
+ _g.sent();
5503
5518
  return [3 /*break*/, 28];
5504
5519
  case 25:
5505
- error_1 = _h.sent();
5520
+ error_1 = _g.sent();
5506
5521
  if (!(error_1 instanceof Error)) {
5507
5522
  throw error_1;
5508
5523
  }
@@ -5522,8 +5537,8 @@
5522
5537
  return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
5523
5538
  case 26:
5524
5539
  // Note: Wait a short time to prevent race conditions
5525
- _h.sent();
5526
- _h.label = 27;
5540
+ _g.sent();
5541
+ _g.label = 27;
5527
5542
  case 27: return [2 /*return*/, $asDeeplyFrozenSerializableJson('Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult', {
5528
5543
  isSuccessful: false,
5529
5544
  errors: __spreadArray([error_1], __read(errors), false).map(serializeError),
@@ -5550,8 +5565,8 @@
5550
5565
  return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
5551
5566
  case 29:
5552
5567
  // Note: Wait a short time to prevent race conditions
5553
- _h.sent();
5554
- _h.label = 30;
5568
+ _g.sent();
5569
+ _g.label = 30;
5555
5570
  case 30: return [2 /*return*/, $asDeeplyFrozenSerializableJson('Successful PipelineExecutorResult', {
5556
5571
  isSuccessful: true,
5557
5572
  errors: errors.map(serializeError),
@@ -5578,7 +5593,7 @@
5578
5593
  */
5579
5594
  function createPipelineExecutor(options) {
5580
5595
  var _this = this;
5581
- var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
5596
+ var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.cacheDirname, cacheDirname = _f === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _f, _g = options.intermediateFilesStrategy, intermediateFilesStrategy = _g === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _g, _h = options.isAutoInstalled, isAutoInstalled = _h === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _h, _j = options.rootDirname, rootDirname = _j === void 0 ? null : _j;
5582
5597
  validatePipeline(pipeline);
5583
5598
  var pipelineIdentification = (function () {
5584
5599
  // Note: This is a ๐Ÿ˜ implementation of [๐Ÿšž]
@@ -5618,6 +5633,9 @@
5618
5633
  isVerbose: isVerbose,
5619
5634
  isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
5620
5635
  rootDirname: rootDirname,
5636
+ cacheDirname: cacheDirname,
5637
+ intermediateFilesStrategy: intermediateFilesStrategy,
5638
+ isAutoInstalled: isAutoInstalled,
5621
5639
  })];
5622
5640
  });
5623
5641
  }); };