@promptbook/core 0.75.3 → 0.75.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +115 -66
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +2 -0
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +2 -1
- package/package.json +1 -1
- package/umd/index.umd.js +123 -74
- package/umd/index.umd.js.map +1 -1
|
@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
|
|
|
24
24
|
import { DEFAULT_REMOTE_URL_PATH } from '../config';
|
|
25
25
|
import { DEFAULT_CSV_SETTINGS } from '../config';
|
|
26
26
|
import { DEFAULT_IS_VERBOSE } from '../config';
|
|
27
|
+
import { SET_IS_VERBOSE } from '../config';
|
|
27
28
|
import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
|
|
28
29
|
import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
|
|
29
30
|
import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
|
|
@@ -136,6 +137,7 @@ export { DEFAULT_REMOTE_URL };
|
|
|
136
137
|
export { DEFAULT_REMOTE_URL_PATH };
|
|
137
138
|
export { DEFAULT_CSV_SETTINGS };
|
|
138
139
|
export { DEFAULT_IS_VERBOSE };
|
|
140
|
+
export { SET_IS_VERBOSE };
|
|
139
141
|
export { DEFAULT_IS_AUTO_INSTALLED };
|
|
140
142
|
export { pipelineJsonToString };
|
|
141
143
|
export { pipelineStringToJson };
|
|
@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
|
|
|
204
204
|
*
|
|
205
205
|
* @public exported from `@promptbook/core`
|
|
206
206
|
*/
|
|
207
|
-
export declare
|
|
207
|
+
export declare let DEFAULT_IS_VERBOSE: boolean;
|
|
208
|
+
/**
|
|
209
|
+
* @@@
|
|
210
|
+
*
|
|
211
|
+
* Note: This is experimental feature
|
|
212
|
+
*
|
|
213
|
+
* @public exported from `@promptbook/core`
|
|
214
|
+
*/
|
|
215
|
+
export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
|
|
208
216
|
/**
|
|
209
217
|
* @@@
|
|
210
218
|
*
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { string_markdown } from '../../../types/typeAliases';
|
|
2
|
+
import type { Scraper } from '../Scraper';
|
|
2
3
|
/**
|
|
3
4
|
* Creates a message with all registered scrapers
|
|
4
5
|
*
|
|
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
|
|
|
6
7
|
*
|
|
7
8
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
8
9
|
*/
|
|
9
|
-
export declare function $registeredScrapersMessage(): string_markdown;
|
|
10
|
+
export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
|
|
10
11
|
/**
|
|
11
12
|
* TODO: [®] DRY Register logic
|
|
12
13
|
*/
|
package/package.json
CHANGED
package/umd/index.umd.js
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
*
|
|
25
25
|
* @see https://github.com/webgptorg/promptbook
|
|
26
26
|
*/
|
|
27
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
27
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.3';
|
|
28
28
|
/**
|
|
29
29
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
30
30
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -836,7 +836,17 @@
|
|
|
836
836
|
*
|
|
837
837
|
* @public exported from `@promptbook/core`
|
|
838
838
|
*/
|
|
839
|
-
|
|
839
|
+
exports.DEFAULT_IS_VERBOSE = false;
|
|
840
|
+
/**
|
|
841
|
+
* @@@
|
|
842
|
+
*
|
|
843
|
+
* Note: This is experimental feature
|
|
844
|
+
*
|
|
845
|
+
* @public exported from `@promptbook/core`
|
|
846
|
+
*/
|
|
847
|
+
function SET_IS_VERBOSE(isVerbose) {
|
|
848
|
+
exports.DEFAULT_IS_VERBOSE = isVerbose;
|
|
849
|
+
}
|
|
840
850
|
/**
|
|
841
851
|
* @@@
|
|
842
852
|
*
|
|
@@ -1602,7 +1612,7 @@
|
|
|
1602
1612
|
return __generator(this, function (_d) {
|
|
1603
1613
|
switch (_d.label) {
|
|
1604
1614
|
case 0:
|
|
1605
|
-
_a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
|
|
1615
|
+
_a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? exports.DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
|
|
1606
1616
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
1607
1617
|
return __generator(this, function (_a) {
|
|
1608
1618
|
if (isVerbose) {
|
|
@@ -4498,7 +4508,7 @@
|
|
|
4498
4508
|
return __generator(this, function (_h) {
|
|
4499
4509
|
switch (_h.label) {
|
|
4500
4510
|
case 0:
|
|
4501
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4511
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? exports.DEFAULT_IS_VERBOSE : _a;
|
|
4502
4512
|
preparedPipeline = options.preparedPipeline;
|
|
4503
4513
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4504
4514
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -4807,7 +4817,7 @@
|
|
|
4807
4817
|
*/
|
|
4808
4818
|
function createPipelineExecutor(options) {
|
|
4809
4819
|
var _this = this;
|
|
4810
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4820
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? exports.DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4811
4821
|
validatePipeline(pipeline);
|
|
4812
4822
|
var pipelineIdentification = (function () {
|
|
4813
4823
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -4869,7 +4879,7 @@
|
|
|
4869
4879
|
return __generator(this, function (_d) {
|
|
4870
4880
|
switch (_d.label) {
|
|
4871
4881
|
case 0:
|
|
4872
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4882
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? exports.DEFAULT_IS_VERBOSE : _a;
|
|
4873
4883
|
if (tools === undefined || tools.llm === undefined) {
|
|
4874
4884
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4875
4885
|
}
|
|
@@ -5059,9 +5069,15 @@
|
|
|
5059
5069
|
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
5060
5070
|
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
5061
5071
|
if (!existingRegistration) {
|
|
5072
|
+
if (exports.DEFAULT_IS_VERBOSE) {
|
|
5073
|
+
console.warn("[\uD83D\uDCE6] Registering `".concat(packageName, ".").concat(className, "` to `").concat(this.registerName, "`"));
|
|
5074
|
+
}
|
|
5062
5075
|
this.storage.push(registered);
|
|
5063
5076
|
}
|
|
5064
5077
|
else {
|
|
5078
|
+
if (exports.DEFAULT_IS_VERBOSE) {
|
|
5079
|
+
console.warn("[\uD83D\uDCE6] Re-registering `".concat(packageName, ".").concat(className, "` to `").concat(this.registerName, "`"));
|
|
5080
|
+
}
|
|
5065
5081
|
this.storage[existingRegistrationIndex] = registered;
|
|
5066
5082
|
}
|
|
5067
5083
|
return {
|
|
@@ -5110,8 +5126,8 @@
|
|
|
5110
5126
|
*
|
|
5111
5127
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
5112
5128
|
*/
|
|
5113
|
-
function $registeredScrapersMessage() {
|
|
5114
|
-
var e_1, _a, e_2, _b;
|
|
5129
|
+
function $registeredScrapersMessage(availableScrapers) {
|
|
5130
|
+
var e_1, _a, e_2, _b, e_3, _c;
|
|
5115
5131
|
/**
|
|
5116
5132
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
5117
5133
|
*/
|
|
@@ -5123,15 +5139,15 @@
|
|
|
5123
5139
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
5124
5140
|
};
|
|
5125
5141
|
try {
|
|
5126
|
-
for (var
|
|
5127
|
-
var
|
|
5142
|
+
for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
5143
|
+
var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
|
|
5128
5144
|
_loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
5129
5145
|
}
|
|
5130
5146
|
}
|
|
5131
5147
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5132
5148
|
finally {
|
|
5133
5149
|
try {
|
|
5134
|
-
if (
|
|
5150
|
+
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
5135
5151
|
}
|
|
5136
5152
|
finally { if (e_1) throw e_1.error; }
|
|
5137
5153
|
}
|
|
@@ -5142,18 +5158,31 @@
|
|
|
5142
5158
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
5143
5159
|
};
|
|
5144
5160
|
try {
|
|
5145
|
-
for (var
|
|
5146
|
-
var
|
|
5161
|
+
for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
|
|
5162
|
+
var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
|
|
5147
5163
|
_loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
5148
5164
|
}
|
|
5149
5165
|
}
|
|
5150
5166
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
5151
5167
|
finally {
|
|
5152
5168
|
try {
|
|
5153
|
-
if (
|
|
5169
|
+
if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
|
|
5154
5170
|
}
|
|
5155
5171
|
finally { if (e_2) throw e_2.error; }
|
|
5156
5172
|
}
|
|
5173
|
+
try {
|
|
5174
|
+
for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
|
|
5175
|
+
var metadata_1 = availableScrapers_1_1.value.metadata;
|
|
5176
|
+
all.push(metadata_1);
|
|
5177
|
+
}
|
|
5178
|
+
}
|
|
5179
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
5180
|
+
finally {
|
|
5181
|
+
try {
|
|
5182
|
+
if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
|
|
5183
|
+
}
|
|
5184
|
+
finally { if (e_3) throw e_3.error; }
|
|
5185
|
+
}
|
|
5157
5186
|
var metadata = all.map(function (metadata) {
|
|
5158
5187
|
var isMetadataAviailable = $scrapersMetadataRegister
|
|
5159
5188
|
.list()
|
|
@@ -5167,42 +5196,44 @@
|
|
|
5167
5196
|
var packageName = _a.packageName, className = _a.className;
|
|
5168
5197
|
return metadata.packageName === packageName && metadata.className === className;
|
|
5169
5198
|
});
|
|
5170
|
-
|
|
5199
|
+
var isAvilableInTools = availableScrapers.some(function (_a) {
|
|
5200
|
+
var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
|
|
5201
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
5202
|
+
});
|
|
5203
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
|
|
5171
5204
|
});
|
|
5172
5205
|
if (metadata.length === 0) {
|
|
5173
|
-
return "No scrapers are available";
|
|
5206
|
+
return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
|
|
5174
5207
|
}
|
|
5175
5208
|
return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
5176
5209
|
.map(function (_a, i) {
|
|
5177
|
-
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
|
|
5178
|
-
var more;
|
|
5179
|
-
// TODO:
|
|
5180
|
-
if (
|
|
5181
|
-
more
|
|
5182
|
-
}
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
more = "(installed)";
|
|
5196
|
-
}
|
|
5197
|
-
else {
|
|
5198
|
-
more = "*(unknown state, looks like a unexpected behavior)*";
|
|
5199
|
-
}
|
|
5210
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
|
|
5211
|
+
var more = [];
|
|
5212
|
+
// TODO: [🧠] Maybe use `documentationUrl`
|
|
5213
|
+
if (isMetadataAviailable) {
|
|
5214
|
+
more.push("\u2B1C Metadata registered");
|
|
5215
|
+
} // not else
|
|
5216
|
+
if (isInstalled) {
|
|
5217
|
+
more.push("\uD83D\uDFE9 Installed");
|
|
5218
|
+
} // not else
|
|
5219
|
+
if (isAvilableInTools) {
|
|
5220
|
+
more.push("\uD83D\uDFE6 Available in tools");
|
|
5221
|
+
} // not else
|
|
5222
|
+
if (!isMetadataAviailable && isInstalled) {
|
|
5223
|
+
more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
|
|
5224
|
+
} // not else
|
|
5225
|
+
if (!isInstalled && isAvilableInTools) {
|
|
5226
|
+
more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
|
|
5227
|
+
} // not else
|
|
5200
5228
|
if (!isAvilableInBrowser) {
|
|
5201
|
-
more
|
|
5229
|
+
more.push("Not usable in browser");
|
|
5202
5230
|
}
|
|
5203
|
-
|
|
5231
|
+
var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
|
|
5232
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
|
|
5233
|
+
.map(function (mimeType) { return "\"".concat(mimeType, "\""); })
|
|
5234
|
+
.join(', ')).concat(moreText);
|
|
5204
5235
|
})
|
|
5205
|
-
.join('\n')), "\n "); });
|
|
5236
|
+
.join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
|
|
5206
5237
|
}
|
|
5207
5238
|
/**
|
|
5208
5239
|
* TODO: [®] DRY Register logic
|
|
@@ -5535,60 +5566,78 @@
|
|
|
5535
5566
|
return __generator(this, function (_c) {
|
|
5536
5567
|
switch (_c.label) {
|
|
5537
5568
|
case 0:
|
|
5538
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5569
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? exports.DEFAULT_IS_VERBOSE : _b;
|
|
5539
5570
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5540
5571
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5541
|
-
var partialPieces, sourceHandler,
|
|
5542
|
-
var e_1,
|
|
5543
|
-
return __generator(this, function (
|
|
5544
|
-
switch (
|
|
5572
|
+
var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
|
|
5573
|
+
var e_1, _a;
|
|
5574
|
+
return __generator(this, function (_b) {
|
|
5575
|
+
switch (_b.label) {
|
|
5545
5576
|
case 0:
|
|
5546
5577
|
partialPieces = null;
|
|
5547
5578
|
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
5548
5579
|
case 1:
|
|
5549
|
-
sourceHandler =
|
|
5550
|
-
|
|
5580
|
+
sourceHandler = _b.sent();
|
|
5581
|
+
scrapers = arrayableToArray(tools.scrapers);
|
|
5582
|
+
_loop_1 = function (scraper) {
|
|
5583
|
+
var partialPiecesUnchecked;
|
|
5584
|
+
return __generator(this, function (_c) {
|
|
5585
|
+
switch (_c.label) {
|
|
5586
|
+
case 0:
|
|
5587
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
5588
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5589
|
+
) {
|
|
5590
|
+
return [2 /*return*/, "continue"];
|
|
5591
|
+
}
|
|
5592
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
5593
|
+
case 1:
|
|
5594
|
+
partialPiecesUnchecked = _c.sent();
|
|
5595
|
+
if (partialPiecesUnchecked !== null) {
|
|
5596
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5597
|
+
return [2 /*return*/, "break"];
|
|
5598
|
+
}
|
|
5599
|
+
console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
|
|
5600
|
+
.split('\n')
|
|
5601
|
+
.map(function (line) { return "> ".concat(line); })
|
|
5602
|
+
.join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
5603
|
+
return [2 /*return*/];
|
|
5604
|
+
}
|
|
5605
|
+
});
|
|
5606
|
+
};
|
|
5607
|
+
_b.label = 2;
|
|
5551
5608
|
case 2:
|
|
5552
|
-
|
|
5553
|
-
|
|
5554
|
-
|
|
5609
|
+
_b.trys.push([2, 7, 8, 9]);
|
|
5610
|
+
scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
|
|
5611
|
+
_b.label = 3;
|
|
5555
5612
|
case 3:
|
|
5556
|
-
if (!!
|
|
5557
|
-
scraper =
|
|
5558
|
-
|
|
5559
|
-
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5560
|
-
) {
|
|
5561
|
-
return [3 /*break*/, 5];
|
|
5562
|
-
}
|
|
5563
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
5613
|
+
if (!!scrapers_1_1.done) return [3 /*break*/, 6];
|
|
5614
|
+
scraper = scrapers_1_1.value;
|
|
5615
|
+
return [5 /*yield**/, _loop_1(scraper)];
|
|
5564
5616
|
case 4:
|
|
5565
|
-
|
|
5566
|
-
if (
|
|
5567
|
-
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5568
|
-
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5617
|
+
state_1 = _b.sent();
|
|
5618
|
+
if (state_1 === "break")
|
|
5569
5619
|
return [3 /*break*/, 6];
|
|
5570
|
-
|
|
5571
|
-
_d.label = 5;
|
|
5620
|
+
_b.label = 5;
|
|
5572
5621
|
case 5:
|
|
5573
|
-
|
|
5622
|
+
scrapers_1_1 = scrapers_1.next();
|
|
5574
5623
|
return [3 /*break*/, 3];
|
|
5575
5624
|
case 6: return [3 /*break*/, 9];
|
|
5576
5625
|
case 7:
|
|
5577
|
-
e_1_1 =
|
|
5626
|
+
e_1_1 = _b.sent();
|
|
5578
5627
|
e_1 = { error: e_1_1 };
|
|
5579
5628
|
return [3 /*break*/, 9];
|
|
5580
5629
|
case 8:
|
|
5581
5630
|
try {
|
|
5582
|
-
if (
|
|
5631
|
+
if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
|
|
5583
5632
|
}
|
|
5584
5633
|
finally { if (e_1) throw e_1.error; }
|
|
5585
5634
|
return [7 /*endfinally*/];
|
|
5586
5635
|
case 9:
|
|
5587
5636
|
if (partialPieces === null) {
|
|
5588
|
-
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge
|
|
5637
|
+
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
|
|
5589
5638
|
.split('\n')
|
|
5590
5639
|
.map(function (line) { return "> ".concat(line); })
|
|
5591
|
-
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
|
|
5640
|
+
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
5592
5641
|
}
|
|
5593
5642
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
5594
5643
|
{
|
|
@@ -5735,7 +5784,7 @@
|
|
|
5735
5784
|
if (isPipelinePrepared(pipeline)) {
|
|
5736
5785
|
return [2 /*return*/, pipeline];
|
|
5737
5786
|
}
|
|
5738
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5787
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? exports.DEFAULT_IS_VERBOSE : _b;
|
|
5739
5788
|
parameters = pipeline.parameters, tasks = pipeline.tasks, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
5740
5789
|
if (tools === undefined || tools.llm === undefined) {
|
|
5741
5790
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -9632,7 +9681,7 @@
|
|
|
9632
9681
|
*/
|
|
9633
9682
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
9634
9683
|
if (options === void 0) { options = {}; }
|
|
9635
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
9684
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? exports.DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
9636
9685
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
9637
9686
|
var registeredItem = $llmToolsRegister
|
|
9638
9687
|
.list()
|
|
@@ -10426,7 +10475,6 @@
|
|
|
10426
10475
|
exports.DEFAULT_EXECUTIONS_CACHE_DIRNAME = DEFAULT_EXECUTIONS_CACHE_DIRNAME;
|
|
10427
10476
|
exports.DEFAULT_INTERMEDIATE_FILES_STRATEGY = DEFAULT_INTERMEDIATE_FILES_STRATEGY;
|
|
10428
10477
|
exports.DEFAULT_IS_AUTO_INSTALLED = DEFAULT_IS_AUTO_INSTALLED;
|
|
10429
|
-
exports.DEFAULT_IS_VERBOSE = DEFAULT_IS_VERBOSE;
|
|
10430
10478
|
exports.DEFAULT_MAX_EXECUTION_ATTEMPTS = DEFAULT_MAX_EXECUTION_ATTEMPTS;
|
|
10431
10479
|
exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
|
|
10432
10480
|
exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
|
|
@@ -10467,6 +10515,7 @@
|
|
|
10467
10515
|
exports.PipelineUrlError = PipelineUrlError;
|
|
10468
10516
|
exports.PrefixStorage = PrefixStorage;
|
|
10469
10517
|
exports.RESERVED_PARAMETER_NAMES = RESERVED_PARAMETER_NAMES;
|
|
10518
|
+
exports.SET_IS_VERBOSE = SET_IS_VERBOSE;
|
|
10470
10519
|
exports.SectionTypes = SectionTypes;
|
|
10471
10520
|
exports.SheetsFormfactorDefinition = SheetsFormfactorDefinition;
|
|
10472
10521
|
exports.TaskTypes = TaskTypes;
|