@promptbook/core 0.75.2 → 0.75.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +175 -70
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +2 -0
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +2 -1
- package/package.json +1 -1
- package/umd/index.umd.js +183 -78
- package/umd/index.umd.js.map +1 -1
|
@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
|
|
|
24
24
|
import { DEFAULT_REMOTE_URL_PATH } from '../config';
|
|
25
25
|
import { DEFAULT_CSV_SETTINGS } from '../config';
|
|
26
26
|
import { DEFAULT_IS_VERBOSE } from '../config';
|
|
27
|
+
import { SET_IS_VERBOSE } from '../config';
|
|
27
28
|
import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
|
|
28
29
|
import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
|
|
29
30
|
import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
|
|
@@ -136,6 +137,7 @@ export { DEFAULT_REMOTE_URL };
|
|
|
136
137
|
export { DEFAULT_REMOTE_URL_PATH };
|
|
137
138
|
export { DEFAULT_CSV_SETTINGS };
|
|
138
139
|
export { DEFAULT_IS_VERBOSE };
|
|
140
|
+
export { SET_IS_VERBOSE };
|
|
139
141
|
export { DEFAULT_IS_AUTO_INSTALLED };
|
|
140
142
|
export { pipelineJsonToString };
|
|
141
143
|
export { pipelineStringToJson };
|
|
@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
|
|
|
204
204
|
*
|
|
205
205
|
* @public exported from `@promptbook/core`
|
|
206
206
|
*/
|
|
207
|
-
export declare
|
|
207
|
+
export declare let DEFAULT_IS_VERBOSE: boolean;
|
|
208
|
+
/**
|
|
209
|
+
* @@@
|
|
210
|
+
*
|
|
211
|
+
* Note: This is experimental feature
|
|
212
|
+
*
|
|
213
|
+
* @public exported from `@promptbook/core`
|
|
214
|
+
*/
|
|
215
|
+
export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
|
|
208
216
|
/**
|
|
209
217
|
* @@@
|
|
210
218
|
*
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { string_markdown } from '../../../types/typeAliases';
|
|
2
|
+
import type { Scraper } from '../Scraper';
|
|
2
3
|
/**
|
|
3
4
|
* Creates a message with all registered scrapers
|
|
4
5
|
*
|
|
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
|
|
|
6
7
|
*
|
|
7
8
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
8
9
|
*/
|
|
9
|
-
export declare function $registeredScrapersMessage(): string_markdown;
|
|
10
|
+
export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
|
|
10
11
|
/**
|
|
11
12
|
* TODO: [®] DRY Register logic
|
|
12
13
|
*/
|
package/package.json
CHANGED
package/umd/index.umd.js
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
*
|
|
25
25
|
* @see https://github.com/webgptorg/promptbook
|
|
26
26
|
*/
|
|
27
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
27
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.3';
|
|
28
28
|
/**
|
|
29
29
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
30
30
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -836,7 +836,17 @@
|
|
|
836
836
|
*
|
|
837
837
|
* @public exported from `@promptbook/core`
|
|
838
838
|
*/
|
|
839
|
-
|
|
839
|
+
exports.DEFAULT_IS_VERBOSE = false;
|
|
840
|
+
/**
|
|
841
|
+
* @@@
|
|
842
|
+
*
|
|
843
|
+
* Note: This is experimental feature
|
|
844
|
+
*
|
|
845
|
+
* @public exported from `@promptbook/core`
|
|
846
|
+
*/
|
|
847
|
+
function SET_IS_VERBOSE(isVerbose) {
|
|
848
|
+
exports.DEFAULT_IS_VERBOSE = isVerbose;
|
|
849
|
+
}
|
|
840
850
|
/**
|
|
841
851
|
* @@@
|
|
842
852
|
*
|
|
@@ -1602,7 +1612,7 @@
|
|
|
1602
1612
|
return __generator(this, function (_d) {
|
|
1603
1613
|
switch (_d.label) {
|
|
1604
1614
|
case 0:
|
|
1605
|
-
_a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
|
|
1615
|
+
_a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? exports.DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
|
|
1606
1616
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
1607
1617
|
return __generator(this, function (_a) {
|
|
1608
1618
|
if (isVerbose) {
|
|
@@ -4498,7 +4508,7 @@
|
|
|
4498
4508
|
return __generator(this, function (_h) {
|
|
4499
4509
|
switch (_h.label) {
|
|
4500
4510
|
case 0:
|
|
4501
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4511
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? exports.DEFAULT_IS_VERBOSE : _a;
|
|
4502
4512
|
preparedPipeline = options.preparedPipeline;
|
|
4503
4513
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4504
4514
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -4807,7 +4817,7 @@
|
|
|
4807
4817
|
*/
|
|
4808
4818
|
function createPipelineExecutor(options) {
|
|
4809
4819
|
var _this = this;
|
|
4810
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4820
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? exports.DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4811
4821
|
validatePipeline(pipeline);
|
|
4812
4822
|
var pipelineIdentification = (function () {
|
|
4813
4823
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -4869,7 +4879,7 @@
|
|
|
4869
4879
|
return __generator(this, function (_d) {
|
|
4870
4880
|
switch (_d.label) {
|
|
4871
4881
|
case 0:
|
|
4872
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4882
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? exports.DEFAULT_IS_VERBOSE : _a;
|
|
4873
4883
|
if (tools === undefined || tools.llm === undefined) {
|
|
4874
4884
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4875
4885
|
}
|
|
@@ -5059,9 +5069,15 @@
|
|
|
5059
5069
|
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
5060
5070
|
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
5061
5071
|
if (!existingRegistration) {
|
|
5072
|
+
if (exports.DEFAULT_IS_VERBOSE) {
|
|
5073
|
+
console.warn("[\uD83D\uDCE6] Registering `".concat(packageName, ".").concat(className, "` to `").concat(this.registerName, "`"));
|
|
5074
|
+
}
|
|
5062
5075
|
this.storage.push(registered);
|
|
5063
5076
|
}
|
|
5064
5077
|
else {
|
|
5078
|
+
if (exports.DEFAULT_IS_VERBOSE) {
|
|
5079
|
+
console.warn("[\uD83D\uDCE6] Re-registering `".concat(packageName, ".").concat(className, "` to `").concat(this.registerName, "`"));
|
|
5080
|
+
}
|
|
5065
5081
|
this.storage[existingRegistrationIndex] = registered;
|
|
5066
5082
|
}
|
|
5067
5083
|
return {
|
|
@@ -5110,50 +5126,63 @@
|
|
|
5110
5126
|
*
|
|
5111
5127
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
5112
5128
|
*/
|
|
5113
|
-
function $registeredScrapersMessage() {
|
|
5114
|
-
var e_1, _a, e_2, _b;
|
|
5129
|
+
function $registeredScrapersMessage(availableScrapers) {
|
|
5130
|
+
var e_1, _a, e_2, _b, e_3, _c;
|
|
5115
5131
|
/**
|
|
5116
5132
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
5117
5133
|
*/
|
|
5118
5134
|
var all = [];
|
|
5119
|
-
var _loop_1 = function (packageName, className) {
|
|
5135
|
+
var _loop_1 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
|
|
5120
5136
|
if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
|
|
5121
5137
|
return "continue";
|
|
5122
5138
|
}
|
|
5123
|
-
all.push({ packageName: packageName, className: className });
|
|
5139
|
+
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
5124
5140
|
};
|
|
5125
5141
|
try {
|
|
5126
|
-
for (var
|
|
5127
|
-
var
|
|
5128
|
-
_loop_1(packageName, className);
|
|
5142
|
+
for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
5143
|
+
var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
|
|
5144
|
+
_loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
5129
5145
|
}
|
|
5130
5146
|
}
|
|
5131
5147
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5132
5148
|
finally {
|
|
5133
5149
|
try {
|
|
5134
|
-
if (
|
|
5150
|
+
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
5135
5151
|
}
|
|
5136
5152
|
finally { if (e_1) throw e_1.error; }
|
|
5137
5153
|
}
|
|
5138
|
-
var _loop_2 = function (packageName, className) {
|
|
5154
|
+
var _loop_2 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
|
|
5139
5155
|
if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
|
|
5140
5156
|
return "continue";
|
|
5141
5157
|
}
|
|
5142
|
-
all.push({ packageName: packageName, className: className });
|
|
5158
|
+
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
5143
5159
|
};
|
|
5144
5160
|
try {
|
|
5145
|
-
for (var
|
|
5146
|
-
var
|
|
5147
|
-
_loop_2(packageName, className);
|
|
5161
|
+
for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
|
|
5162
|
+
var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
|
|
5163
|
+
_loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
5148
5164
|
}
|
|
5149
5165
|
}
|
|
5150
5166
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
5151
5167
|
finally {
|
|
5152
5168
|
try {
|
|
5153
|
-
if (
|
|
5169
|
+
if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
|
|
5154
5170
|
}
|
|
5155
5171
|
finally { if (e_2) throw e_2.error; }
|
|
5156
5172
|
}
|
|
5173
|
+
try {
|
|
5174
|
+
for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
|
|
5175
|
+
var metadata_1 = availableScrapers_1_1.value.metadata;
|
|
5176
|
+
all.push(metadata_1);
|
|
5177
|
+
}
|
|
5178
|
+
}
|
|
5179
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
5180
|
+
finally {
|
|
5181
|
+
try {
|
|
5182
|
+
if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
|
|
5183
|
+
}
|
|
5184
|
+
finally { if (e_3) throw e_3.error; }
|
|
5185
|
+
}
|
|
5157
5186
|
var metadata = all.map(function (metadata) {
|
|
5158
5187
|
var isMetadataAviailable = $scrapersMetadataRegister
|
|
5159
5188
|
.list()
|
|
@@ -5167,38 +5196,44 @@
|
|
|
5167
5196
|
var packageName = _a.packageName, className = _a.className;
|
|
5168
5197
|
return metadata.packageName === packageName && metadata.className === className;
|
|
5169
5198
|
});
|
|
5170
|
-
|
|
5199
|
+
var isAvilableInTools = availableScrapers.some(function (_a) {
|
|
5200
|
+
var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
|
|
5201
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
5202
|
+
});
|
|
5203
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
|
|
5171
5204
|
});
|
|
5172
5205
|
if (metadata.length === 0) {
|
|
5173
|
-
return "No scrapers are available";
|
|
5206
|
+
return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
|
|
5174
5207
|
}
|
|
5175
5208
|
return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
5176
5209
|
.map(function (_a, i) {
|
|
5177
|
-
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
|
|
5178
|
-
var more;
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
more
|
|
5185
|
-
}
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(
|
|
5210
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
|
|
5211
|
+
var more = [];
|
|
5212
|
+
// TODO: [🧠] Maybe use `documentationUrl`
|
|
5213
|
+
if (isMetadataAviailable) {
|
|
5214
|
+
more.push("\u2B1C Metadata registered");
|
|
5215
|
+
} // not else
|
|
5216
|
+
if (isInstalled) {
|
|
5217
|
+
more.push("\uD83D\uDFE9 Installed");
|
|
5218
|
+
} // not else
|
|
5219
|
+
if (isAvilableInTools) {
|
|
5220
|
+
more.push("\uD83D\uDFE6 Available in tools");
|
|
5221
|
+
} // not else
|
|
5222
|
+
if (!isMetadataAviailable && isInstalled) {
|
|
5223
|
+
more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
|
|
5224
|
+
} // not else
|
|
5225
|
+
if (!isInstalled && isAvilableInTools) {
|
|
5226
|
+
more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
|
|
5227
|
+
} // not else
|
|
5228
|
+
if (!isAvilableInBrowser) {
|
|
5229
|
+
more.push("Not usable in browser");
|
|
5230
|
+
}
|
|
5231
|
+
var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
|
|
5232
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
|
|
5233
|
+
.map(function (mimeType) { return "\"".concat(mimeType, "\""); })
|
|
5234
|
+
.join(', ')).concat(moreText);
|
|
5200
5235
|
})
|
|
5201
|
-
.join('\n')), "\n "); });
|
|
5236
|
+
.join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
|
|
5202
5237
|
}
|
|
5203
5238
|
/**
|
|
5204
5239
|
* TODO: [®] DRY Register logic
|
|
@@ -5531,57 +5566,78 @@
|
|
|
5531
5566
|
return __generator(this, function (_c) {
|
|
5532
5567
|
switch (_c.label) {
|
|
5533
5568
|
case 0:
|
|
5534
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5569
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? exports.DEFAULT_IS_VERBOSE : _b;
|
|
5535
5570
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5536
5571
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5537
|
-
var partialPieces, sourceHandler,
|
|
5538
|
-
var e_1,
|
|
5539
|
-
return __generator(this, function (
|
|
5540
|
-
switch (
|
|
5572
|
+
var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
|
|
5573
|
+
var e_1, _a;
|
|
5574
|
+
return __generator(this, function (_b) {
|
|
5575
|
+
switch (_b.label) {
|
|
5541
5576
|
case 0:
|
|
5542
5577
|
partialPieces = null;
|
|
5543
5578
|
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
5544
5579
|
case 1:
|
|
5545
|
-
sourceHandler =
|
|
5546
|
-
|
|
5580
|
+
sourceHandler = _b.sent();
|
|
5581
|
+
scrapers = arrayableToArray(tools.scrapers);
|
|
5582
|
+
_loop_1 = function (scraper) {
|
|
5583
|
+
var partialPiecesUnchecked;
|
|
5584
|
+
return __generator(this, function (_c) {
|
|
5585
|
+
switch (_c.label) {
|
|
5586
|
+
case 0:
|
|
5587
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
5588
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5589
|
+
) {
|
|
5590
|
+
return [2 /*return*/, "continue"];
|
|
5591
|
+
}
|
|
5592
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
5593
|
+
case 1:
|
|
5594
|
+
partialPiecesUnchecked = _c.sent();
|
|
5595
|
+
if (partialPiecesUnchecked !== null) {
|
|
5596
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5597
|
+
return [2 /*return*/, "break"];
|
|
5598
|
+
}
|
|
5599
|
+
console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
|
|
5600
|
+
.split('\n')
|
|
5601
|
+
.map(function (line) { return "> ".concat(line); })
|
|
5602
|
+
.join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
5603
|
+
return [2 /*return*/];
|
|
5604
|
+
}
|
|
5605
|
+
});
|
|
5606
|
+
};
|
|
5607
|
+
_b.label = 2;
|
|
5547
5608
|
case 2:
|
|
5548
|
-
|
|
5549
|
-
|
|
5550
|
-
|
|
5609
|
+
_b.trys.push([2, 7, 8, 9]);
|
|
5610
|
+
scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
|
|
5611
|
+
_b.label = 3;
|
|
5551
5612
|
case 3:
|
|
5552
|
-
if (!!
|
|
5553
|
-
scraper =
|
|
5554
|
-
|
|
5555
|
-
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5556
|
-
) {
|
|
5557
|
-
return [3 /*break*/, 5];
|
|
5558
|
-
}
|
|
5559
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
5613
|
+
if (!!scrapers_1_1.done) return [3 /*break*/, 6];
|
|
5614
|
+
scraper = scrapers_1_1.value;
|
|
5615
|
+
return [5 /*yield**/, _loop_1(scraper)];
|
|
5560
5616
|
case 4:
|
|
5561
|
-
|
|
5562
|
-
if (
|
|
5563
|
-
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5564
|
-
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5617
|
+
state_1 = _b.sent();
|
|
5618
|
+
if (state_1 === "break")
|
|
5565
5619
|
return [3 /*break*/, 6];
|
|
5566
|
-
|
|
5567
|
-
_d.label = 5;
|
|
5620
|
+
_b.label = 5;
|
|
5568
5621
|
case 5:
|
|
5569
|
-
|
|
5622
|
+
scrapers_1_1 = scrapers_1.next();
|
|
5570
5623
|
return [3 /*break*/, 3];
|
|
5571
5624
|
case 6: return [3 /*break*/, 9];
|
|
5572
5625
|
case 7:
|
|
5573
|
-
e_1_1 =
|
|
5626
|
+
e_1_1 = _b.sent();
|
|
5574
5627
|
e_1 = { error: e_1_1 };
|
|
5575
5628
|
return [3 /*break*/, 9];
|
|
5576
5629
|
case 8:
|
|
5577
5630
|
try {
|
|
5578
|
-
if (
|
|
5631
|
+
if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
|
|
5579
5632
|
}
|
|
5580
5633
|
finally { if (e_1) throw e_1.error; }
|
|
5581
5634
|
return [7 /*endfinally*/];
|
|
5582
5635
|
case 9:
|
|
5583
5636
|
if (partialPieces === null) {
|
|
5584
|
-
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge
|
|
5637
|
+
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
|
|
5638
|
+
.split('\n')
|
|
5639
|
+
.map(function (line) { return "> ".concat(line); })
|
|
5640
|
+
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
5585
5641
|
}
|
|
5586
5642
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
5587
5643
|
{
|
|
@@ -5728,7 +5784,7 @@
|
|
|
5728
5784
|
if (isPipelinePrepared(pipeline)) {
|
|
5729
5785
|
return [2 /*return*/, pipeline];
|
|
5730
5786
|
}
|
|
5731
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5787
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? exports.DEFAULT_IS_VERBOSE : _b;
|
|
5732
5788
|
parameters = pipeline.parameters, tasks = pipeline.tasks, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
5733
5789
|
if (tools === undefined || tools.llm === undefined) {
|
|
5734
5790
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -9625,7 +9681,7 @@
|
|
|
9625
9681
|
*/
|
|
9626
9682
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
9627
9683
|
if (options === void 0) { options = {}; }
|
|
9628
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
9684
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? exports.DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
9629
9685
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
9630
9686
|
var registeredItem = $llmToolsRegister
|
|
9631
9687
|
.list()
|
|
@@ -10115,7 +10171,56 @@
|
|
|
10115
10171
|
* @public exported from `@promptbook/core`
|
|
10116
10172
|
*/
|
|
10117
10173
|
function isPipelineInterfacesEqual(pipelineInterface1, pipelineInterface2) {
|
|
10118
|
-
|
|
10174
|
+
var e_1, _a, e_2, _b;
|
|
10175
|
+
try {
|
|
10176
|
+
for (var _c = __values(['inputParameters', 'outputParameters']), _d = _c.next(); !_d.done; _d = _c.next()) {
|
|
10177
|
+
var whichParameters = _d.value;
|
|
10178
|
+
var parameters1 = pipelineInterface1[whichParameters]; // <- Note: `isPipelineInterfacesEqual` is just temporary solution, no need to fix this
|
|
10179
|
+
var parameters2 = pipelineInterface2[whichParameters];
|
|
10180
|
+
if (parameters1.length !== parameters2.length) {
|
|
10181
|
+
return false;
|
|
10182
|
+
}
|
|
10183
|
+
var _loop_1 = function (parameter) {
|
|
10184
|
+
var matchingParameter = parameters2.find(function (_a) {
|
|
10185
|
+
var name = _a.name;
|
|
10186
|
+
return name === parameter.name;
|
|
10187
|
+
});
|
|
10188
|
+
if (!matchingParameter) {
|
|
10189
|
+
return { value: false };
|
|
10190
|
+
}
|
|
10191
|
+
// Note: Do not compare description, it is not relevant for compatibility
|
|
10192
|
+
if (matchingParameter.isInput !== parameter.isInput) {
|
|
10193
|
+
return { value: false };
|
|
10194
|
+
}
|
|
10195
|
+
if (matchingParameter.isOutput !== parameter.isOutput) {
|
|
10196
|
+
return { value: false };
|
|
10197
|
+
}
|
|
10198
|
+
};
|
|
10199
|
+
try {
|
|
10200
|
+
for (var parameters1_1 = (e_2 = void 0, __values(parameters1)), parameters1_1_1 = parameters1_1.next(); !parameters1_1_1.done; parameters1_1_1 = parameters1_1.next()) {
|
|
10201
|
+
var parameter = parameters1_1_1.value;
|
|
10202
|
+
var state_1 = _loop_1(parameter);
|
|
10203
|
+
if (typeof state_1 === "object")
|
|
10204
|
+
return state_1.value;
|
|
10205
|
+
}
|
|
10206
|
+
}
|
|
10207
|
+
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
10208
|
+
finally {
|
|
10209
|
+
try {
|
|
10210
|
+
if (parameters1_1_1 && !parameters1_1_1.done && (_b = parameters1_1.return)) _b.call(parameters1_1);
|
|
10211
|
+
}
|
|
10212
|
+
finally { if (e_2) throw e_2.error; }
|
|
10213
|
+
}
|
|
10214
|
+
}
|
|
10215
|
+
}
|
|
10216
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
10217
|
+
finally {
|
|
10218
|
+
try {
|
|
10219
|
+
if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
|
|
10220
|
+
}
|
|
10221
|
+
finally { if (e_1) throw e_1.error; }
|
|
10222
|
+
}
|
|
10223
|
+
return true;
|
|
10119
10224
|
}
|
|
10120
10225
|
|
|
10121
10226
|
/**
|
|
@@ -10370,7 +10475,6 @@
|
|
|
10370
10475
|
exports.DEFAULT_EXECUTIONS_CACHE_DIRNAME = DEFAULT_EXECUTIONS_CACHE_DIRNAME;
|
|
10371
10476
|
exports.DEFAULT_INTERMEDIATE_FILES_STRATEGY = DEFAULT_INTERMEDIATE_FILES_STRATEGY;
|
|
10372
10477
|
exports.DEFAULT_IS_AUTO_INSTALLED = DEFAULT_IS_AUTO_INSTALLED;
|
|
10373
|
-
exports.DEFAULT_IS_VERBOSE = DEFAULT_IS_VERBOSE;
|
|
10374
10478
|
exports.DEFAULT_MAX_EXECUTION_ATTEMPTS = DEFAULT_MAX_EXECUTION_ATTEMPTS;
|
|
10375
10479
|
exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
|
|
10376
10480
|
exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
|
|
@@ -10411,6 +10515,7 @@
|
|
|
10411
10515
|
exports.PipelineUrlError = PipelineUrlError;
|
|
10412
10516
|
exports.PrefixStorage = PrefixStorage;
|
|
10413
10517
|
exports.RESERVED_PARAMETER_NAMES = RESERVED_PARAMETER_NAMES;
|
|
10518
|
+
exports.SET_IS_VERBOSE = SET_IS_VERBOSE;
|
|
10414
10519
|
exports.SectionTypes = SectionTypes;
|
|
10415
10520
|
exports.SheetsFormfactorDefinition = SheetsFormfactorDefinition;
|
|
10416
10521
|
exports.TaskTypes = TaskTypes;
|