@promptbook/node 0.75.2 → 0.75.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +108 -68
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +2 -0
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +2 -1
- package/package.json +2 -2
- package/umd/index.umd.js +108 -68
- package/umd/index.umd.js.map +1 -1
package/esm/index.es.js
CHANGED
|
@@ -26,7 +26,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
26
26
|
*
|
|
27
27
|
* @see https://github.com/webgptorg/promptbook
|
|
28
28
|
*/
|
|
29
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
29
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.3';
|
|
30
30
|
/**
|
|
31
31
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
32
32
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -4752,50 +4752,63 @@ var $scrapersRegister = new $Register('scraper_constructors');
|
|
|
4752
4752
|
*
|
|
4753
4753
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
4754
4754
|
*/
|
|
4755
|
-
function $registeredScrapersMessage() {
|
|
4756
|
-
var e_1, _a, e_2, _b;
|
|
4755
|
+
function $registeredScrapersMessage(availableScrapers) {
|
|
4756
|
+
var e_1, _a, e_2, _b, e_3, _c;
|
|
4757
4757
|
/**
|
|
4758
4758
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
4759
4759
|
*/
|
|
4760
4760
|
var all = [];
|
|
4761
|
-
var _loop_1 = function (packageName, className) {
|
|
4761
|
+
var _loop_1 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
|
|
4762
4762
|
if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
|
|
4763
4763
|
return "continue";
|
|
4764
4764
|
}
|
|
4765
|
-
all.push({ packageName: packageName, className: className });
|
|
4765
|
+
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
4766
4766
|
};
|
|
4767
4767
|
try {
|
|
4768
|
-
for (var
|
|
4769
|
-
var
|
|
4770
|
-
_loop_1(packageName, className);
|
|
4768
|
+
for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
4769
|
+
var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
|
|
4770
|
+
_loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
4771
4771
|
}
|
|
4772
4772
|
}
|
|
4773
4773
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4774
4774
|
finally {
|
|
4775
4775
|
try {
|
|
4776
|
-
if (
|
|
4776
|
+
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
4777
4777
|
}
|
|
4778
4778
|
finally { if (e_1) throw e_1.error; }
|
|
4779
4779
|
}
|
|
4780
|
-
var _loop_2 = function (packageName, className) {
|
|
4780
|
+
var _loop_2 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
|
|
4781
4781
|
if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
|
|
4782
4782
|
return "continue";
|
|
4783
4783
|
}
|
|
4784
|
-
all.push({ packageName: packageName, className: className });
|
|
4784
|
+
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
4785
4785
|
};
|
|
4786
4786
|
try {
|
|
4787
|
-
for (var
|
|
4788
|
-
var
|
|
4789
|
-
_loop_2(packageName, className);
|
|
4787
|
+
for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
|
|
4788
|
+
var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
|
|
4789
|
+
_loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
4790
4790
|
}
|
|
4791
4791
|
}
|
|
4792
4792
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
4793
4793
|
finally {
|
|
4794
4794
|
try {
|
|
4795
|
-
if (
|
|
4795
|
+
if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
|
|
4796
4796
|
}
|
|
4797
4797
|
finally { if (e_2) throw e_2.error; }
|
|
4798
4798
|
}
|
|
4799
|
+
try {
|
|
4800
|
+
for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
|
|
4801
|
+
var metadata_1 = availableScrapers_1_1.value.metadata;
|
|
4802
|
+
all.push(metadata_1);
|
|
4803
|
+
}
|
|
4804
|
+
}
|
|
4805
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
4806
|
+
finally {
|
|
4807
|
+
try {
|
|
4808
|
+
if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
|
|
4809
|
+
}
|
|
4810
|
+
finally { if (e_3) throw e_3.error; }
|
|
4811
|
+
}
|
|
4799
4812
|
var metadata = all.map(function (metadata) {
|
|
4800
4813
|
var isMetadataAviailable = $scrapersMetadataRegister
|
|
4801
4814
|
.list()
|
|
@@ -4809,38 +4822,44 @@ function $registeredScrapersMessage() {
|
|
|
4809
4822
|
var packageName = _a.packageName, className = _a.className;
|
|
4810
4823
|
return metadata.packageName === packageName && metadata.className === className;
|
|
4811
4824
|
});
|
|
4812
|
-
|
|
4825
|
+
var isAvilableInTools = availableScrapers.some(function (_a) {
|
|
4826
|
+
var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
|
|
4827
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
4828
|
+
});
|
|
4829
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
|
|
4813
4830
|
});
|
|
4814
4831
|
if (metadata.length === 0) {
|
|
4815
|
-
return "No scrapers are available";
|
|
4832
|
+
return spaceTrim$1("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
|
|
4816
4833
|
}
|
|
4817
4834
|
return spaceTrim$1(function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
4818
4835
|
.map(function (_a, i) {
|
|
4819
|
-
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
|
|
4820
|
-
var more;
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
more
|
|
4827
|
-
}
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(
|
|
4836
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
|
|
4837
|
+
var more = [];
|
|
4838
|
+
// TODO: [🧠] Maybe use `documentationUrl`
|
|
4839
|
+
if (isMetadataAviailable) {
|
|
4840
|
+
more.push("\u2B1C Metadata registered");
|
|
4841
|
+
} // not else
|
|
4842
|
+
if (isInstalled) {
|
|
4843
|
+
more.push("\uD83D\uDFE9 Installed");
|
|
4844
|
+
} // not else
|
|
4845
|
+
if (isAvilableInTools) {
|
|
4846
|
+
more.push("\uD83D\uDFE6 Available in tools");
|
|
4847
|
+
} // not else
|
|
4848
|
+
if (!isMetadataAviailable && isInstalled) {
|
|
4849
|
+
more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
|
|
4850
|
+
} // not else
|
|
4851
|
+
if (!isInstalled && isAvilableInTools) {
|
|
4852
|
+
more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
|
|
4853
|
+
} // not else
|
|
4854
|
+
if (!isAvilableInBrowser) {
|
|
4855
|
+
more.push("Not usable in browser");
|
|
4856
|
+
}
|
|
4857
|
+
var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
|
|
4858
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
|
|
4859
|
+
.map(function (mimeType) { return "\"".concat(mimeType, "\""); })
|
|
4860
|
+
.join(', ')).concat(moreText);
|
|
4842
4861
|
})
|
|
4843
|
-
.join('\n')), "\n "); });
|
|
4862
|
+
.join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
|
|
4844
4863
|
}
|
|
4845
4864
|
/**
|
|
4846
4865
|
* TODO: [®] DRY Register logic
|
|
@@ -5176,54 +5195,75 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5176
5195
|
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5177
5196
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5178
5197
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5179
|
-
var partialPieces, sourceHandler,
|
|
5180
|
-
var e_1,
|
|
5181
|
-
return __generator(this, function (
|
|
5182
|
-
switch (
|
|
5198
|
+
var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
|
|
5199
|
+
var e_1, _a;
|
|
5200
|
+
return __generator(this, function (_b) {
|
|
5201
|
+
switch (_b.label) {
|
|
5183
5202
|
case 0:
|
|
5184
5203
|
partialPieces = null;
|
|
5185
5204
|
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
5186
5205
|
case 1:
|
|
5187
|
-
sourceHandler =
|
|
5188
|
-
|
|
5206
|
+
sourceHandler = _b.sent();
|
|
5207
|
+
scrapers = arrayableToArray(tools.scrapers);
|
|
5208
|
+
_loop_1 = function (scraper) {
|
|
5209
|
+
var partialPiecesUnchecked;
|
|
5210
|
+
return __generator(this, function (_c) {
|
|
5211
|
+
switch (_c.label) {
|
|
5212
|
+
case 0:
|
|
5213
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
5214
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5215
|
+
) {
|
|
5216
|
+
return [2 /*return*/, "continue"];
|
|
5217
|
+
}
|
|
5218
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
5219
|
+
case 1:
|
|
5220
|
+
partialPiecesUnchecked = _c.sent();
|
|
5221
|
+
if (partialPiecesUnchecked !== null) {
|
|
5222
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5223
|
+
return [2 /*return*/, "break"];
|
|
5224
|
+
}
|
|
5225
|
+
console.warn(spaceTrim$1(function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
|
|
5226
|
+
.split('\n')
|
|
5227
|
+
.map(function (line) { return "> ".concat(line); })
|
|
5228
|
+
.join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
5229
|
+
return [2 /*return*/];
|
|
5230
|
+
}
|
|
5231
|
+
});
|
|
5232
|
+
};
|
|
5233
|
+
_b.label = 2;
|
|
5189
5234
|
case 2:
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5235
|
+
_b.trys.push([2, 7, 8, 9]);
|
|
5236
|
+
scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
|
|
5237
|
+
_b.label = 3;
|
|
5193
5238
|
case 3:
|
|
5194
|
-
if (!!
|
|
5195
|
-
scraper =
|
|
5196
|
-
|
|
5197
|
-
// <- TODO: [🦔] Implement mime-type wildcards
|
|
5198
|
-
) {
|
|
5199
|
-
return [3 /*break*/, 5];
|
|
5200
|
-
}
|
|
5201
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
5239
|
+
if (!!scrapers_1_1.done) return [3 /*break*/, 6];
|
|
5240
|
+
scraper = scrapers_1_1.value;
|
|
5241
|
+
return [5 /*yield**/, _loop_1(scraper)];
|
|
5202
5242
|
case 4:
|
|
5203
|
-
|
|
5204
|
-
if (
|
|
5205
|
-
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5206
|
-
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5243
|
+
state_1 = _b.sent();
|
|
5244
|
+
if (state_1 === "break")
|
|
5207
5245
|
return [3 /*break*/, 6];
|
|
5208
|
-
|
|
5209
|
-
_d.label = 5;
|
|
5246
|
+
_b.label = 5;
|
|
5210
5247
|
case 5:
|
|
5211
|
-
|
|
5248
|
+
scrapers_1_1 = scrapers_1.next();
|
|
5212
5249
|
return [3 /*break*/, 3];
|
|
5213
5250
|
case 6: return [3 /*break*/, 9];
|
|
5214
5251
|
case 7:
|
|
5215
|
-
e_1_1 =
|
|
5252
|
+
e_1_1 = _b.sent();
|
|
5216
5253
|
e_1 = { error: e_1_1 };
|
|
5217
5254
|
return [3 /*break*/, 9];
|
|
5218
5255
|
case 8:
|
|
5219
5256
|
try {
|
|
5220
|
-
if (
|
|
5257
|
+
if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
|
|
5221
5258
|
}
|
|
5222
5259
|
finally { if (e_1) throw e_1.error; }
|
|
5223
5260
|
return [7 /*endfinally*/];
|
|
5224
5261
|
case 9:
|
|
5225
5262
|
if (partialPieces === null) {
|
|
5226
|
-
throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n Cannot scrape knowledge
|
|
5263
|
+
throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
|
|
5264
|
+
.split('\n')
|
|
5265
|
+
.map(function (line) { return "> ".concat(line); })
|
|
5266
|
+
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
5227
5267
|
}
|
|
5228
5268
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
5229
5269
|
{
|