@promptbook/website-crawler 0.75.2 → 0.75.4

package/esm/index.es.js

@@ -24,7 +24,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  *
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.75.1';
+var PROMPTBOOK_ENGINE_VERSION = '0.75.3';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2972,50 +2972,63 @@ var $scrapersRegister = new $Register('scraper_constructors');
  *
  * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
  */
-function $registeredScrapersMessage() {
-    var e_1, _a, e_2, _b;
+function $registeredScrapersMessage(availableScrapers) {
+    var e_1, _a, e_2, _b, e_3, _c;
     /**
      * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
      */
     var all = [];
-    var _loop_1 = function (packageName, className) {
+    var _loop_1 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
         if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
             return "continue";
         }
-        all.push({ packageName: packageName, className: className });
+        all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
     };
     try {
-        for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
-            var _e = _d.value, packageName = _e.packageName, className = _e.className;
-            _loop_1(packageName, className);
+        for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
+            var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
+            _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
         }
     }
     catch (e_1_1) { e_1 = { error: e_1_1 }; }
     finally {
         try {
-            if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
+            if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
         }
         finally { if (e_1) throw e_1.error; }
     }
-    var _loop_2 = function (packageName, className) {
+    var _loop_2 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
         if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
             return "continue";
         }
-        all.push({ packageName: packageName, className: className });
+        all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
     };
     try {
-        for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
-            var _h = _g.value, packageName = _h.packageName, className = _h.className;
-            _loop_2(packageName, className);
+        for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
+            var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
+            _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
         }
     }
     catch (e_2_1) { e_2 = { error: e_2_1 }; }
     finally {
         try {
-            if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+            if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
         }
         finally { if (e_2) throw e_2.error; }
     }
+    try {
+        for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
+            var metadata_1 = availableScrapers_1_1.value.metadata;
+            all.push(metadata_1);
+        }
+    }
+    catch (e_3_1) { e_3 = { error: e_3_1 }; }
+    finally {
+        try {
+            if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
+        }
+        finally { if (e_3) throw e_3.error; }
+    }
     var metadata = all.map(function (metadata) {
         var isMetadataAviailable = $scrapersMetadataRegister
             .list()
@@ -3029,38 +3042,44 @@ function $registeredScrapersMessage() {
             var packageName = _a.packageName, className = _a.className;
             return metadata.packageName === packageName && metadata.className === className;
         });
-        return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
+        var isAvilableInTools = availableScrapers.some(function (_a) {
+            var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
+            return metadata.packageName === packageName && metadata.className === className;
+        });
+        return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
     });
     if (metadata.length === 0) {
-        return "No scrapers are available";
+        return spaceTrim$1("\n            **No scrapers are available**\n\n            This is a unexpected behavior, you are probably using some broken version of Promptbook\n            At least there should be available the metadata of the scrapers\n        ");
     }
     return spaceTrim$1(function (block) { return "\n            Available scrapers are:\n            ".concat(block(metadata
         .map(function (_a, i) {
-        var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
-        var more;
-        if (just(false)) {
-            more = '';
-        }
-        else if (!isMetadataAviailable && !isInstalled) {
-            // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
-            more = "(not installed and no metadata, looks like a unexpected behavior)";
-        }
-        else if (isMetadataAviailable && !isInstalled) {
-            // TODO: [�][�]
-            more = "(not installed)";
-        }
-        else if (!isMetadataAviailable && isInstalled) {
-            more = "(no metadata, looks like a unexpected behavior)";
-        }
-        else if (isMetadataAviailable && isInstalled) {
-            more = "(installed)";
-        }
-        else {
-            more = "(unknown state, looks like a unexpected behavior)";
+        var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
+        var more = [];
+        // TODO: [🧠] Maybe use `documentationUrl`
+        if (isMetadataAviailable) {
+            more.push("\u2B1C Metadata registered");
+        } // not else
+        if (isInstalled) {
+            more.push("\uD83D\uDFE9 Installed");
+        } // not else
+        if (isAvilableInTools) {
+            more.push("\uD83D\uDFE6 Available in tools");
+        } // not else
+        if (!isMetadataAviailable && isInstalled) {
+            more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
+        } // not else
+        if (!isInstalled && isAvilableInTools) {
+            more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
+        } // not else
+        if (!isAvilableInBrowser) {
+            more.push("Not usable in browser");
         }
-        return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(more);
+        var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
+        return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
+            .map(function (mimeType) { return "\"".concat(mimeType, "\""); })
+            .join(', ')).concat(moreText);
     })
-        .join('\n')), "\n        "); });
+        .join('\n')), "\n\n            Legend:\n            - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n            - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n            - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n        "); });
 }
 /**
  * TODO: [®] DRY Register logic
@@ -3308,54 +3327,75 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
                     _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
                     knowledgePreparedUnflatten = new Array(knowledgeSources.length);
                     return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
-                            var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
-                            var e_1, _c;
-                            return __generator(this, function (_d) {
-                                switch (_d.label) {
+                            var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
+                            var e_1, _a;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
                                     case 0:
                                         partialPieces = null;
                                         return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                     case 1:
-                                        sourceHandler = _d.sent();
-                                        _d.label = 2;
+                                        sourceHandler = _b.sent();
+                                        scrapers = arrayableToArray(tools.scrapers);
+                                        _loop_1 = function (scraper) {
+                                            var partialPiecesUnchecked;
+                                            return __generator(this, function (_c) {
+                                                switch (_c.label) {
+                                                    case 0:
+                                                        if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                                                        // <- TODO: [🦔] Implement mime-type wildcards
+                                                        ) {
+                                                            return [2 /*return*/, "continue"];
+                                                        }
+                                                        return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                                    case 1:
+                                                        partialPiecesUnchecked = _c.sent();
+                                                        if (partialPiecesUnchecked !== null) {
+                                                            partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
+                                                            return [2 /*return*/, "break"];
+                                                        }
+                                                        console.warn(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n                        \n                        The source:\n                        > ").concat(block(knowledgeSource.sourceContent
+                                                            .split('\n')
+                                                            .map(function (line) { return "> ".concat(line); })
+                                                            .join('\n')), "\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
+                                                        return [2 /*return*/];
+                                                }
+                                            });
+                                        };
+                                        _b.label = 2;
                                     case 2:
-                                        _d.trys.push([2, 7, 8, 9]);
-                                        _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
-                                        _d.label = 3;
+                                        _b.trys.push([2, 7, 8, 9]);
+                                        scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
+                                        _b.label = 3;
                                     case 3:
-                                        if (!!_b.done) return [3 /*break*/, 6];
-                                        scraper = _b.value;
-                                        if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                                        // <- TODO: [🦔] Implement mime-type wildcards
-                                        ) {
-                                            return [3 /*break*/, 5];
-                                        }
-                                        return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                        if (!!scrapers_1_1.done) return [3 /*break*/, 6];
+                                        scraper = scrapers_1_1.value;
+                                        return [5 /*yield**/, _loop_1(scraper)];
                                     case 4:
-                                        partialPiecesUnchecked = _d.sent();
-                                        if (partialPiecesUnchecked !== null) {
-                                            partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
-                                            // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                                        state_1 = _b.sent();
+                                        if (state_1 === "break")
                                             return [3 /*break*/, 6];
-                                        }
-                                        _d.label = 5;
+                                        _b.label = 5;
                                     case 5:
-                                        _b = _a.next();
+                                        scrapers_1_1 = scrapers_1.next();
                                         return [3 /*break*/, 3];
                                     case 6: return [3 /*break*/, 9];
                                     case 7:
-                                        e_1_1 = _d.sent();
+                                        e_1_1 = _b.sent();
                                         e_1 = { error: e_1_1 };
                                         return [3 /*break*/, 9];
                                     case 8:
                                         try {
-                                            if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
+                                            if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
                                         }
                                         finally { if (e_1) throw e_1.error; }
                                         return [7 /*endfinally*/];
                                     case 9:
                                         if (partialPieces === null) {
-                                            throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage()), "\n\n\n                    "); }));
+                                            throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge\n                        \n                        The source:\n                        > ".concat(block(knowledgeSource.sourceContent
+                                                .split('\n')
+                                                .map(function (line) { return "> ".concat(line); })
+                                                .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
                                         }
                                         pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
                                                 {