@promptbook/website-crawler 0.75.3 → 0.75.5

package/esm/index.es.js

@@ -24,7 +24,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  *
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.75.2';
+var PROMPTBOOK_ENGINE_VERSION = '0.75.4';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2092,6 +2092,42 @@ var PipelineExecutionError = /** @class */ (function (_super) {
     return PipelineExecutionError;
 }(Error));
 
+/**
+ * This error indicates problems parsing the format value
+ *
+ * For example, when the format value is not a valid JSON or CSV
+ * This is not thrown directly but in extended classes
+ *
+ * @public exported from `@promptbook/core`
+ */
+var AbstractFormatError = /** @class */ (function (_super) {
+    __extends(AbstractFormatError, _super);
+    // Note: To allow instanceof do not put here error `name`
+    // public readonly name = 'AbstractFormatError';
+    function AbstractFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        Object.setPrototypeOf(_this, AbstractFormatError.prototype);
+        return _this;
+    }
+    return AbstractFormatError;
+}(Error));
+
+/**
+ * This error indicates problem with parsing of CSV
+ *
+ * @public exported from `@promptbook/core`
+ */
+var CsvFormatError = /** @class */ (function (_super) {
+    __extends(CsvFormatError, _super);
+    function CsvFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'CsvFormatError';
+        Object.setPrototypeOf(_this, CsvFormatError.prototype);
+        return _this;
+    }
+    return CsvFormatError;
+}(AbstractFormatError));
+
 /**
  * This error indicates that the pipeline collection cannot be propperly loaded
  *
@@ -2149,10 +2185,14 @@ var LimitReachedError = /** @class */ (function (_super) {
  * @public exported from `@promptbook/core`
  */
 var ERRORS = {
-    ExpectError: ExpectError,
+    AbstractFormatError: AbstractFormatError,
+    CsvFormatError: CsvFormatError,
     CollectionError: CollectionError,
     EnvironmentMismatchError: EnvironmentMismatchError,
+    ExpectError: ExpectError,
+    KnowledgeScrapeError: KnowledgeScrapeError,
     LimitReachedError: LimitReachedError,
+    MissingToolsError: MissingToolsError,
     NotFoundError: NotFoundError,
     NotYetImplementedError: NotYetImplementedError,
     ParseError: ParseError,
@@ -2972,8 +3012,8 @@ var $scrapersRegister = new $Register('scraper_constructors');
  *
  * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
  */
-function $registeredScrapersMessage() {
-    var e_1, _a, e_2, _b;
+function $registeredScrapersMessage(availableScrapers) {
+    var e_1, _a, e_2, _b, e_3, _c;
     /**
      * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
      */
@@ -2985,15 +3025,15 @@ function $registeredScrapersMessage() {
         all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
     };
     try {
-        for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
-            var _e = _d.value, packageName = _e.packageName, className = _e.className, mimeTypes = _e.mimeTypes, documentationUrl = _e.documentationUrl, isAvilableInBrowser = _e.isAvilableInBrowser;
+        for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
+            var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
             _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
         }
     }
     catch (e_1_1) { e_1 = { error: e_1_1 }; }
     finally {
         try {
-            if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
+            if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
         }
         finally { if (e_1) throw e_1.error; }
     }
@@ -3004,18 +3044,31 @@ function $registeredScrapersMessage() {
         all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
     };
     try {
-        for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
-            var _h = _g.value, packageName = _h.packageName, className = _h.className, mimeTypes = _h.mimeTypes, documentationUrl = _h.documentationUrl, isAvilableInBrowser = _h.isAvilableInBrowser;
+        for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
+            var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
             _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
         }
     }
     catch (e_2_1) { e_2 = { error: e_2_1 }; }
     finally {
         try {
-            if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+            if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
         }
         finally { if (e_2) throw e_2.error; }
     }
+    try {
+        for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
+            var metadata_1 = availableScrapers_1_1.value.metadata;
+            all.push(metadata_1);
+        }
+    }
+    catch (e_3_1) { e_3 = { error: e_3_1 }; }
+    finally {
+        try {
+            if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
+        }
+        finally { if (e_3) throw e_3.error; }
+    }
     var metadata = all.map(function (metadata) {
         var isMetadataAviailable = $scrapersMetadataRegister
             .list()
@@ -3029,42 +3082,44 @@ function $registeredScrapersMessage() {
             var packageName = _a.packageName, className = _a.className;
             return metadata.packageName === packageName && metadata.className === className;
         });
-        return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
+        var isAvilableInTools = availableScrapers.some(function (_a) {
+            var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
+            return metadata.packageName === packageName && metadata.className === className;
+        });
+        return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
     });
     if (metadata.length === 0) {
-        return "No scrapers are available";
+        return spaceTrim$1("\n            **No scrapers are available**\n\n            This is a unexpected behavior, you are probably using some broken version of Promptbook\n            At least there should be available the metadata of the scrapers\n        ");
     }
     return spaceTrim$1(function (block) { return "\n            Available scrapers are:\n            ".concat(block(metadata
         .map(function (_a, i) {
-        var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
-        var more;
-        // TODO: Use documentationUrl
-        if (just(false)) {
-            more = '';
-        }
-        else if (!isMetadataAviailable && !isInstalled) {
-            // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
-            more = "*(not installed and no metadata, looks like a unexpected behavior)*";
-        }
-        else if (isMetadataAviailable && !isInstalled) {
-            // TODO: [�][�]
-            more = "*(not installed)*";
-        }
-        else if (!isMetadataAviailable && isInstalled) {
-            more = "*(no metadata, looks like a unexpected behavior)*";
-        }
-        else if (isMetadataAviailable && isInstalled) {
-            more = "(installed)";
-        }
-        else {
-            more = "*(unknown state, looks like a unexpected behavior)*";
-        }
+        var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
+        var more = [];
+        // TODO: [🧠] Maybe use `documentationUrl`
+        if (isMetadataAviailable) {
+            more.push("\u2B1C Metadata registered");
+        } // not else
+        if (isInstalled) {
+            more.push("\uD83D\uDFE9 Installed");
+        } // not else
+        if (isAvilableInTools) {
+            more.push("\uD83D\uDFE6 Available in tools");
+        } // not else
+        if (!isMetadataAviailable && isInstalled) {
+            more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
+        } // not else
+        if (!isInstalled && isAvilableInTools) {
+            more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
+        } // not else
         if (!isAvilableInBrowser) {
-            more += " *(not available in browser)*";
+            more.push("Not usable in browser");
         }
-        return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes.join(', '), " ").concat(more);
+        var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
+        return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
+            .map(function (mimeType) { return "\"".concat(mimeType, "\""); })
+            .join(', ')).concat(moreText);
     })
-        .join('\n')), "\n        "); });
+        .join('\n')), "\n\n            Legend:\n            - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n            - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n            - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n        "); });
 }
 /**
  * TODO: [®] DRY Register logic
@@ -3312,57 +3367,75 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
                     _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
                     knowledgePreparedUnflatten = new Array(knowledgeSources.length);
                     return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
-                            var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
-                            var e_1, _c;
-                            return __generator(this, function (_d) {
-                                switch (_d.label) {
+                            var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
+                            var e_1, _a;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
                                     case 0:
                                         partialPieces = null;
                                         return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                     case 1:
-                                        sourceHandler = _d.sent();
-                                        _d.label = 2;
+                                        sourceHandler = _b.sent();
+                                        scrapers = arrayableToArray(tools.scrapers);
+                                        _loop_1 = function (scraper) {
+                                            var partialPiecesUnchecked;
+                                            return __generator(this, function (_c) {
+                                                switch (_c.label) {
+                                                    case 0:
+                                                        if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                                                        // <- TODO: [🦔] Implement mime-type wildcards
+                                                        ) {
+                                                            return [2 /*return*/, "continue"];
+                                                        }
+                                                        return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                                    case 1:
+                                                        partialPiecesUnchecked = _c.sent();
+                                                        if (partialPiecesUnchecked !== null) {
+                                                            partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
+                                                            return [2 /*return*/, "break"];
+                                                        }
+                                                        console.warn(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n                        \n                        The source:\n                        > ").concat(block(knowledgeSource.sourceContent
+                                                            .split('\n')
+                                                            .map(function (line) { return "> ".concat(line); })
+                                                            .join('\n')), "\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
+                                                        return [2 /*return*/];
+                                                }
+                                            });
+                                        };
+                                        _b.label = 2;
                                     case 2:
-                                        _d.trys.push([2, 7, 8, 9]);
-                                        _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
-                                        _d.label = 3;
+                                        _b.trys.push([2, 7, 8, 9]);
+                                        scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
+                                        _b.label = 3;
                                     case 3:
-                                        if (!!_b.done) return [3 /*break*/, 6];
-                                        scraper = _b.value;
-                                        if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                                        // <- TODO: [🦔] Implement mime-type wildcards
-                                        ) {
-                                            return [3 /*break*/, 5];
-                                        }
-                                        return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                        if (!!scrapers_1_1.done) return [3 /*break*/, 6];
+                                        scraper = scrapers_1_1.value;
+                                        return [5 /*yield**/, _loop_1(scraper)];
                                     case 4:
-                                        partialPiecesUnchecked = _d.sent();
-                                        if (partialPiecesUnchecked !== null) {
-                                            partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
-                                            // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                                        state_1 = _b.sent();
+                                        if (state_1 === "break")
                                             return [3 /*break*/, 6];
-                                        }
-                                        _d.label = 5;
+                                        _b.label = 5;
                                     case 5:
-                                        _b = _a.next();
+                                        scrapers_1_1 = scrapers_1.next();
                                         return [3 /*break*/, 3];
                                     case 6: return [3 /*break*/, 9];
                                     case 7:
-                                        e_1_1 = _d.sent();
+                                        e_1_1 = _b.sent();
                                         e_1 = { error: e_1_1 };
                                         return [3 /*break*/, 9];
                                     case 8:
                                         try {
-                                            if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
+                                            if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
                                         }
                                         finally { if (e_1) throw e_1.error; }
                                         return [7 /*endfinally*/];
                                     case 9:
                                         if (partialPieces === null) {
-                                            throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge from source:\n                        \n                        > ".concat(block(knowledgeSource.sourceContent
+                                            throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge\n                        \n                        The source:\n                        > ".concat(block(knowledgeSource.sourceContent
                                                 .split('\n')
                                                 .map(function (line) { return "> ".concat(line); })
-                                                .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage()), "\n\n\n                    "); }));
+                                                .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
                                         }
                                         pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
                                                 {
@@ -3788,42 +3861,6 @@ function union() {
     return union;
 }
 
-/**
- * This error indicates problems parsing the format value
- *
- * For example, when the format value is not a valid JSON or CSV
- * This is not thrown directly but in extended classes
- *
- * @public exported from `@promptbook/core`
- */
-var AbstractFormatError = /** @class */ (function (_super) {
-    __extends(AbstractFormatError, _super);
-    // Note: To allow instanceof do not put here error `name`
-    // public readonly name = 'AbstractFormatError';
-    function AbstractFormatError(message) {
-        var _this = _super.call(this, message) || this;
-        Object.setPrototypeOf(_this, AbstractFormatError.prototype);
-        return _this;
-    }
-    return AbstractFormatError;
-}(Error));
-
-/**
- * This error indicates problem with parsing of CSV
- *
- * @public exported from `@promptbook/core`
- */
-var CsvFormatError = /** @class */ (function (_super) {
-    __extends(CsvFormatError, _super);
-    function CsvFormatError(message) {
-        var _this = _super.call(this, message) || this;
-        _this.name = 'CsvFormatError';
-        Object.setPrototypeOf(_this, CsvFormatError.prototype);
-        return _this;
-    }
-    return CsvFormatError;
-}(AbstractFormatError));
-
 /**
  * @@@
  *
@@ -3864,7 +3901,7 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                Error(s) from CSV parsing:\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n                                The CSV data:\n                                ").concat(block(value), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
                                         var _a, _b;
@@ -3902,7 +3939,7 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim$1(function (block) { return "\n                                CSV parsing error\n\n                                Error(s) from CSV parsing:\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n                                The CSV data:\n                                ").concat(block(value), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
                                         var _this = this;