@promptbook/website-crawler 0.75.2 → 0.75.3

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/website-crawler",
-    "version": "0.75.2",
+    "version": "0.75.3",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "--note-0": "    <- [🐊]",
     "private": false,
@@ -54,7 +54,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.75.2"
+        "@promptbook/core": "0.75.3"
     },
     "dependencies": {
         "@mozilla/readability": "0.5.0",

package/umd/index.umd.js

@@ -22,7 +22,7 @@
      *
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.75.1';
+    var PROMPTBOOK_ENGINE_VERSION = '0.75.2';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2976,16 +2976,16 @@
          * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
          */
         var all = [];
-        var _loop_1 = function (packageName, className) {
+        var _loop_1 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
             if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
                 return "continue";
             }
-            all.push({ packageName: packageName, className: className });
+            all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
         };
         try {
             for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
-                var _e = _d.value, packageName = _e.packageName, className = _e.className;
-                _loop_1(packageName, className);
+                var _e = _d.value, packageName = _e.packageName, className = _e.className, mimeTypes = _e.mimeTypes, documentationUrl = _e.documentationUrl, isAvilableInBrowser = _e.isAvilableInBrowser;
+                _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
             }
         }
         catch (e_1_1) { e_1 = { error: e_1_1 }; }
@@ -2995,16 +2995,16 @@
             }
             finally { if (e_1) throw e_1.error; }
         }
-        var _loop_2 = function (packageName, className) {
+        var _loop_2 = function (packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser) {
             if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
                 return "continue";
             }
-            all.push({ packageName: packageName, className: className });
+            all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
         };
         try {
             for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
-                var _h = _g.value, packageName = _h.packageName, className = _h.className;
-                _loop_2(packageName, className);
+                var _h = _g.value, packageName = _h.packageName, className = _h.className, mimeTypes = _h.mimeTypes, documentationUrl = _h.documentationUrl, isAvilableInBrowser = _h.isAvilableInBrowser;
+                _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
             }
         }
         catch (e_2_1) { e_2 = { error: e_2_1 }; }
@@ -3034,29 +3034,33 @@
         }
         return spaceTrim__default["default"](function (block) { return "\n            Available scrapers are:\n            ".concat(block(metadata
             .map(function (_a, i) {
-            var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
+            var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
             var more;
+            // TODO: Use documentationUrl
             if (just(false)) {
                 more = '';
             }
             else if (!isMetadataAviailable && !isInstalled) {
                 // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
-                more = "(not installed and no metadata, looks like a unexpected behavior)";
+                more = "*(not installed and no metadata, looks like a unexpected behavior)*";
             }
             else if (isMetadataAviailable && !isInstalled) {
                 // TODO: [�][�]
-                more = "(not installed)";
+                more = "*(not installed)*";
             }
             else if (!isMetadataAviailable && isInstalled) {
-                more = "(no metadata, looks like a unexpected behavior)";
+                more = "*(no metadata, looks like a unexpected behavior)*";
             }
             else if (isMetadataAviailable && isInstalled) {
                 more = "(installed)";
             }
             else {
-                more = "(unknown state, looks like a unexpected behavior)";
+                more = "*(unknown state, looks like a unexpected behavior)*";
             }
-            return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(more);
+            if (!isAvilableInBrowser) {
+                more += " *(not available in browser)*";
+            }
+            return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes.join(', '), " ").concat(more);
         })
             .join('\n')), "\n        "); });
     }
@@ -3353,7 +3357,10 @@
                                             return [7 /*endfinally*/];
                                         case 9:
                                             if (partialPieces === null) {
-                                                throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n                        Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage()), "\n\n\n                    "); }));
+                                                throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n                        Cannot scrape knowledge from source:\n                        \n                        > ".concat(block(knowledgeSource.sourceContent
+                                                    .split('\n')
+                                                    .map(function (line) { return "> ".concat(line); })
+                                                    .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage()), "\n\n\n                    "); }));
                                             }
                                             pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
                                                     {