@promptbook/markdown-utils 0.75.3 → 0.75.4

package/esm/typings/src/_packages/core.index.d.ts

@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
 import { DEFAULT_REMOTE_URL_PATH } from '../config';
 import { DEFAULT_CSV_SETTINGS } from '../config';
 import { DEFAULT_IS_VERBOSE } from '../config';
+import { SET_IS_VERBOSE } from '../config';
 import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
 import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
 import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
@@ -136,6 +137,7 @@ export { DEFAULT_REMOTE_URL };
 export { DEFAULT_REMOTE_URL_PATH };
 export { DEFAULT_CSV_SETTINGS };
 export { DEFAULT_IS_VERBOSE };
+export { SET_IS_VERBOSE };
 export { DEFAULT_IS_AUTO_INSTALLED };
 export { pipelineJsonToString };
 export { pipelineStringToJson };

package/esm/typings/src/config.d.ts

@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
  *
  * @public exported from `@promptbook/core`
  */
-export declare const DEFAULT_IS_VERBOSE = false;
+export declare let DEFAULT_IS_VERBOSE: boolean;
+/**
+ * @@@
+ *
+ * Note: This is experimental feature
+ *
+ * @public exported from `@promptbook/core`
+ */
+export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
 /**
  * @@@
  *

package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts

@@ -1,4 +1,5 @@
 import type { string_markdown } from '../../../types/typeAliases';
+import type { Scraper } from '../Scraper';
 /**
  * Creates a message with all registered scrapers
  *
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
  *
  * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
  */
-export declare function $registeredScrapersMessage(): string_markdown;
+export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
 /**
  * TODO: [®] DRY Register logic
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/markdown-utils",
-    "version": "0.75.3",
+    "version": "0.75.4",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "--note-0": "    <- [🐊]",
     "private": false,

package/umd/index.umd.js

@@ -22,7 +22,7 @@
      *
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.75.2';
+    var PROMPTBOOK_ENGINE_VERSION = '0.75.3';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -3044,8 +3044,8 @@
      *
      * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
      */
-    function $registeredScrapersMessage() {
-        var e_1, _a, e_2, _b;
+    function $registeredScrapersMessage(availableScrapers) {
+        var e_1, _a, e_2, _b, e_3, _c;
         /**
          * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
          */
@@ -3057,15 +3057,15 @@
             all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
         };
         try {
-            for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
-                var _e = _d.value, packageName = _e.packageName, className = _e.className, mimeTypes = _e.mimeTypes, documentationUrl = _e.documentationUrl, isAvilableInBrowser = _e.isAvilableInBrowser;
+            for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
+                var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
                 _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
             }
         }
         catch (e_1_1) { e_1 = { error: e_1_1 }; }
         finally {
             try {
-                if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
+                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
             }
             finally { if (e_1) throw e_1.error; }
         }
@@ -3076,18 +3076,31 @@
             all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
         };
         try {
-            for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
-                var _h = _g.value, packageName = _h.packageName, className = _h.className, mimeTypes = _h.mimeTypes, documentationUrl = _h.documentationUrl, isAvilableInBrowser = _h.isAvilableInBrowser;
+            for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
+                var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
                 _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
             }
         }
         catch (e_2_1) { e_2 = { error: e_2_1 }; }
         finally {
             try {
-                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+                if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
             }
             finally { if (e_2) throw e_2.error; }
         }
+        try {
+            for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
+                var metadata_1 = availableScrapers_1_1.value.metadata;
+                all.push(metadata_1);
+            }
+        }
+        catch (e_3_1) { e_3 = { error: e_3_1 }; }
+        finally {
+            try {
+                if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
+            }
+            finally { if (e_3) throw e_3.error; }
+        }
         var metadata = all.map(function (metadata) {
             var isMetadataAviailable = $scrapersMetadataRegister
                 .list()
@@ -3101,42 +3114,44 @@
                 var packageName = _a.packageName, className = _a.className;
                 return metadata.packageName === packageName && metadata.className === className;
             });
-            return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
+            var isAvilableInTools = availableScrapers.some(function (_a) {
+                var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
+                return metadata.packageName === packageName && metadata.className === className;
+            });
+            return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
         });
         if (metadata.length === 0) {
-            return "No scrapers are available";
+            return spaceTrim__default["default"]("\n            **No scrapers are available**\n\n            This is a unexpected behavior, you are probably using some broken version of Promptbook\n            At least there should be available the metadata of the scrapers\n        ");
         }
         return spaceTrim__default["default"](function (block) { return "\n            Available scrapers are:\n            ".concat(block(metadata
             .map(function (_a, i) {
-            var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
-            var more;
-            // TODO: Use documentationUrl
-            if (just(false)) {
-                more = '';
-            }
-            else if (!isMetadataAviailable && !isInstalled) {
-                // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
-                more = "*(not installed and no metadata, looks like a unexpected behavior)*";
-            }
-            else if (isMetadataAviailable && !isInstalled) {
-                // TODO: [�][�]
-                more = "*(not installed)*";
-            }
-            else if (!isMetadataAviailable && isInstalled) {
-                more = "*(no metadata, looks like a unexpected behavior)*";
-            }
-            else if (isMetadataAviailable && isInstalled) {
-                more = "(installed)";
-            }
-            else {
-                more = "*(unknown state, looks like a unexpected behavior)*";
-            }
+            var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
+            var more = [];
+            // TODO: [🧠] Maybe use `documentationUrl`
+            if (isMetadataAviailable) {
+                more.push("\u2B1C Metadata registered");
+            } // not else
+            if (isInstalled) {
+                more.push("\uD83D\uDFE9 Installed");
+            } // not else
+            if (isAvilableInTools) {
+                more.push("\uD83D\uDFE6 Available in tools");
+            } // not else
+            if (!isMetadataAviailable && isInstalled) {
+                more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
+            } // not else
+            if (!isInstalled && isAvilableInTools) {
+                more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
+            } // not else
             if (!isAvilableInBrowser) {
-                more += " *(not available in browser)*";
+                more.push("Not usable in browser");
             }
-            return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes.join(', '), " ").concat(more);
+            var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
+            return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
+                .map(function (mimeType) { return "\"".concat(mimeType, "\""); })
+                .join(', ')).concat(moreText);
         })
-            .join('\n')), "\n        "); });
+            .join('\n')), "\n\n            Legend:\n            - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n            - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n            - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n        "); });
     }
     /**
      * TODO: [®] DRY Register logic
@@ -3384,57 +3399,75 @@
                         _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
                         knowledgePreparedUnflatten = new Array(knowledgeSources.length);
                         return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
-                                var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
-                                var e_1, _c;
-                                return __generator(this, function (_d) {
-                                    switch (_d.label) {
+                                var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
+                                var e_1, _a;
+                                return __generator(this, function (_b) {
+                                    switch (_b.label) {
                                         case 0:
                                             partialPieces = null;
                                             return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                         case 1:
-                                            sourceHandler = _d.sent();
-                                            _d.label = 2;
+                                            sourceHandler = _b.sent();
+                                            scrapers = arrayableToArray(tools.scrapers);
+                                            _loop_1 = function (scraper) {
+                                                var partialPiecesUnchecked;
+                                                return __generator(this, function (_c) {
+                                                    switch (_c.label) {
+                                                        case 0:
+                                                            if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                                                            // <- TODO: [🦔] Implement mime-type wildcards
+                                                            ) {
+                                                                return [2 /*return*/, "continue"];
+                                                            }
+                                                            return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                                        case 1:
+                                                            partialPiecesUnchecked = _c.sent();
+                                                            if (partialPiecesUnchecked !== null) {
+                                                                partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
+                                                                return [2 /*return*/, "break"];
+                                                            }
+                                                            console.warn(spaceTrim__default["default"](function (block) { return "\n                        Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n                        \n                        The source:\n                        > ").concat(block(knowledgeSource.sourceContent
+                                                                .split('\n')
+                                                                .map(function (line) { return "> ".concat(line); })
+                                                                .join('\n')), "\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
+                                                            return [2 /*return*/];
+                                                    }
+                                                });
+                                            };
+                                            _b.label = 2;
                                         case 2:
-                                            _d.trys.push([2, 7, 8, 9]);
-                                            _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
-                                            _d.label = 3;
+                                            _b.trys.push([2, 7, 8, 9]);
+                                            scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
+                                            _b.label = 3;
                                         case 3:
-                                            if (!!_b.done) return [3 /*break*/, 6];
-                                            scraper = _b.value;
-                                            if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                                            // <- TODO: [🦔] Implement mime-type wildcards
-                                            ) {
-                                                return [3 /*break*/, 5];
-                                            }
-                                            return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                            if (!!scrapers_1_1.done) return [3 /*break*/, 6];
+                                            scraper = scrapers_1_1.value;
+                                            return [5 /*yield**/, _loop_1(scraper)];
                                         case 4:
-                                            partialPiecesUnchecked = _d.sent();
-                                            if (partialPiecesUnchecked !== null) {
-                                                partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
-                                                // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                                            state_1 = _b.sent();
+                                            if (state_1 === "break")
                                                 return [3 /*break*/, 6];
-                                            }
-                                            _d.label = 5;
+                                            _b.label = 5;
                                         case 5:
-                                            _b = _a.next();
+                                            scrapers_1_1 = scrapers_1.next();
                                             return [3 /*break*/, 3];
                                         case 6: return [3 /*break*/, 9];
                                         case 7:
-                                            e_1_1 = _d.sent();
+                                            e_1_1 = _b.sent();
                                             e_1 = { error: e_1_1 };
                                             return [3 /*break*/, 9];
                                         case 8:
                                             try {
-                                                if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
+                                                if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
                                             }
                                             finally { if (e_1) throw e_1.error; }
                                             return [7 /*endfinally*/];
                                         case 9:
                                             if (partialPieces === null) {
-                                                throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n                        Cannot scrape knowledge from source:\n                        \n                        > ".concat(block(knowledgeSource.sourceContent
+                                                throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n                        Cannot scrape knowledge\n                        \n                        The source:\n                        > ".concat(block(knowledgeSource.sourceContent
                                                     .split('\n')
                                                     .map(function (line) { return "> ".concat(line); })
-                                                    .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage()), "\n\n\n                    "); }));
+                                                    .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
                                             }
                                             pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
                                                     {