@promptbook/core 0.75.3 → 0.75.5

package/esm/index.es.js

@@ -22,7 +22,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  *
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.75.2';
+var PROMPTBOOK_ENGINE_VERSION = '0.75.4';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -835,6 +835,16 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
  * @public exported from `@promptbook/core`
  */
 var DEFAULT_IS_VERBOSE = false;
+/**
+ * @@@
+ *
+ * Note: This is experimental feature
+ *
+ * @public exported from `@promptbook/core`
+ */
+function SET_IS_VERBOSE(isVerbose) {
+    DEFAULT_IS_VERBOSE = isVerbose;
+}
 /**
  * @@@
  *
@@ -2343,6 +2353,42 @@ function joinLlmExecutionTools() {
 
 var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./books/prepare-knowledge-from-markdown.book.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./books/prepare-knowledge-keywords.book.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./books/prepare-knowledge-title.book.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book.md",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n-   Your output format is JSON object\n-   Write just the JSON object, no other text should be present\n-   It contains the following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./books/prepare-persona.book.md"}];
 
+/**
+ * This error indicates problems parsing the format value
+ *
+ * For example, when the format value is not a valid JSON or CSV
+ * This is not thrown directly but in extended classes
+ *
+ * @public exported from `@promptbook/core`
+ */
+var AbstractFormatError = /** @class */ (function (_super) {
+    __extends(AbstractFormatError, _super);
+    // Note: To allow instanceof do not put here error `name`
+    // public readonly name = 'AbstractFormatError';
+    function AbstractFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        Object.setPrototypeOf(_this, AbstractFormatError.prototype);
+        return _this;
+    }
+    return AbstractFormatError;
+}(Error));
+
+/**
+ * This error indicates problem with parsing of CSV
+ *
+ * @public exported from `@promptbook/core`
+ */
+var CsvFormatError = /** @class */ (function (_super) {
+    __extends(CsvFormatError, _super);
+    function CsvFormatError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'CsvFormatError';
+        Object.setPrototypeOf(_this, CsvFormatError.prototype);
+        return _this;
+    }
+    return CsvFormatError;
+}(AbstractFormatError));
+
 /**
  * This error indicates that the pipeline collection cannot be propperly loaded
  *
@@ -2394,6 +2440,22 @@ var ExpectError = /** @class */ (function (_super) {
     return ExpectError;
 }(Error));
 
+/**
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
+ *
+ * @public exported from `@promptbook/core`
+ */
+var KnowledgeScrapeError = /** @class */ (function (_super) {
+    __extends(KnowledgeScrapeError, _super);
+    function KnowledgeScrapeError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'KnowledgeScrapeError';
+        Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
+        return _this;
+    }
+    return KnowledgeScrapeError;
+}(Error));
+
 /**
  * This error type indicates that some limit was reached
  *
@@ -2432,10 +2494,14 @@ var NotYetImplementedError = /** @class */ (function (_super) {
  * @public exported from `@promptbook/core`
  */
 var ERRORS = {
-    ExpectError: ExpectError,
+    AbstractFormatError: AbstractFormatError,
+    CsvFormatError: CsvFormatError,
     CollectionError: CollectionError,
     EnvironmentMismatchError: EnvironmentMismatchError,
+    ExpectError: ExpectError,
+    KnowledgeScrapeError: KnowledgeScrapeError,
     LimitReachedError: LimitReachedError,
+    MissingToolsError: MissingToolsError,
     NotFoundError: NotFoundError,
     NotYetImplementedError: NotYetImplementedError,
     ParseError: ParseError,
@@ -2748,42 +2814,6 @@ function union() {
     return union;
 }
 
-/**
- * This error indicates problems parsing the format value
- *
- * For example, when the format value is not a valid JSON or CSV
- * This is not thrown directly but in extended classes
- *
- * @public exported from `@promptbook/core`
- */
-var AbstractFormatError = /** @class */ (function (_super) {
-    __extends(AbstractFormatError, _super);
-    // Note: To allow instanceof do not put here error `name`
-    // public readonly name = 'AbstractFormatError';
-    function AbstractFormatError(message) {
-        var _this = _super.call(this, message) || this;
-        Object.setPrototypeOf(_this, AbstractFormatError.prototype);
-        return _this;
-    }
-    return AbstractFormatError;
-}(Error));
-
-/**
- * This error indicates problem with parsing of CSV
- *
- * @public exported from `@promptbook/core`
- */
-var CsvFormatError = /** @class */ (function (_super) {
-    __extends(CsvFormatError, _super);
-    function CsvFormatError(message) {
-        var _this = _super.call(this, message) || this;
-        _this.name = 'CsvFormatError';
-        Object.setPrototypeOf(_this, CsvFormatError.prototype);
-        return _this;
-    }
-    return CsvFormatError;
-}(AbstractFormatError));
-
 /**
  * @@@
  *
@@ -2824,7 +2854,7 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new CsvFormatError(spaceTrim(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim(function (block) { return "\n                                CSV parsing error\n\n                                Error(s) from CSV parsing:\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n                                The CSV data:\n                                ").concat(block(value), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
                                         var _a, _b;
@@ -2862,7 +2892,7 @@ var CsvFormatDefinition = {
                             case 0:
                                 csv = parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
                                 if (csv.errors.length !== 0) {
-                                    throw new CsvFormatError(spaceTrim(function (block) { return "\n                                CSV parsing error\n\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n                            "); }));
+                                    throw new CsvFormatError(spaceTrim(function (block) { return "\n                                CSV parsing error\n\n                                Error(s) from CSV parsing:\n                                ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n                                The CSV data:\n                                ").concat(block(value), "\n                            "); }));
                                 }
                                 return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
                                         var _this = this;
@@ -4922,22 +4952,6 @@ function preparePersona(personaDescription, tools, options) {
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
  */
 
-/**
- * This error indicates that the promptbook can not retrieve knowledge from external sources
- *
- * @public exported from `@promptbook/core`
- */
-var KnowledgeScrapeError = /** @class */ (function (_super) {
-    __extends(KnowledgeScrapeError, _super);
-    function KnowledgeScrapeError(message) {
-        var _this = _super.call(this, message) || this;
-        _this.name = 'KnowledgeScrapeError';
-        Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
-        return _this;
-    }
-    return KnowledgeScrapeError;
-}(Error));
-
 /**
  * @@@
  *
@@ -5057,9 +5071,15 @@ var $Register = /** @class */ (function () {
         var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
         var existingRegistration = this.storage[existingRegistrationIndex];
         if (!existingRegistration) {
+            if (DEFAULT_IS_VERBOSE) {
+                console.warn("[\uD83D\uDCE6] Registering `".concat(packageName, ".").concat(className, "` to `").concat(this.registerName, "`"));
+            }
             this.storage.push(registered);
         }
         else {
+            if (DEFAULT_IS_VERBOSE) {
+                console.warn("[\uD83D\uDCE6] Re-registering `".concat(packageName, ".").concat(className, "` to `").concat(this.registerName, "`"));
+            }
             this.storage[existingRegistrationIndex] = registered;
         }
         return {
@@ -5108,8 +5128,8 @@ var $scrapersRegister = new $Register('scraper_constructors');
  *
  * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
  */
-function $registeredScrapersMessage() {
-    var e_1, _a, e_2, _b;
+function $registeredScrapersMessage(availableScrapers) {
+    var e_1, _a, e_2, _b, e_3, _c;
     /**
      * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
      */
@@ -5121,15 +5141,15 @@ function $registeredScrapersMessage() {
         all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
     };
     try {
-        for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
-            var _e = _d.value, packageName = _e.packageName, className = _e.className, mimeTypes = _e.mimeTypes, documentationUrl = _e.documentationUrl, isAvilableInBrowser = _e.isAvilableInBrowser;
+        for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
+            var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
             _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
         }
     }
     catch (e_1_1) { e_1 = { error: e_1_1 }; }
     finally {
         try {
-            if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
+            if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
         }
         finally { if (e_1) throw e_1.error; }
     }
@@ -5140,18 +5160,31 @@ function $registeredScrapersMessage() {
         all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
     };
     try {
-        for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
-            var _h = _g.value, packageName = _h.packageName, className = _h.className, mimeTypes = _h.mimeTypes, documentationUrl = _h.documentationUrl, isAvilableInBrowser = _h.isAvilableInBrowser;
+        for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
+            var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
             _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
         }
     }
     catch (e_2_1) { e_2 = { error: e_2_1 }; }
     finally {
         try {
-            if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+            if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
         }
         finally { if (e_2) throw e_2.error; }
     }
+    try {
+        for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
+            var metadata_1 = availableScrapers_1_1.value.metadata;
+            all.push(metadata_1);
+        }
+    }
+    catch (e_3_1) { e_3 = { error: e_3_1 }; }
+    finally {
+        try {
+            if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
+        }
+        finally { if (e_3) throw e_3.error; }
+    }
     var metadata = all.map(function (metadata) {
         var isMetadataAviailable = $scrapersMetadataRegister
             .list()
@@ -5165,42 +5198,44 @@ function $registeredScrapersMessage() {
             var packageName = _a.packageName, className = _a.className;
             return metadata.packageName === packageName && metadata.className === className;
         });
-        return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
+        var isAvilableInTools = availableScrapers.some(function (_a) {
+            var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
+            return metadata.packageName === packageName && metadata.className === className;
+        });
+        return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
     });
     if (metadata.length === 0) {
-        return "No scrapers are available";
+        return spaceTrim("\n            **No scrapers are available**\n\n            This is a unexpected behavior, you are probably using some broken version of Promptbook\n            At least there should be available the metadata of the scrapers\n        ");
     }
     return spaceTrim(function (block) { return "\n            Available scrapers are:\n            ".concat(block(metadata
         .map(function (_a, i) {
-        var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
-        var more;
-        // TODO: Use documentationUrl
-        if (just(false)) {
-            more = '';
-        }
-        else if (!isMetadataAviailable && !isInstalled) {
-            // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
-            more = "*(not installed and no metadata, looks like a unexpected behavior)*";
-        }
-        else if (isMetadataAviailable && !isInstalled) {
-            // TODO: [�][�]
-            more = "*(not installed)*";
-        }
-        else if (!isMetadataAviailable && isInstalled) {
-            more = "*(no metadata, looks like a unexpected behavior)*";
-        }
-        else if (isMetadataAviailable && isInstalled) {
-            more = "(installed)";
-        }
-        else {
-            more = "*(unknown state, looks like a unexpected behavior)*";
-        }
+        var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
+        var more = [];
+        // TODO: [🧠] Maybe use `documentationUrl`
+        if (isMetadataAviailable) {
+            more.push("\u2B1C Metadata registered");
+        } // not else
+        if (isInstalled) {
+            more.push("\uD83D\uDFE9 Installed");
+        } // not else
+        if (isAvilableInTools) {
+            more.push("\uD83D\uDFE6 Available in tools");
+        } // not else
+        if (!isMetadataAviailable && isInstalled) {
+            more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
+        } // not else
+        if (!isInstalled && isAvilableInTools) {
+            more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
+        } // not else
         if (!isAvilableInBrowser) {
-            more += " *(not available in browser)*";
+            more.push("Not usable in browser");
         }
-        return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes.join(', '), " ").concat(more);
+        var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
+        return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
+            .map(function (mimeType) { return "\"".concat(mimeType, "\""); })
+            .join(', ')).concat(moreText);
     })
-        .join('\n')), "\n        "); });
+        .join('\n')), "\n\n            Legend:\n            - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n            - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n            - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n        "); });
 }
 /**
  * TODO: [®] DRY Register logic
@@ -5536,57 +5571,75 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
                     _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
                     knowledgePreparedUnflatten = new Array(knowledgeSources.length);
                     return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
-                            var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
-                            var e_1, _c;
-                            return __generator(this, function (_d) {
-                                switch (_d.label) {
+                            var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
+                            var e_1, _a;
+                            return __generator(this, function (_b) {
+                                switch (_b.label) {
                                     case 0:
                                         partialPieces = null;
                                         return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                     case 1:
-                                        sourceHandler = _d.sent();
-                                        _d.label = 2;
+                                        sourceHandler = _b.sent();
+                                        scrapers = arrayableToArray(tools.scrapers);
+                                        _loop_1 = function (scraper) {
+                                            var partialPiecesUnchecked;
+                                            return __generator(this, function (_c) {
+                                                switch (_c.label) {
+                                                    case 0:
+                                                        if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
+                                                        // <- TODO: [🦔] Implement mime-type wildcards
+                                                        ) {
+                                                            return [2 /*return*/, "continue"];
+                                                        }
+                                                        return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                                    case 1:
+                                                        partialPiecesUnchecked = _c.sent();
+                                                        if (partialPiecesUnchecked !== null) {
+                                                            partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
+                                                            return [2 /*return*/, "break"];
+                                                        }
+                                                        console.warn(spaceTrim(function (block) { return "\n                        Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n                        \n                        The source:\n                        > ").concat(block(knowledgeSource.sourceContent
+                                                            .split('\n')
+                                                            .map(function (line) { return "> ".concat(line); })
+                                                            .join('\n')), "\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
+                                                        return [2 /*return*/];
+                                                }
+                                            });
+                                        };
+                                        _b.label = 2;
                                     case 2:
-                                        _d.trys.push([2, 7, 8, 9]);
-                                        _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
-                                        _d.label = 3;
+                                        _b.trys.push([2, 7, 8, 9]);
+                                        scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
+                                        _b.label = 3;
                                     case 3:
-                                        if (!!_b.done) return [3 /*break*/, 6];
-                                        scraper = _b.value;
-                                        if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
-                                        // <- TODO: [🦔] Implement mime-type wildcards
-                                        ) {
-                                            return [3 /*break*/, 5];
-                                        }
-                                        return [4 /*yield*/, scraper.scrape(sourceHandler)];
+                                        if (!!scrapers_1_1.done) return [3 /*break*/, 6];
+                                        scraper = scrapers_1_1.value;
+                                        return [5 /*yield**/, _loop_1(scraper)];
                                     case 4:
-                                        partialPiecesUnchecked = _d.sent();
-                                        if (partialPiecesUnchecked !== null) {
-                                            partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
-                                            // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
+                                        state_1 = _b.sent();
+                                        if (state_1 === "break")
                                             return [3 /*break*/, 6];
-                                        }
-                                        _d.label = 5;
+                                        _b.label = 5;
                                     case 5:
-                                        _b = _a.next();
+                                        scrapers_1_1 = scrapers_1.next();
                                         return [3 /*break*/, 3];
                                     case 6: return [3 /*break*/, 9];
                                     case 7:
-                                        e_1_1 = _d.sent();
+                                        e_1_1 = _b.sent();
                                         e_1 = { error: e_1_1 };
                                         return [3 /*break*/, 9];
                                     case 8:
                                         try {
-                                            if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
+                                            if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
                                         }
                                         finally { if (e_1) throw e_1.error; }
                                         return [7 /*endfinally*/];
                                     case 9:
                                         if (partialPieces === null) {
-                                            throw new KnowledgeScrapeError(spaceTrim(function (block) { return "\n                        Cannot scrape knowledge from source:\n                        \n                        > ".concat(block(knowledgeSource.sourceContent
+                                            throw new KnowledgeScrapeError(spaceTrim(function (block) { return "\n                        Cannot scrape knowledge\n                        \n                        The source:\n                        > ".concat(block(knowledgeSource.sourceContent
                                                 .split('\n')
                                                 .map(function (line) { return "> ".concat(line); })
-                                                .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage()), "\n\n\n                    "); }));
+                                                .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
                                         }
                                         pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
                                                 {
@@ -9082,6 +9135,25 @@ var CallbackInterfaceTools = /** @class */ (function () {
     return CallbackInterfaceTools;
 }());
 
+/**
+ * This error indicates @@@
+ *
+ * @public exported from `@promptbook/core`
+ */
+var BoilerplateError = /** @class */ (function (_super) {
+    __extends(BoilerplateError, _super);
+    function BoilerplateError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'BoilerplateError';
+        Object.setPrototypeOf(_this, BoilerplateError.prototype);
+        return _this;
+    }
+    return BoilerplateError;
+}(Error));
+/**
+ * TODO: @@@ Do not forget to add the error into `0-index.ts` ERRORS
+ */
+
 /**
  * Pretty print an embedding vector for logging
  *
@@ -10405,5 +10477,5 @@ var PrefixStorage = /** @class */ (function () {
     return PrefixStorage;
 }());
 
-export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_CSV_SETTINGS, DEFAULT_EXECUTIONS_CACHE_DIRNAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TITLE, ERRORS, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, PROMPTBOOK_ENGINE_VERSION, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _DocumentScraperMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, cacheLlmTools, collectionToJson, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, pipelineJsonToString, pipelineStringToJson, pipelineStringToJsonSync, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, stringifyPipelineJson, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline };
+export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_CSV_SETTINGS, DEFAULT_EXECUTIONS_CACHE_DIRNAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TITLE, ERRORS, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, PROMPTBOOK_ENGINE_VERSION, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _DocumentScraperMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, cacheLlmTools, collectionToJson, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, pipelineJsonToString, pipelineStringToJson, pipelineStringToJsonSync, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, stringifyPipelineJson, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline };
 //# sourceMappingURL=index.es.js.map