@promptbook/core 0.72.0-6 → 0.72.0-8

package/umd/index.umd.js

@@ -1,13 +1,14 @@
 (function (global, factory) {
-    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('moment')) :
-    typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'moment'], factory) :
-    (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.moment));
-})(this, (function (exports, spaceTrim, prettier, parserHtml, waitasecond, papaparse, hexEncoder, sha256, moment) { 'use strict';
+    typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('fs/promises'), require('path'), require('child_process'), require('colors'), require('waitasecond'), require('crypto-js'), require('crypto-js/enc-hex'), require('papaparse'), require('@mozilla/readability'), require('jsdom'), require('showdown'), require('mime-types'), require('crypto-js/sha256'), require('moment')) :
+    typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'fs/promises', 'path', 'child_process', 'colors', 'waitasecond', 'crypto-js', 'crypto-js/enc-hex', 'papaparse', '@mozilla/readability', 'jsdom', 'showdown', 'mime-types', 'crypto-js/sha256', 'moment'], factory) :
+    (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.promises, global.path, global.child_process, global.colors, global.waitasecond, global.cryptoJs, global.hexEncoder, global.papaparse, global.readability, global.jsdom, global.showdown, global.mimeTypes, global.sha256, global.moment));
+})(this, (function (exports, spaceTrim, prettier, parserHtml, promises, path, child_process, colors, waitasecond, cryptoJs, hexEncoder, papaparse, readability, jsdom, showdown, mimeTypes, sha256, moment) { 'use strict';
 
     function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
     var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
     var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
+    var colors__default = /*#__PURE__*/_interopDefaultLegacy(colors);
     var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
     var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
     var moment__default = /*#__PURE__*/_interopDefaultLegacy(moment);
@@ -16,7 +17,7 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.72.0-5';
+    var PROMPTBOOK_VERSION = '0.72.0-7';
     // TODO: [main] !!!! List here all the versions and annotate + put into script
 
     /*! *****************************************************************************
@@ -688,9 +689,19 @@
     /**
      * Where to store the cache of executions for promptbook CLI
      *
+     * Note: When the folder does not exist, it is created recursively
+     *
      * @public exported from `@promptbook/core`
      */
     var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
+    /**
+     * Where to store the scrape cache
+     *
+     * Note: When the folder does not exist, it is created recursively
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
     /**
      * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
      *
@@ -1220,11 +1231,11 @@
      *     >  ex port function validatePipeline(promptbook: really_unknown): asserts promptbook is PipelineJson {
      */
     /**
-     * TODO: [🐣][main] !!!! Validate that all samples match expectations
-     * TODO: [🐣][🐝][main] !!!! Validate that knowledge is valid (non-void)
-     * TODO: [🐣][main] !!!! Validate that persona can be used only with CHAT variant
-     * TODO: [🐣][main] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
-     * TODO: [🐣][main] !!!! Validate that reserved parameter is not used as joker
+     * TODO: [🧳][main] !!!! Validate that all samples match expectations
+     * TODO: [🧳][🐝][main] !!!! Validate that knowledge is valid (non-void)
+     * TODO: [🧳][main] !!!! Validate that persona can be used only with CHAT variant
+     * TODO: [🧳][main] !!!! Validate that parameter with reserved name not used RESERVED_PARAMETER_NAMES
+     * TODO: [🧳][main] !!!! Validate that reserved parameter is not used as joker
      * TODO: [🧠] Validation not only logic itself but imports around - files and websites and rerefenced pipelines exists
      * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
      */
@@ -1639,6 +1650,22 @@
         // <- [🅱]
     ];
 
+    /**
+     * This error indicates that the promptbook can not retrieve knowledge from external sources
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var KnowledgeScrapeError = /** @class */ (function (_super) {
+        __extends(KnowledgeScrapeError, _super);
+        function KnowledgeScrapeError(message) {
+            var _this = _super.call(this, message) || this;
+            _this.name = 'KnowledgeScrapeError';
+            Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
+            return _this;
+        }
+        return KnowledgeScrapeError;
+    }(Error));
+
     /**
      * Async version of Array.forEach
      *
@@ -1646,6 +1673,7 @@
      * @param options - Options for the function
      * @param callbackfunction - Function to call for each item
      * @public exported from `@promptbook/utils`
+     * @deprecated [🪂] Use queues instead
      */
     function forEachAsync(array, options, callbackfunction) {
         return __awaiter(this, void 0, void 0, function () {
@@ -1715,59 +1743,246 @@
     }
 
     /**
-     * Represents the usage with no resources consumed
+     * This error type indicates that some tools are missing for pipeline execution or preparation
      *
      * @public exported from `@promptbook/core`
      */
-    var ZERO_USAGE = $deepFreeze({
-        price: { value: 0 },
-        input: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
-        },
-        output: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
-        },
-    });
+    var MissingToolsError = /** @class */ (function (_super) {
+        __extends(MissingToolsError, _super);
+        function MissingToolsError(message) {
+            var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n                "); })) || this;
+            _this.name = 'MissingToolsError';
+            Object.setPrototypeOf(_this, MissingToolsError.prototype);
+            return _this;
+        }
+        return MissingToolsError;
+    }(Error));
+
     /**
-     * Represents the usage with unknown resources consumed
+     * Detects if the code is running in a Node.js environment
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    var $isRunningInNode = new Function("\n    try {\n        return this === global;\n    } catch (e) {\n        return false;\n    }\n");
+
+    /**
+     * This error type indicates that you try to use a feature that is not available in the current environment
      *
      * @public exported from `@promptbook/core`
      */
-    var UNCERTAIN_USAGE = $deepFreeze({
-        price: { value: 0, isUncertain: true },
-        input: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
-        },
-        output: {
-            tokensCount: { value: 0, isUncertain: true },
-            charactersCount: { value: 0, isUncertain: true },
-            wordsCount: { value: 0, isUncertain: true },
-            sentencesCount: { value: 0, isUncertain: true },
-            linesCount: { value: 0, isUncertain: true },
-            paragraphsCount: { value: 0, isUncertain: true },
-            pagesCount: { value: 0, isUncertain: true },
-        },
-    });
+    var EnvironmentMismatchError = /** @class */ (function (_super) {
+        __extends(EnvironmentMismatchError, _super);
+        function EnvironmentMismatchError(message) {
+            var _this = _super.call(this, message) || this;
+            _this.name = 'EnvironmentMismatchError';
+            Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
+            return _this;
+        }
+        return EnvironmentMismatchError;
+    }(Error));
 
-    var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n-   Your output format is JSON object\n-   Write just the JSON object, no other text should be present\n-   It contains the following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
+    /**
+     * Normalize options for `execCommand` and `execCommands`
+     *
+     * @private internal utility of `execCommand` and `execCommands`
+     */
+    function execCommandNormalizeOptions(options) {
+        var _a;
+        var _b, _c, _d;
+        var command;
+        var cwd;
+        var crashOnError;
+        var args = [];
+        var timeout;
+        if (typeof options === 'string') {
+            // TODO: [1] DRY default values
+            command = options;
+            cwd = process.cwd();
+            crashOnError = true;
+            timeout = Infinity;
+        }
+        else {
+            /*
+            TODO:
+            if ((options as any).commands !== undefined) {
+                commands = (options as any).commands;
+            } else {
+                commands = [(options as any).command];
+            }
+            */
+            // TODO: [1] DRY default values
+            command = options.command;
+            cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
+            crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
+            timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
+        }
+        // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
+        var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
+            .map(function (_a) {
+            var _b = __read(_a, 1), match = _b[0];
+            return match;
+        })
+            .filter(function (arg) { return arg !== ''; });
+        if (_.length > 1) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            _a = __read(_), command = _a[0], args = _a.slice(1);
+        }
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        if (options.args) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
+        }
+        var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
+        if (['ts-node'].includes(humanReadableCommand)) {
+            humanReadableCommand += " ".concat(args[1]);
+        }
+        return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
+    }
+    // TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
+
+    /**
+     * Run one command in a shell
+     *
+     * Note: There are 2 similar functions in the codebase:
+     * - `$execCommand` which runs a single command
+     * - `$execCommands` which runs multiple commands
+     *
+     * @public exported from `@promptbook/node`
+     */
+    function $execCommand(options) {
+        if (!$isRunningInNode()) {
+            throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
+        }
+        return new Promise(
+        //            <- TODO: [🧱] Implement in a functional (not new Class) way
+        function (resolve, reject) {
+            // eslint-disable-next-line prefer-const
+            var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
+            if (timeout !== Infinity) {
+                // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
+                waitasecond.forTime(timeout).then(function () {
+                    if (crashOnError) {
+                        reject(new Error("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms")));
+                    }
+                    else {
+                        console.warn("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms but continues running"));
+                        resolve('Command exceeded time limit');
+                    }
+                });
+            }
+            if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
+                command = "".concat(command, ".cmd");
+            }
+            // !!!!!! Verbose mode - to all consoles
+            console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
+            try {
+                var commandProcess = child_process.spawn(command, args, { cwd: cwd, shell: true });
+                commandProcess.on('message', function (message) {
+                    console.info({ message: message });
+                });
+                var output_1 = [];
+                commandProcess.stdout.on('data', function (stdout) {
+                    output_1.push(stdout.toString());
+                    console.info(stdout.toString());
+                });
+                commandProcess.stderr.on('data', function (stderr) {
+                    output_1.push(stderr.toString());
+                    if (stderr.toString().trim()) {
+                        console.warn(stderr.toString());
+                    }
+                });
+                var finishWithCode = function (code) {
+                    if (code !== 0) {
+                        if (crashOnError) {
+                            reject(new Error(output_1.join('\n').trim() ||
+                                "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
+                        }
+                        else {
+                            console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
+                            resolve(spaceTrim.spaceTrim(output_1.join('\n')));
+                        }
+                    }
+                    else {
+                        resolve(spaceTrim.spaceTrim(output_1.join('\n')));
+                    }
+                };
+                commandProcess.on('close', finishWithCode);
+                commandProcess.on('exit', finishWithCode);
+                commandProcess.on('disconnect', function () {
+                    // Note: Unexpected disconnection should always result in rejection
+                    reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
+                });
+                commandProcess.on('error', function (error) {
+                    if (crashOnError) {
+                        reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
+                    }
+                    else {
+                        console.warn(error);
+                        resolve(spaceTrim.spaceTrim(output_1.join('\n')));
+                    }
+                });
+            }
+            catch (error) {
+                // Note: Unexpected error in sync code should always result in rejection
+                reject(error);
+            }
+        });
+    }
+    /**
+     * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
+     */
+
+    /**
+     * Checks if the file exists
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
+     *
+     * @private within the repository
+     */
+    function $isFileExisting(filename) {
+        return __awaiter(this, void 0, void 0, function () {
+            var isReadAccessAllowed, isFile;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!$isRunningInNode()) {
+                            throw new EnvironmentMismatchError('Function `$isFileExisting` works only in Node environment.js');
+                        }
+                        return [4 /*yield*/, promises.access(filename, promises.constants.R_OK)
+                                .then(function () { return true; })
+                                .catch(function () { return false; })];
+                    case 1:
+                        isReadAccessAllowed = _a.sent();
+                        if (!isReadAccessAllowed) {
+                            return [2 /*return*/, false];
+                        }
+                        return [4 /*yield*/, promises.stat(filename)
+                                .then(function (fileStat) { return fileStat.isFile(); })
+                                .catch(function () { return false; })];
+                    case 2:
+                        isFile = _a.sent();
+                        return [2 /*return*/, isFile];
+                }
+            });
+        });
+    }
+    /**
+     * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
+     * TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
+     * TODO: [🖇] What about symlinks?
+     */
+
+    /**
+     * Get the file extension from a file name
+     *
+     * @private within the repository
+     */
+    function getFileExtension(value) {
+        var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
+        return match ? match[1].toLowerCase() : null;
+    }
 
     var defaultDiacriticsRemovalMap = [
         {
@@ -2061,10 +2276,6 @@
                     charType = 'NUMBER';
                     normalizedChar = char;
                 }
-                else if (/^\/$/.test(char)) {
-                    charType = 'SLASH';
-                    normalizedChar = char;
-                }
                 else {
                     charType = 'OTHER';
                     normalizedChar = '-';
@@ -2109,6 +2320,32 @@
         return text;
     }
 
+    /**
+     * Tests if given string is valid URL.
+     *
+     * Note: This does not check if the file exists only if the path is valid
+     * @public exported from `@promptbook/utils`
+     */
+    function isValidFilePath(filename) {
+        if (typeof filename !== 'string') {
+            return false;
+        }
+        var filenameSlashes = filename.split('\\').join('/');
+        // Absolute Unix path: /hello.txt
+        if (/^(\/)/i.test(filenameSlashes)) {
+            return true;
+        }
+        // Absolute Windows path: /hello.txt
+        if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
+            return true;
+        }
+        // Relative path: ./hello.txt
+        if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * @@@
      *
@@ -2118,20 +2355,121 @@
      * @public exported from `@promptbook/utils`
      */
     function titleToName(value) {
-        if (value.startsWith('http://') || value.startsWith('https://')) {
-            // TODO: Maybe check against some list unallowed characters
-            return value;
+        if (isValidUrl(value)) {
+            value = value.replace(/^https?:\/\//, '');
+            value = value.replace(/\.html$/, '');
         }
-        if (value.startsWith('./') || value.startsWith('../')) {
-            // TODO: Maybe check against some list unallowed characters
-            return value;
+        else if (isValidFilePath(value)) {
+            value = path.basename(value);
+            // Note: Keeping extension in the name
         }
+        value = value.split('/').join('-');
         value = removeEmojis(value);
         value = normalizeToKebabCase(value);
         // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
         return value;
     }
 
+    /**
+     * @@@
+     *
+     * @private for `FileCacheStorage`
+     */
+    function nameToSubfolderPath(name) {
+        return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
+    }
+
+    /**
+     * Just marks a place of place where should be something implemented
+     * No side effects.
+     *
+     * Note: It can be usefull suppressing eslint errors of unused variables
+     *
+     * @param value any values
+     * @returns void
+     * @private within the repository
+     */
+    function TODO_USE() {
+        var value = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            value[_i] = arguments[_i];
+        }
+    }
+
+    /**
+     * Create a filename for intermediate cache for scrapers
+     *
+     * Note: It also checks if directory exists and creates it if not
+     *
+     * @private as internal utility for scrapers
+     */
+    function getScraperIntermediateSource(source, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        sourceFilename = source.filename, url = source.url;
+                        rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
+                        hash = cryptoJs.SHA256(
+                        //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
+                        hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
+                            .toString( /* hex */)
+                            .substring(0, 20);
+                        semanticName = normalizeToKebabCase(titleToName((sourceFilename || url || '').split('intermediate').join(''))).substring(0, 20);
+                        pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
+                        name = pieces.join('-').split('--').join('-');
+                        // <- TODO: Use MAX_FILENAME_LENGTH
+                        TODO_USE(rootDirname); // <- TODO: !!!!!!
+                        cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
+                            cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
+                            .join('/') +
+                            '.' +
+                            extension;
+                        return [4 /*yield*/, promises.mkdir(path.dirname(cacheFilename), { recursive: true })];
+                    case 1:
+                        _a.sent();
+                        isDestroyed = true;
+                        fileHandler = {
+                            filename: cacheFilename,
+                            get isDestroyed() {
+                                return isDestroyed;
+                            },
+                            destroy: function () {
+                                return __awaiter(this, void 0, void 0, function () {
+                                    return __generator(this, function (_a) {
+                                        switch (_a.label) {
+                                            case 0:
+                                                if (!isCacheCleaned) return [3 /*break*/, 2];
+                                                if (isVerbose) {
+                                                    console.info('legacyDocumentScraper: Clening cache');
+                                                }
+                                                return [4 /*yield*/, promises.rm(cacheFilename)];
+                                            case 1:
+                                                _a.sent();
+                                                _a.label = 2;
+                                            case 2:
+                                                isDestroyed = true;
+                                                return [2 /*return*/];
+                                        }
+                                    });
+                                });
+                            },
+                        };
+                        return [2 /*return*/, fileHandler];
+                }
+            });
+        });
+    }
+    /**
+     * Note: Not using `FileCacheStorage` for two reasons:
+     * 1) Need to store more than serialized JSONs
+     * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
+     * TODO: [🐱‍🐉][🧠] Make some smart crop
+     */
+
+    var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n-   Your output format is JSON object\n-   Write just the JSON object, no other text should be present\n-   It contains the following keys:\n    -   `modelName`: The name of the model to use\n    -   `systemMessage`: The system message to provide context to the model\n    -   `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
+
     /**
      * This error indicates errors during the execution of the pipeline
      *
@@ -2164,22 +2502,6 @@
         return CollectionError;
     }(Error));
 
-    /**
-     * This error type indicates that you try to use a feature that is not available in the current environment
-     *
-     * @public exported from `@promptbook/core`
-     */
-    var EnvironmentMismatchError = /** @class */ (function (_super) {
-        __extends(EnvironmentMismatchError, _super);
-        function EnvironmentMismatchError(message) {
-            var _this = _super.call(this, message) || this;
-            _this.name = 'EnvironmentMismatchError';
-            Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
-            return _this;
-        }
-        return EnvironmentMismatchError;
-    }(Error));
-
     /**
      * This error occurs when some expectation is not met in the execution of the pipeline
      *
@@ -2679,19 +3001,72 @@
      */
 
     /**
-     * Function `addUsage` will add multiple usages into one
-     *
-     * Note: If you provide 0 values, it returns ZERO_USAGE
+     * Represents the usage with no resources consumed
      *
      * @public exported from `@promptbook/core`
      */
-    function addUsage() {
-        var usageItems = [];
-        for (var _i = 0; _i < arguments.length; _i++) {
-            usageItems[_i] = arguments[_i];
-        }
-        return usageItems.reduce(function (acc, item) {
-            var e_1, _a, e_2, _b;
+    var ZERO_USAGE = $deepFreeze({
+        price: { value: 0 },
+        input: {
+            tokensCount: { value: 0 },
+            charactersCount: { value: 0 },
+            wordsCount: { value: 0 },
+            sentencesCount: { value: 0 },
+            linesCount: { value: 0 },
+            paragraphsCount: { value: 0 },
+            pagesCount: { value: 0 },
+        },
+        output: {
+            tokensCount: { value: 0 },
+            charactersCount: { value: 0 },
+            wordsCount: { value: 0 },
+            sentencesCount: { value: 0 },
+            linesCount: { value: 0 },
+            paragraphsCount: { value: 0 },
+            pagesCount: { value: 0 },
+        },
+    });
+    /**
+     * Represents the usage with unknown resources consumed
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var UNCERTAIN_USAGE = $deepFreeze({
+        price: { value: 0, isUncertain: true },
+        input: {
+            tokensCount: { value: 0, isUncertain: true },
+            charactersCount: { value: 0, isUncertain: true },
+            wordsCount: { value: 0, isUncertain: true },
+            sentencesCount: { value: 0, isUncertain: true },
+            linesCount: { value: 0, isUncertain: true },
+            paragraphsCount: { value: 0, isUncertain: true },
+            pagesCount: { value: 0, isUncertain: true },
+        },
+        output: {
+            tokensCount: { value: 0, isUncertain: true },
+            charactersCount: { value: 0, isUncertain: true },
+            wordsCount: { value: 0, isUncertain: true },
+            sentencesCount: { value: 0, isUncertain: true },
+            linesCount: { value: 0, isUncertain: true },
+            paragraphsCount: { value: 0, isUncertain: true },
+            pagesCount: { value: 0, isUncertain: true },
+        },
+    });
+
+    /**
+     * Function `addUsage` will add multiple usages into one
+     *
+     * Note: If you provide 0 values, it returns ZERO_USAGE
+     *
+     * @public exported from `@promptbook/core`
+     */
+    function addUsage() {
+        var usageItems = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            usageItems[_i] = arguments[_i];
+        }
+        return usageItems.reduce(function (acc, item) {
+            var e_1, _a, e_2, _b;
             var _c;
             acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
             try {
@@ -2955,23 +3330,6 @@
         return union;
     }
 
-    /**
-     * Just marks a place of place where should be something implemented
-     * No side effects.
-     *
-     * Note: It can be usefull suppressing eslint errors of unused variables
-     *
-     * @param value any values
-     * @returns void
-     * @private within the repository
-     */
-    function TODO_USE() {
-        var value = [];
-        for (var _i = 0; _i < arguments.length; _i++) {
-            value[_i] = arguments[_i];
-        }
-    }
-
     /**
      * This error indicates problems parsing the format value
      *
@@ -3015,7 +3373,7 @@
      */
     var MANDATORY_CSV_SETTINGS = Object.freeze({
         header: true,
-        // encoding: 'utf8',
+        // encoding: 'utf-8',
     });
 
     /**
@@ -4469,25 +4827,26 @@
      */
     function executePipeline(options) {
         return __awaiter(this, void 0, void 0, function () {
-            var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, isVerbose, preparedPipeline, llmTools, errors, warnings, executionReport, isReturned, _a, _b, parameter, e_1_1, _loop_1, _c, _d, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
-            var e_1, _e, e_2, _f;
-            return __generator(this, function (_g) {
-                switch (_g.label) {
+            var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, settings, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, llmTools, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
+            var e_1, _f, e_2, _g;
+            return __generator(this, function (_h) {
+                switch (_h.label) {
                     case 0:
                         inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, settings = options.settings;
-                        maxParallelCount = settings.maxParallelCount, isVerbose = settings.isVerbose;
+                        maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
                         preparedPipeline = options.preparedPipeline;
                         llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
                         if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
                         return [4 /*yield*/, preparePipeline(pipeline, {
                                 llmTools: llmTools,
+                                rootDirname: rootDirname,
                                 isVerbose: isVerbose,
                                 maxParallelCount: maxParallelCount,
                             })];
                     case 1:
-                        preparedPipeline = _g.sent();
+                        preparedPipeline = _h.sent();
                         setPreparedPipeline(preparedPipeline);
-                        _g.label = 2;
+                        _h.label = 2;
                     case 2:
                         errors = [];
                         warnings = [];
@@ -4500,17 +4859,17 @@
                             promptExecutions: [],
                         };
                         isReturned = false;
-                        _g.label = 3;
+                        _h.label = 3;
                     case 3:
-                        _g.trys.push([3, 9, 10, 11]);
-                        _a = __values(preparedPipeline.parameters.filter(function (_a) {
+                        _h.trys.push([3, 9, 10, 11]);
+                        _b = __values(preparedPipeline.parameters.filter(function (_a) {
                             var isInput = _a.isInput;
                             return isInput;
-                        })), _b = _a.next();
-                        _g.label = 4;
+                        })), _c = _b.next();
+                        _h.label = 4;
                     case 4:
-                        if (!!_b.done) return [3 /*break*/, 8];
-                        parameter = _b.value;
+                        if (!!_c.done) return [3 /*break*/, 8];
+                        parameter = _c.value;
                         if (!(inputParameters[parameter.name] === undefined)) return [3 /*break*/, 7];
                         isReturned = true;
                         if (!(onProgress !== undefined)) return [3 /*break*/, 6];
@@ -4518,8 +4877,8 @@
                         return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
                     case 5:
                         // Note: Wait a short time to prevent race conditions
-                        _g.sent();
-                        _g.label = 6;
+                        _h.sent();
+                        _h.label = 6;
                     case 6: return [2 /*return*/, $asDeeplyFrozenSerializableJson("Unuccessful PipelineExecutorResult (with missing parameter {".concat(parameter.name, "}) PipelineExecutorResult"), {
                             isSuccessful: false,
                             errors: __spreadArray([
@@ -4532,24 +4891,24 @@
                             preparedPipeline: preparedPipeline,
                         })];
                     case 7:
-                        _b = _a.next();
+                        _c = _b.next();
                         return [3 /*break*/, 4];
                     case 8: return [3 /*break*/, 11];
                     case 9:
-                        e_1_1 = _g.sent();
+                        e_1_1 = _h.sent();
                         e_1 = { error: e_1_1 };
                         return [3 /*break*/, 11];
                     case 10:
                         try {
-                            if (_b && !_b.done && (_e = _a.return)) _e.call(_a);
+                            if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
                         }
                         finally { if (e_1) throw e_1.error; }
                         return [7 /*endfinally*/];
                     case 11:
                         _loop_1 = function (parameterName) {
                             var parameter;
-                            return __generator(this, function (_h) {
-                                switch (_h.label) {
+                            return __generator(this, function (_j) {
+                                switch (_j.label) {
                                     case 0:
                                         parameter = preparedPipeline.parameters.find(function (_a) {
                                             var name = _a.name;
@@ -4566,8 +4925,8 @@
                                         return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
                                     case 2:
                                         // Note: Wait a short time to prevent race conditions
-                                        _h.sent();
-                                        _h.label = 3;
+                                        _j.sent();
+                                        _j.label = 3;
                                     case 3: return [2 /*return*/, { value: $asDeeplyFrozenSerializableJson(spaceTrim.spaceTrim(function (block) { return "\n                        Unuccessful PipelineExecutorResult (with extra parameter {".concat(parameter.name, "}) PipelineExecutorResult\n\n                        ").concat(block(pipelineIdentification), "\n                    "); }), {
                                                 isSuccessful: false,
                                                 errors: __spreadArray([
@@ -4583,39 +4942,39 @@
                                 }
                             });
                         };
-                        _g.label = 12;
+                        _h.label = 12;
                     case 12:
-                        _g.trys.push([12, 17, 18, 19]);
-                        _c = __values(Object.keys(inputParameters)), _d = _c.next();
-                        _g.label = 13;
+                        _h.trys.push([12, 17, 18, 19]);
+                        _d = __values(Object.keys(inputParameters)), _e = _d.next();
+                        _h.label = 13;
                     case 13:
-                        if (!!_d.done) return [3 /*break*/, 16];
-                        parameterName = _d.value;
+                        if (!!_e.done) return [3 /*break*/, 16];
+                        parameterName = _e.value;
                         return [5 /*yield**/, _loop_1(parameterName)];
                     case 14:
-                        state_1 = _g.sent();
+                        state_1 = _h.sent();
                         if (typeof state_1 === "object")
                             return [2 /*return*/, state_1.value];
-                        _g.label = 15;
+                        _h.label = 15;
                     case 15:
-                        _d = _c.next();
+                        _e = _d.next();
                         return [3 /*break*/, 13];
                     case 16: return [3 /*break*/, 19];
                     case 17:
-                        e_2_1 = _g.sent();
+                        e_2_1 = _h.sent();
                         e_2 = { error: e_2_1 };
                         return [3 /*break*/, 19];
                     case 18:
                         try {
-                            if (_d && !_d.done && (_f = _c.return)) _f.call(_c);
+                            if (_e && !_e.done && (_g = _d.return)) _g.call(_d);
                         }
                         finally { if (e_2) throw e_2.error; }
                         return [7 /*endfinally*/];
                     case 19:
                         parametersToPass = inputParameters;
-                        _g.label = 20;
+                        _h.label = 20;
                     case 20:
-                        _g.trys.push([20, 25, , 28]);
+                        _h.trys.push([20, 25, , 28]);
                         resovedParameterNames_1 = preparedPipeline.parameters
                             .filter(function (_a) {
                             var isInput = _a.isInput;
@@ -4630,8 +4989,8 @@
                         loopLimit = LOOP_LIMIT;
                         _loop_2 = function () {
                             var currentTemplate, work_1;
-                            return __generator(this, function (_j) {
-                                switch (_j.label) {
+                            return __generator(this, function (_k) {
+                                switch (_k.label) {
                                     case 0:
                                         if (loopLimit-- < 0) {
                                             // Note: Really UnexpectedError not LimitReachedError - this should be catched during validatePipeline
@@ -4657,7 +5016,7 @@
                                         if (!!currentTemplate) return [3 /*break*/, 3];
                                         /* [🤹‍♂️] */ return [4 /*yield*/, Promise.race(resolving_1)];
                                     case 2:
-                                        /* [🤹‍♂️] */ _j.sent();
+                                        /* [🤹‍♂️] */ _k.sent();
                                         return [3 /*break*/, 4];
                                     case 3:
                                         unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
@@ -4692,24 +5051,24 @@
                                         // <- Note: Errors are catched here [3]
                                         //    TODO: BUT if in multiple templates are errors, only the first one is catched so maybe we should catch errors here and save them to errors array here
                                         resolving_1.push(work_1);
-                                        _j.label = 4;
+                                        _k.label = 4;
                                     case 4: return [2 /*return*/];
                                 }
                             });
                         };
-                        _g.label = 21;
+                        _h.label = 21;
                     case 21:
                         if (!(unresovedTemplates_1.length > 0)) return [3 /*break*/, 23];
                         return [5 /*yield**/, _loop_2()];
                     case 22:
-                        _g.sent();
+                        _h.sent();
                         return [3 /*break*/, 21];
                     case 23: return [4 /*yield*/, Promise.all(resolving_1)];
                     case 24:
-                        _g.sent();
+                        _h.sent();
                         return [3 /*break*/, 28];
                     case 25:
-                        error_1 = _g.sent();
+                        error_1 = _h.sent();
                         if (!(error_1 instanceof Error)) {
                             throw error_1;
                         }
@@ -4729,8 +5088,8 @@
                         return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
                     case 26:
                         // Note: Wait a short time to prevent race conditions
-                        _g.sent();
-                        _g.label = 27;
+                        _h.sent();
+                        _h.label = 27;
                     case 27: return [2 /*return*/, $asDeeplyFrozenSerializableJson('Unuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult', {
                             isSuccessful: false,
                             errors: __spreadArray([error_1], __read(errors), false).map(serializeError),
@@ -4757,8 +5116,8 @@
                         return [4 /*yield*/, waitasecond.forTime(IMMEDIATE_TIME)];
                     case 29:
                         // Note: Wait a short time to prevent race conditions
-                        _g.sent();
-                        _g.label = 30;
+                        _h.sent();
+                        _h.label = 30;
                     case 30: return [2 /*return*/, $asDeeplyFrozenSerializableJson('Successful PipelineExecutorResult', {
                             isSuccessful: true,
                             errors: errors.map(serializeError),
@@ -4770,211 +5129,884 @@
                         })];
                 }
             });
-        });
-    }
+        });
+    }
+    /**
+     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     */
+
+    /**
+     * Creates executor function from pipeline and execution tools.
+     *
+     * @returns The executor function
+     * @throws {PipelineLogicError} on logical error in the pipeline
+     * @public exported from `@promptbook/core`
+     */
+    function createPipelineExecutor(options) {
+        var _this = this;
+        var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
+        var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
+        validatePipeline(pipeline);
+        var pipelineIdentification = (function () {
+            // Note: This is a 😐 implementation of [🚞]
+            var _ = [];
+            if (pipeline.sourceFile !== undefined) {
+                _.push("File: ".concat(pipeline.sourceFile));
+            }
+            if (pipeline.pipelineUrl !== undefined) {
+                _.push("Url: ".concat(pipeline.pipelineUrl));
+            }
+            return _.join('\n');
+        })();
+        var preparedPipeline;
+        if (isPipelinePrepared(pipeline)) {
+            preparedPipeline = pipeline;
+        }
+        else if (isNotPreparedWarningSupressed !== true) {
+            console.warn(spaceTrim.spaceTrim(function (block) { return "\n                    Pipeline is not prepared\n\n                    ".concat(block(pipelineIdentification), "\n\n                    It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n                    But it is recommended to prepare the pipeline during collection preparation\n\n                    @see more at https://ptbk.io/prepare-pipeline\n                "); }));
+        }
+        var runCount = 0;
+        var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                runCount++;
+                return [2 /*return*/, /* not await */ executePipeline({
+                        pipeline: pipeline,
+                        preparedPipeline: preparedPipeline,
+                        setPreparedPipeline: function (newPreparedPipeline) {
+                            preparedPipeline = newPreparedPipeline;
+                        },
+                        inputParameters: inputParameters,
+                        tools: tools,
+                        onProgress: onProgress,
+                        pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(pipelineIdentification), "\n                    ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n                "); }),
+                        settings: {
+                            maxExecutionAttempts: maxExecutionAttempts,
+                            maxParallelCount: maxParallelCount,
+                            csvSettings: csvSettings,
+                            isVerbose: isVerbose,
+                            isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
+                            rootDirname: rootDirname,
+                        },
+                    })];
+            });
+        }); };
+        return pipelineExecutor;
+    }
+    /**
+     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     */
+
+    /**
+     * Scraper for markdown files
+     *
+     * @see `documentationUrl` for more details
+     * @public exported from `@promptbook/core`
+     */
+    var markdownScraper = {
+        /**
+         * Mime types that this scraper can handle
+         */
+        mimeTypes: ['text/markdown', 'text/plain'],
+        /**
+         * Link to documentation
+         */
+        documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+        /**
+         * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
+         */
+        scrape: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, knowledgeContent, result, outputParameters, knowledgePiecesRaw, knowledgeTextPieces, knowledge;
+                var _f, _g, _h;
+                var _this = this;
+                return __generator(this, function (_j) {
+                    switch (_j.label) {
+                        case 0:
+                            llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
+                            if (llmTools === undefined) {
+                                throw new MissingToolsError('LLM tools are required for scraping external files');
+                                // <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
+                            }
+                            TODO_USE(maxParallelCount); // <- [🪂]
+                            collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
+                            _c = createPipelineExecutor;
+                            _f = {};
+                            return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
+                        case 1:
+                            prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
+                                    _f.tools = {
+                                        llm: llmTools,
+                                    },
+                                    _f)]);
+                            _d = createPipelineExecutor;
+                            _g = {};
+                            return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
+                        case 2:
+                            prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
+                                    _g.tools = {
+                                        llm: llmTools,
+                                    },
+                                    _g)]);
+                            _e = createPipelineExecutor;
+                            _h = {};
+                            return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
+                        case 3:
+                            prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
+                                    _h.tools = {
+                                        llm: llmTools,
+                                    },
+                                    _h)]);
+                            return [4 /*yield*/, source.asText()];
+                        case 4:
+                            knowledgeContent = _j.sent();
+                            return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
+                        case 5:
+                            result = _j.sent();
+                            assertsExecutionSuccessful(result);
+                            outputParameters = result.outputParameters;
+                            knowledgePiecesRaw = outputParameters.knowledgePieces;
+                            knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
+                            //                                                               <- TODO: [main] !!!!! Smarter split and filter out empty pieces
+                            if (isVerbose) {
+                                console.info('knowledgeTextPieces:', knowledgeTextPieces);
+                            }
+                            return [4 /*yield*/, Promise.all(
+                                // TODO: [🪂] !! Do not send all at once but in chunks
+                                knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
+                                    var name, title, knowledgePieceContent, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
+                                    return __generator(this, function (_c) {
+                                        switch (_c.label) {
+                                            case 0:
+                                                name = "piece-".concat(i);
+                                                title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
+                                                knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
+                                                keywords = [];
+                                                index = [];
+                                                _c.label = 1;
+                                            case 1:
+                                                _c.trys.push([1, 7, , 8]);
+                                                return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
+                                            case 2:
+                                                titleResult = _c.sent();
+                                                _a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
+                                                title = spaceTrim__default["default"](titleRaw) /* <- TODO: Maybe do in pipeline */;
+                                                name = titleToName(title);
+                                                return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
+                                            case 3:
+                                                keywordsResult = _c.sent();
+                                                _b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
+                                                keywords = (keywordsRaw || '')
+                                                    .split(',')
+                                                    .map(function (keyword) { return keyword.trim(); })
+                                                    .filter(function (keyword) { return keyword !== ''; });
+                                                if (isVerbose) {
+                                                    console.info("Keywords for \"".concat(title, "\":"), keywords);
+                                                }
+                                                if (!!llmTools.callEmbeddingModel) return [3 /*break*/, 4];
+                                                // TODO: [🟥] Detect browser / node and make it colorfull
+                                                console.error('No callEmbeddingModel function provided');
+                                                return [3 /*break*/, 6];
+                                            case 4: return [4 /*yield*/, llmTools.callEmbeddingModel({
+                                                    title: "Embedding for ".concat(title) /* <- Note: No impact on embedding result itself, just for logging */,
+                                                    parameters: {},
+                                                    content: knowledgePieceContent,
+                                                    modelRequirements: {
+                                                        modelVariant: 'EMBEDDING',
+                                                    },
+                                                })];
+                                            case 5:
+                                                embeddingResult = _c.sent();
+                                                index.push({
+                                                    modelName: embeddingResult.modelName,
+                                                    position: embeddingResult.content,
+                                                });
+                                                _c.label = 6;
+                                            case 6: return [3 /*break*/, 8];
+                                            case 7:
+                                                error_1 = _c.sent();
+                                                // Note: Here is expected error:
+                                                //     > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
+                                                if (!(error_1 instanceof PipelineExecutionError)) {
+                                                    throw error_1;
+                                                }
+                                                // TODO: [🟥] Detect browser / node and make it colorfull
+                                                console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
+                                                return [3 /*break*/, 8];
+                                            case 8: return [2 /*return*/, {
+                                                    name: name,
+                                                    title: title,
+                                                    content: knowledgePieceContent,
+                                                    keywords: keywords,
+                                                    index: index,
+                                                    // <- TODO: [☀] sources,
+                                                }];
+                                        }
+                                    });
+                                }); }))];
+                        case 6:
+                            knowledge = _j.sent();
+                            return [2 /*return*/, knowledge];
+                    }
+                });
+            });
+        },
+    } /* TODO: [🦷] as const */;
+    /**
+     * TODO: [🦖] Make some system for putting scrapers to separete packages
+     * TODO: [🪂] Do it in parallel 11:11
+     * TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+     */
+
+    /**
+     * Scraper of .docx and .odt files
+     *
+     * @see `documentationUrl` for more details
+     * @public exported from `@promptbook/core`
+     */
+    var documentScraper = {
+        /**
+         * Mime types that this scraper can handle
+         */
+        mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
+        /**
+         * Link to documentation
+         */
+        documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+        /**
+         * Convert the `.docx` or `.odt`  to `.md` file and returns intermediate source
+         *
+         * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+         */
+        $convert: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, command_1;
+                return __generator(this, function (_e) {
+                    switch (_e.label) {
+                        case 0:
+                            _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
+                            if (!$isRunningInNode()) {
+                                throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
+                            }
+                            if (externalProgramsPaths.pandocPath === undefined) {
+                                throw new MissingToolsError('Pandoc is required for scraping .docx files');
+                            }
+                            if (source.filename === null) {
+                                // TODO: [🧠] Maybe save file as temporary
+                                throw new KnowledgeScrapeError('When parsing .docx file, it must be real file in the file system');
+                            }
+                            extension = getFileExtension(source.filename);
+                            return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                    rootDirname: rootDirname,
+                                    cacheDirname: cacheDirname,
+                                    isCacheCleaned: isCacheCleaned,
+                                    extension: 'md',
+                                    isVerbose: isVerbose,
+                                })];
+                        case 1:
+                            cacheFilehandler = _e.sent();
+                            return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        case 2:
+                            if (!!(_e.sent())) return [3 /*break*/, 5];
+                            command_1 = "\"".concat(externalProgramsPaths.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
+                            // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
+                            return [4 /*yield*/, $execCommand(command_1)];
+                        case 3:
+                            // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
+                            _e.sent();
+                            return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        case 4:
+                            // Note: [0]
+                            if (!(_e.sent())) {
+                                throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n                    File that was supposed to be created by Pandoc does not exist for unknown reason\n\n                    Expected file:\n                    ".concat(block(cacheFilehandler.filename), "\n\n                    Command:\n                    > ").concat(block(command_1), "\n\n                "); }));
+                            }
+                            _e.label = 5;
+                        case 5: return [2 /*return*/, cacheFilehandler];
+                    }
+                });
+            });
+        },
+        /**
+         * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+         */
+        scrape: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var cacheFilehandler, markdownSource, knowledge;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, documentScraper.$convert(source, options)];
+                        case 1:
+                            cacheFilehandler = _a.sent();
+                            markdownSource = {
+                                source: source.source,
+                                filename: cacheFilehandler.filename,
+                                url: null,
+                                mimeType: 'text/markdown',
+                                asText: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, promises.readFile(cacheFilehandler.filename, 'utf-8')];
+                                                case 1: 
+                                                // Note: [0] In $convert we check that the file exists
+                                                return [2 /*return*/, _a.sent()];
+                                            }
+                                        });
+                                    });
+                                },
+                                asJson: function () {
+                                    throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+                                },
+                                asBlob: function () {
+                                    throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
+                                },
+                            };
+                            knowledge = markdownScraper.scrape(markdownSource, options);
+                            return [4 /*yield*/, cacheFilehandler.destroy()];
+                        case 2:
+                            _a.sent();
+                            return [2 /*return*/, knowledge];
+                    }
+                });
+            });
+        },
+    } /* TODO: [🦷] as const */;
+    /**
+     * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+     * TODO: [🦖] Make some system for putting scrapers to separete packages
+     * TODO: [🪂] Do it in parallel 11:11
+     * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+     */
+
+    /**
+     * Scraper for .docx files
+     *
+     * @see `documentationUrl` for more details
+     * @public exported from `@promptbook/core`
+     */
+    var legacyDocumentScraper = {
+        /**
+         * Mime types that this scraper can handle
+         */
+        mimeTypes: ['application/msword', 'text/rtf'],
+        /**
+         * Link to documentation
+         */
+        documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+        /**
+         * Convert the `.doc` or `.rtf`  to `.doc` file and returns intermediate source
+         *
+         * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+         */
+        $convert: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
+                return __generator(this, function (_e) {
+                    switch (_e.label) {
+                        case 0:
+                            _a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
+                            if (!$isRunningInNode()) {
+                                throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
+                            }
+                            if (externalProgramsPaths.libreOfficePath === undefined) {
+                                throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
+                            }
+                            if (source.filename === null) {
+                                // TODO: [🧠] Maybe save file as temporary
+                                throw new KnowledgeScrapeError('When parsing .doc or .rtf file, it must be real file in the file system');
+                            }
+                            extension = getFileExtension(source.filename);
+                            return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                    rootDirname: rootDirname,
+                                    cacheDirname: cacheDirname,
+                                    isCacheCleaned: isCacheCleaned,
+                                    extension: 'docx',
+                                    isVerbose: isVerbose,
+                                })];
+                        case 1:
+                            cacheFilehandler = _e.sent();
+                            if (isVerbose) {
+                                console.info("documentScraper: Converting .".concat(extension, " -> .docx"));
+                            }
+                            return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        case 2:
+                            if (!!(_e.sent())) return [3 /*break*/, 8];
+                            documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
+                                .split('\\')
+                                .join('/');
+                            command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\"  --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
+                            // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
+                            return [4 /*yield*/, $execCommand(command_1)];
+                        case 3:
+                            // TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
+                            _e.sent();
+                            return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
+                        case 4:
+                            files_1 = _e.sent();
+                            if (files_1.length !== 1) {
+                                throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n                        Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n                        The temporary folder:\n                        ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n                        Command:\n                        > ").concat(block(command_1), "\n                    "); }));
+                            }
+                            file = files_1[0];
+                            return [4 /*yield*/, promises.rename(path.join(documentSourceOutdirPathForLibreOffice_1, file), cacheFilehandler.filename)];
+                        case 5:
+                            _e.sent();
+                            return [4 /*yield*/, promises.rmdir(documentSourceOutdirPathForLibreOffice_1)];
+                        case 6:
+                            _e.sent();
+                            return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
+                        case 7:
+                            if (!(_e.sent())) {
+                                throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n                            File that was supposed to be created by LibreOffice does not exist for unknown reason\n\n                            Expected file:\n                            ".concat(block(cacheFilehandler.filename), "\n\n                            The temporary folder:\n                            ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n                            Command:\n                            > ").concat(block(command_1), "\n\n                        "); }));
+                            }
+                            _e.label = 8;
+                        case 8: return [2 /*return*/, cacheFilehandler];
+                    }
+                });
+            });
+        },
+        /**
+         * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
+         */
+        scrape: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var cacheFilehandler, markdownSource, knowledge;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, legacyDocumentScraper.$convert(source, options)];
+                        case 1:
+                            cacheFilehandler = _a.sent();
+                            markdownSource = {
+                                source: source.source,
+                                filename: cacheFilehandler.filename,
+                                url: null,
+                                mimeType: 'text/markdown',
+                                asText: function () {
+                                    throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asText`');
+                                },
+                                asJson: function () {
+                                    throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
+                                },
+                                asBlob: function () {
+                                    throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
+                                },
+                            };
+                            knowledge = documentScraper.scrape(markdownSource, options);
+                            return [4 /*yield*/, cacheFilehandler.destroy()];
+                        case 2:
+                            _a.sent();
+                            return [2 /*return*/, knowledge];
+                    }
+                });
+            });
+        },
+    } /* TODO: [🦷] as const */;
+    /**
+     * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+     * TODO: [🦖] Make some system for putting scrapers to separete packages
+     * TODO: [🪂] Do it in parallel 11:11
+     * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+     */
+
+    /**
+     * Scraper for .docx files
+     *
+     * @see `documentationUrl` for more details
+     * @public exported from `@promptbook/core`
+     */
+    var pdfScraper = {
+        /**
+         * Mime types that this scraper can handle
+         */
+        mimeTypes: ['application/pdf'],
+        /**
+         * Link to documentation
+         */
+        documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+        /**
+         * Converts the `.pdf` file to `.md` file and returns intermediate source
+         */
+        $convert: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                return __generator(this, function (_a) {
+                    TODO_USE(source);
+                    TODO_USE(options);
+                    throw new NotYetImplementedError('PDF conversion not yet implemented');
+                });
+            });
+        },
+        /**
+         * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
+         */
+        scrape: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                return __generator(this, function (_a) {
+                    TODO_USE(source);
+                    TODO_USE(options);
+                    /*
+                    const {
+                        externalProgramsPaths = {},
+                        cacheDirname = SCRAPE_CACHE_DIRNAME,
+                        isCacheCleaned = false,
+                        isVerbose = IS_VERBOSE,
+                    } = options;
+                    */
+                    throw new NotYetImplementedError('PDF scraping not yet implemented');
+                });
+            });
+        },
+    } /* TODO: [🦷] as const */;
+    /**
+     * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+     * TODO: [🦖] Make some system for putting scrapers to separete packages
+     * TODO: [🪂] Do it in parallel 11:11
+     * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+     */
+
+    /**
+     * A converter instance that uses showdown and highlight extensions
+     *
+     * @type {Converter}
+     * @private for markdown and html knowledge scrapers
+     */
+    var markdownConverter = new showdown.Converter({
+        flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
+        /*
+        > import showdownHighlight from 'showdown-highlight';
+        > extensions: [
+        >     showdownHighlight({
+        >         // Whether to add the classes to the <pre> tag, default is false
+        >         pre: true,
+        >         // Whether to use hljs' auto language detection, default is true
+        >         auto_detection: true,
+        >     }),
+        > ],
+        */
+    });
+    /**
+     * TODO: !!!!!! Figure out better name not to confuse with `Converter`
+     * TODO: !!!!!! Lazy-make converter
+     */
+
+    /**
+     * Scraper for .docx files
+     *
+     * @see `documentationUrl` for more details
+     * @public exported from `@promptbook/core`
+     */
+    var websiteScraper = {
+        /**
+         * Mime types that this scraper can handle
+         */
+        mimeTypes: ['text/html'],
+        /**
+         * Link to documentation
+         */
+        documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+        /**
+         * Convert the website  to `.md` file and returns intermediate source
+         *
+         * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+         */
+        $convert: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var 
+                // TODO: [🧠] Maybe in node use headless browser not just JSDOM
+                // externalProgramsPaths = {},
+                rootDirname, _a, cacheDirname, _b, isCacheCleaned, _c, isVerbose, jsdom$1, _d, reader, article, html, i, cacheFilehandler, markdown;
+                return __generator(this, function (_e) {
+                    switch (_e.label) {
+                        case 0:
+                            rootDirname = options.rootDirname, _a = options.cacheDirname, cacheDirname = _a === void 0 ? SCRAPE_CACHE_DIRNAME : _a, _b = options.isCacheCleaned, isCacheCleaned = _b === void 0 ? false : _b, _c = options.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
+                            // TODO: !!!!!! Does this work in browser? Make it work.
+                            if (source.url === null) {
+                                throw new KnowledgeScrapeError('Website scraper requires URL');
+                            }
+                            _d = jsdom.JSDOM.bind;
+                            return [4 /*yield*/, source.asText()];
+                        case 1:
+                            jsdom$1 = new (_d.apply(jsdom.JSDOM, [void 0, _e.sent(), {
+                                    url: source.url,
+                                }]))();
+                            reader = new readability.Readability(jsdom$1.window.document);
+                            article = reader.parse();
+                            console.log(article);
+                            return [4 /*yield*/, waitasecond.forTime(10000)];
+                        case 2:
+                            _e.sent();
+                            html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
+                            // Note: Unwrap html such as it is convertable by `markdownConverter`
+                            for (i = 0; i < 2; i++) {
+                                html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
+                            }
+                            if (html.includes('<div')) {
+                                html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
+                            }
+                            return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                    rootDirname: rootDirname,
+                                    cacheDirname: cacheDirname,
+                                    isCacheCleaned: isCacheCleaned,
+                                    extension: 'html',
+                                    isVerbose: isVerbose,
+                                })];
+                        case 3:
+                            cacheFilehandler = _e.sent();
+                            return [4 /*yield*/, promises.writeFile(cacheFilehandler.filename, html, 'utf-8')];
+                        case 4:
+                            _e.sent();
+                            markdown = markdownConverter.makeMarkdown(html, jsdom$1.window.document);
+                            return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
+                    }
+                });
+            });
+        },
+        /**
+         * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
+         */
+        scrape: function (source, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var cacheFilehandler, markdownSource, knowledge;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, websiteScraper.$convert(source, options)];
+                        case 1:
+                            cacheFilehandler = _a.sent();
+                            markdownSource = {
+                                source: source.source,
+                                filename: cacheFilehandler.filename,
+                                url: null,
+                                mimeType: 'text/markdown',
+                                asText: function () {
+                                    return cacheFilehandler.markdown;
+                                },
+                                asJson: function () {
+                                    throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+                                },
+                                asBlob: function () {
+                                    throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
+                                },
+                            };
+                            knowledge = markdownScraper.scrape(markdownSource, options);
+                            return [4 /*yield*/, cacheFilehandler.destroy()];
+                        case 2:
+                            _a.sent();
+                            return [2 /*return*/, knowledge];
+                    }
+                });
+            });
+        },
+    } /* TODO: [🦷] as const */;
     /**
-     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     * TODO: !!!!!! Put into separate package
+     * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
+     * TODO: [🦖] Make some system for putting scrapers to separete packages
+     * TODO: [🪂] Do it in parallel 11:11
+     * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
      */
 
+    // TODO: [🦖] !!!!!! Pass scrapers as dependency,
     /**
-     * Creates executor function from pipeline and execution tools.
+     * @@@
      *
-     * @returns The executor function
-     * @throws {PipelineLogicError} on logical error in the pipeline
-     * @public exported from `@promptbook/core`
+     * @private because this will be replaced by a system of one scraper per package [🦖]
+     * TODO: [🦖] System for scrapers NOT public exported from `@promptbook/core`
      */
-    function createPipelineExecutor(options) {
-        var _this = this;
-        var pipeline = options.pipeline, tools = options.tools, _a = options.settings, settings = _a === void 0 ? {} : _a;
-        var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f;
-        validatePipeline(pipeline);
-        var pipelineIdentification = (function () {
-            // Note: This is a 😐 implementation of [🚞]
-            var _ = [];
-            if (pipeline.sourceFile !== undefined) {
-                _.push("File: ".concat(pipeline.sourceFile));
-            }
-            if (pipeline.pipelineUrl !== undefined) {
-                _.push("Url: ".concat(pipeline.pipelineUrl));
-            }
-            return _.join('\n');
-        })();
-        var preparedPipeline;
-        if (isPipelinePrepared(pipeline)) {
-            preparedPipeline = pipeline;
-        }
-        else if (isNotPreparedWarningSupressed !== true) {
-            console.warn(spaceTrim.spaceTrim(function (block) { return "\n                    Pipeline is not prepared\n\n                    ".concat(block(pipelineIdentification), "\n\n                    It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n                    But it is recommended to prepare the pipeline during collection preparation\n\n                    @see more at https://ptbk.io/prepare-pipeline\n                "); }));
-        }
-        var runCount = 0;
-        var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                runCount++;
-                return [2 /*return*/, /* not await */ executePipeline({
-                        pipeline: pipeline,
-                        preparedPipeline: preparedPipeline,
-                        setPreparedPipeline: function (newPreparedPipeline) {
-                            preparedPipeline = newPreparedPipeline;
-                        },
-                        inputParameters: inputParameters,
-                        tools: tools,
-                        onProgress: onProgress,
-                        pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(pipelineIdentification), "\n                    ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n                "); }),
-                        settings: {
-                            maxExecutionAttempts: maxExecutionAttempts,
-                            maxParallelCount: maxParallelCount,
-                            csvSettings: csvSettings,
-                            isVerbose: isVerbose,
-                            isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
-                        },
-                    })];
-            });
-        }); };
-        return pipelineExecutor;
+    var SCRAPERS = [
+        markdownScraper,
+        documentScraper,
+        legacyDocumentScraper,
+        pdfScraper,
+        websiteScraper,
+        // <- Note: [♓️] This is the order of the scrapers for knowledge, BUT consider some better (more explicit) way to do this
+    ];
+
+    /**
+     * Creates unique name for the source
+     *
+     * @private within the repository
+     */
+    function sourceContentToName(sourceContent) {
+        // TODO: !!!!!! Better name for source than gibberish hash
+        var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
+            //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
+            .toString( /* hex */)
+            .substring(0, 20);
+        //    <- TODO: [🥬] Make some system for hashes and ids of promptbook
+        var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
+        var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
+        var name = pieces.join('-').split('--').join('-');
+        // <- TODO: Use MAX_FILENAME_LENGTH
+        return name;
     }
     /**
-     * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
+     * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
+     */
+
+    /**
+     * Convert file extension to mime type
+     *
+     * @private within the repository
      */
+    function extensionToMimeType(value) {
+        return mimeTypes.lookup(value) || 'application/octet-stream';
+    }
 
     /**
      * @@@
      *
-     * @public exported from `@promptbook/core`
+     * @private for scraper utilities
      */
-    function prepareKnowledgeFromMarkdown(knowledgeContent /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
+    function makeKnowledgeSourceHandler(knowledgeSource, options) {
+        var _a;
         return __awaiter(this, void 0, void 0, function () {
-            var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgePiecesRaw, knowledgeTextPieces, knowledge;
-            var _f, _g, _h;
-            var _this = this;
-            return __generator(this, function (_j) {
-                switch (_j.label) {
+            var sourceContent, name, _b, _c, rootDirname, _d, isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
+            return __generator(this, function (_e) {
+                switch (_e.label) {
                     case 0:
-                        llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
-                        TODO_USE(maxParallelCount); // <- [🪂]
-                        collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
-                        _c = createPipelineExecutor;
-                        _f = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
+                        sourceContent = knowledgeSource.sourceContent;
+                        name = knowledgeSource.name;
+                        _b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
+                        TODO_USE(isVerbose);
+                        if (!name) {
+                            name = sourceContentToName(sourceContent);
+                        }
+                        if (!isValidUrl(sourceContent)) return [3 /*break*/, 2];
+                        url = sourceContent;
+                        return [4 /*yield*/, fetch(url)];
                     case 1:
-                        prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
-                                _f.tools = {
-                                    llm: llmTools,
+                        response_1 = _e.sent();
+                        mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
+                        return [2 /*return*/, {
+                                source: name,
+                                filename: null,
+                                url: url,
+                                mimeType: mimeType,
+                                asBlob: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.blob()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
+                                    });
                                 },
-                                _f)]);
-                        _d = createPipelineExecutor;
-                        _g = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
-                    case 2:
-                        prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
-                                _g.tools = {
-                                    llm: llmTools,
+                                asJson: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.json()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
+                                    });
                                 },
-                                _g)]);
-                        _e = createPipelineExecutor;
-                        _h = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
-                    case 3:
-                        prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
-                                _h.tools = {
-                                    llm: llmTools,
+                                asText: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, response_1.text()];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    return [2 /*return*/, content];
+                                            }
+                                        });
+                                    });
                                 },
-                                _h)]);
-                        return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
-                    case 4:
-                        result = _j.sent();
-                        assertsExecutionSuccessful(result);
-                        outputParameters = result.outputParameters;
-                        knowledgePiecesRaw = outputParameters.knowledgePieces;
-                        knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
-                        //                                                               <- TODO: [main] !!!!! Smarter split and filter out empty pieces
-                        if (isVerbose) {
-                            console.info('knowledgeTextPieces:', knowledgeTextPieces);
+                            }];
+                    case 2:
+                        if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
+                        if (!$isRunningInNode()) {
+                            throw new EnvironmentMismatchError('Importing knowledge source file works only in Node.js environment');
                         }
-                        return [4 /*yield*/, Promise.all(
-                            // TODO: [🪂] !! Do not send all at once but in chunks
-                            knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
-                                var name, title, knowledgePieceContent, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
-                                return __generator(this, function (_c) {
-                                    switch (_c.label) {
-                                        case 0:
-                                            name = "piece-".concat(i);
-                                            title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
-                                            knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
-                                            keywords = [];
-                                            index = [];
-                                            _c.label = 1;
-                                        case 1:
-                                            _c.trys.push([1, 7, , 8]);
-                                            return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
-                                        case 2:
-                                            titleResult = _c.sent();
-                                            _a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
-                                            title = spaceTrim__default["default"](titleRaw) /* <- TODO: Maybe do in pipeline */;
-                                            name = titleToName(title);
-                                            return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
-                                        case 3:
-                                            keywordsResult = _c.sent();
-                                            _b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
-                                            keywords = (keywordsRaw || '')
-                                                .split(',')
-                                                .map(function (keyword) { return keyword.trim(); })
-                                                .filter(function (keyword) { return keyword !== ''; });
-                                            if (isVerbose) {
-                                                console.info("Keywords for \"".concat(title, "\":"), keywords);
+                        if (rootDirname === null) {
+                            throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
+                            //          <- TODO: [🧠] What is the best error type here`
+                        }
+                        filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
+                        fileExtension = getFileExtension(filename_1);
+                        mimeType_1 = extensionToMimeType(fileExtension || '');
+                        return [4 /*yield*/, $isFileExisting(filename_1)];
+                    case 3:
+                        if (!(_e.sent())) {
+                            throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(filename_1), "\n                      "); }));
+                        }
+                        // TODO: !!!!!! Test security file - file is scoped to the project (maybe do this in `filesystemTools`)
+                        return [2 /*return*/, {
+                                source: name,
+                                filename: filename_1,
+                                url: null,
+                                mimeType: mimeType_1,
+                                asBlob: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var content;
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, promises.readFile(filename_1)];
+                                                case 1:
+                                                    content = _a.sent();
+                                                    //  <- Note: Its OK to use sync in tooling for tests
+                                                    return [2 /*return*/, new Blob([
+                                                            content,
+                                                            // <- TODO: !!!!!! Maybe not working
+                                                        ], { type: mimeType_1 })];
                                             }
-                                            if (!!llmTools.callEmbeddingModel) return [3 /*break*/, 4];
-                                            // TODO: [🟥] Detect browser / node and make it colorfull
-                                            console.error('No callEmbeddingModel function provided');
-                                            return [3 /*break*/, 6];
-                                        case 4: return [4 /*yield*/, llmTools.callEmbeddingModel({
-                                                title: "Embedding for ".concat(title) /* <- Note: No impact on embedding result itself, just for logging */,
-                                                parameters: {},
-                                                content: knowledgePieceContent,
-                                                modelRequirements: {
-                                                    modelVariant: 'EMBEDDING',
-                                                },
-                                            })];
-                                        case 5:
-                                            embeddingResult = _c.sent();
-                                            index.push({
-                                                modelName: embeddingResult.modelName,
-                                                position: embeddingResult.content,
-                                            });
-                                            _c.label = 6;
-                                        case 6: return [3 /*break*/, 8];
-                                        case 7:
-                                            error_1 = _c.sent();
-                                            // Note: Here is expected error:
-                                            //     > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
-                                            if (!(error_1 instanceof PipelineExecutionError)) {
-                                                throw error_1;
+                                        });
+                                    });
+                                },
+                                asJson: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        var _a, _b;
+                                        return __generator(this, function (_c) {
+                                            switch (_c.label) {
+                                                case 0:
+                                                    _b = (_a = JSON).parse;
+                                                    return [4 /*yield*/, promises.readFile(filename_1, 'utf-8')];
+                                                case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
                                             }
-                                            // TODO: [🟥] Detect browser / node and make it colorfull
-                                            console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
-                                            return [3 /*break*/, 8];
-                                        case 8: return [2 /*return*/, {
-                                                name: name,
-                                                title: title,
-                                                content: knowledgePieceContent,
-                                                keywords: keywords,
-                                                index: index,
-                                                // <- TODO: [☀] sources,
-                                            }];
-                                    }
-                                });
-                            }); }))];
-                    case 5:
-                        knowledge = _j.sent();
-                        return [2 /*return*/, knowledge];
+                                        });
+                                    });
+                                },
+                                asText: function () {
+                                    return __awaiter(this, void 0, void 0, function () {
+                                        return __generator(this, function (_a) {
+                                            switch (_a.label) {
+                                                case 0: return [4 /*yield*/, promises.readFile(filename_1, 'utf-8')];
+                                                case 1: return [2 /*return*/, _a.sent()];
+                                            }
+                                        });
+                                    });
+                                },
+                            }];
+                    case 4: return [2 /*return*/, {
+                            source: name,
+                            filename: null,
+                            url: null,
+                            mimeType: 'text/markdown',
+                            asText: function () {
+                                return knowledgeSource.sourceContent;
+                            },
+                            asJson: function () {
+                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+                            },
+                            asBlob: function () {
+                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
+                            },
+                        }];
                 }
             });
         });
     }
-    /**
-     * TODO: [🐝][🔼][main] !!! Export via `@promptbook/markdown`
-     * TODO: [🪂] Do it in parallel 11:11
-     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
-     */
 
     /**
      * Prepares the knowle
@@ -4984,21 +6016,64 @@
      */
     function prepareKnowledgePieces(knowledgeSources, options) {
         return __awaiter(this, void 0, void 0, function () {
-            var _a, maxParallelCount, knowledgePrepared;
+            var _a, maxParallelCount, rootDirname, _b, isVerbose, knowledgePreparedUnflatten, knowledgePrepared;
             var _this = this;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
+            return __generator(this, function (_c) {
+                switch (_c.label) {
                     case 0:
-                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
-                        knowledgePrepared = [];
-                        return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
-                                var partialPieces, pieces;
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.sourceContent, // <- TODO: [🐝][main] !!! Unhardcode markdown, detect which type it is - BE AWARE of big package size
-                                            options)];
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
+                        knowledgePreparedUnflatten = new Array(knowledgeSources.length);
+                        return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
+                                var partialPieces, sourceHandler, SCRAPERS_1, SCRAPERS_1_1, scraper, partialPiecesUnchecked, e_1_1, pieces;
+                                var e_1, _a;
+                                return __generator(this, function (_b) {
+                                    switch (_b.label) {
+                                        case 0:
+                                            partialPieces = null;
+                                            return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
                                         case 1:
-                                            partialPieces = _a.sent();
+                                            sourceHandler = _b.sent();
+                                            _b.label = 2;
+                                        case 2:
+                                            _b.trys.push([2, 7, 8, 9]);
+                                            SCRAPERS_1 = __values(SCRAPERS), SCRAPERS_1_1 = SCRAPERS_1.next();
+                                            _b.label = 3;
+                                        case 3:
+                                            if (!!SCRAPERS_1_1.done) return [3 /*break*/, 6];
+                                            scraper = SCRAPERS_1_1.value;
+                                            if (!scraper.mimeTypes.includes(sourceHandler.mimeType)
+                                            // <- TODO: [🦔] Implement mime-type wildcards
+                                            ) {
+                                                return [3 /*break*/, 5];
+                                            }
+                                            return [4 /*yield*/, scraper.scrape(sourceHandler, options)];
+                                        case 4:
+                                            partialPiecesUnchecked = _b.sent();
+                                            if (partialPiecesUnchecked !== null) {
+                                                partialPieces = partialPiecesUnchecked;
+                                                return [3 /*break*/, 6];
+                                            }
+                                            _b.label = 5;
+                                        case 5:
+                                            SCRAPERS_1_1 = SCRAPERS_1.next();
+                                            return [3 /*break*/, 3];
+                                        case 6: return [3 /*break*/, 9];
+                                        case 7:
+                                            e_1_1 = _b.sent();
+                                            e_1 = { error: e_1_1 };
+                                            return [3 /*break*/, 9];
+                                        case 8:
+                                            try {
+                                                if (SCRAPERS_1_1 && !SCRAPERS_1_1.done && (_a = SCRAPERS_1.return)) _a.call(SCRAPERS_1);
+                                            }
+                                            finally { if (e_1) throw e_1.error; }
+                                            return [7 /*endfinally*/];
+                                        case 9:
+                                            if (partialPieces === null) {
+                                                throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n                        Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        Available scrapers:\n                        ").concat(block(SCRAPERS.flatMap(function (scraper) { return scraper.mimeTypes; })
+                                                    .map(function (mimeType) { return "- ".concat(mimeType); })
+                                                    .join('\n')), "\n\n\n                    "); }));
+                                            }
                                             pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
                                                     {
                                                         name: knowledgeSource.name,
@@ -5006,13 +6081,14 @@
                                                         // <- TODO: [❎]
                                                     },
                                                 ] })); });
-                                            knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
+                                            knowledgePreparedUnflatten[index] = pieces;
                                             return [2 /*return*/];
                                     }
                                 });
                             }); })];
                     case 1:
-                        _b.sent();
+                        _c.sent();
+                        knowledgePrepared = knowledgePreparedUnflatten.flat();
                         return [2 /*return*/, knowledgePrepared];
                 }
             });
@@ -5029,7 +6105,7 @@
     >
     > export async function prepareKnowledgePieces(
     >   knowledge: PrepareKnowledgeKnowledge,
-    >   options: PrepareOptions,
+    >   options: PrepareAndScrapeOptions,
     > ):
     */
     /**
@@ -5142,6 +6218,9 @@
                 switch (_d.label) {
                     case 0:
                         llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
+                        if (llmTools === undefined) {
+                            throw new MissingToolsError('LLM tools are required for preparing persona');
+                        }
                         collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
                         _b = createPipelineExecutor;
                         _c = {};
@@ -5285,7 +6364,7 @@
      */
     function preparePipeline(pipeline, options) {
         return __awaiter(this, void 0, void 0, function () {
-            var llmTools, _a, maxParallelCount, _b, isVerbose, parameters, templates, 
+            var llmTools, rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates, 
             /*
             <- TODO: [🧠][🪑] `promptbookVersion` */
             knowledgeSources /*
@@ -5298,8 +6377,11 @@
                         if (isPipelinePrepared(pipeline)) {
                             return [2 /*return*/, pipeline];
                         }
-                        llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
+                        llmTools = options.llmTools, rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
                         parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
+                        if (llmTools === undefined) {
+                            throw new MissingToolsError('LLM tools are required for preparing the pipeline');
+                        }
                         llmToolsWithUsage = countTotalUsage(llmTools);
                         currentPreparation = {
                             id: 1,
@@ -5321,6 +6403,7 @@
                                     switch (_a.label) {
                                         case 0: return [4 /*yield*/, preparePersona(persona.description, {
                                                 llmTools: llmToolsWithUsage,
+                                                rootDirname: rootDirname,
                                                 maxParallelCount: maxParallelCount /* <- TODO:  [🪂] */,
                                                 isVerbose: isVerbose,
                                             })];
@@ -5335,11 +6418,7 @@
                     case 1:
                         _c.sent();
                         knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
-                        return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, {
-                                llmTools: llmToolsWithUsage,
-                                maxParallelCount: maxParallelCount /* <- TODO:  [🪂] */,
-                                isVerbose: isVerbose,
-                            })];
+                        return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, options), { llmTools: llmToolsWithUsage, rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO:  [🪂] */, isVerbose: isVerbose }))];
                     case 2:
                         partialknowledgePiecesPrepared = _c.sent();
                         knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
@@ -5349,6 +6428,7 @@
                                 knowledgePiecesCount: knowledgePiecesPrepared.length,
                             }, {
                                 llmTools: llmToolsWithUsage,
+                                rootDirname: rootDirname,
                                 maxParallelCount: maxParallelCount /* <- TODO:  [🪂] */,
                                 isVerbose: isVerbose,
                             })];
@@ -5371,36 +6451,10 @@
      *       @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format
      */
 
-    /**
-     * Tests if given string is valid URL.
-     *
-     * Note: This does not check if the file exists only if the path is valid
-     * @public exported from `@promptbook/utils`
-     */
-    function isValidFilePath(filePath) {
-        if (typeof filePath !== 'string') {
-            return false;
-        }
-        var filePathSlashes = filePath.split('\\').join('/');
-        // Absolute Unix path: /hello.txt
-        if (/^(\/)/i.test(filePathSlashes)) {
-            return true;
-        }
-        // Absolute Windows path: /hello.txt
-        if (/^([A-Z]{1,2}:\/?)\//i.test(filePathSlashes)) {
-            return true;
-        }
-        // Relative path: ./hello.txt
-        if (/^(\.\.?\/)+/i.test(filePathSlashes)) {
-            return true;
-        }
-        return false;
-    }
-
     /**
      * Parses the knowledge command
      *
-     * @see ./KNOWLEDGE-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var knowledgeCommandParser = {
@@ -5418,7 +6472,7 @@
          */
         description: "Tells promptbook which external knowledge to use",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/41',
         /**
@@ -5462,11 +6516,8 @@
          */
         $applyToPipelineJson: function (command, $pipelineJson) {
             var sourceContent = command.sourceContent;
-            var name = 'source-' + sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(sourceContent))).toString( /* hex */);
-            //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
-            //    <- TODO: This should be replaced with a better name later in preparation (done with some propper LLM summarization)
             $pipelineJson.knowledgeSources.push({
-                name: name,
+                name: sourceContentToName(sourceContent),
                 sourceContent: sourceContent,
             });
         },
@@ -5496,7 +6547,7 @@
     /**
      * Parses the template command
      *
-     * @see ./TEMPLATE-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var templateCommandParser = {
@@ -5532,7 +6583,7 @@
          */
         description: "What should the code template template do",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/64',
         /**
@@ -5757,7 +6808,7 @@
     /**
      * Parses the expect command
      *
-     * @see ./EXPECT-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var expectCommandParser = {
@@ -5775,7 +6826,7 @@
          */
         description: spaceTrim__default["default"]("\n        Expect command describes the desired output of the template (after post-processing)\n        It can set limits for the maximum/minimum length of the output, measured in characters, words, sentences, paragraphs or some other shape of the output.\n    "),
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/30',
         /**
@@ -5935,10 +6986,6 @@
                     charType = 'NUMBER';
                     normalizedChar = char;
                 }
-                else if (/^\/$/.test(char)) {
-                    charType = 'SLASH';
-                    normalizedChar = char;
-                }
                 else {
                     charType = 'OTHER';
                     normalizedChar = '_';
@@ -6145,7 +7192,7 @@
      *
      * Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.ptbk.md` file
      *
-     * @see ./FOREACH-README.md for more details <- TODO: @@@ Write theese README files OR remove this link + add annotation here (to all commands)
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var foreachCommandParser = {
@@ -6167,7 +7214,7 @@
          */
         description: "@@",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/148',
         /**
@@ -6302,7 +7349,7 @@
     /**
      * Parses the format command
      *
-     * @see ./FORMAT-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var formatCommandParser = {
@@ -6320,7 +7367,7 @@
          */
         description: spaceTrim__default["default"]("\n        Format command describes the desired output of the template (after post-processing)\n        It can set limits for the maximum/minimum length of the output, measured in characters, words, sentences, paragraphs or some other shape of the output.\n    "),
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/30',
         /**
@@ -6376,7 +7423,7 @@
     /**
      * Parses the joker command
      *
-     * @see ./JOKER-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var jokerCommandParser = {
@@ -6394,7 +7441,7 @@
          */
         description: "Joker parameter is used instead of executing the template result if jokers value meets the expectations requirements",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/66',
         /**
@@ -6455,7 +7502,7 @@
     /**
      * Parses the model command
      *
-     * @see ./MODEL-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var modelCommandParser = {
@@ -6473,7 +7520,7 @@
          */
         description: "Tells which `modelRequirements` (for example which model) to use for the prompt template execution",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/67',
         /**
@@ -6555,7 +7602,11 @@
             // TODO: [🚜] DRY
             if ($templateJson.modelRequirements[command.key] !== undefined) {
                 if ($templateJson.modelRequirements[command.key] === command.value) {
-                    console.warn("Multiple commands `MODEL ".concat(command.key, " ").concat(command.value, "` in the template \"").concat($templateJson.title || $templateJson.name, "\""));
+                    console.warn("Multiple commands `MODEL ".concat({
+                        modelName: 'NAME',
+                        modelVariant: 'VARIANT',
+                        maxTokens: '???',
+                    }[command.key], " ").concat(command.value, "` in the template \"").concat($templateJson.title || $templateJson.name, "\""));
                 }
                 else {
                     throw new ParseError(spaceTrim__default["default"]("\n                              Redefinition of MODEL `".concat(command.key, "` in the template \"").concat($templateJson.title || $templateJson.name, "\"\n\n                              You have used:\n                              - MODEL ").concat(command.key, " ").concat($templateJson.modelRequirements[command.key], "\n                              - MODEL ").concat(command.key, " ").concat(command.value, "\n                          ")));
@@ -6598,7 +7649,7 @@
     /**
      * Parses the parameter command
      *
-     * @see ./PARAMETER-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var parameterCommandParser = {
@@ -6623,7 +7674,7 @@
          */
         description: "Describes one parameter of the template",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/68',
         /**
@@ -6708,7 +7759,7 @@
     /**
      * Parses the persona command
      *
-     * @see ./PERSONA-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var personaCommandParser = {
@@ -6730,7 +7781,7 @@
          */
         description: "Persona command is used to specify who the system is, it will be transformed into system message, top_t,...",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/22',
         /**
@@ -6846,7 +7897,7 @@
     /**
      * Parses the postprocess command
      *
-     * @see ./POSTPROCESS-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var postprocessCommandParser = {
@@ -6865,7 +7916,7 @@
          */
         description: "Defines the postprocess function to be used on the result from LLM and before the result is validated",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/31',
         /**
@@ -6926,7 +7977,7 @@
     /**
      * Parses the PROMPTBOOK_VERSION command
      *
-     * @see ./PROMPTBOOK_VERSION-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var promptbookVersionCommandParser = {
@@ -6945,7 +7996,7 @@
          */
         description: "Which version of the promptbook is the .ptbk.md using",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/69',
         /**
@@ -7004,7 +8055,7 @@
     /**
      * Parses the url command
      *
-     * @see ./URL-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var urlCommandParser = {
@@ -7027,7 +8078,7 @@
          */
         description: "Declares unique URL for the pipeline",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/70',
         /**
@@ -7107,7 +8158,7 @@
     /**
      * Parses the action command
      *
-     * @see ./ACTION-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var actionCommandParser = {
@@ -7125,7 +8176,7 @@
          */
         description: "Actions influences from the pipeline or template into external world. Like turning on a light, sending an email, etc.",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/72',
         /**
@@ -7177,7 +8228,7 @@
     /**
      * Parses the instrument command
      *
-     * @see ./INSTRUMENT-README.md for more details
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var instrumentCommandParser = {
@@ -7195,7 +8246,7 @@
          */
         description: "Instrument command is used to specify the instrument to be used in the pipeline or template like search, calculate, etc.",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/71',
         /**
@@ -7250,7 +8301,7 @@
      *
      * Note: @@@ This command is used as boilerplate for new commands - it should NOT be used in any `.ptbk.md` file
      *
-     * @see ./BOILERPLATE-README.md for more details <- TODO: @@@ Write theese README files OR remove this link + add annotation here (to all commands)
+     * @see `documentationUrl` for more details
      * @private within the commands folder
      */
     var boilerplateCommandParser = {
@@ -7272,7 +8323,7 @@
          */
         description: "@@",
         /**
-         * Link to discussion
+         * Link to documentation
          */
         documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
         /**
@@ -8122,16 +9173,17 @@
      * @public exported from `@promptbook/core`
      */
     function pipelineStringToJson(pipelineString, options) {
-        if (options === void 0) { options = { llmTools: null }; }
         return __awaiter(this, void 0, void 0, function () {
             var llmTools, pipelineJson;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
-                        llmTools = options.llmTools;
+                        llmTools = (options || {}).llmTools;
                         pipelineJson = pipelineStringToJsonSync(pipelineString);
-                        if (!(llmTools !== null)) return [3 /*break*/, 2];
-                        return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
+                        if (!(llmTools !== undefined)) return [3 /*break*/, 2];
+                        return [4 /*yield*/, preparePipeline(pipelineJson, options || {
+                                rootDirname: null,
+                            })];
                     case 1:
                         pipelineJson = _a.sent();
                         _a.label = 2;
@@ -8339,6 +9391,38 @@
      * TODO: [🍙] Make some standard order of json properties
      */
 
+    /**
+     * Delagates the user interaction to a async callback function
+     * You need to provide your own implementation of this callback function and its bind to UI.
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var CallbackInterfaceTools = /** @class */ (function () {
+        function CallbackInterfaceTools(options) {
+            this.options = options;
+        }
+        /**
+         * Trigger the custom callback function
+         */
+        CallbackInterfaceTools.prototype.promptDialog = function (options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var answer;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, this.options.callback(options)];
+                        case 1:
+                            answer = _a.sent();
+                            if (this.options.isVerbose) {
+                                console.info(spaceTrim.spaceTrim(function (block) { return "\n                        \uD83D\uDCD6 ".concat(block(options.promptTitle), "\n                        \uD83D\uDC64 ").concat(block(answer), "\n                    "); }));
+                            }
+                            return [2 /*return*/, answer];
+                    }
+                });
+            });
+        };
+        return CallbackInterfaceTools;
+    }());
+
     /**
      * Pretty print an embedding vector for logging
      *
@@ -8414,38 +9498,6 @@
      * TODO: [🏛] Maybe make some markdown builder
      */
 
-    /**
-     * Delagates the user interaction to a async callback function
-     * You need to provide your own implementation of this callback function and its bind to UI.
-     *
-     * @public exported from `@promptbook/core`
-     */
-    var CallbackInterfaceTools = /** @class */ (function () {
-        function CallbackInterfaceTools(options) {
-            this.options = options;
-        }
-        /**
-         * Trigger the custom callback function
-         */
-        CallbackInterfaceTools.prototype.promptDialog = function (options) {
-            return __awaiter(this, void 0, void 0, function () {
-                var answer;
-                return __generator(this, function (_a) {
-                    switch (_a.label) {
-                        case 0: return [4 /*yield*/, this.options.callback(options)];
-                        case 1:
-                            answer = _a.sent();
-                            if (this.options.isVerbose) {
-                                console.info(spaceTrim.spaceTrim(function (block) { return "\n                        \uD83D\uDCD6 ".concat(block(options.promptTitle), "\n                        \uD83D\uDC64 ").concat(block(answer), "\n                    "); }));
-                            }
-                            return [2 /*return*/, answer];
-                    }
-                });
-            });
-        };
-        return CallbackInterfaceTools;
-    }());
-
     /**
      * @@@
      *
@@ -9413,6 +10465,7 @@
     exports.ExecutionReportStringOptionsDefaults = ExecutionReportStringOptionsDefaults;
     exports.ExpectError = ExpectError;
     exports.IS_VERBOSE = IS_VERBOSE;
+    exports.KnowledgeScrapeError = KnowledgeScrapeError;
     exports.LimitReachedError = LimitReachedError;
     exports.MANDATORY_CSV_SETTINGS = MANDATORY_CSV_SETTINGS;
     exports.MAX_EXECUTION_ATTEMPTS = MAX_EXECUTION_ATTEMPTS;
@@ -9422,6 +10475,7 @@
     exports.MAX_PARALLEL_COUNT = MAX_PARALLEL_COUNT;
     exports.MODEL_VARIANTS = MODEL_VARIANTS;
     exports.MemoryStorage = MemoryStorage;
+    exports.MissingToolsError = MissingToolsError;
     exports.NotFoundError = NotFoundError;
     exports.NotYetImplementedError = NotYetImplementedError;
     exports.PIPELINE_COLLECTION_BASE_FILENAME = PIPELINE_COLLECTION_BASE_FILENAME;
@@ -9432,6 +10486,7 @@
     exports.PipelineUrlError = PipelineUrlError;
     exports.PrefixStorage = PrefixStorage;
     exports.RESERVED_PARAMETER_NAMES = RESERVED_PARAMETER_NAMES;
+    exports.SCRAPE_CACHE_DIRNAME = SCRAPE_CACHE_DIRNAME;
     exports.TemplateTypes = TemplateTypes;
     exports.TextFormatDefinition = TextFormatDefinition;
     exports.UNCERTAIN_USAGE = UNCERTAIN_USAGE;
@@ -9452,16 +10507,19 @@
     exports.createLlmToolsFromConfiguration = createLlmToolsFromConfiguration;
     exports.createPipelineExecutor = createPipelineExecutor;
     exports.createSubcollection = createSubcollection;
+    exports.documentScraper = documentScraper;
     exports.embeddingVectorToString = embeddingVectorToString;
     exports.executionReportJsonToString = executionReportJsonToString;
     exports.isPassingExpectations = isPassingExpectations;
     exports.isPipelinePrepared = isPipelinePrepared;
     exports.joinLlmExecutionTools = joinLlmExecutionTools;
+    exports.legacyDocumentScraper = legacyDocumentScraper;
     exports.limitTotalUsage = limitTotalUsage;
+    exports.markdownScraper = markdownScraper;
+    exports.pdfScraper = pdfScraper;
     exports.pipelineJsonToString = pipelineJsonToString;
     exports.pipelineStringToJson = pipelineStringToJson;
     exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
-    exports.prepareKnowledgeFromMarkdown = prepareKnowledgeFromMarkdown;
     exports.prepareKnowledgePieces = prepareKnowledgePieces;
     exports.preparePersona = preparePersona;
     exports.preparePipeline = preparePipeline;
@@ -9472,6 +10530,7 @@
     exports.usageToHuman = usageToHuman;
     exports.usageToWorktime = usageToWorktime;
     exports.validatePipeline = validatePipeline;
+    exports.websiteScraper = websiteScraper;
 
     Object.defineProperty(exports, '__esModule', { value: true });