@promptbook/cli 0.84.0-0 → 0.84.0-10

package/esm/index.es.js

@@ -12,8 +12,7 @@ import parserHtml from 'prettier/parser-html';
 import { unparse, parse } from 'papaparse';
 import { SHA256 } from 'crypto-js';
 import { lookup } from 'mime-types';
-import { exec as exec$2, spawn } from 'child_process';
-import { promisify } from 'util';
+import { spawn } from 'child_process';
 import glob from 'glob-promise';
 import prompts from 'prompts';
 import moment from 'moment';
@@ -39,7 +38,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.83.0';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0-9';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -203,7 +202,7 @@ var NAME = "Promptbook";
  *
  * @public exported from `@promptbook/core`
  */
-var ADMIN_EMAIL = 'me@pavolhejny.com';
+var ADMIN_EMAIL = 'pavol@ptbk.io';
 /**
  * Name of the responsible person for the Promptbook on GitHub
  *
@@ -813,7 +812,7 @@ var ORDER_OF_PIPELINE_JSON = [
  *
  * @private within the repository
  */
-var REPLACING_NONCE = 'u$k42k%!V2zo34w7Fu#@QUHYPW';
+var REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
  * @@@
  *
@@ -1916,7 +1915,7 @@ function countTotalUsage(llmTools) {
 var NotYetImplementedError = /** @class */ (function (_super) {
     __extends(NotYetImplementedError, _super);
     function NotYetImplementedError(message) {
-        var _this = _super.call(this, spaceTrim$1(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This feature is not implemented yet but it will be soon.\n\n                    If you want speed up the implementation or just read more, look here:\n                    https://github.com/webgptorg/promptbook\n\n                    Or contact us on me@pavolhejny.com\n\n                "); })) || this;
+        var _this = _super.call(this, spaceTrim$1(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This feature is not implemented yet but it will be soon.\n\n                    If you want speed up the implementation or just read more, look here:\n                    https://github.com/webgptorg/promptbook\n\n                    Or contact us on pavol@ptbk.io\n\n                "); })) || this;
         _this.name = 'NotYetImplementedError';
         Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
         return _this;
@@ -10393,30 +10392,187 @@ function compilePipeline(pipelineString, tools, options) {
  * TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
  */
 
-// Note: We want to use the `exec` as async function
-var exec$1 = promisify(exec$2);
+/**
+ * Normalize options for `execCommand` and `execCommands`
+ *
+ * Note: `$` is used to indicate that this function behaves differently according to `process.platform`
+ *
+ * @private internal utility of `execCommand` and `execCommands`
+ */
+function $execCommandNormalizeOptions(options) {
+    var _a;
+    var _b, _c, _d, _e;
+    var command;
+    var cwd;
+    var crashOnError;
+    var args = [];
+    var timeout;
+    var isVerbose;
+    if (typeof options === 'string') {
+        // TODO: [1] DRY default values
+        command = options;
+        cwd = process.cwd();
+        crashOnError = true;
+        timeout = Infinity; // <- TODO: [⏳]
+        isVerbose = DEFAULT_IS_VERBOSE;
+    }
+    else {
+        /*
+        TODO:
+        if ((options as any).commands !== undefined) {
+            commands = (options as any).commands;
+        } else {
+            commands = [(options as any).command];
+        }
+        */
+        // TODO: [1] DRY default values
+        command = options.command;
+        cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
+        crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
+        timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
+        isVerbose = (_e = options.isVerbose) !== null && _e !== void 0 ? _e : DEFAULT_IS_VERBOSE;
+    }
+    // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
+    var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
+        .map(function (_a) {
+        var _b = __read(_a, 1), match = _b[0];
+        return match;
+    })
+        .filter(function (arg) { return arg !== ''; });
+    if (_.length > 1) {
+        _a = __read(_), command = _a[0], args = _a.slice(1);
+    }
+    if (options.args) {
+        args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
+    }
+    var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
+    if (['ts-node'].includes(humanReadableCommand)) {
+        humanReadableCommand += " ".concat(args[1]);
+    }
+    if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
+        command = "".concat(command, ".cmd");
+    }
+    return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout, isVerbose: isVerbose };
+}
+// TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
+
+/**
+ * Run one command in a shell
+ *
+ *
+ * Note: There are 2 similar functions in the codebase:
+ * - `$execCommand` which runs a single command
+ * - `$execCommands` which runs multiple commands
+ * Note: `$` is used to indicate that this function is not a pure function - it runs a command in a shell
+ *
+ * @public exported from `@promptbook/node`
+ */
+function $execCommand(options) {
+    if (!$isRunningInNode()) {
+        throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
+    }
+    return new Promise(function (resolve, reject) {
+        // eslint-disable-next-line prefer-const
+        var _a = $execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
+        if (timeout !== Infinity) {
+            // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
+            forTime(timeout).then(function () {
+                if (crashOnError) {
+                    reject(new Error("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms")));
+                }
+                else {
+                    console.warn("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms but continues running"));
+                    resolve('Command exceeded time limit');
+                }
+            });
+        }
+        if (isVerbose) {
+            console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
+        }
+        try {
+            var commandProcess = spawn(command, args, { cwd: cwd, shell: true });
+            if (isVerbose) {
+                commandProcess.on('message', function (message) {
+                    console.info({ message: message });
+                });
+            }
+            var output_1 = [];
+            commandProcess.stdout.on('data', function (stdout) {
+                output_1.push(stdout.toString());
+                if (isVerbose) {
+                    console.info(stdout.toString());
+                }
+            });
+            commandProcess.stderr.on('data', function (stderr) {
+                output_1.push(stderr.toString());
+                if (isVerbose && stderr.toString().trim()) {
+                    console.warn(stderr.toString());
+                }
+            });
+            var finishWithCode = function (code) {
+                if (code !== 0) {
+                    if (crashOnError) {
+                        reject(new Error(output_1.join('\n').trim() ||
+                            "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
+                    }
+                    else {
+                        if (isVerbose) {
+                            console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
+                        }
+                        resolve(spaceTrim$1(output_1.join('\n')));
+                    }
+                }
+                else {
+                    resolve(spaceTrim$1(output_1.join('\n')));
+                }
+            };
+            commandProcess.on('close', finishWithCode);
+            commandProcess.on('exit', finishWithCode);
+            commandProcess.on('disconnect', function () {
+                // Note: Unexpected disconnection should always result in rejection
+                reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
+            });
+            commandProcess.on('error', function (error) {
+                if (crashOnError) {
+                    reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
+                }
+                else {
+                    if (isVerbose) {
+                        console.warn(error);
+                    }
+                    resolve(spaceTrim$1(output_1.join('\n')));
+                }
+            });
+        }
+        catch (error) {
+            // Note: Unexpected error in sync code should always result in rejection
+            reject(error);
+        }
+    });
+}
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
 /**
  * @@@
  *
  * @private within the repository
  */
 function locateAppOnLinux(_a) {
-    var appName = _a.appName, linuxWhich = _a.linuxWhich;
+    var linuxWhich = _a.linuxWhich;
     return __awaiter(this, void 0, void 0, function () {
-        var _b, stderr, stdout, error_1;
-        return __generator(this, function (_c) {
-            switch (_c.label) {
+        var result, error_1;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
                 case 0:
-                    _c.trys.push([0, 2, , 3]);
-                    return [4 /*yield*/, exec$1("which ".concat(linuxWhich))];
+                    _b.trys.push([0, 2, , 3]);
+                    return [4 /*yield*/, $execCommand({ crashOnError: true, command: "which ".concat(linuxWhich) })];
                 case 1:
-                    _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
-                    if (!stderr && stdout) {
-                        return [2 /*return*/, stdout.trim()];
-                    }
-                    throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr));
+                    result = _b.sent();
+                    return [2 /*return*/, result.trim()];
                 case 2:
-                    error_1 = _c.sent();
+                    error_1 = _b.sent();
                     if (!(error_1 instanceof Error)) {
                         throw error_1;
                     }
@@ -10463,43 +10619,41 @@ function isExecutable(path, fs) {
 //       @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 var userhome = require('userhome');
-// Note: We want to use the `exec` as async function
-var exec = promisify(exec$2);
 /**
  * @@@
  *
  * @private within the repository
  */
 function locateAppOnMacOs(_a) {
-    var appName = _a.appName, macOsName = _a.macOsName;
+    var macOsName = _a.macOsName;
     return __awaiter(this, void 0, void 0, function () {
-        var toExec, regPath, altPath, _b, stderr, stdout, error_1;
-        return __generator(this, function (_c) {
-            switch (_c.label) {
+        var toExec, regPath, altPath, result, error_1;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
                 case 0:
-                    _c.trys.push([0, 6, , 7]);
+                    _b.trys.push([0, 6, , 7]);
                     toExec = "/Contents/MacOS/".concat(macOsName);
                     regPath = "/Applications/".concat(macOsName, ".app") + toExec;
                     altPath = userhome(regPath.slice(1));
                     return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())];
                 case 1:
-                    if (!_c.sent()) return [3 /*break*/, 2];
+                    if (!_b.sent()) return [3 /*break*/, 2];
                     return [2 /*return*/, regPath];
                 case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())];
                 case 3:
-                    if (_c.sent()) {
+                    if (_b.sent()) {
                         return [2 /*return*/, altPath];
                     }
-                    _c.label = 4;
-                case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))];
+                    _b.label = 4;
+                case 4: return [4 /*yield*/, $execCommand({
+                        crashOnError: true,
+                        command: "mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"),
+                    })];
                 case 5:
-                    _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
-                    if (!stderr && stdout) {
-                        return [2 /*return*/, stdout.trim() + toExec];
-                    }
-                    throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr));
+                    result = _b.sent();
+                    return [2 /*return*/, result.trim() + toExec];
                 case 6:
-                    error_1 = _c.sent();
+                    error_1 = _b.sent();
                     if (!(error_1 instanceof Error)) {
                         throw error_1;
                     }
@@ -10600,7 +10754,7 @@ function locateApp(options) {
     }
     else if (process.platform === 'darwin') {
         if (macOsName) {
-            return locateAppOnMacOs({ appName: appName, macOsName: macOsName });
+            return locateAppOnMacOs({ macOsName: macOsName });
         }
         else {
             throw new Error("".concat(appName, " is not available on macOS."));
@@ -10608,7 +10762,7 @@ function locateApp(options) {
     }
     else {
         if (linuxWhich) {
-            return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich });
+            return locateAppOnLinux({ linuxWhich: linuxWhich });
         }
         else {
             throw new Error("".concat(appName, " is not available on Linux."));
@@ -15815,173 +15969,11 @@ var _OpenAiAssistantRegistration = $llmToolsRegister.register(createOpenAiAssist
  */
 
 /**
- * Normalize options for `execCommand` and `execCommands`
- *
- * Note: `$` is used to indicate that this function behaves differently according to `process.platform`
- *
- * @private internal utility of `execCommand` and `execCommands`
- */
-function $execCommandNormalizeOptions(options) {
-    var _a;
-    var _b, _c, _d, _e;
-    var command;
-    var cwd;
-    var crashOnError;
-    var args = [];
-    var timeout;
-    var isVerbose;
-    if (typeof options === 'string') {
-        // TODO: [1] DRY default values
-        command = options;
-        cwd = process.cwd();
-        crashOnError = true;
-        timeout = Infinity; // <- TODO: [⏳]
-        isVerbose = DEFAULT_IS_VERBOSE;
-    }
-    else {
-        /*
-        TODO:
-        if ((options as any).commands !== undefined) {
-            commands = (options as any).commands;
-        } else {
-            commands = [(options as any).command];
-        }
-        */
-        // TODO: [1] DRY default values
-        command = options.command;
-        cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
-        crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
-        timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
-        isVerbose = (_e = options.isVerbose) !== null && _e !== void 0 ? _e : DEFAULT_IS_VERBOSE;
-    }
-    // TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
-    var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
-        .map(function (_a) {
-        var _b = __read(_a, 1), match = _b[0];
-        return match;
-    })
-        .filter(function (arg) { return arg !== ''; });
-    if (_.length > 1) {
-        _a = __read(_), command = _a[0], args = _a.slice(1);
-    }
-    if (options.args) {
-        args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
-    }
-    var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
-    if (['ts-node'].includes(humanReadableCommand)) {
-        humanReadableCommand += " ".concat(args[1]);
-    }
-    if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
-        command = "".concat(command, ".cmd");
-    }
-    return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout, isVerbose: isVerbose };
-}
-// TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
-
-/**
- * Run one command in a shell
- *
+ * Create a filename for intermediate cache for scrapers
  *
- * Note: There are 2 similar functions in the codebase:
- * - `$execCommand` which runs a single command
- * - `$execCommands` which runs multiple commands
- * Note: `$` is used to indicate that this function is not a pure function - it runs a command in a shell
+ * Note: It also checks if directory exists and creates it if not
  *
- * @public exported from `@promptbook/node`
- */
-function $execCommand(options) {
-    if (!$isRunningInNode()) {
-        throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
-    }
-    return new Promise(function (resolve, reject) {
-        // eslint-disable-next-line prefer-const
-        var _a = $execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
-        if (timeout !== Infinity) {
-            // TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
-            forTime(timeout).then(function () {
-                if (crashOnError) {
-                    reject(new Error("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms")));
-                }
-                else {
-                    console.warn("Command \"".concat(humanReadableCommand, "\" exceeded time limit of ").concat(timeout, "ms but continues running"));
-                    resolve('Command exceeded time limit');
-                }
-            });
-        }
-        if (isVerbose) {
-            console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
-        }
-        try {
-            var commandProcess = spawn(command, args, { cwd: cwd, shell: true });
-            if (isVerbose) {
-                commandProcess.on('message', function (message) {
-                    console.info({ message: message });
-                });
-            }
-            var output_1 = [];
-            commandProcess.stdout.on('data', function (stdout) {
-                output_1.push(stdout.toString());
-                if (isVerbose) {
-                    console.info(stdout.toString());
-                }
-            });
-            commandProcess.stderr.on('data', function (stderr) {
-                output_1.push(stderr.toString());
-                if (isVerbose && stderr.toString().trim()) {
-                    console.warn(stderr.toString());
-                }
-            });
-            var finishWithCode = function (code) {
-                if (code !== 0) {
-                    if (crashOnError) {
-                        reject(new Error(output_1.join('\n').trim() ||
-                            "Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
-                    }
-                    else {
-                        if (isVerbose) {
-                            console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
-                        }
-                        resolve(spaceTrim$1(output_1.join('\n')));
-                    }
-                }
-                else {
-                    resolve(spaceTrim$1(output_1.join('\n')));
-                }
-            };
-            commandProcess.on('close', finishWithCode);
-            commandProcess.on('exit', finishWithCode);
-            commandProcess.on('disconnect', function () {
-                // Note: Unexpected disconnection should always result in rejection
-                reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
-            });
-            commandProcess.on('error', function (error) {
-                if (crashOnError) {
-                    reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
-                }
-                else {
-                    if (isVerbose) {
-                        console.warn(error);
-                    }
-                    resolve(spaceTrim$1(output_1.join('\n')));
-                }
-            });
-        }
-        catch (error) {
-            // Note: Unexpected error in sync code should always result in rejection
-            reject(error);
-        }
-    });
-}
-/**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
- */
-
-/**
- * Create a filename for intermediate cache for scrapers
- *
- * Note: It also checks if directory exists and creates it if not
- *
- * @private as internal utility for scrapers
+ * @private as internal utility for scrapers
  */
 function getScraperIntermediateSource(source, options) {
     return __awaiter(this, void 0, void 0, function () {
@@ -16246,6 +16238,207 @@ var MarkdownScraper = /** @class */ (function () {
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
  */
 
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+var boilerplateScraperMetadata = $deepFreeze({
+    title: 'Boilerplate scraper',
+    packageName: '@promptbook/boilerplate',
+    className: 'BoilerplateScraper',
+    mimeTypes: [
+        '@@@/@@@',
+        // <- TODO: @@@ Add compatible mime types with Boilerplate scraper
+    ],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@@',
+    isAvilableInBrowser: false,
+    // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
+    requiredExecutables: [
+    /* @@@ 'Pandoc' */
+    ],
+}); /* <- Note: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/wizzard`
+ * @public exported from `@promptbook/cli`
+ */
+var _BoilerplateScraperMetadataRegistration = $scrapersMetadataRegister.register(boilerplateScraperMetadata);
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
+/**
+ * Scraper of @@@ files
+ *
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/boilerplate`
+ */
+var BoilerplateScraper = /** @class */ (function () {
+    function BoilerplateScraper(tools, options) {
+        this.tools = tools;
+        this.options = options;
+        this.markdownScraper = new MarkdownScraper(tools, options);
+    }
+    Object.defineProperty(BoilerplateScraper.prototype, "metadata", {
+        /**
+         * Metadata of the scraper which includes title, mime types, etc.
+         */
+        get: function () {
+            return boilerplateScraperMetadata;
+        },
+        enumerable: false,
+        configurable: true
+    });
+    /**
+     * Convert the `.@@@`  to `.md` file and returns intermediate source
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    BoilerplateScraper.prototype.$convert = function (source) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function () {
+            var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
+            return __generator(this, function (_g) {
+                switch (_g.label) {
+                    case 0:
+                        _b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
+                        // TODO: @@@ Preserve or delete
+                        if (!$isRunningInNode()) {
+                            throw new KnowledgeScrapeError('BoilerplateScraper is only supported in Node environment');
+                        }
+                        // TODO: @@@ Preserve or delete
+                        if (this.tools.fs === undefined) {
+                            throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools');
+                            //          <- TODO: [🧠] What is the best error type here`
+                        }
+                        // TODO: @@@ Preserve, delete or modify
+                        if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
+                            throw new MissingToolsError('Pandoc is required for scraping .docx files');
+                        }
+                        // TODO: @@@ Preserve, delete or modify
+                        if (source.filename === null) {
+                            // TODO: [🧠] Maybe save file as temporary
+                            throw new KnowledgeScrapeError('When parsing .@@@ file, it must be real file in the file system');
+                        }
+                        extension = getFileExtension(source.filename);
+                        return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                rootDirname: rootDirname,
+                                cacheDirname: cacheDirname,
+                                intermediateFilesStrategy: intermediateFilesStrategy,
+                                extension: 'md',
+                                isVerbose: isVerbose,
+                            })];
+                    case 1:
+                        cacheFilehandler = _g.sent();
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
+                    case 2:
+                        if (!!(_g.sent())) return [3 /*break*/, 5];
+                        command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
+                        return [4 /*yield*/, $execCommand(command_1)];
+                    case 3:
+                        _g.sent();
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
+                    case 4:
+                        // Note: [0]
+                        if (!(_g.sent())) {
+                            throw new UnexpectedError(spaceTrim(function (block) { return "\n                    File that was supposed to be created by Pandoc does not exist for unknown reason\n\n                    Expected file:\n                    ".concat(block(cacheFilehandler.filename), "\n\n                    Command:\n                    > ").concat(block(command_1), "\n\n                "); }));
+                        }
+                        _g.label = 5;
+                    case 5: return [2 /*return*/, cacheFilehandler];
+                }
+            });
+        });
+    };
+    /**
+     * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    BoilerplateScraper.prototype.scrape = function (source) {
+        return __awaiter(this, void 0, void 0, function () {
+            var cacheFilehandler, markdownSource, knowledge;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.$convert(source)];
+                    case 1:
+                        cacheFilehandler = _a.sent();
+                        markdownSource = {
+                            source: source.source,
+                            filename: cacheFilehandler.filename,
+                            url: null,
+                            mimeType: 'text/markdown',
+                            asText: function () {
+                                return __awaiter(this, void 0, void 0, function () {
+                                    return __generator(this, function (_a) {
+                                        switch (_a.label) {
+                                            case 0: return [4 /*yield*/, readFile(cacheFilehandler.filename, 'utf-8')];
+                                            case 1: 
+                                            // Note: [0] In $convert we check that the file exists
+                                            return [2 /*return*/, _a.sent()];
+                                        }
+                                    });
+                                });
+                            },
+                            asJson: function () {
+                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+                            },
+                            /*
+                            TODO: [🥽]
+                                > asBlob() {
+                                >     throw new UnexpectedError(
+                                >         'Did not expect that `markdownScraper` would need to get the content `asBlob`',
+                                >     );
+                                > },
+                            */
+                        };
+                        knowledge = this.markdownScraper.scrape(markdownSource);
+                        return [4 /*yield*/, cacheFilehandler.destroy()];
+                    case 2:
+                        _a.sent();
+                        return [2 /*return*/, knowledge];
+                }
+            });
+        });
+    };
+    return BoilerplateScraper;
+}());
+/**
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+ * TODO: [🪂] Do it in parallel
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ * @@@ Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * Constructor of `BoilerplateScraper`
+ *
+ * @public exported from `@promptbook/boilerplate`
+ */
+var createBoilerplateScraper = Object.assign(function (tools, options) {
+    return new BoilerplateScraper(tools, options);
+}, boilerplateScraperMetadata); /* <- Note: [🤛] */
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */
+
+/**
+ * Registration of known scraper
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/boilerplate`
+ * @public exported from `@promptbook/wizzard`
+ * @public exported from `@promptbook/cli`
+ */
+var _BoilerplateScraperRegistration = $scrapersRegister.register(createBoilerplateScraper);
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * Metadata of the scraper
  *
@@ -16345,7 +16538,7 @@ var DocumentScraper = /** @class */ (function () {
                     case 4:
                         // Note: [0]
                         if (!(_g.sent())) {
-                            throw new UnexpectedError(spaceTrim(function (block) { return "\n                    File that was supposed to be created by Pandoc does not exist for unknown reason\n\n                    Expected file:\n                    ".concat(block(cacheFilehandler.filename), "\n\n                    Command:\n                    > ").concat(block(command_1), "\n\n                "); }));
+                            throw new UnexpectedError(spaceTrim(function (block) { return "\n                            File that was supposed to be created by Pandoc does not exist for unknown reason\n\n                            Expected file:\n                            ".concat(block(cacheFilehandler.filename), "\n\n                            Command:\n                            > ").concat(block(command_1), "\n\n                        "); }));
                         }
                         _g.label = 5;
                     case 5: return [2 /*return*/, cacheFilehandler];
@@ -16520,7 +16713,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
                     case 4:
                         files_1 = _g.sent();
                         if (files_1.length !== 1) {
-                            throw new UnexpectedError(spaceTrim(function (block) { return "\n                        Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n                        The temporary folder:\n                        ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n                        Command:\n                        > ").concat(block(command_1), "\n                    "); }));
+                            throw new UnexpectedError(spaceTrim(function (block) { return "\n                            Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n                            The temporary folder:\n                            ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n                            Command:\n                            > ").concat(block(command_1), "\n                        "); }));
                         }
                         file = files_1[0];
                         return [4 /*yield*/, rename(join(documentSourceOutdirPathForLibreOffice_1, file), cacheFilehandler.filename)];
@@ -16670,6 +16863,216 @@ var _MarkdownScraperRegistration = $scrapersRegister.register(createMarkdownScra
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+var markitdownScraperMetadata = $deepFreeze({
+    title: 'Markitdown scraper',
+    packageName: '@promptbook/markitdown',
+    className: 'MarkitdownScraper',
+    mimeTypes: [
+        'application/pdf',
+        // TODO: Make priority for scrapers and than allow all mime types here:
+        // 'text/html',
+        // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    ],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+    isAvilableInBrowser: false,
+    // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
+    requiredExecutables: [],
+}); /* <- Note: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/wizzard`
+ * @public exported from `@promptbook/cli`
+ */
+var _MarkitdownScraperMetadataRegistration = $scrapersMetadataRegister.register(markitdownScraperMetadata);
+/**
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
+/**
+ * Integration of Markitdown by Microsoft into Promptbook
+ *
+ * @see https://github.com/microsoft/markitdown
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/markitdown`
+ * @public exported from `@promptbook/pdf`
+ */
+var MarkitdownScraper = /** @class */ (function () {
+    function MarkitdownScraper(tools, options) {
+        this.tools = tools;
+        this.options = options;
+        this.markdownScraper = new MarkdownScraper(tools, options);
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        var MarkItDown = require('markitdown-ts').MarkItDown;
+        // <- TODO: !!! Use Markitdown directly not through this package
+        // <- Note: !!!!!!!
+        this.markitdown = new MarkItDown();
+    }
+    Object.defineProperty(MarkitdownScraper.prototype, "metadata", {
+        /**
+         * Metadata of the scraper which includes title, mime types, etc.
+         */
+        get: function () {
+            return markitdownScraperMetadata;
+        },
+        enumerable: false,
+        configurable: true
+    });
+    /**
+     * Convert the documents to `.md` file and returns intermediate source
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    MarkitdownScraper.prototype.$convert = function (source) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, _b, rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, cacheFilehandler, src, result;
+            return __generator(this, function (_f) {
+                switch (_f.label) {
+                    case 0:
+                        _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
+                        if (!$isRunningInNode()) {
+                            throw new KnowledgeScrapeError('MarkitdownScraper is only supported in Node environment');
+                        }
+                        if (this.tools.fs === undefined) {
+                            throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools');
+                            //          <- TODO: [🧠] What is the best error type here`
+                        }
+                        return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                rootDirname: rootDirname,
+                                cacheDirname: cacheDirname,
+                                intermediateFilesStrategy: intermediateFilesStrategy,
+                                extension: 'md',
+                                isVerbose: isVerbose,
+                            })];
+                    case 1:
+                        cacheFilehandler = _f.sent();
+                        return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
+                    case 2:
+                        if (!!(_f.sent())) return [3 /*break*/, 5];
+                        src = source.filename || source.url || null;
+                        console.log('!!!', { src: src, source: source, cacheFilehandler: cacheFilehandler });
+                        if (src === null) {
+                            throw new UnexpectedError('Source has no filename or url');
+                        }
+                        return [4 /*yield*/, this.markitdown.convert(src, {
+                            // TODO: !!!!!! Pass when sacraping Youtube
+                            // enableYoutubeTranscript: true,
+                            // youtubeTranscriptLanguage: 'en',
+                            })];
+                    case 3:
+                        result = _f.sent();
+                        if (result === null || result === undefined) {
+                            throw new Error("Markitdown could not convert the \"".concat(source.source, "\""));
+                            // <- TODO: !!! Make MarkitdownError
+                        }
+                        console.log('!!!', { result: result, cacheFilehandler: cacheFilehandler });
+                        return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, result.text_content)];
+                    case 4:
+                        _f.sent();
+                        _f.label = 5;
+                    case 5: return [2 /*return*/, cacheFilehandler];
+                }
+            });
+        });
+    };
+    /**
+     * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    MarkitdownScraper.prototype.scrape = function (source) {
+        return __awaiter(this, void 0, void 0, function () {
+            var cacheFilehandler, markdownSource, knowledge;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.$convert(source)];
+                    case 1:
+                        cacheFilehandler = _a.sent();
+                        markdownSource = {
+                            source: source.source,
+                            filename: cacheFilehandler.filename,
+                            url: null,
+                            mimeType: 'text/markdown',
+                            asText: function () {
+                                return __awaiter(this, void 0, void 0, function () {
+                                    return __generator(this, function (_a) {
+                                        switch (_a.label) {
+                                            case 0: return [4 /*yield*/, readFile(cacheFilehandler.filename, 'utf-8')];
+                                            case 1: 
+                                            // Note: [0] In $convert we check that the file exists
+                                            return [2 /*return*/, _a.sent()];
+                                        }
+                                    });
+                                });
+                            },
+                            asJson: function () {
+                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+                            },
+                            /*
+                            TODO: [🥽]
+                                > asBlob() {
+                                >     throw new UnexpectedError(
+                                >         'Did not expect that `markdownScraper` would need to get the content `asBlob`',
+                                >     );
+                                > },
+                            */
+                        };
+                        knowledge = this.markdownScraper.scrape(markdownSource);
+                        return [4 /*yield*/, cacheFilehandler.destroy()];
+                    case 2:
+                        _a.sent();
+                        return [2 /*return*/, knowledge];
+                }
+            });
+        });
+    };
+    return MarkitdownScraper;
+}());
+/**
+ * TODO: [🧠][🌜] Export only from `@promptbook/markitdown` or `@promptbook/pdf` NOT both
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
+ * TODO: [🪂] Do it in parallel
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * Constructor of `MarkitdownScraper`
+ *
+ * @public exported from `@promptbook/markitdown`
+ * @public exported from `@promptbook/pdf`
+ */
+var createMarkitdownScraper = Object.assign(function (tools, options) {
+    return new MarkitdownScraper(tools, options);
+}, markitdownScraperMetadata); /* <- Note: [🤛] */
+/**
+ * TODO: [🧠][🌜] Export only from `@promptbook/markitdown` or `@promptbook/pdf` NOT both
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */
+
+/**
+ * Registration of known scraper
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/markitdown`
+ * @public exported from `@promptbook/pdf`
+ * @public exported from `@promptbook/wizzard`
+ * @public exported from `@promptbook/cli`
+ */
+var _MarkitdownScraperRegistration = $scrapersRegister.register(createMarkitdownScraper);
+/**
+ * TODO: [🧠][🌜] Export only from `@promptbook/markitdown` or `@promptbook/pdf` NOT both
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ * Note: [💞] Ignore a discrepancy between file name and entity name
+ */
+
 /**
  * Metadata of the scraper
  *
@@ -16679,7 +17082,7 @@ var pdfScraperMetadata = $deepFreeze({
     title: 'Pdf scraper',
     packageName: '@promptbook/pdf',
     className: 'PdfScraper',
-    mimeTypes: ['application/pdf'],
+    mimeTypes: ['application/pdf-DISABLED'],
     documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
     isAvilableInBrowser: false,
     // <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
@@ -16990,5 +17393,5 @@ var _WebsiteScraperRegistration = $scrapersRegister.register(createWebsiteScrape
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
-export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION, _AnthropicClaudeMetadataRegistration, _AnthropicClaudeRegistration, _AzureOpenAiMetadataRegistration, _AzureOpenAiRegistration, _CLI, _DocumentScraperMetadataRegistration, _DocumentScraperRegistration, _GoogleMetadataRegistration, _GoogleRegistration, _LegacyDocumentScraperMetadataRegistration, _LegacyDocumentScraperRegistration, _MarkdownScraperMetadataRegistration, _MarkdownScraperRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiAssistantRegistration, _OpenAiMetadataRegistration, _OpenAiRegistration, _PdfScraperMetadataRegistration, _PdfScraperRegistration, _WebsiteScraperMetadataRegistration, _WebsiteScraperRegistration };
+export { BOOK_LANGUAGE_VERSION, PROMPTBOOK_ENGINE_VERSION, _AnthropicClaudeMetadataRegistration, _AnthropicClaudeRegistration, _AzureOpenAiMetadataRegistration, _AzureOpenAiRegistration, _BoilerplateScraperMetadataRegistration, _BoilerplateScraperRegistration, _CLI, _DocumentScraperMetadataRegistration, _DocumentScraperRegistration, _GoogleMetadataRegistration, _GoogleRegistration, _LegacyDocumentScraperMetadataRegistration, _LegacyDocumentScraperRegistration, _MarkdownScraperMetadataRegistration, _MarkdownScraperRegistration, _MarkitdownScraperMetadataRegistration, _MarkitdownScraperRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiAssistantRegistration, _OpenAiMetadataRegistration, _OpenAiRegistration, _PdfScraperMetadataRegistration, _PdfScraperRegistration, _WebsiteScraperMetadataRegistration, _WebsiteScraperRegistration };
 //# sourceMappingURL=index.es.js.map