@promptbook/cli 0.72.0-15 → 0.72.0-23

package/esm/index.es.js

@@ -10,6 +10,8 @@ import { unparse, parse } from 'papaparse';
 import { SHA256 } from 'crypto-js';
 import hexEncoder from 'crypto-js/enc-hex';
 import { lookup } from 'mime-types';
+import { exec as exec$2, spawn } from 'child_process';
+import { promisify } from 'util';
 import * as dotenv from 'dotenv';
 import sha256 from 'crypto-js/sha256';
 import glob from 'glob-promise';
@@ -17,16 +19,12 @@ import { io } from 'socket.io-client';
 import Anthropic from '@anthropic-ai/sdk';
 import { OpenAIClient, AzureKeyCredential } from '@azure/openai';
 import OpenAI from 'openai';
-import { spawn } from 'child_process';
-import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
-import { Converter } from 'showdown';
 
 // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-14';
+var PROMPTBOOK_VERSION = '0.72.0-22';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -4927,7 +4925,6 @@ var $scrapersRegister = new $Register('scraper_constructors');
  * TODO: [®] DRY Register logic
  */
 
-// TODO: !!!!!!last - Maybe delete this function
 /**
  * Creates a message with all registered scrapers
  *
@@ -5200,7 +5197,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     return __awaiter(this, void 0, void 0, function () {
         var sourceContent, name, _b, _c, rootDirname, _d, 
         // <- TODO: process.cwd() if running in Node.js
-        isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType_1;
+        isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
         return __generator(this, function (_e) {
             switch (_e.label) {
                 case 0:
@@ -5222,19 +5219,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                             filename: null,
                             url: url,
                             mimeType: mimeType,
-                            asBlob: function () {
-                                return __awaiter(this, void 0, void 0, function () {
-                                    var content;
-                                    return __generator(this, function (_a) {
-                                        switch (_a.label) {
-                                            case 0: return [4 /*yield*/, response_1.blob()];
-                                            case 1:
-                                                content = _a.sent();
-                                                return [2 /*return*/, content];
-                                        }
-                                    });
-                                });
-                            },
+                            /*
+                            TODO: [🥽]
+                                > async asBlob() {
+                                >     // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
+                                >     const content = await response.blob();
+                                >     return content;
+                                > },
+                            */
                             asJson: function () {
                                 return __awaiter(this, void 0, void 0, function () {
                                     var content;
@@ -5274,34 +5266,31 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                     }
                     filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
                     fileExtension = getFileExtension(filename_1);
-                    mimeType_1 = extensionToMimeType(fileExtension || '');
+                    mimeType = extensionToMimeType(fileExtension || '');
                     return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
                 case 3:
                     if (!(_e.sent())) {
                         throw new NotFoundError(spaceTrim$1(function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(filename_1), "\n                      "); }));
                     }
-                    // TODO: !!!!!! Test security file - file is scoped to the project (maybe do this in `filesystemTools`)
+                    // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
                     return [2 /*return*/, {
                             source: name,
                             filename: filename_1,
                             url: null,
-                            mimeType: mimeType_1,
-                            asBlob: function () {
-                                return __awaiter(this, void 0, void 0, function () {
-                                    var content;
-                                    return __generator(this, function (_a) {
-                                        switch (_a.label) {
-                                            case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
-                                            case 1:
-                                                content = _a.sent();
-                                                return [2 /*return*/, new Blob([
-                                                        content,
-                                                        // <- TODO: !!!!!! Test that this is working
-                                                    ], { type: mimeType_1 })];
-                                        }
-                                    });
-                                });
-                            },
+                            mimeType: mimeType,
+                            /*
+                            TODO: [🥽]
+                                > async asBlob() {
+                                >     const content = await tools.fs!.readFile(filename);
+                                >     return new Blob(
+                                >         [
+                                >             content,
+                                >             // <- TODO: [🥽] This is NOT tested, test it
+                                >         ],
+                                >         { type: mimeType },
+                                >     );
+                                > },
+                            */
                             asJson: function () {
                                 return __awaiter(this, void 0, void 0, function () {
                                     var _a, _b;
@@ -5337,9 +5326,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                         asJson: function () {
                             throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
                         },
-                        asBlob: function () {
-                            throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
-                        },
+                        /*
+                        TODO: [🥽]
+                            > asBlob() {
+                            >     throw new UnexpectedError(
+                            >         'Did not expect that `markdownScraper` would need to get the content `asBlob`',
+                            >     );
+                            > },
+                        */
                     }];
             }
         });
@@ -5664,6 +5658,7 @@ var knowledgeCommandParser = {
         'KNOWLEDGE ./hejny-cv.md',
         'KNOWLEDGE ./hejny-cv.pdf',
         'KNOWLEDGE ./hejny-cv.docx',
+        //          <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
     ],
     /**
      * Parses the KNOWLEDGE command
@@ -8372,6 +8367,301 @@ function pipelineStringToJson(pipelineString, tools, options) {
  * TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
  */
 
+// Note: We want to use the `exec` as async function
+var exec$1 = promisify(exec$2);
+/**
+ * @@@
+ *
+ * @private within the repository
+ */
+function locateAppOnLinux(_a) {
+    var appName = _a.appName, linuxWhich = _a.linuxWhich;
+    return __awaiter(this, void 0, void 0, function () {
+        var _b, stderr, stdout;
+        return __generator(this, function (_c) {
+            switch (_c.label) {
+                case 0: return [4 /*yield*/, exec$1("which ".concat(linuxWhich))];
+                case 1:
+                    _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
+                    if (!stderr && stdout) {
+                        return [2 /*return*/, stdout.trim()];
+                    }
+                    throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr));
+            }
+        });
+    });
+}
+/**
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+function $provideFilesystemForNode(options) {
+    if (!$isRunningInNode()) {
+        throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
+    }
+    var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
+    TODO_USE(isVerbose);
+    return {
+        stat: stat,
+        access: access,
+        constants: constants,
+        readFile: readFile,
+        readdir: readdir,
+    };
+}
+/**
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * Checks if the file is executable
+ *
+ * @private within the repository
+ */
+function isExecutable(path, fs) {
+    return __awaiter(this, void 0, void 0, function () {
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    _a.trys.push([0, 2, , 3]);
+                    return [4 /*yield*/, fs.access(path, fs.constants.X_OK)];
+                case 1:
+                    _a.sent();
+                    return [2 /*return*/, true];
+                case 2:
+                    _a.sent();
+                    return [2 /*return*/, false];
+                case 3: return [2 /*return*/];
+            }
+        });
+    });
+}
+/**
+ * Note: Not [~🟢~] because it is not directly dependent on `fs
+ * TODO: [🖇] What about symlinks?
+ */
+
+// Note: Module `userhome` has no types available, so it is imported using `require`
+//       @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+var userhome = require('userhome');
+// Note: We want to use the `exec` as async function
+var exec = promisify(exec$2);
+/**
+ * @@@
+ *
+ * @private within the repository
+ */
+function locateAppOnMacOs(_a) {
+    var appName = _a.appName, macOsName = _a.macOsName;
+    return __awaiter(this, void 0, void 0, function () {
+        var toExec, regPath, altPath, _b, stderr, stdout;
+        return __generator(this, function (_c) {
+            switch (_c.label) {
+                case 0:
+                    toExec = "/Contents/MacOS/".concat(macOsName);
+                    regPath = "/Applications/".concat(macOsName, ".app") + toExec;
+                    altPath = userhome(regPath.slice(1));
+                    return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())];
+                case 1:
+                    if (!_c.sent()) return [3 /*break*/, 2];
+                    return [2 /*return*/, regPath];
+                case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())];
+                case 3:
+                    if (_c.sent()) {
+                        return [2 /*return*/, altPath];
+                    }
+                    _c.label = 4;
+                case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))];
+                case 5:
+                    _b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
+                    if (!stderr && stdout) {
+                        return [2 /*return*/, stdout.trim() + toExec];
+                    }
+                    throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr));
+            }
+        });
+    });
+}
+/**
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * @@@
+ *
+ * @private within the repository
+ */
+function locateAppOnWindows(_a) {
+    var appName = _a.appName, windowsSuffix = _a.windowsSuffix;
+    return __awaiter(this, void 0, void 0, function () {
+        var prefixes, prefixes_1, prefixes_1_1, prefix, path, e_1_1;
+        var e_1, _b;
+        return __generator(this, function (_c) {
+            switch (_c.label) {
+                case 0:
+                    prefixes = [
+                        process.env.LOCALAPPDATA,
+                        join(process.env.LOCALAPPDATA || '', 'Programs'),
+                        process.env.PROGRAMFILES,
+                        process.env['PROGRAMFILES(X86)'],
+                    ];
+                    _c.label = 1;
+                case 1:
+                    _c.trys.push([1, 6, 7, 8]);
+                    prefixes_1 = __values(prefixes), prefixes_1_1 = prefixes_1.next();
+                    _c.label = 2;
+                case 2:
+                    if (!!prefixes_1_1.done) return [3 /*break*/, 5];
+                    prefix = prefixes_1_1.value;
+                    path = prefix + windowsSuffix;
+                    return [4 /*yield*/, isExecutable(path, $provideFilesystemForNode())];
+                case 3:
+                    if (_c.sent()) {
+                        return [2 /*return*/, path];
+                    }
+                    _c.label = 4;
+                case 4:
+                    prefixes_1_1 = prefixes_1.next();
+                    return [3 /*break*/, 2];
+                case 5: return [3 /*break*/, 8];
+                case 6:
+                    e_1_1 = _c.sent();
+                    e_1 = { error: e_1_1 };
+                    return [3 /*break*/, 8];
+                case 7:
+                    try {
+                        if (prefixes_1_1 && !prefixes_1_1.done && (_b = prefixes_1.return)) _b.call(prefixes_1);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                    return [7 /*endfinally*/];
+                case 8: throw new Error("Can not locate app ".concat(appName, " on Windows."));
+            }
+        });
+    });
+}
+/**
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * Locates an application on the system
+ *
+ * @private within the repository
+ */
+function locateApp(options) {
+    if (!$isRunningInNode()) {
+        throw new EnvironmentMismatchError('Locating apps works only in Node.js environment');
+    }
+    var appName = options.appName, linuxWhich = options.linuxWhich, windowsSuffix = options.windowsSuffix, macOsName = options.macOsName;
+    if (process.platform === 'win32') {
+        if (windowsSuffix) {
+            return locateAppOnWindows({ appName: appName, windowsSuffix: windowsSuffix });
+        }
+        else {
+            throw new Error("".concat(appName, " is not available on Windows."));
+        }
+    }
+    else if (process.platform === 'darwin') {
+        if (macOsName) {
+            return locateAppOnMacOs({ appName: appName, macOsName: macOsName });
+        }
+        else {
+            throw new Error("".concat(appName, " is not available on macOS."));
+        }
+    }
+    else {
+        if (linuxWhich) {
+            return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich });
+        }
+        else {
+            throw new Error("".concat(appName, " is not available on Linux."));
+        }
+    }
+}
+/**
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * @@@
+ *
+ * @private within the repository
+ */
+function locateLibreoffice() {
+    return locateApp({
+        appName: 'Libreoffice',
+        linuxWhich: 'libreoffice',
+        windowsSuffix: '\\LibreOffice\\program\\soffice.exe',
+        macOsName: 'LibreOffice',
+    });
+}
+/**
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * @@@
+ *
+ * @private within the repository
+ */
+function locatePandoc() {
+    return locateApp({
+        appName: 'Pandoc',
+        linuxWhich: 'pandoc',
+        windowsSuffix: '\\Pandoc\\pandoc.exe',
+        macOsName: 'Pandoc',
+    });
+}
+/**
+ * TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents`
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/node`
+ */
+function $provideExecutablesForNode(options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, _b, isAutoInstalled, _c, isVerbose;
+        var _d;
+        return __generator(this, function (_e) {
+            switch (_e.label) {
+                case 0:
+                    if (!$isRunningInNode()) {
+                        throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
+                    }
+                    _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
+                    TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables
+                    TODO_USE(isVerbose);
+                    _d = {};
+                    return [4 /*yield*/, locatePandoc()];
+                case 1:
+                    _d.pandocPath = _e.sent();
+                    return [4 /*yield*/, locateLibreoffice()];
+                case 2: return [2 /*return*/, (_d.libreOfficePath = _e.sent(),
+                        _d)];
+            }
+        });
+    });
+}
+/**
+ * TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
 /**
  * @@@
  *
@@ -8602,57 +8892,6 @@ function $provideLlmToolsFromEnv(options) {
  * TODO: [®] DRY Register logic
  */
 
-/**
- * @@@
- *
- * @public exported from `@promptbook/node`
- */
-function $provideExecutablesForNode(options) {
-    return __awaiter(this, void 0, void 0, function () {
-        var _a, _b, isAutoInstalled, _c, isVerbose;
-        return __generator(this, function (_d) {
-            if (!$isRunningInNode()) {
-                throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
-            }
-            _a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
-            TODO_USE(isAutoInstalled);
-            TODO_USE(isVerbose);
-            return [2 /*return*/, {
-                    // TODO: !!!!!! use `locate-app` library here
-                    pandocPath: 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe',
-                    libreOfficePath: 'C:/Program Files/LibreOffice/program/swriter.exe',
-                }];
-        });
-    });
-}
-/**
- * TODO: [🧠] THis should be maybe in different folder
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
- */
-
-/**
- * @@@
- *
- * @public exported from `@promptbook/node`
- */
-function $provideFilesystemForNode(options) {
-    if (!$isRunningInNode()) {
-        throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
-    }
-    var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
-    TODO_USE(isVerbose);
-    return {
-        stat: stat,
-        access: access,
-        constants: constants,
-        readFile: readFile,
-        readdir: readdir,
-    };
-}
-/**
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
- */
-
 /**
  * @@@
  *
@@ -12732,7 +12971,7 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
                             assistant_id: this.assistantId,
                             thread: {
                                 messages: [
-                                    // TODO: !!!!!! Allow threads to be passed
+                                    // TODO:  [🗯] !!!!!! Allow threads to be passed
                                     { role: 'user', content: rawPromptContent },
                                 ],
                             },
@@ -12742,7 +12981,6 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
                         if (this.options.isVerbose) {
                             console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
                         }
-                        console.log('!!!!!! OpenAI client', client);
                         return [4 /*yield*/, client.beta.threads.createAndRunStream(rawRequest)];
                     case 2:
                         stream = _g.sent();
@@ -12760,7 +12998,7 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
                                 messageDelta.content[0].type === 'text') {
                                 console.info('messageDelta', (_a = messageDelta.content[0].text) === null || _a === void 0 ? void 0 : _a.value);
                             }
-                            // TODO: !!!!!! report progress
+                            // <- TODO: [🐚] Make streaming and running tasks working
                         });
                         stream.on('messageCreated', function (message) {
                             if (_this.options.isVerbose) {
@@ -12792,14 +13030,16 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
                         // eslint-disable-next-line prefer-const
                         complete = getCurrentIsoDate();
                         usage = UNCERTAIN_USAGE;
-                        // TODO: !!!!!!> = computeOpenAiUsage(content, resultContent || '', rawResponse);
+                        // <- TODO: [🥘] Compute real usage for assistant
+                        //       ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse);
                         if (resultContent === null) {
                             throw new PipelineExecutionError('No response message from OpenAI');
                         }
                         return [2 /*return*/, $asDeeplyFrozenSerializableJson('OpenAiAssistantExecutionTools ChatPromptResult', {
                                 content: resultContent,
                                 modelName: 'assistant',
-                                //          <- TODO: !!!!!! Can we detect really used model: rawResponse.model || modelName,
+                                // <- TODO: [🥘] Detect used model in assistant
+                                //       ?> model: rawResponse.model || modelName,
                                 timing: {
                                     start: start,
                                     complete: complete,
@@ -12817,7 +13057,6 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
     return OpenAiAssistantExecutionTools;
 }(OpenAiExecutionTools));
 /**
- * TODO: !!!!!! DO not use colors - can be used in browser
  * TODO: [🧠][🧙‍♂️] Maybe there can be some wizzard for thoose who want to use just OpenAI
  * TODO: Maybe make custom OpenAiError
  * TODO: [🧠][🈁] Maybe use `isDeterministic` from options
@@ -13436,9 +13675,14 @@ var DocumentScraper = /** @class */ (function () {
                             asJson: function () {
                                 throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
                             },
-                            asBlob: function () {
-                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
-                            },
+                            /*
+                            TODO: [🥽]
+                                > asBlob() {
+                                >     throw new UnexpectedError(
+                                >         'Did not expect that `markdownScraper` would need to get the content `asBlob`',
+                                >     );
+                                > },
+                            */
                         };
                         knowledge = this.markdownScraper.scrape(markdownSource);
                         return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13604,9 +13848,14 @@ var LegacyDocumentScraper = /** @class */ (function () {
                             asJson: function () {
                                 throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
                             },
-                            asBlob: function () {
-                                throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
-                            },
+                            /*
+                            TODO: [🥽]
+                                > asBlob() {
+                                >     throw new UnexpectedError(
+                                >         'Did not expect that `documentScraper` would need to get the content `asBlob`',
+                                >     );
+                                > },
+                            */
                         };
                         knowledge = this.documentScraper.scrape(markdownSource);
                         return [4 /*yield*/, cacheFilehandler.destroy()];
@@ -13811,195 +14060,5 @@ var _PdfScraperRegistration = $scrapersRegister.register(createPdfScraper);
  * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
  */
 
-/**
- * Metadata of the scraper
- *
- * @private within the scraper directory
- */
-var websiteScraperMetadata = $deepFreeze({
-    title: 'Website scraper',
-    packageName: '@promptbook/website-crawler',
-    className: 'WebsiteScraper',
-    mimeTypes: ['text/html'],
-    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
-    isAvilableInBrowser: false,
-    requiredExecutables: [],
-}); /* <- TODO: [🤛] */
-/**
- * Registration of known scraper metadata
- *
- * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
- *
- * @public exported from `@promptbook/core`
- * @public exported from `@promptbook/cli`
- */
-var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
-
-/**
- * Create a new showdown converter instance
- *
- * @private utility of `WebsiteScraper`
- */
-function createShowdownConverter() {
-    return new Converter({
-        flavor: 'github',
-        /*
-        > import showdownHighlight from 'showdown-highlight';
-        > extensions: [
-        >     showdownHighlight({
-        >         // Whether to add the classes to the <pre> tag, default is false
-        >         pre: true,
-        >         // Whether to use hljs' auto language detection, default is true
-        >         auto_detection: true,
-        >     }),
-        > ],
-        */
-    });
-}
-
-/**
- * Scraper for websites
- *
- * @see `documentationUrl` for more details
- * @public exported from `@promptbook/website-crawler`
- */
-var WebsiteScraper = /** @class */ (function () {
-    function WebsiteScraper(tools, options) {
-        this.tools = tools;
-        this.options = options;
-        this.markdownScraper = new MarkdownScraper(tools, options);
-        this.showdownConverter = createShowdownConverter();
-    }
-    Object.defineProperty(WebsiteScraper.prototype, "metadata", {
-        /**
-         * Metadata of the scraper which includes title, mime types, etc.
-         */
-        get: function () {
-            return websiteScraperMetadata;
-        },
-        enumerable: false,
-        configurable: true
-    });
-    /**
-     * Convert the website  to `.md` file and returns intermediate source
-     *
-     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
-     */
-    WebsiteScraper.prototype.$convert = function (source) {
-        return __awaiter(this, void 0, void 0, function () {
-            var _a, _b, 
-            // TODO: [🧠] Maybe in node use headless browser not just JSDOM
-            rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
-            return __generator(this, function (_g) {
-                switch (_g.label) {
-                    case 0:
-                        _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
-                        // TODO: !!!!!! Does this work in browser? Make it work.
-                        if (source.url === null) {
-                            throw new KnowledgeScrapeError('Website scraper requires URL');
-                        }
-                        _f = JSDOM.bind;
-                        return [4 /*yield*/, source.asText()];
-                    case 1:
-                        jsdom = new (_f.apply(JSDOM, [void 0, _g.sent(), {
-                                url: source.url,
-                            }]))();
-                        reader = new Readability(jsdom.window.document);
-                        article = reader.parse();
-                        html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
-                        // Note: Unwrap html such as it is convertable by `markdownConverter`
-                        for (i = 0; i < 2; i++) {
-                            html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
-                        }
-                        if (html.includes('<div')) {
-                            html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
-                        }
-                        return [4 /*yield*/, getScraperIntermediateSource(source, {
-                                rootDirname: rootDirname,
-                                cacheDirname: cacheDirname,
-                                intermediateFilesStrategy: intermediateFilesStrategy,
-                                extension: 'html',
-                                isVerbose: isVerbose,
-                            })];
-                    case 2:
-                        cacheFilehandler = _g.sent();
-                        return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
-                    case 3:
-                        _g.sent();
-                        markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
-                        return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
-                }
-            });
-        });
-    };
-    /**
-     * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
-     */
-    WebsiteScraper.prototype.scrape = function (source) {
-        return __awaiter(this, void 0, void 0, function () {
-            var cacheFilehandler, markdownSource, knowledge;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.$convert(source)];
-                    case 1:
-                        cacheFilehandler = _a.sent();
-                        markdownSource = {
-                            source: source.source,
-                            filename: cacheFilehandler.filename,
-                            url: null,
-                            mimeType: 'text/markdown',
-                            asText: function () {
-                                return cacheFilehandler.markdown;
-                            },
-                            asJson: function () {
-                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
-                            },
-                            asBlob: function () {
-                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
-                            },
-                        };
-                        knowledge = this.markdownScraper.scrape(markdownSource);
-                        return [4 /*yield*/, cacheFilehandler.destroy()];
-                    case 2:
-                        _a.sent();
-                        return [2 /*return*/, knowledge];
-                }
-            });
-        });
-    };
-    return WebsiteScraper;
-}());
-/**
- * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
- * TODO: [🪂] Do it in parallel 11:11
- * Note: No need to aggregate usage here, it is done by intercepting the llmTools
- * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
- */
-
-/**
- * @@@
- *
- * @public exported from `@promptbook/website-crawler`
- */
-var createWebsiteScraper = Object.assign(function (tools, options) {
-    return new WebsiteScraper(tools, options);
-}, websiteScraperMetadata); /* <- TODO: [🤛] */
-/**
- * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
- */
-
-/**
- * Registration of known scraper
- *
- * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
- *
- * @public exported from `@promptbook/website-crawler`
- * @public exported from `@promptbook/cli`
- */
-var _WebsiteScraperRegistration = $scrapersRegister.register(createWebsiteScraper);
-/**
- * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
- */
-
-export { PROMPTBOOK_VERSION, _AnthropicClaudeMetadataRegistration, _AnthropicClaudeRegistration, _AzureOpenAiMetadataRegistration, _AzureOpenAiRegistration, _CLI, _DocumentScraperMetadataRegistration, _DocumentScraperRegistration, _LegacyDocumentScraperMetadataRegistration, _LegacyDocumentScraperRegistration, _MarkdownScraperMetadataRegistration, _MarkdownScraperRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiAssistantRegistration, _OpenAiMetadataRegistration, _OpenAiRegistration, _PdfScraperMetadataRegistration, _PdfScraperRegistration, _WebsiteScraperMetadataRegistration, _WebsiteScraperRegistration };
+export { PROMPTBOOK_VERSION, _AnthropicClaudeMetadataRegistration, _AnthropicClaudeRegistration, _AzureOpenAiMetadataRegistration, _AzureOpenAiRegistration, _CLI, _DocumentScraperMetadataRegistration, _DocumentScraperRegistration, _LegacyDocumentScraperMetadataRegistration, _LegacyDocumentScraperRegistration, _MarkdownScraperMetadataRegistration, _MarkdownScraperRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiAssistantRegistration, _OpenAiMetadataRegistration, _OpenAiRegistration, _PdfScraperMetadataRegistration, _PdfScraperRegistration };
 //# sourceMappingURL=index.es.js.map