@promptbook/cli 0.72.0-30 → 0.72.0-32

package/esm/index.es.js

@@ -2,7 +2,7 @@ import commander from 'commander';
 import spaceTrim$1, { spaceTrim } from 'spacetrim';
 import colors from 'colors';
 import { forTime } from 'waitasecond';
-import { stat, access, constants, readFile, readdir, writeFile, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
+import { stat, access, constants, readFile, writeFile, readdir, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
 import { join, basename, dirname } from 'path';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
@@ -27,7 +27,7 @@ import { Converter } from 'showdown';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-29';
+var PROMPTBOOK_VERSION = '0.72.0-31';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -8415,6 +8415,7 @@ function $provideFilesystemForNode(options) {
         access: access,
         constants: constants,
         readFile: readFile,
+        writeFile: writeFile,
         readdir: readdir,
     };
 }
@@ -14120,13 +14121,26 @@ var WebsiteScraper = /** @class */ (function () {
         this.tools = tools;
         this.options = options;
         this.markdownScraper = new MarkdownScraper(tools, options);
-        // TODO: !!!!!! Remove
+        this.showdownConverter = createShowdownConverter();
+        /**/
+        // TODO:  [🏄] !!!!!! Remove or describe why it is here
         TODO_USE(Readability);
         TODO_USE(Converter);
         TODO_USE(JSDOM);
-        TODO_USE(new JSDOM());
         TODO_USE(createShowdownConverter);
-        this.showdownConverter = createShowdownConverter();
+        /**/
+        /**/
+        var jsdom = new JSDOM();
+        var reader = new Readability(jsdom.window.document);
+        keepUnused(reader);
+        /**/
+        /**/
+        keepUnused(DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_VERBOSE, DEFAULT_SCRAPE_CACHE_DIRNAME);
+        keepUnused(EnvironmentMismatchError);
+        keepUnused(KnowledgeScrapeError);
+        keepUnused(getScraperIntermediateSource);
+        keepUnused();
+        /**/
     }
     Object.defineProperty(WebsiteScraper.prototype, "metadata", {
         /**
@@ -14145,10 +14159,55 @@ var WebsiteScraper = /** @class */ (function () {
      */
     WebsiteScraper.prototype.$convert = function (source) {
         return __awaiter(this, void 0, void 0, function () {
-            var markdown;
-            return __generator(this, function (_a) {
-                markdown = "";
-                return [2 /*return*/, __assign(__assign({}, source), { markdown: markdown, destroy: function () { } })];
+            var _a, _b, 
+            // TODO: [🧠] Maybe in node use headless browser not just JSDOM
+            rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
+            return __generator(this, function (_g) {
+                switch (_g.label) {
+                    case 0:
+                        _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
+                        // [🏄] !!!!!!!
+                        keepUnused(rootDirname, cacheDirname, intermediateFilesStrategy, isVerbose);
+                        if (source.url === null) {
+                            throw new KnowledgeScrapeError('Website scraper requires URL');
+                        }
+                        if (this.tools.fs === undefined) {
+                            throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
+                        }
+                        _f = JSDOM.bind;
+                        return [4 /*yield*/, source.asText()];
+                    case 1:
+                        jsdom = new (_f.apply(JSDOM, [void 0, _g.sent(), {
+                                // <- TODO: !!!!!!! Problem with build is probbably in `new JSDOM();`
+                                url: source.url,
+                            }]))();
+                        // [🏄] !!!!!!!
+                        keepUnused(jsdom);
+                        reader = new Readability(jsdom.window.document);
+                        article = reader.parse();
+                        html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
+                        // Note: Unwrap html such as it is convertable by `markdownConverter`
+                        for (i = 0; i < 2; i++) {
+                            html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
+                        }
+                        if (html.includes('<div')) {
+                            html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
+                        }
+                        return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                rootDirname: rootDirname,
+                                cacheDirname: cacheDirname,
+                                intermediateFilesStrategy: intermediateFilesStrategy,
+                                extension: 'html',
+                                isVerbose: isVerbose,
+                            })];
+                    case 2:
+                        cacheFilehandler = _g.sent();
+                        return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8')];
+                    case 3:
+                        _g.sent();
+                        markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
+                        return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
+                }
             });
         });
     };