@promptbook/cli 0.72.0-30 → 0.72.0-32

package/esm/typings/src/execution/FilesystemTools.d.ts

@@ -3,7 +3,7 @@ import type fs from 'fs/promises';
 /**
  * Container for all the tools needed to manipulate with filesystem
  */
-export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'stat' | 'readdir'>;
+export type FilesystemTools = Pick<typeof fs, 'access' | 'constants' | 'readFile' | 'writeFile' | 'stat' | 'readdir'>;
 /**
  * TODO: Implement destroyable pattern to free resources
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/cli",
-    "version": "0.72.0-30",
+    "version": "0.72.0-32",
     "description": "Supercharge your use of large language models",
     "private": false,
     "sideEffects": false,

package/umd/index.umd.js

@@ -39,7 +39,7 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.72.0-29';
+    var PROMPTBOOK_VERSION = '0.72.0-31';
     // TODO: [main] !!!! List here all the versions and annotate + put into script
 
     /*! *****************************************************************************
@@ -8427,6 +8427,7 @@
             access: promises.access,
             constants: promises.constants,
             readFile: promises.readFile,
+            writeFile: promises.writeFile,
             readdir: promises.readdir,
         };
     }
@@ -14132,13 +14133,26 @@
             this.tools = tools;
             this.options = options;
             this.markdownScraper = new MarkdownScraper(tools, options);
-            // TODO: !!!!!! Remove
+            this.showdownConverter = createShowdownConverter();
+            /**/
+            // TODO:  [🏄] !!!!!! Remove or describe why it is here
             TODO_USE(readability.Readability);
             TODO_USE(showdown.Converter);
             TODO_USE(jsdom.JSDOM);
-            TODO_USE(new jsdom.JSDOM());
             TODO_USE(createShowdownConverter);
-            this.showdownConverter = createShowdownConverter();
+            /**/
+            /**/
+            var jsdom$1 = new jsdom.JSDOM();
+            var reader = new readability.Readability(jsdom$1.window.document);
+            keepUnused(reader);
+            /**/
+            /**/
+            keepUnused(DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_VERBOSE, DEFAULT_SCRAPE_CACHE_DIRNAME);
+            keepUnused(EnvironmentMismatchError);
+            keepUnused(KnowledgeScrapeError);
+            keepUnused(getScraperIntermediateSource);
+            keepUnused();
+            /**/
         }
         Object.defineProperty(WebsiteScraper.prototype, "metadata", {
             /**
@@ -14157,10 +14171,55 @@
          */
         WebsiteScraper.prototype.$convert = function (source) {
             return __awaiter(this, void 0, void 0, function () {
-                var markdown;
-                return __generator(this, function (_a) {
-                    markdown = "";
-                    return [2 /*return*/, __assign(__assign({}, source), { markdown: markdown, destroy: function () { } })];
+                var _a, _b, 
+                // TODO: [🧠] Maybe in node use headless browser not just JSDOM
+                rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
+                return __generator(this, function (_g) {
+                    switch (_g.label) {
+                        case 0:
+                            _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
+                            // [🏄] !!!!!!!
+                            keepUnused(rootDirname, cacheDirname, intermediateFilesStrategy, isVerbose);
+                            if (source.url === null) {
+                                throw new KnowledgeScrapeError('Website scraper requires URL');
+                            }
+                            if (this.tools.fs === undefined) {
+                                throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
+                            }
+                            _f = jsdom.JSDOM.bind;
+                            return [4 /*yield*/, source.asText()];
+                        case 1:
+                            jsdom$1 = new (_f.apply(jsdom.JSDOM, [void 0, _g.sent(), {
+                                    // <- TODO: !!!!!!! Problem with build is probbably in `new JSDOM();`
+                                    url: source.url,
+                                }]))();
+                            // [🏄] !!!!!!!
+                            keepUnused(jsdom$1);
+                            reader = new readability.Readability(jsdom$1.window.document);
+                            article = reader.parse();
+                            html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
+                            // Note: Unwrap html such as it is convertable by `markdownConverter`
+                            for (i = 0; i < 2; i++) {
+                                html = html.replace(/<div\s*(?:id="readability-page-\d+"\s+class="page")?>(.*)<\/div>/is, '$1');
+                            }
+                            if (html.includes('<div')) {
+                                html = (article === null || article === void 0 ? void 0 : article.textContent) || '';
+                            }
+                            return [4 /*yield*/, getScraperIntermediateSource(source, {
+                                    rootDirname: rootDirname,
+                                    cacheDirname: cacheDirname,
+                                    intermediateFilesStrategy: intermediateFilesStrategy,
+                                    extension: 'html',
+                                    isVerbose: isVerbose,
+                                })];
+                        case 2:
+                            cacheFilehandler = _g.sent();
+                            return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8')];
+                        case 3:
+                            _g.sent();
+                            markdown = this.showdownConverter.makeMarkdown(html, jsdom$1.window.document);
+                            return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
+                    }
                 });
             });
         };