@promptbook/website-crawler 0.71.0-16 → 0.71.0-18

package/README.md

@@ -2,7 +2,7 @@
 
 # ![Promptbook logo - cube with letters P and B](./other/design/logo-h1.png) Promptbook
 
-Supercharge your use of large language models
+Build responsible, controlled and transparent applications on top of LLM models!
 
 
 
@@ -26,6 +26,10 @@ Supercharge your use of large language models
 
 
 
+<blockquote style="color: #ff8811">
+    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
+</blockquote>
+
 ## 📦 Package `@promptbook/website-crawler`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).

package/esm/index.es.js

@@ -1,13 +1,13 @@
 import spaceTrim$1, { spaceTrim } from 'spacetrim';
 import { Readability } from '@mozilla/readability';
-import { mkdir, rm, readFile, writeFile } from 'fs/promises';
+import { mkdir, rm, writeFile } from 'fs/promises';
 import { JSDOM } from 'jsdom';
-import { forTime } from 'waitasecond';
 import { SHA256 } from 'crypto-js';
 import hexEncoder from 'crypto-js/enc-hex';
 import { basename, join, dirname } from 'path';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
+import { forTime } from 'waitasecond';
 import { lookup } from 'mime-types';
 import { unparse, parse } from 'papaparse';
 import { Converter } from 'showdown';
@@ -16,7 +16,7 @@ import { Converter } from 'showdown';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.71.0-15';
+var PROMPTBOOK_VERSION = '0.71.0-17';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -2527,8 +2527,7 @@ function countTotalUsage(llmTools) {
  * Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
  *
  * Note: Internal utility of `joinLlmExecutionTools` but exposed type
- * @public exported from `@promptbook/types`
- * TODO: !!!!!! Export as runtime class not just type
+ * @public exported from `@promptbook/core`
  */
 var MultipleLlmExecutionTools = /** @class */ (function () {
     /**
@@ -2915,7 +2914,7 @@ var $scrapersRegister = new $Register('scraper_constructors');
  * TODO: [®] DRY Register logic
  */
 
-// TODO: !!!!!! Maybe delete this function
+// TODO: !!!!!!last - Maybe delete this function
 /**
  * Creates a message with all registered scrapers
  *
@@ -3023,7 +3022,6 @@ function $registeredScrapersMessage() {
  * @private within the repository
  */
 function sourceContentToName(sourceContent) {
-    // TODO: !!!!!! Better name for source than gibberish hash
     var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
         //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
         .toString( /* hex */)
@@ -3200,7 +3198,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                                                 content = _a.sent();
                                                 return [2 /*return*/, new Blob([
                                                         content,
-                                                        // <- TODO: !!!!!! Maybe not working
+                                                        // <- TODO: !!!!!! Test that this is working
                                                     ], { type: mimeType_1 })];
                                         }
                                     });
@@ -3213,7 +3211,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                                         switch (_c.label) {
                                             case 0:
                                                 _b = (_a = JSON).parse;
-                                                return [4 /*yield*/, readFile(filename_1, 'utf-8')];
+                                                return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
                                             case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
                                         }
                                     });
@@ -3223,7 +3221,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                                 return __awaiter(this, void 0, void 0, function () {
                                     return __generator(this, function (_a) {
                                         switch (_a.label) {
-                                            case 0: return [4 /*yield*/, readFile(filename_1, 'utf-8')];
+                                            case 0: return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
                                             case 1: return [2 /*return*/, _a.sent()];
                                         }
                                     });
@@ -5778,32 +5776,29 @@ var MarkdownScraper = /** @class */ (function () {
  */
 
 /**
- * A converter instance that uses showdown and highlight extensions
+ * Create a new showdown converter instance
  *
- * @type {Converter}
- * @private for markdown and html knowledge scrapers
- */
-var markdownConverter = new Converter({
-    flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
-    /*
-    > import showdownHighlight from 'showdown-highlight';
-    > extensions: [
-    >     showdownHighlight({
-    >         // Whether to add the classes to the <pre> tag, default is false
-    >         pre: true,
-    >         // Whether to use hljs' auto language detection, default is true
-    >         auto_detection: true,
-    >     }),
-    > ],
-    */
-});
-/**
- * TODO: !!!!!! Figure out better name not to confuse with `Converter`
- * TODO: !!!!!! Lazy-make converter
+ * @private utility of `WebsiteScraper`
  */
+function createShowdownConverter() {
+    return new Converter({
+        flavor: 'github',
+        /*
+        > import showdownHighlight from 'showdown-highlight';
+        > extensions: [
+        >     showdownHighlight({
+        >         // Whether to add the classes to the <pre> tag, default is false
+        >         pre: true,
+        >         // Whether to use hljs' auto language detection, default is true
+        >         auto_detection: true,
+        >     }),
+        > ],
+        */
+    });
+}
 
 /**
- * Scraper for .docx files
+ * Scraper for websites
  *
  * @see `documentationUrl` for more details
  * @public exported from `@promptbook/website-crawler`
@@ -5813,6 +5808,7 @@ var WebsiteScraper = /** @class */ (function () {
         this.tools = tools;
         this.options = options;
         this.markdownScraper = new MarkdownScraper(tools, options);
+        this.showdownConverter = createShowdownConverter();
     }
     Object.defineProperty(WebsiteScraper.prototype, "metadata", {
         /**
@@ -5833,7 +5829,6 @@ var WebsiteScraper = /** @class */ (function () {
         return __awaiter(this, void 0, void 0, function () {
             var _a, _b, 
             // TODO: [🧠] Maybe in node use headless browser not just JSDOM
-            // externalProgramsPaths = {},
             rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
             return __generator(this, function (_g) {
                 switch (_g.label) {
@@ -5851,10 +5846,6 @@ var WebsiteScraper = /** @class */ (function () {
                             }]))();
                         reader = new Readability(jsdom.window.document);
                         article = reader.parse();
-                        console.log(article);
-                        return [4 /*yield*/, forTime(10000)];
-                    case 2:
-                        _g.sent();
                         html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
                         // Note: Unwrap html such as it is convertable by `markdownConverter`
                         for (i = 0; i < 2; i++) {
@@ -5870,12 +5861,12 @@ var WebsiteScraper = /** @class */ (function () {
                                 extension: 'html',
                                 isVerbose: isVerbose,
                             })];
-                    case 3:
+                    case 2:
                         cacheFilehandler = _g.sent();
                         return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
-                    case 4:
+                    case 3:
                         _g.sent();
-                        markdown = markdownConverter.makeMarkdown(html, jsdom.window.document);
+                        markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
                         return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
                 }
             });
@@ -5919,7 +5910,6 @@ var WebsiteScraper = /** @class */ (function () {
     return WebsiteScraper;
 }());
 /**
- * TODO: !!!!!! Put into separate package
  * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
  * TODO: [🪂] Do it in parallel 11:11
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools