@promptbook/website-crawler 0.72.0-28 → 0.72.0-29

package/esm/index.es.js

@@ -15,7 +15,7 @@ import { unparse, parse } from 'papaparse';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-27';
+var PROMPTBOOK_VERSION = '0.72.0-28';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -632,22 +632,6 @@ var websiteScraperMetadata = $deepFreeze({
  */
 $scrapersMetadataRegister.register(websiteScraperMetadata);
 
-/**
- * This error indicates that the promptbook can not retrieve knowledge from external sources
- *
- * @public exported from `@promptbook/core`
- */
-var KnowledgeScrapeError = /** @class */ (function (_super) {
-    __extends(KnowledgeScrapeError, _super);
-    function KnowledgeScrapeError(message) {
-        var _this = _super.call(this, message) || this;
-        _this.name = 'KnowledgeScrapeError';
-        Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
-        return _this;
-    }
-    return KnowledgeScrapeError;
-}(Error));
-
 /**
  * Just marks a place of place where should be something implemented
  * No side effects.
@@ -2838,6 +2822,22 @@ function preparePersona(personaDescription, tools, options) {
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
  */
 
+/**
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
+ *
+ * @public exported from `@promptbook/core`
+ */
+var KnowledgeScrapeError = /** @class */ (function (_super) {
+    __extends(KnowledgeScrapeError, _super);
+    function KnowledgeScrapeError(message) {
+        var _this = _super.call(this, message) || this;
+        _this.name = 'KnowledgeScrapeError';
+        Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
+        return _this;
+    }
+    return KnowledgeScrapeError;
+}(Error));
+
 /**
  * @@@
  *
@@ -5695,6 +5695,28 @@ var MarkdownScraper = /** @class */ (function () {
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
  */
 
+/**
+ * Create a new showdown converter instance
+ *
+ * @private utility of `WebsiteScraper`
+ */
+function createShowdownConverter() {
+    return new Converter({
+        flavor: 'github',
+        /*
+        > import showdownHighlight from 'showdown-highlight';
+        > extensions: [
+        >     showdownHighlight({
+        >         // Whether to add the classes to the <pre> tag, default is false
+        >         pre: true,
+        >         // Whether to use hljs' auto language detection, default is true
+        >         auto_detection: true,
+        >     }),
+        > ],
+        */
+    });
+}
+
 /**
  * Scraper for websites
  *
@@ -5706,9 +5728,12 @@ var WebsiteScraper = /** @class */ (function () {
         this.tools = tools;
         this.options = options;
         this.markdownScraper = new MarkdownScraper(tools, options);
+        // TODO: !!!!!! Remove
         TODO_USE(Readability);
         TODO_USE(Converter);
         TODO_USE(JSDOM);
+        TODO_USE(createShowdownConverter);
+        this.showdownConverter = createShowdownConverter();
     }
     Object.defineProperty(WebsiteScraper.prototype, "metadata", {
         /**
@@ -5729,9 +5754,6 @@ var WebsiteScraper = /** @class */ (function () {
         return __awaiter(this, void 0, void 0, function () {
             var markdown;
             return __generator(this, function (_a) {
-                if (source.url === null) {
-                    throw new KnowledgeScrapeError('Website scraper requires URL');
-                }
                 markdown = "";
                 return [2 /*return*/, __assign(__assign({}, source), { markdown: markdown, destroy: function () { } })];
             });