@promptbook/cli 0.72.0-23 → 0.72.0-27

package/esm/index.es.js

@@ -24,7 +24,7 @@ import OpenAI from 'openai';
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-22';
+var PROMPTBOOK_VERSION = '0.72.0-26';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -14060,5 +14060,142 @@ var _PdfScraperRegistration = $scrapersRegister.register(createPdfScraper);
  * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
  */
 
-export { PROMPTBOOK_VERSION, _AnthropicClaudeMetadataRegistration, _AnthropicClaudeRegistration, _AzureOpenAiMetadataRegistration, _AzureOpenAiRegistration, _CLI, _DocumentScraperMetadataRegistration, _DocumentScraperRegistration, _LegacyDocumentScraperMetadataRegistration, _LegacyDocumentScraperRegistration, _MarkdownScraperMetadataRegistration, _MarkdownScraperRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiAssistantRegistration, _OpenAiMetadataRegistration, _OpenAiRegistration, _PdfScraperMetadataRegistration, _PdfScraperRegistration };
+/**
+ * Metadata of the scraper
+ *
+ * @private within the scraper directory
+ */
+var websiteScraperMetadata = $deepFreeze({
+    title: 'Website scraper',
+    packageName: '@promptbook/website-crawler',
+    className: 'WebsiteScraper',
+    mimeTypes: ['text/html'],
+    documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
+    isAvilableInBrowser: false,
+    requiredExecutables: [],
+}); /* <- TODO: [🤛] */
+/**
+ * Registration of known scraper metadata
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/core`
+ * @public exported from `@promptbook/cli`
+ */
+var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
+
+/**
+ * Scraper for websites
+ *
+ * @see `documentationUrl` for more details
+ * @public exported from `@promptbook/website-crawler`
+ */
+var WebsiteScraper = /** @class */ (function () {
+    function WebsiteScraper(tools, options) {
+        this.tools = tools;
+        this.options = options;
+        this.markdownScraper = new MarkdownScraper(tools, options);
+    }
+    Object.defineProperty(WebsiteScraper.prototype, "metadata", {
+        /**
+         * Metadata of the scraper which includes title, mime types, etc.
+         */
+        get: function () {
+            return websiteScraperMetadata;
+        },
+        enumerable: false,
+        configurable: true
+    });
+    /**
+     * Convert the website  to `.md` file and returns intermediate source
+     *
+     * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
+     */
+    WebsiteScraper.prototype.$convert = function (source) {
+        return __awaiter(this, void 0, void 0, function () {
+            var markdown;
+            return __generator(this, function (_a) {
+                if (source.url === null) {
+                    throw new KnowledgeScrapeError('Website scraper requires URL');
+                }
+                markdown = "";
+                return [2 /*return*/, __assign(__assign({}, source), { markdown: markdown, destroy: function () { } })];
+            });
+        });
+    };
+    /**
+     * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
+     */
+    WebsiteScraper.prototype.scrape = function (source) {
+        return __awaiter(this, void 0, void 0, function () {
+            var cacheFilehandler, markdownSource, knowledge;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.$convert(source)];
+                    case 1:
+                        cacheFilehandler = _a.sent();
+                        markdownSource = {
+                            source: source.source,
+                            filename: cacheFilehandler.filename,
+                            url: null,
+                            mimeType: 'text/markdown',
+                            asText: function () {
+                                return cacheFilehandler.markdown;
+                            },
+                            asJson: function () {
+                                throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
+                            },
+                            /*
+                            TODO: [🥽]
+                                > asBlob() {
+                                >     throw new UnexpectedError(
+                                >         'Did not expect that `markdownScraper` would need to get the content `asBlob`',
+                                >     );
+                                > },
+                            */
+                        };
+                        knowledge = this.markdownScraper.scrape(markdownSource);
+                        return [4 /*yield*/, cacheFilehandler.destroy()];
+                    case 2:
+                        _a.sent();
+                        return [2 /*return*/, knowledge];
+                }
+            });
+        });
+    };
+    return WebsiteScraper;
+}());
+/**
+ * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
+ * TODO: [🪂] Do it in parallel 11:11
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
+ */
+
+/**
+ * @@@
+ *
+ * @public exported from `@promptbook/website-crawler`
+ */
+var createWebsiteScraper = Object.assign(function (tools, options) {
+    return new WebsiteScraper(tools, options);
+}, websiteScraperMetadata); /* <- TODO: [🤛] */
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */
+
+/**
+ * Registration of known scraper
+ *
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
+ *
+ * @public exported from `@promptbook/website-crawler`
+ * @public exported from `@promptbook/cli`
+ */
+var _WebsiteScraperRegistration = $scrapersRegister.register(createWebsiteScraper);
+/**
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
+ */
+
+export { PROMPTBOOK_VERSION, _AnthropicClaudeMetadataRegistration, _AnthropicClaudeRegistration, _AzureOpenAiMetadataRegistration, _AzureOpenAiRegistration, _CLI, _DocumentScraperMetadataRegistration, _DocumentScraperRegistration, _LegacyDocumentScraperMetadataRegistration, _LegacyDocumentScraperRegistration, _MarkdownScraperMetadataRegistration, _MarkdownScraperRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiAssistantRegistration, _OpenAiMetadataRegistration, _OpenAiRegistration, _PdfScraperMetadataRegistration, _PdfScraperRegistration, _WebsiteScraperMetadataRegistration, _WebsiteScraperRegistration };
 //# sourceMappingURL=index.es.js.map