@botpress/runtime 1.6.4 → 1.6.6

package/dist/internal.js

@@ -48,7 +48,7 @@ var init_define_BUILD = __esm({
 var define_PACKAGE_VERSIONS_default;
 var init_define_PACKAGE_VERSIONS = __esm({
   "<define:__PACKAGE_VERSIONS__>"() {
-    define_PACKAGE_VERSIONS_default = { runtime: "1.6.4", adk: "not-installed", sdk: "4.17.3", llmz: "0.0.27", zai: "2.4.0", cognitive: "0.2.0" };
+    define_PACKAGE_VERSIONS_default = { runtime: "1.6.6", adk: "not-installed", sdk: "4.17.3", llmz: "0.0.27", zai: "2.4.0", cognitive: "0.2.0" };
   }
 });
 
@@ -25291,7 +25291,7 @@ var require_follow_redirects = __commonJS({
       var currentUrlParts = parseUrl(this._currentUrl);
       var currentHost = currentHostHeader || currentUrlParts.host;
       var currentUrl = /^\w+:/.test(location) ? this._currentUrl : url2.format(Object.assign(currentUrlParts, { host: currentHost }));
-      var redirectUrl = resolveUrl(location, currentUrl);
+      var redirectUrl = resolveUrl2(location, currentUrl);
       debug("redirecting to", redirectUrl.href);
       this._isRedirect = true;
       spreadUrlObject(redirectUrl, this._options);
@@ -25375,7 +25375,7 @@ var require_follow_redirects = __commonJS({
       }
       return parsed;
     }
-    function resolveUrl(relative, base) {
+    function resolveUrl2(relative, base) {
       return useNativeURL ? new URL2(relative, base) : parseUrl(url2.resolve(base, relative));
     }
     function validateUrl(input) {
@@ -44688,6 +44688,95 @@ var XMLParser = class {
   }
 };
 
+// src/primitives/data-sources/html-fetch.ts
+init_define_BUILD();
+init_define_PACKAGE_VERSIONS();
+function extractHtmlMetadata(html) {
+  const metadata = {};
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  if (titleMatch && titleMatch[1]) {
+    metadata.title = titleMatch[1].trim();
+  }
+  let descriptionMatch = html.match(
+    /<meta\s+(?:name|property)=["'](?:description|og:description)["']\s+content="([^"]+)"/i
+  );
+  if (!descriptionMatch) {
+    descriptionMatch = html.match(
+      /<meta\s+(?:name|property)=["'](?:description|og:description)["']\s+content='([^']+)'/i
+    );
+  }
+  if (descriptionMatch && descriptionMatch[1]) {
+    metadata.description = descriptionMatch[1].trim();
+  }
+  const faviconPatterns = [
+    // rel first, double quotes
+    /<link\s+[^>]*rel="(?:icon|shortcut icon|apple-touch-icon)"[^>]*href="([^"]+)"/i,
+    // rel first, single quotes
+    /<link\s+[^>]*rel='(?:icon|shortcut icon|apple-touch-icon)'[^>]*href='([^']+)'/i,
+    // href first, double quotes
+    /<link\s+[^>]*href="([^"]+)"[^>]*rel="(?:icon|shortcut icon|apple-touch-icon)"/i,
+    // href first, single quotes
+    /<link\s+[^>]*href='([^']+)'[^>]*rel='(?:icon|shortcut icon|apple-touch-icon)'/i
+  ];
+  for (const pattern of faviconPatterns) {
+    const faviconMatch = html.match(pattern);
+    if (faviconMatch && faviconMatch[1]) {
+      metadata.favicon = faviconMatch[1].trim();
+      break;
+    }
+  }
+  if (!metadata.favicon) {
+    metadata.favicon = "/favicon.ico";
+  }
+  return metadata;
+}
+function resolveUrl(url2, baseUrl) {
+  if (url2.startsWith("http://") || url2.startsWith("https://")) {
+    return url2;
+  }
+  try {
+    const base = new URL(baseUrl);
+    return new URL(url2, base.origin).href;
+  } catch {
+    return url2;
+  }
+}
+async function fetchHtml(url2, options) {
+  const userAgent = options?.userAgent || "Mozilla/5.0 (compatible; BotpressBot/1.0)";
+  const fetchOptions = {
+    headers: {
+      "User-Agent": userAgent
+    }
+  };
+  if (options?.timeout) {
+    fetchOptions.signal = AbortSignal.timeout(options.timeout);
+  }
+  const response = await fetch(url2, fetchOptions);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url2}: ${response.status} ${response.statusText}`);
+  }
+  const contentType = response.headers.get("content-type") || "text/html";
+  const content = await response.text();
+  const isHtml = contentType.includes("text/html") || contentType.includes("application/xhtml");
+  if (!isHtml) {
+    return {
+      url: url2,
+      contentType,
+      content
+    };
+  }
+  const extracted = extractHtmlMetadata(content);
+  if (extracted.favicon) {
+    extracted.favicon = resolveUrl(extracted.favicon, url2);
+  }
+  return {
+    url: url2,
+    contentType,
+    content,
+    metadata: extracted
+  };
+}
+
 // src/primitives/data-sources/source-website.ts
 var State = z20.object({
   urls: z20.array(
@@ -44707,6 +44796,7 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
   urls;
   filterFn;
   customFetch;
+  fetchStrategy;
   maxPages;
   maxDepth;
   transformFn;
@@ -44717,7 +44807,16 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     this.sitemapUrl = options.sitemapUrl ?? void 0;
     this.urls = options.urls ?? void 0;
     this.filterFn = "filter" in options ? options.filter : void 0;
-    this.customFetch = options.fetch ?? void 0;
+    if (typeof options.fetch === "string") {
+      this.fetchStrategy = options.fetch;
+      this.customFetch = void 0;
+    } else if (typeof options.fetch === "function") {
+      this.customFetch = options.fetch;
+      this.fetchStrategy = "node:fetch";
+    } else {
+      this.fetchStrategy = "node:fetch";
+      this.customFetch = void 0;
+    }
     this.maxPages = Math.max(1, Math.min(("maxPages" in options ? options.maxPages : void 0) ?? 5e4, 5e4));
     this.maxDepth = Math.max(1, Math.min(("maxDepth" in options ? options.maxDepth : void 0) ?? 20, 20));
   }
@@ -44725,51 +44824,82 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     return !!adk.project.integrations.get("browser");
   }
   /**
-   * Fetch content from a URL with fallback strategy
+   * Convert HtmlMetadata to FetchResult metadata format
+   */
+  convertMetadata(metadata) {
+    const result = {};
+    if (metadata.title) {
+      result[WellKnownMetadata.knowledge.TITLE] = metadata.title;
+    }
+    if (metadata.description) {
+      result[WellKnownMetadata.knowledge.DESCRIPTION] = metadata.description;
+    }
+    if (metadata.favicon) {
+      result[WellKnownMetadata.knowledge.FAVICON] = metadata.favicon;
+    }
+    return result;
+  }
+  /**
+   * Default fetch implementation using Node's built-in fetch
+   */
+  async defaultFetch(url2) {
+    const result = await fetchHtml(url2, {
+      timeout: 3e4
+    });
+    if (!result.metadata) {
+      return {
+        url: result.url,
+        contentType: result.contentType,
+        content: result.content
+      };
+    }
+    return {
+      url: result.url,
+      contentType: result.contentType,
+      content: result.content,
+      metadata: this.convertMetadata(result.metadata)
+    };
+  }
+  /**
+   * Fetch content from a URL for sitemap parsing (raw content needed)
    */
   async fetchSitemap(url2) {
     if (this.customFetch) {
       try {
         return await this.customFetch(url2);
       } catch (err) {
-        console.warn(`Custom fetch failed for ${url2}, falling back...`);
+        console.warn(`Custom fetch failed for ${url2}, falling back to ${this.fetchStrategy}...`);
       }
     }
-    if (!this.isBrowserIntegrationAvailable()) {
-      throw new Error(
-        `The 'browser' integration is not installed and is required for crawling website. Please provide a custom fetch function or install the 'browser' integration.`
-      );
-    }
-    const output2 = await adk.project.integrations.get("browser")?.actions.browsePages({
-      urls: [url2],
-      timeout: 3e4,
-      waitFor: 500
-    });
-    const result = output2?.results[0];
-    if (!result || !result.content) {
-      throw new Error(`Failed to fetch content from ${url2}`);
+    if (this.fetchStrategy === "integration:browser") {
+      return this.fetchWithBrowserIntegration(url2, { raw: true });
+    } else {
+      return this.defaultFetch(url2);
     }
-    return {
-      url: result.url,
-      contentType: "application/html",
-      content: result.raw
-    };
   }
   /**
-   * Fetch content from a URL with fallback strategy
+   * Fetch content from a URL for indexing (with metadata extraction)
    */
   async fetchUrl(url2) {
     if (this.customFetch) {
       try {
         return await this.customFetch(url2);
       } catch (err) {
-        console.warn(`Custom fetch failed for ${url2}, falling back...`);
+        console.warn(`Custom fetch failed for ${url2}, falling back to ${this.fetchStrategy}...`);
       }
     }
+    if (this.fetchStrategy === "integration:browser") {
+      return this.fetchWithBrowserIntegration(url2, { raw: false });
+    } else {
+      return this.defaultFetch(url2);
+    }
+  }
+  /**
+   * Fetch content using the browser integration
+   */
+  async fetchWithBrowserIntegration(url2, options) {
     if (!this.isBrowserIntegrationAvailable()) {
-      throw new Error(
-        `The 'browser' integration is not installed and is required for crawling website. Please provide a custom fetch function or install the 'browser' integration.`
-      );
+      throw new Error(`The 'browser' integration is not installed. Please install it or use fetch: 'node:fetch'.`);
     }
     const output2 = await adk.project.integrations.get("browser")?.actions.browsePages({
       urls: [url2],
@@ -44780,6 +44910,13 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     if (!result || !result.content) {
       throw new Error(`Failed to fetch content from ${url2}`);
     }
+    if (options.raw && result.raw) {
+      return {
+        url: result.url,
+        contentType: "application/html",
+        content: result.raw
+      };
+    }
     return {
       url: result.url,
       contentType: "text/markdown",
@@ -45008,6 +45145,13 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
         const toRemove = existingFiles.filter(
           (f) => !discoveredUrls.find((u) => u.loc === f.metadata?.[WellKnownMetadata.knowledge.URL])
         );
+        if (existingFiles.length > 0 && toRemove.length >= existingFiles.length * 0.8) {
+          console.error(
+            `Warning: All existing files (${existingFiles.length}) are scheduled for removal. Please check if the sitemap URL is correct and the website is accessible. We will try again in 5 minutes.`
+          );
+          await step2.sleep("retry wait", 5 * 60 * 1e3);
+          throw new Error("Aborting sync due to potential misconfiguration (all files to be removed)");
+        }
         const toFetch = [];
         let skippedUnchanged = 0;
         for (const url2 of discoveredUrls) {