@botpress/runtime 1.6.4 → 1.6.6

package/dist/primitives/data-sources/html-fetch.d.ts

@@ -0,0 +1,57 @@
+/**
+ * HTML fetching and metadata extraction utilities
+ */
+export type HtmlMetadata = {
+    title?: string;
+    description?: string;
+    favicon?: string;
+};
+export type FetchHtmlResult = {
+    url: string;
+    contentType: string;
+    content: string;
+    metadata?: HtmlMetadata;
+};
+/**
+ * Extract metadata from HTML content using regex patterns
+ *
+ * @param html - The HTML content to parse
+ * @returns Extracted metadata including title, description, and favicon
+ */
+export declare function extractHtmlMetadata(html: string): HtmlMetadata;
+/**
+ * Resolve a potentially relative URL to an absolute URL
+ *
+ * @param url - The URL to resolve (may be relative)
+ * @param baseUrl - The base URL to resolve against
+ * @returns The absolute URL, or the original URL if resolution fails
+ */
+export declare function resolveUrl(url: string, baseUrl: string): string;
+/**
+ * Fetch content from a URL and extract metadata if HTML
+ *
+ * This function safely handles both HTML and non-HTML content (XML, JSON, text, etc.).
+ * Metadata extraction only occurs for HTML content types. For other content types
+ * (like sitemap.xml, robots.txt, RSS feeds), it returns the raw content without
+ * attempting metadata extraction.
+ *
+ * @param url - The URL to fetch
+ * @param options - Optional fetch options
+ * @returns Fetch result with content and extracted metadata (HTML only)
+ *
+ * @example
+ * // Fetching HTML - extracts metadata
+ * const html = await fetchHtml('https://example.com')
+ * console.log(html.metadata?.title) // "Example Domain"
+ *
+ * @example
+ * // Fetching XML - no metadata extraction
+ * const xml = await fetchHtml('https://example.com/sitemap.xml')
+ * console.log(xml.content) // Raw XML content
+ * console.log(xml.metadata) // undefined
+ */
+export declare function fetchHtml(url: string, options?: {
+    userAgent?: string;
+    timeout?: number;
+}): Promise<FetchHtmlResult>;
+//# sourceMappingURL=html-fetch.d.ts.map

package/dist/primitives/data-sources/html-fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-fetch.d.ts","sourceRoot":"","sources":["../../../src/primitives/data-sources/html-fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GAAG;IACzB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,YAAY,CAAA;CACxB,CAAA;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAoD9D;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAa/D;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IACR,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,GACA,OAAO,CAAC,eAAe,CAAC,CA+C1B"}

package/dist/primitives/data-sources/source-website.d.ts

@@ -16,16 +16,68 @@ type FetchResult = {
         [WellKnownMetadata.knowledge.FAVICON]?: string;
     };
 };
+/**
+ * Fetch strategy for retrieving web content
+ *
+ * - 'node:fetch': Uses Node's built-in fetch (fast, no dependencies, works for static HTML)
+ * - 'integration:browser': Uses browser integration (slower, requires browser integration, handles JavaScript/SPAs)
+ */
+type FetchStrategy = 'node:fetch' | 'integration:browser';
+/**
+ * Fetch option can be:
+ * - A strategy string: 'node:fetch' or 'integration:browser'
+ * - A custom function: for special authentication, headers, or processing
+ */
+type FetchOption = FetchStrategy | ((url: string) => Promise<FetchResult> | FetchResult);
 type WebsiteSourceOptions = {
     id?: string;
     filter?: (context: SitemapFilterContext) => boolean;
-    fetch?: (url: string) => Promise<FetchResult> | FetchResult;
+    /**
+     * Fetch method to use for retrieving web pages
+     *
+     * Options:
+     * - 'node:fetch': Fast, uses Node's built-in fetch (best for static HTML sites) **[DEFAULT]**
+     * - 'integration:browser': Slower, uses browser integration (best for JavaScript/SPAs)
+     * - Custom function: Provide your own fetch implementation (for auth, special headers, etc.)
+     * - undefined: Defaults to 'node:fetch'
+     *
+     * @default 'node:fetch'
+     *
+     * @example
+     * // Use Node's built-in fetch (default, can be omitted)
+     * { fetch: 'node:fetch' }
+     *
+     * @example
+     * // Use browser integration for JavaScript-heavy sites
+     * { fetch: 'integration:browser' }
+     *
+     * @example
+     * // Custom fetch with authentication
+     * {
+     *   fetch: async (url) => {
+     *     const response = await fetch(url, {
+     *       headers: { Authorization: 'Bearer token' }
+     *     })
+     *     return {
+     *       url,
+     *       contentType: 'text/html',
+     *       content: await response.text()
+     *     }
+     *   }
+     * }
+     */
+    fetch?: FetchOption;
     maxPages?: number;
     maxDepth?: number;
 };
 type UrlsSourceOptions = {
     id?: string;
-    fetch?: (url: string) => Promise<FetchResult> | FetchResult;
+    /**
+     * Fetch method to use for retrieving web pages
+     *
+     * See WebsiteSourceOptions.fetch for detailed documentation
+     */
+    fetch?: FetchOption;
 };
 export declare class WebsiteSource extends DataSource {
     private mode;
@@ -34,19 +86,32 @@ export declare class WebsiteSource extends DataSource {
     private urls;
     private filterFn;
     private customFetch;
+    private fetchStrategy;
     private maxPages;
     private maxDepth;
     private transformFn;
     private constructor();
     private isBrowserIntegrationAvailable;
     /**
-     * Fetch content from a URL with fallback strategy
+     * Convert HtmlMetadata to FetchResult metadata format
+     */
+    private convertMetadata;
+    /**
+     * Default fetch implementation using Node's built-in fetch
+     */
+    private defaultFetch;
+    /**
+     * Fetch content from a URL for sitemap parsing (raw content needed)
      */
     private fetchSitemap;
     /**
-     * Fetch content from a URL with fallback strategy
+     * Fetch content from a URL for indexing (with metadata extraction)
      */
     private fetchUrl;
+    /**
+     * Fetch content using the browser integration
+     */
+    private fetchWithBrowserIntegration;
     /**
      * Parse sitemap XML content
      */

package/dist/primitives/data-sources/source-website.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"source-website.d.ts","sourceRoot":"","sources":["../../../src/primitives/data-sources/source-website.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAA4B,MAAM,eAAe,CAAA;AAEpE,OAAO,EAAiB,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAKlE,KAAK,oBAAoB,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,KAAK,WAAW,GAAG;IACjB,GAAG,EAAE,MAAM,CAAA;IACX,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE;QACT,CAAC,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAA;QAC5C,CAAC,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAA;QAClD,CAAC,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAA;KAC/C,CAAA;CACF,CAAA;AAED,KAAK,oBAAoB,GAAG;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAA;IACnD,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,GAAG,WAAW,CAAA;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,KAAK,iBAAiB,GAAG;IACvB,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,GAAG,WAAW,CAAA;CAC5D,CAAA;AAsCD,qBAAa,aAAc,SAAQ,UAAU;IAC3C,OAAO,CAAC,IAAI,CAAmB;IAC/B,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,UAAU,CAAoB;IACtC,OAAO,CAAC,IAAI,CAAsB;IAClC,OAAO,CAAC,QAAQ,CAA0D;IAC1E,OAAO,CAAC,WAAW,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,WAAW,CAAuF;IAE1G,OAAO;IAoBP,OAAO,CAAC,6BAA6B;IAIrC;;OAEG;YACW,YAAY;IA8C1B;;OAEG;YACW,QAAQ;IAkDtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAqDvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAcvB;;OAEG;YACW,uBAAuB;IAoDrC;;OAEG;YACW,YAAY;IAiH1B,IAAW,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uDAyLtB;IAED,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAKtF,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAKzF,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,iBAAsB,GAAG,aAAa;CAIhF"}
+{"version":3,"file":"source-website.d.ts","sourceRoot":"","sources":["../../../src/primitives/data-sources/source-website.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAA4B,MAAM,eAAe,CAAA;AAEpE,OAAO,EAAiB,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAMlE,KAAK,oBAAoB,GAAG;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,KAAK,WAAW,GAAG;IACjB,GAAG,EAAE,MAAM,CAAA;IACX,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE;QACT,CAAC,iBAAiB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAA;QAC5C,CAAC,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAA;QAClD,CAAC,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAA;KAC/C,CAAA;CACF,CAAA;AAED;;;;;GAKG;AACH,KAAK,aAAa,GAAG,YAAY,GAAG,qBAAqB,CAAA;AAEzD;;;;GAIG;AACH,KAAK,WAAW,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC,CAAA;AAExF,KAAK,oBAAoB,GAAG;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAA;IACnD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiCG;IACH,KAAK,CAAC,EAAE,WAAW,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,KAAK,iBAAiB,GAAG;IACvB,EAAE,CAAC,EAAE,MAAM,CAAA;IACX;;;;OAIG;IACH,KAAK,CAAC,EAAE,WAAW,CAAA;CACpB,CAAA;AAsCD,qBAAa,aAAc,SAAQ,UAAU;IAC3C,OAAO,CAAC,IAAI,CAAmB;IAC/B,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,UAAU,CAAoB;IACtC,OAAO,CAAC,IAAI,CAAsB;IAClC,OAAO,CAAC,QAAQ,CAA0D;IAC1E,OAAO,CAAC,WAAW,CAAmE;IACtF,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,WAAW,CAAuF;IAE1G,OAAO;IAmCP,OAAO,CAAC,6BAA6B;IAIrC;;OAEG;IACH,OAAO,CAAC,eAAe;IAgBvB;;OAEG;YACW,YAAY;IAsB1B;;OAEG;YACW,YAAY;IAkB1B;;OAEG;YACW,QAAQ;IAkBtB;;OAEG;YACW,2BAA2B;IAgDzC;;OAEG;IACH,OAAO,CAAC,eAAe;IAqDvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAcvB;;OAEG;YACW,uBAAuB;IAoDrC;;OAEG;YACW,YAAY;IAiH1B,IAAW,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uDAiMtB;IAED,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAKtF,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAKzF,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,iBAAsB,GAAG,aAAa;CAIhF"}

package/dist/runtime.js

@@ -48,7 +48,7 @@ var init_define_BUILD = __esm({
 var define_PACKAGE_VERSIONS_default;
 var init_define_PACKAGE_VERSIONS = __esm({
   "<define:__PACKAGE_VERSIONS__>"() {
-    define_PACKAGE_VERSIONS_default = { runtime: "1.6.4", adk: "not-installed", sdk: "4.17.3", llmz: "0.0.27", zai: "2.4.0", cognitive: "0.2.0" };
+    define_PACKAGE_VERSIONS_default = { runtime: "1.6.6", adk: "not-installed", sdk: "4.17.3", llmz: "0.0.27", zai: "2.4.0", cognitive: "0.2.0" };
   }
 });
 
@@ -25291,7 +25291,7 @@ var require_follow_redirects = __commonJS({
       var currentUrlParts = parseUrl(this._currentUrl);
       var currentHost = currentHostHeader || currentUrlParts.host;
       var currentUrl = /^\w+:/.test(location) ? this._currentUrl : url2.format(Object.assign(currentUrlParts, { host: currentHost }));
-      var redirectUrl = resolveUrl(location, currentUrl);
+      var redirectUrl = resolveUrl2(location, currentUrl);
       debug("redirecting to", redirectUrl.href);
       this._isRedirect = true;
       spreadUrlObject(redirectUrl, this._options);
@@ -25375,7 +25375,7 @@ var require_follow_redirects = __commonJS({
       }
       return parsed;
     }
-    function resolveUrl(relative, base) {
+    function resolveUrl2(relative, base) {
       return useNativeURL ? new URL2(relative, base) : parseUrl(url2.resolve(base, relative));
     }
     function validateUrl(input) {
@@ -42113,6 +42113,95 @@ var XMLParser = class {
   }
 };
 
+// src/primitives/data-sources/html-fetch.ts
+init_define_BUILD();
+init_define_PACKAGE_VERSIONS();
+function extractHtmlMetadata(html) {
+  const metadata = {};
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  if (titleMatch && titleMatch[1]) {
+    metadata.title = titleMatch[1].trim();
+  }
+  let descriptionMatch = html.match(
+    /<meta\s+(?:name|property)=["'](?:description|og:description)["']\s+content="([^"]+)"/i
+  );
+  if (!descriptionMatch) {
+    descriptionMatch = html.match(
+      /<meta\s+(?:name|property)=["'](?:description|og:description)["']\s+content='([^']+)'/i
+    );
+  }
+  if (descriptionMatch && descriptionMatch[1]) {
+    metadata.description = descriptionMatch[1].trim();
+  }
+  const faviconPatterns = [
+    // rel first, double quotes
+    /<link\s+[^>]*rel="(?:icon|shortcut icon|apple-touch-icon)"[^>]*href="([^"]+)"/i,
+    // rel first, single quotes
+    /<link\s+[^>]*rel='(?:icon|shortcut icon|apple-touch-icon)'[^>]*href='([^']+)'/i,
+    // href first, double quotes
+    /<link\s+[^>]*href="([^"]+)"[^>]*rel="(?:icon|shortcut icon|apple-touch-icon)"/i,
+    // href first, single quotes
+    /<link\s+[^>]*href='([^']+)'[^>]*rel='(?:icon|shortcut icon|apple-touch-icon)'/i
+  ];
+  for (const pattern of faviconPatterns) {
+    const faviconMatch = html.match(pattern);
+    if (faviconMatch && faviconMatch[1]) {
+      metadata.favicon = faviconMatch[1].trim();
+      break;
+    }
+  }
+  if (!metadata.favicon) {
+    metadata.favicon = "/favicon.ico";
+  }
+  return metadata;
+}
+function resolveUrl(url2, baseUrl) {
+  if (url2.startsWith("http://") || url2.startsWith("https://")) {
+    return url2;
+  }
+  try {
+    const base = new URL(baseUrl);
+    return new URL(url2, base.origin).href;
+  } catch {
+    return url2;
+  }
+}
+async function fetchHtml(url2, options) {
+  const userAgent = options?.userAgent || "Mozilla/5.0 (compatible; BotpressBot/1.0)";
+  const fetchOptions = {
+    headers: {
+      "User-Agent": userAgent
+    }
+  };
+  if (options?.timeout) {
+    fetchOptions.signal = AbortSignal.timeout(options.timeout);
+  }
+  const response = await fetch(url2, fetchOptions);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url2}: ${response.status} ${response.statusText}`);
+  }
+  const contentType = response.headers.get("content-type") || "text/html";
+  const content = await response.text();
+  const isHtml = contentType.includes("text/html") || contentType.includes("application/xhtml");
+  if (!isHtml) {
+    return {
+      url: url2,
+      contentType,
+      content
+    };
+  }
+  const extracted = extractHtmlMetadata(content);
+  if (extracted.favicon) {
+    extracted.favicon = resolveUrl(extracted.favicon, url2);
+  }
+  return {
+    url: url2,
+    contentType,
+    content,
+    metadata: extracted
+  };
+}
+
 // src/primitives/data-sources/source-website.ts
 var State = z6.object({
   urls: z6.array(
@@ -42132,6 +42221,7 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
   urls;
   filterFn;
   customFetch;
+  fetchStrategy;
   maxPages;
   maxDepth;
   transformFn;
@@ -42142,7 +42232,16 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     this.sitemapUrl = options.sitemapUrl ?? void 0;
     this.urls = options.urls ?? void 0;
     this.filterFn = "filter" in options ? options.filter : void 0;
-    this.customFetch = options.fetch ?? void 0;
+    if (typeof options.fetch === "string") {
+      this.fetchStrategy = options.fetch;
+      this.customFetch = void 0;
+    } else if (typeof options.fetch === "function") {
+      this.customFetch = options.fetch;
+      this.fetchStrategy = "node:fetch";
+    } else {
+      this.fetchStrategy = "node:fetch";
+      this.customFetch = void 0;
+    }
     this.maxPages = Math.max(1, Math.min(("maxPages" in options ? options.maxPages : void 0) ?? 5e4, 5e4));
     this.maxDepth = Math.max(1, Math.min(("maxDepth" in options ? options.maxDepth : void 0) ?? 20, 20));
   }
@@ -42150,51 +42249,82 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     return !!adk.project.integrations.get("browser");
   }
   /**
-   * Fetch content from a URL with fallback strategy
+   * Convert HtmlMetadata to FetchResult metadata format
+   */
+  convertMetadata(metadata) {
+    const result = {};
+    if (metadata.title) {
+      result[WellKnownMetadata.knowledge.TITLE] = metadata.title;
+    }
+    if (metadata.description) {
+      result[WellKnownMetadata.knowledge.DESCRIPTION] = metadata.description;
+    }
+    if (metadata.favicon) {
+      result[WellKnownMetadata.knowledge.FAVICON] = metadata.favicon;
+    }
+    return result;
+  }
+  /**
+   * Default fetch implementation using Node's built-in fetch
+   */
+  async defaultFetch(url2) {
+    const result = await fetchHtml(url2, {
+      timeout: 3e4
+    });
+    if (!result.metadata) {
+      return {
+        url: result.url,
+        contentType: result.contentType,
+        content: result.content
+      };
+    }
+    return {
+      url: result.url,
+      contentType: result.contentType,
+      content: result.content,
+      metadata: this.convertMetadata(result.metadata)
+    };
+  }
+  /**
+   * Fetch content from a URL for sitemap parsing (raw content needed)
    */
   async fetchSitemap(url2) {
     if (this.customFetch) {
       try {
         return await this.customFetch(url2);
       } catch (err) {
-        console.warn(`Custom fetch failed for ${url2}, falling back...`);
+        console.warn(`Custom fetch failed for ${url2}, falling back to ${this.fetchStrategy}...`);
       }
     }
-    if (!this.isBrowserIntegrationAvailable()) {
-      throw new Error(
-        `The 'browser' integration is not installed and is required for crawling website. Please provide a custom fetch function or install the 'browser' integration.`
-      );
-    }
-    const output2 = await adk.project.integrations.get("browser")?.actions.browsePages({
-      urls: [url2],
-      timeout: 3e4,
-      waitFor: 500
-    });
-    const result = output2?.results[0];
-    if (!result || !result.content) {
-      throw new Error(`Failed to fetch content from ${url2}`);
+    if (this.fetchStrategy === "integration:browser") {
+      return this.fetchWithBrowserIntegration(url2, { raw: true });
+    } else {
+      return this.defaultFetch(url2);
     }
-    return {
-      url: result.url,
-      contentType: "application/html",
-      content: result.raw
-    };
   }
   /**
-   * Fetch content from a URL with fallback strategy
+   * Fetch content from a URL for indexing (with metadata extraction)
    */
   async fetchUrl(url2) {
     if (this.customFetch) {
       try {
         return await this.customFetch(url2);
       } catch (err) {
-        console.warn(`Custom fetch failed for ${url2}, falling back...`);
+        console.warn(`Custom fetch failed for ${url2}, falling back to ${this.fetchStrategy}...`);
       }
     }
+    if (this.fetchStrategy === "integration:browser") {
+      return this.fetchWithBrowserIntegration(url2, { raw: false });
+    } else {
+      return this.defaultFetch(url2);
+    }
+  }
+  /**
+   * Fetch content using the browser integration
+   */
+  async fetchWithBrowserIntegration(url2, options) {
     if (!this.isBrowserIntegrationAvailable()) {
-      throw new Error(
-        `The 'browser' integration is not installed and is required for crawling website. Please provide a custom fetch function or install the 'browser' integration.`
-      );
+      throw new Error(`The 'browser' integration is not installed. Please install it or use fetch: 'node:fetch'.`);
     }
     const output2 = await adk.project.integrations.get("browser")?.actions.browsePages({
       urls: [url2],
@@ -42205,6 +42335,13 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     if (!result || !result.content) {
       throw new Error(`Failed to fetch content from ${url2}`);
     }
+    if (options.raw && result.raw) {
+      return {
+        url: result.url,
+        contentType: "application/html",
+        content: result.raw
+      };
+    }
     return {
       url: result.url,
       contentType: "text/markdown",
@@ -42433,6 +42570,13 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
         const toRemove = existingFiles.filter(
           (f) => !discoveredUrls.find((u) => u.loc === f.metadata?.[WellKnownMetadata.knowledge.URL])
         );
+        if (existingFiles.length > 0 && toRemove.length >= existingFiles.length * 0.8) {
+          console.error(
+            `Warning: All existing files (${existingFiles.length}) are scheduled for removal. Please check if the sitemap URL is correct and the website is accessible. We will try again in 5 minutes.`
+          );
+          await step2.sleep("retry wait", 5 * 60 * 1e3);
+          throw new Error("Aborting sync due to potential misconfiguration (all files to be removed)");
+        }
         const toFetch = [];
         let skippedUnchanged = 0;
         for (const url2 of discoveredUrls) {