@botpress/adk-cli 1.6.5 → 1.6.6

package/dist/cli.js

@@ -316257,6 +316257,83 @@ function isLeafTag(obj, options) {
   }
   return false;
 }
+function extractHtmlMetadata(html) {
+  const metadata = {};
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  if (titleMatch && titleMatch[1]) {
+    metadata.title = titleMatch[1].trim();
+  }
+  let descriptionMatch = html.match(/<meta\s+(?:name|property)=["'](?:description|og:description)["']\s+content="([^"]+)"/i);
+  if (!descriptionMatch) {
+    descriptionMatch = html.match(/<meta\s+(?:name|property)=["'](?:description|og:description)["']\s+content='([^']+)'/i);
+  }
+  if (descriptionMatch && descriptionMatch[1]) {
+    metadata.description = descriptionMatch[1].trim();
+  }
+  const faviconPatterns = [
+    /<link\s+[^>]*rel="(?:icon|shortcut icon|apple-touch-icon)"[^>]*href="([^"]+)"/i,
+    /<link\s+[^>]*rel='(?:icon|shortcut icon|apple-touch-icon)'[^>]*href='([^']+)'/i,
+    /<link\s+[^>]*href="([^"]+)"[^>]*rel="(?:icon|shortcut icon|apple-touch-icon)"/i,
+    /<link\s+[^>]*href='([^']+)'[^>]*rel='(?:icon|shortcut icon|apple-touch-icon)'/i
+  ];
+  for (const pattern of faviconPatterns) {
+    const faviconMatch = html.match(pattern);
+    if (faviconMatch && faviconMatch[1]) {
+      metadata.favicon = faviconMatch[1].trim();
+      break;
+    }
+  }
+  if (!metadata.favicon) {
+    metadata.favicon = "/favicon.ico";
+  }
+  return metadata;
+}
+function resolveUrl(url22, baseUrl) {
+  if (url22.startsWith("http://") || url22.startsWith("https://")) {
+    return url22;
+  }
+  try {
+    const base3 = new URL(baseUrl);
+    return new URL(url22, base3.origin).href;
+  } catch {
+    return url22;
+  }
+}
+async function fetchHtml(url22, options) {
+  const userAgent = options?.userAgent || "Mozilla/5.0 (compatible; BotpressBot/1.0)";
+  const fetchOptions = {
+    headers: {
+      "User-Agent": userAgent
+    }
+  };
+  if (options?.timeout) {
+    fetchOptions.signal = AbortSignal.timeout(options.timeout);
+  }
+  const response = await fetch(url22, fetchOptions);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url22}: ${response.status} ${response.statusText}`);
+  }
+  const contentType = response.headers.get("content-type") || "text/html";
+  const content = await response.text();
+  const isHtml = contentType.includes("text/html") || contentType.includes("application/xhtml");
+  if (!isHtml) {
+    return {
+      url: url22,
+      contentType,
+      content
+    };
+  }
+  const extracted = extractHtmlMetadata(content);
+  if (extracted.favicon) {
+    extracted.favicon = resolveUrl(extracted.favicon, url22);
+  }
+  return {
+    url: url22,
+    contentType,
+    content,
+    metadata: extracted
+  };
+}
 function debugLog(...args) {
   if (DEBUG_ENABLED) {
     console.log(...args);
@@ -319533,7 +319610,7 @@ var init_internal = __esm(() => {
   });
   init_define_PACKAGE_VERSIONS = __esm2({
     "<define:__PACKAGE_VERSIONS__>"() {
-      define_PACKAGE_VERSIONS_default = { runtime: "1.6.5", adk: "not-installed", sdk: "4.17.3", llmz: "0.0.27", zai: "2.4.0", cognitive: "0.2.0" };
+      define_PACKAGE_VERSIONS_default = { runtime: "1.6.6", adk: "not-installed", sdk: "4.17.3", llmz: "0.0.27", zai: "2.4.0", cognitive: "0.2.0" };
     }
   });
   init_globalThis = __esm2({
@@ -343606,7 +343683,7 @@ function print() { __p += __j.call(arguments, '') }
         var currentUrlParts = parseUrl(this._currentUrl);
         var currentHost = currentHostHeader || currentUrlParts.host;
         var currentUrl = /^\w+:/.test(location) ? this._currentUrl : url22.format(Object.assign(currentUrlParts, { host: currentHost }));
-        var redirectUrl = resolveUrl(location, currentUrl);
+        var redirectUrl = resolveUrl2(location, currentUrl);
         debug("redirecting to", redirectUrl.href);
         this._isRedirect = true;
         spreadUrlObject(redirectUrl, this._options);
@@ -343689,7 +343766,7 @@ function print() { __p += __j.call(arguments, '') }
         }
         return parsed;
       }
-      function resolveUrl(relative, base3) {
+      function resolveUrl2(relative, base3) {
         return useNativeURL ? new URL2(relative, base3) : parseUrl(url22.resolve(base3, relative));
       }
       function validateUrl(input) {
@@ -355555,6 +355632,8 @@ ${iteration.status.execution_error.stack}`;
   init_define_BUILD();
   init_define_PACKAGE_VERSIONS();
   METADATA_SYMBOL2 = XmlNode.getMetaDataSymbol();
+  init_define_BUILD();
+  init_define_PACKAGE_VERSIONS();
   State = X.object({
     urls: X.array(X.object({
       loc: X.string(),
@@ -355571,6 +355650,7 @@ ${iteration.status.execution_error.stack}`;
     urls;
     filterFn;
     customFetch;
+    fetchStrategy;
     maxPages;
     maxDepth;
     transformFn;
@@ -355581,49 +355661,84 @@ ${iteration.status.execution_error.stack}`;
       this.sitemapUrl = options.sitemapUrl ?? undefined;
       this.urls = options.urls ?? undefined;
       this.filterFn = "filter" in options ? options.filter : undefined;
-      this.customFetch = options.fetch ?? undefined;
+      if (typeof options.fetch === "string") {
+        this.fetchStrategy = options.fetch;
+        this.customFetch = undefined;
+      } else if (typeof options.fetch === "function") {
+        this.customFetch = options.fetch;
+        this.fetchStrategy = "node:fetch";
+      } else {
+        this.fetchStrategy = "node:fetch";
+        this.customFetch = undefined;
+      }
       this.maxPages = Math.max(1, Math.min(("maxPages" in options ? options.maxPages : undefined) ?? 50000, 50000));
       this.maxDepth = Math.max(1, Math.min(("maxDepth" in options ? options.maxDepth : undefined) ?? 20, 20));
     }
     isBrowserIntegrationAvailable() {
       return !!adk.project.integrations.get("browser");
     }
+    convertMetadata(metadata) {
+      const result = {};
+      if (metadata.title) {
+        result[WellKnownMetadata.knowledge.TITLE] = metadata.title;
+      }
+      if (metadata.description) {
+        result[WellKnownMetadata.knowledge.DESCRIPTION] = metadata.description;
+      }
+      if (metadata.favicon) {
+        result[WellKnownMetadata.knowledge.FAVICON] = metadata.favicon;
+      }
+      return result;
+    }
+    async defaultFetch(url22) {
+      const result = await fetchHtml(url22, {
+        timeout: 30000
+      });
+      if (!result.metadata) {
+        return {
+          url: result.url,
+          contentType: result.contentType,
+          content: result.content
+        };
+      }
+      return {
+        url: result.url,
+        contentType: result.contentType,
+        content: result.content,
+        metadata: this.convertMetadata(result.metadata)
+      };
+    }
     async fetchSitemap(url22) {
       if (this.customFetch) {
         try {
           return await this.customFetch(url22);
         } catch (err) {
-          console.warn(`Custom fetch failed for ${url22}, falling back...`);
+          console.warn(`Custom fetch failed for ${url22}, falling back to ${this.fetchStrategy}...`);
         }
       }
-      if (!this.isBrowserIntegrationAvailable()) {
-        throw new Error(`The 'browser' integration is not installed and is required for crawling website. Please provide a custom fetch function or install the 'browser' integration.`);
-      }
-      const output2 = await adk.project.integrations.get("browser")?.actions.browsePages({
-        urls: [url22],
-        timeout: 30000,
-        waitFor: 500
-      });
-      const result = output2?.results[0];
-      if (!result || !result.content) {
-        throw new Error(`Failed to fetch content from ${url22}`);
+      if (this.fetchStrategy === "integration:browser") {
+        return this.fetchWithBrowserIntegration(url22, { raw: true });
+      } else {
+        return this.defaultFetch(url22);
       }
-      return {
-        url: result.url,
-        contentType: "application/html",
-        content: result.raw
-      };
     }
     async fetchUrl(url22) {
       if (this.customFetch) {
         try {
           return await this.customFetch(url22);
         } catch (err) {
-          console.warn(`Custom fetch failed for ${url22}, falling back...`);
+          console.warn(`Custom fetch failed for ${url22}, falling back to ${this.fetchStrategy}...`);
         }
       }
+      if (this.fetchStrategy === "integration:browser") {
+        return this.fetchWithBrowserIntegration(url22, { raw: false });
+      } else {
+        return this.defaultFetch(url22);
+      }
+    }
+    async fetchWithBrowserIntegration(url22, options) {
       if (!this.isBrowserIntegrationAvailable()) {
-        throw new Error(`The 'browser' integration is not installed and is required for crawling website. Please provide a custom fetch function or install the 'browser' integration.`);
+        throw new Error(`The 'browser' integration is not installed. Please install it or use fetch: 'node:fetch'.`);
       }
       const output2 = await adk.project.integrations.get("browser")?.actions.browsePages({
         urls: [url22],
@@ -355634,6 +355749,13 @@ ${iteration.status.execution_error.stack}`;
       if (!result || !result.content) {
         throw new Error(`Failed to fetch content from ${url22}`);
       }
+      if (options.raw && result.raw) {
+        return {
+          url: result.url,
+          contentType: "application/html",
+          content: result.raw
+        };
+      }
       return {
         url: result.url,
         contentType: "text/markdown",
@@ -371664,7 +371786,7 @@ class AgentProjectGenerator {
         deploy: "adk deploy"
       },
       dependencies: {
-        "@botpress/runtime": "^1.6.5"
+        "@botpress/runtime": "^1.6.6"
       },
       devDependencies: {
         typescript: "^5.0.0"
@@ -380871,7 +380993,7 @@ var init_Separator = __esm(async () => {
 var require_package3 = __commonJS((exports, module) => {
   module.exports = {
     name: "@botpress/adk",
-    version: "1.6.5",
+    version: "1.6.6",
     description: "Core ADK library for building AI agents on Botpress",
     type: "module",
     main: "dist/index.js",
@@ -380918,7 +381040,7 @@ var require_package3 = __commonJS((exports, module) => {
       "@botpress/cli": "^4.20",
       "@botpress/client": "^1.27.0",
       "@botpress/cognitive": "^0.2.0",
-      "@botpress/runtime": "^1.6.5",
+      "@botpress/runtime": "^1.6.6",
       "@botpress/sdk": "^4.17.3",
       "@bpinternal/yargs-extra": "^0.0.21",
       "@parcel/watcher": "^2.5.1",
@@ -382306,7 +382428,7 @@ function checkRuntimeVersion(agentRoot) {
 `));
   }
 }
-var semver2, EXPECTED_RUNTIME_VERSION = "1.6.5";
+var semver2, EXPECTED_RUNTIME_VERSION = "1.6.6";
 var init_runtime_version_check = __esm(() => {
   init_source();
   semver2 = __toESM(require_semver2(), 1);
@@ -397905,7 +398027,7 @@ if (!checkNodeVersion(true)) {
 }
 var __filename2 = fileURLToPath9(import.meta.url);
 var __dirname5 = dirname3(__filename2);
-var CLI_VERSION = "1.6.5";
+var CLI_VERSION = "1.6.6";
 program.name("adk").description("Botpress Agent Development Kit (ADK) - CLI for building AI agents").version(CLI_VERSION).option("--no-cache", "Disable caching for integration lookups").configureHelp({
   formatHelp: () => formatHelp(program, CLI_VERSION)
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botpress/adk-cli",
-  "version": "1.6.5",
+  "version": "1.6.6",
   "description": "Command-line interface for the Botpress Agent Development Kit (ADK)",
   "type": "module",
   "main": "dist/index.js",
@@ -40,9 +40,9 @@
     "url": "https://github.com/botpress/adk"
   },
   "dependencies": {
-    "@botpress/adk": "^1.6.5",
+    "@botpress/adk": "^1.6.6",
     "@botpress/cli": "^4.20",
-    "@botpress/runtime": "^1.6.5",
+    "@botpress/runtime": "^1.6.6",
     "adm-zip": "^0.5.16",
     "chalk": "^5.4.1",
     "clipboardy": "^4.0.0",