@botpress/runtime 1.7.18 → 1.8.0

package/dist/internal.js

@@ -48,7 +48,7 @@ var init_define_BUILD = __esm({
 var define_PACKAGE_VERSIONS_default;
 var init_define_PACKAGE_VERSIONS = __esm({
   "<define:__PACKAGE_VERSIONS__>"() {
-    define_PACKAGE_VERSIONS_default = { runtime: "1.7.18", adk: "1.7.18", sdk: "4.19.0", llmz: "0.0.33", zai: "2.5.0", cognitive: "0.2.0" };
+    define_PACKAGE_VERSIONS_default = { runtime: "1.8.0", adk: "1.8.0", sdk: "4.19.0", llmz: "0.0.33", zai: "2.5.0", cognitive: "0.2.0" };
   }
 });
 
@@ -43438,6 +43438,15 @@ var TableSource = class _TableSource extends DataSource {
     this.table = table;
     this.transformFn = options.transform ?? void 0;
   }
+  /** Get serializable configuration for change detection */
+  getConfig() {
+    return {
+      id: this.id,
+      type: this.type,
+      tableName: this.table.name,
+      transformFn: this.transformFn?.toString() || null
+    };
+  }
   get syncWorkflow() {
     return createSyncWorkflow({
       type: "table",
@@ -45051,6 +45060,27 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     this.maxPages = Math.max(1, Math.min(("maxPages" in options ? options.maxPages : void 0) ?? 5e4, 5e4));
     this.maxDepth = Math.max(1, Math.min(("maxDepth" in options ? options.maxDepth : void 0) ?? 20, 20));
   }
+  /** Get serializable configuration for change detection (only user-provided options) */
+  getConfig() {
+    const config = {
+      mode: this.mode,
+      fetchStrategy: this.fetchStrategy
+    };
+    if (this.mode === "website") {
+      config.baseUrl = this.baseUrl;
+      config.maxPages = this.maxPages;
+      config.maxDepth = this.maxDepth;
+    } else if (this.mode === "sitemap") {
+      config.sitemapUrl = this.sitemapUrl;
+      config.maxPages = this.maxPages;
+    } else if (this.mode === "urls") {
+      config.urls = this.urls;
+    }
+    if (this.filterFn) {
+      config.filterFn = this.filterFn.toString();
+    }
+    return config;
+  }
   isBrowserIntegrationAvailable() {
     return !!adk.project.integrations.get("browser");
   }
@@ -45370,28 +45400,28 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
             tags
           }).collect()
         );
-        const existingFileMap = new Map(
-          existingFiles.map((f) => [f.metadata?.[WellKnownMetadata.knowledge.URL], f])
-        );
-        const toRemove = existingFiles.filter(
-          (f) => !discoveredUrls.find((u) => u.loc === f.metadata?.[WellKnownMetadata.knowledge.URL])
-        );
-        if (existingFiles.length > 0 && toRemove.length >= existingFiles.length * 0.8) {
-          console.error(
-            `Warning: All existing files (${existingFiles.length}) are scheduled for removal. Please check if the sitemap URL is correct and the website is accessible. We will try again in 5 minutes.`
+        if (input.force && existingFiles.length > 0) {
+          console.warn(
+            `\u26A0\uFE0F  Website source configuration changed - deleting ${existingFiles.length} existing files and recrawling`
+          );
+          await step2.map(
+            "deleting all existing files for recrawl",
+            existingFiles,
+            (f) => client.deleteFile({ id: f.id }).catch(() => null),
+            { concurrency: 5 }
           );
-          await step2.sleep("retry wait", 5 * 60 * 1e3);
-          throw new Error("Aborting sync due to potential misconfiguration (all files to be removed)");
+          console.log(`\u2705 Deleted ${existingFiles.length} files, starting fresh crawl`);
         }
+        const existingFileMap = input.force ? /* @__PURE__ */ new Map() : new Map(existingFiles.map((f) => [f.metadata?.[WellKnownMetadata.knowledge.URL], f]));
+        const toRemove = input.force ? [] : existingFiles.filter(
+          (f) => !discoveredUrls.find((u) => u.loc === f.metadata?.[WellKnownMetadata.knowledge.URL])
+        );
         const toFetch = [];
         let skippedUnchanged = 0;
         for (const url2 of discoveredUrls) {
           const existing = existingFileMap.get(url2.loc);
           if (!existing) {
             toFetch.push(url2);
-          } else if (input.force) {
-            console.log(`Force re-indexing page: ${url2.loc}`);
-            toFetch.push(url2);
           } else {
             const existingMetadata = existing.metadata;
             if (url2.lastmod && existingMetadata?.lastmod !== url2.lastmod) {
@@ -45505,7 +45535,17 @@ var WebsiteSource = class _WebsiteSource extends DataSource {
     return new _WebsiteSource(id, "sitemap", { ...options, sitemapUrl });
   }
   static fromUrls(urls, options = {}) {
-    const id = options.id || `urls_${urls.length}_pages`;
+    let defaultId = `urls_${urls.length}_pages`;
+    if (urls.length > 0) {
+      try {
+        const firstUrl = new URL(urls[0]);
+        const domain = firstUrl.hostname.replace(/^www\./, "").replace(/\./g, "_");
+        const urlsHash = urls.slice().sort().join("|").split("").reduce((hash, char) => (hash << 5) - hash + char.charCodeAt(0) | 0, 0).toString(16).replace("-", "").slice(0, 8);
+        defaultId = `urls_${domain}_${urlsHash}`;
+      } catch {
+      }
+    }
+    const id = options.id || defaultId;
     return new _WebsiteSource(id, "urls", { ...options, urls });
   }
 };
@@ -45515,12 +45555,29 @@ init_define_BUILD();
 init_define_PACKAGE_VERSIONS();
 import { z as z23 } from "@botpress/sdk";
 var DirectorySource = class _DirectorySource extends DataSource {
-  directoryPath;
-  filterFn;
+  _directoryPath;
+  _filterFn;
   constructor(id, directoryPath, options = {}) {
     super(id, "directory");
-    this.directoryPath = directoryPath;
-    this.filterFn = options.filter ?? void 0;
+    this._directoryPath = directoryPath;
+    this._filterFn = options.filter ?? void 0;
+  }
+  /** Get the directory path for this source */
+  get directoryPath() {
+    return this._directoryPath;
+  }
+  /** Get the filter function for this source */
+  get filterFn() {
+    return this._filterFn;
+  }
+  /** Get serializable configuration for change detection */
+  getConfig() {
+    return {
+      id: this.id,
+      type: this.type,
+      directoryPath: this._directoryPath,
+      filterFn: this._filterFn?.toString() || null
+    };
   }
   get syncWorkflow() {
     return createSyncWorkflow({
@@ -45667,6 +45724,17 @@ var DirectorySource = class _DirectorySource extends DataSource {
   }
 };
 
+// src/primitives/data-sources/index.ts
+function isDirectorySource(source) {
+  return source.type === "directory";
+}
+function isWebsiteSource(source) {
+  return source.type === "website";
+}
+function isTableSource(source) {
+  return source.type === "table";
+}
+
 // src/primitives/knowledge.ts
 init_define_BUILD();
 init_define_PACKAGE_VERSIONS();
@@ -46003,6 +46071,9 @@ var DataSource2;
   DataSource3.Table = TableSource;
   DataSource3.Website = WebsiteSource;
   DataSource3.Directory = DirectorySource;
+  DataSource3.isDirectory = isDirectorySource;
+  DataSource3.isWebsite = isWebsiteSource;
+  DataSource3.isTable = isTableSource;
 })(DataSource2 || (DataSource2 = {}));
 
 // src/workers/index.ts