@botpress/runtime 1.10.4 → 1.11.0

package/dist/runtime.js

@@ -48,7 +48,7 @@ var init_define_BUILD = __esm({
 var define_PACKAGE_VERSIONS_default;
 var init_define_PACKAGE_VERSIONS = __esm({
   "<define:__PACKAGE_VERSIONS__>"() {
-    define_PACKAGE_VERSIONS_default = { runtime: "1.10.4", adk: "1.10.4", sdk: "4.20.2", llmz: "0.0.33", zai: "2.5.0", cognitive: "0.2.0" };
+    define_PACKAGE_VERSIONS_default = { runtime: "1.11.0", adk: "1.11.0", sdk: "4.20.2", llmz: "0.0.33", zai: "2.5.0", cognitive: "0.2.0" };
   }
 });
 
@@ -16000,9 +16000,9 @@ var require_combined_stream = __commonJS({
   }
 });
 
-// ../../node_modules/mime-db/db.json
+// ../../node_modules/form-data/node_modules/mime-types/node_modules/mime-db/db.json
 var require_db = __commonJS({
-  "../../node_modules/mime-db/db.json"(exports2, module) {
+  "../../node_modules/form-data/node_modules/mime-types/node_modules/mime-db/db.json"(exports2, module) {
     module.exports = {
       "application/1d-interleaved-parityfec": {
         source: "iana"
@@ -24525,18 +24525,18 @@ var require_db = __commonJS({
   }
 });
 
-// ../../node_modules/mime-db/index.js
+// ../../node_modules/form-data/node_modules/mime-types/node_modules/mime-db/index.js
 var require_mime_db = __commonJS({
-  "../../node_modules/mime-db/index.js"(exports2, module) {
+  "../../node_modules/form-data/node_modules/mime-types/node_modules/mime-db/index.js"(exports2, module) {
     init_define_BUILD();
     init_define_PACKAGE_VERSIONS();
     module.exports = require_db();
   }
 });
 
-// ../../node_modules/mime-types/index.js
+// ../../node_modules/form-data/node_modules/mime-types/index.js
 var require_mime_types = __commonJS({
-  "../../node_modules/mime-types/index.js"(exports2) {
+  "../../node_modules/form-data/node_modules/mime-types/index.js"(exports2) {
     "use strict";
     init_define_BUILD();
     init_define_PACKAGE_VERSIONS();
@@ -33751,15 +33751,14 @@ var init_constants = __esm({
     WellKnownTags = {
       knowledge: {
         /**
-         * All knowledge base have this tag (with value "true") to identify them as knowledge-related records.
-         * @example "knowledge": "true"
+         * All knowledge base have this tag (with value "knowledge-base") to identify them as knowledge-related records.
+         * @example "source": "knowledge-base"
          */
-        KNOWLEDGE: "knowledge",
+        KNOWLEDGE: "source",
         /**
          * The ID of the knowledge base the record belongs to.
-         * This is the ID of the Knowledge Base primitive
+         * This is the ID of the Knowledge Base primitive from Botpress.
          * @example "kbId": "kb_01K6RT9T39KF7K0A7R7D71TDZ1"
-         * @deprecated Use KNOWLEDGE_BASE_NAME for now, as we will be moving to IDs later.
          */
         KNOWLEDGE_BASE_ID: "kbId",
         /**
@@ -33769,15 +33768,15 @@ var init_constants = __esm({
         KNOWLEDGE_BASE_NAME: "kbName",
         /**
          * The ID of the Data Source the record was ingested from.
-         * @example "sourceId": "docs"
+         * @example "dsId": "docs"
          */
-        KNOWLEDGE_SOURCE_ID: "sourceId",
+        KNOWLEDGE_SOURCE_ID: "dsId",
         /**
          * The type of the Data Source the record was ingested from.
-         * Possible values are: "directory", "table", "api", etc.
-         * @example "sourceType": "directory"
+         * Possible values are: "document", "rich-text", "web-page", etc.
+         * @example "dsType": "document"
          */
-        KNOWLEDGE_SOURCE_TYPE: "sourceType"
+        KNOWLEDGE_SOURCE_TYPE: "dsType"
       }
     };
     WellKnownMetadata = {
@@ -33874,7 +33873,7 @@ If the question is not related to the knowledge bases, do NOT use this tool.`.tr
               includeBreadcrumb: true,
               contextDepth: 4,
               tags: {
-                [WellKnownTags.knowledge.KNOWLEDGE]: "true",
+                [WellKnownTags.knowledge.KNOWLEDGE]: "knowledge-base",
                 [WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME]: kbNames
               }
             });
@@ -33909,11 +33908,14 @@ If the question is not related to the knowledge bases, do NOT use this tool.`.tr
               if (tags[WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME]) {
                 citationMetadata.knowledgeBase = tags[WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME];
               }
+              if (tags[WellKnownTags.knowledge.KNOWLEDGE_BASE_ID]) {
+                citationMetadata.knowledgeBaseId = tags[WellKnownTags.knowledge.KNOWLEDGE_BASE_ID];
+              }
               if (tags[WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE]) {
-                citationMetadata.sourceType = tags[WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE];
+                citationMetadata.dsType = tags[WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE];
               }
               if (tags[WellKnownTags.knowledge.KNOWLEDGE_SOURCE_ID]) {
-                citationMetadata.sourceId = tags[WellKnownTags.knowledge.KNOWLEDGE_SOURCE_ID];
+                citationMetadata.dsId = tags[WellKnownTags.knowledge.KNOWLEDGE_SOURCE_ID];
               }
               for (const key in citationMetadata) {
                 let value = citationMetadata[key];
@@ -34442,8 +34444,9 @@ var init_source_base = __esm({
       size: z6.number()
     });
     SyncInput = z6.object({
-      sourceId: z6.string(),
+      dsId: z6.string(),
       kbName: z6.string(),
+      kbId: z6.string(),
       force: z6.boolean().optional().describe("Force re-indexing even if files haven't changed").default(false)
     });
     SyncOutput = z6.object({
@@ -34460,18 +34463,16 @@ var init_source_base = __esm({
       state: props.state,
       timeout: "120m",
       async handler(execProps) {
-        const { kbName, sourceId } = execProps.input;
+        const { kbName, kbId, dsId } = execProps.input;
         const kb = adk.project.knowledge.find((kb2) => kb2.name === kbName);
         if (!kb) {
           throw new Error(`Knowledge base '${kbName}' not found`);
         }
-        const source = kb.sources.find((s) => s.id === sourceId);
+        const source = kb.sources.find((s) => s.id === dsId);
         if (!source) {
-          throw new Error(`Data source with ID '${sourceId}' not found in knowledge base '${kbName}'`);
+          throw new Error(`Data source with ID '${dsId}' not found in knowledge base '${kbName}'`);
         }
-        console.log(
-          `\u{1F504} Starting sync for data source '${sourceId}' of type '${source.type}' in knowledge base '${kbName}'`
-        );
+        console.log(`\u{1F504} Starting sync for data source '${dsId}' of type '${source.type}' in knowledge base '${kbName}' (${kbId})`);
         return await props.handler.bind(source)(execProps);
       }
     });
@@ -36162,6 +36163,7 @@ var init_source_website = __esm({
     init_constants();
     init_fxp();
     init_library();
+    init_context3();
     init_html_fetch();
     State = z8.object({
       urls: z8.array(
@@ -36178,6 +36180,7 @@ var init_source_website = __esm({
       mode;
       baseUrl;
       sitemapUrl;
+      llmsTxtUrl;
       urls;
       filterFn;
       customFetch;
@@ -36186,10 +36189,11 @@ var init_source_website = __esm({
       maxDepth;
       transformFn;
       constructor(id, mode, options) {
-        super(id, "website");
+        super(id, "web-page");
         this.mode = mode;
         this.baseUrl = options.baseUrl ?? void 0;
         this.sitemapUrl = options.sitemapUrl ?? void 0;
+        this.llmsTxtUrl = options.llmsTxtUrl ?? void 0;
         this.urls = options.urls ?? void 0;
         this.filterFn = "filter" in options ? options.filter : void 0;
         if (typeof options.fetch === "string") {
@@ -36218,6 +36222,9 @@ var init_source_website = __esm({
         } else if (this.mode === "sitemap") {
           config.sitemapUrl = this.sitemapUrl;
           config.maxPages = this.maxPages;
+        } else if (this.mode === "llms-txt") {
+          config.llmsTxtUrl = this.llmsTxtUrl;
+          config.maxPages = this.maxPages;
         } else if (this.mode === "urls") {
           config.urls = this.urls;
         }
@@ -36269,7 +36276,7 @@ var init_source_website = __esm({
       /**
        * Fetch content from a URL for sitemap parsing (raw content needed)
        */
-      async fetchSitemap(url2) {
+      async fetchRaw(url2) {
         if (this.customFetch) {
           try {
             return await this.customFetch(url2);
@@ -36387,6 +36394,21 @@ var init_source_website = __esm({
         }
         return { urls };
       }
+      parseLlmsTxt(content) {
+        const urls = [];
+        const lines = content.split("\n");
+        const urlRegex = /https?:\/\/[^\s)]+\.md/g;
+        for (const line of lines) {
+          const matches = line.matchAll(urlRegex);
+          for (const match2 of matches) {
+            const [url2] = match2;
+            if (url2) {
+              urls.push(url2);
+            }
+          }
+        }
+        return { urls };
+      }
       /**
        * Discover URLs from a website using browser integration
        */
@@ -36444,6 +36466,28 @@ var init_source_website = __esm({
         if (this.mode === "website") {
           return this.discoverUrlsFromWebsite(step2);
         }
+        if (this.mode === "llms-txt") {
+          if (!this.llmsTxtUrl) {
+            throw new Error("No llms.txt URL provided");
+          }
+          const { content } = await step2("fetch llms.txt", () => this.fetchRaw(this.llmsTxtUrl));
+          const { urls } = this.parseLlmsTxt(content);
+          console.log(`Parsed ${urls.length} URLs from llms.txt`);
+          const filteredUrls = [];
+          for (const url2 of urls) {
+            if (filteredUrls.length >= this.maxPages) {
+              console.log(`Reached maxPages limit (${this.maxPages}), stopping`);
+              break;
+            }
+            const filterContext = { url: url2 };
+            if (!this.filterFn || this.filterFn(filterContext)) {
+              filteredUrls.push({ loc: url2 });
+            } else {
+              console.log(`Skipped URL (filtered): ${url2}`);
+            }
+          }
+          return filteredUrls;
+        }
         if (!this.sitemapUrl) {
           throw new Error("No sitemap URL provided");
         }
@@ -36457,7 +36501,7 @@ var init_source_website = __esm({
           }
           await step2(`processing sitemap ${item.url}`, async () => {
             try {
-              const { content, contentType } = await this.fetchSitemap(item.url);
+              const { content, contentType } = await this.fetchRaw(item.url);
               console.log(`Fetched sitemap ${item.url} (${content.length} bytes), processing... ${contentType}`);
               console.log(content, contentType);
               try {
@@ -36526,15 +36570,16 @@ var init_source_website = __esm({
             console.log(
               `Starting sync for WebsiteSource [${this.id}] in mode [${this.mode}, maxPages=${this.maxPages}, maxDepth=${this.maxDepth}, baseUrl=${this.baseUrl}, sitemapUrl=${this.sitemapUrl}]`
             );
-            console.log(`Using knowledge base: ${input.kbName}, force reindex: ${!!input.force}, ${input.sourceId}`);
+            console.log(`Using knowledge base: ${input.kbName}, force reindex: ${!!input.force}, ${input.dsId}`);
             if (input.force) {
               console.log("\u{1F504} FORCE MODE: Re-indexing all files regardless of changes");
             }
             const tags = {
-              [WellKnownTags.knowledge.KNOWLEDGE]: "true",
+              [WellKnownTags.knowledge.KNOWLEDGE]: "knowledge-base",
+              [WellKnownTags.knowledge.KNOWLEDGE_BASE_ID]: input.kbId,
+              [WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME]: input.kbName,
               [WellKnownTags.knowledge.KNOWLEDGE_SOURCE_ID]: this.id,
-              [WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE]: this.type,
-              [WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME]: input.kbName
+              [WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE]: this.type
             };
             const discoveredUrls = await step2("discover urls from sitemap", () => this.discoverUrls(step2, state));
             console.log(`Discovered ${discoveredUrls.length} URLs from sitemap`);
@@ -36580,6 +36625,7 @@ var init_source_website = __esm({
             console.log(
               `To fetch: ${toFetch.length}, To remove: ${toRemove.length}, Skipped (unchanged): ${skippedUnchanged}${input.force ? " [FORCE MODE]" : ""}`
             );
+            console.log(`[SYNC DEBUG] Starting 'deleting removed urls' step...`);
             const deleted = await step2.map(
               "deleting removed urls",
               toRemove,
@@ -36593,6 +36639,7 @@ var init_source_website = __esm({
               ),
               { concurrency: 5 }
             );
+            console.log(`[SYNC DEBUG] Deleted ${deleted.length} URLs, starting fetch phase...`);
             const fetchAndIndex = async (sitemapUrl) => {
               try {
                 const {
@@ -36639,8 +36686,8 @@ var init_source_website = __esm({
                     ...fetchMetadata?.[WellKnownMetadata.knowledge.FAVICON] && {
                       [WellKnownMetadata.knowledge.FAVICON]: fetchMetadata[WellKnownMetadata.knowledge.FAVICON]
                     },
-                    sourceId: this.id,
-                    sourceType: this.type
+                    dsId: this.id,
+                    dsType: this.type
                   }
                 });
                 return {
@@ -36654,13 +36701,53 @@ var init_source_website = __esm({
                 return null;
               }
             };
+            console.log(`[SYNC DEBUG] Starting 'fetching and indexing pages' for ${toFetch.slice(0, this.maxPages).length} URLs...`);
             const indexed = await step2.map(
               "fetching and indexing pages",
               toFetch.slice(0, this.maxPages),
               (url2) => fetchAndIndex(url2),
               { concurrency: 20, maxAttempts: 2 }
             );
+            console.log(`[SYNC DEBUG] Fetch complete. ${indexed.length} attempted, ${indexed.filter((f) => f !== null).length} successful`);
             const successful = indexed.filter((f) => f !== null);
+            console.log(`[SYNC DEBUG] Starting dsData registration...`);
+            await step2("register web page source", async () => {
+              try {
+                const botId = context2.get("botId");
+                const getStateResult = await client._inner.getState({ id: botId, type: "bot", name: "dsData" }).catch(() => ({ state: null }));
+                const freshState = getStateResult.state;
+                const existingPayload = freshState?.payload || {};
+                const kbPayload = existingPayload[input.kbId] || {};
+                const websiteUrl = this.baseUrl || this.sitemapUrl || this.urls?.[0] || "";
+                const getTitle = () => {
+                  if (!websiteUrl) return "Website";
+                  try {
+                    const urlWithProtocol = websiteUrl.includes("://") ? websiteUrl : `https://${websiteUrl}`;
+                    return new URL(urlWithProtocol).hostname;
+                  } catch {
+                    return websiteUrl;
+                  }
+                };
+                kbPayload[this.id] = {
+                  type: "web-page",
+                  title: getTitle(),
+                  createdOn: kbPayload[this.id]?.createdOn || Date.now(),
+                  // Preserve original creation time
+                  data: { websiteUrl, pages: [], indexingJobs: [] }
+                };
+                existingPayload[input.kbId] = kbPayload;
+                await client._inner.setState({
+                  id: botId,
+                  type: "bot",
+                  name: "dsData",
+                  payload: existingPayload
+                });
+                console.log(`Registered web page source "${this.id}" in dsData for KB ${input.kbId}`);
+              } catch (err) {
+                console.warn("Failed to register web page source in dsData state:", err);
+              }
+            });
+            console.log(`[SYNC DEBUG] \u2705 WebsiteSource sync complete for "${this.id}". Processed: ${discoveredUrls.length}, Added: ${successful.length}, Deleted: ${deleted.length}`);
             return {
               processed: discoveredUrls.length,
               deleted,
@@ -36679,6 +36766,10 @@ var init_source_website = __esm({
         const id = options.id || `sitemap_${sitemapUrl.replace(/https?:\/\//, "").replace(/\//g, "_")}`;
         return new _WebsiteSource(id, "sitemap", { ...options, sitemapUrl });
       }
+      static fromLlmsTxt(llmsTxtUrl, options = {}) {
+        const id = options.id || `llmstxt_${llmsTxtUrl.replace(/https?:\/\//, "").replace(/\//g, "_")}`;
+        return new _WebsiteSource(id, "llms-txt", { ...options, llmsTxtUrl });
+      }
       static fromUrls(urls, options = {}) {
         let defaultId = `urls_${urls.length}_pages`;
         if (urls.length > 0) {
@@ -43498,7 +43589,7 @@ var init_source_directory = __esm({
       _directoryPath;
       _filterFn;
       constructor(id, directoryPath, options = {}) {
-        super(id, "directory");
+        super(id, "document");
         this._directoryPath = directoryPath;
         this._filterFn = options.filter ?? void 0;
       }
@@ -43543,10 +43634,11 @@ var init_source_directory = __esm({
             const crypto3 = await import("crypto");
             const directory = path4.resolve(adk.environment.agent.directory, this.directoryPath);
             const tags = {
-              [WellKnownTags.knowledge.KNOWLEDGE]: "true",
+              [WellKnownTags.knowledge.KNOWLEDGE]: "knowledge-base",
+              [WellKnownTags.knowledge.KNOWLEDGE_BASE_ID]: input.kbId,
+              [WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME]: input.kbName,
               [WellKnownTags.knowledge.KNOWLEDGE_SOURCE_ID]: this.id,
-              [WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE]: this.type,
-              [WellKnownTags.knowledge.KNOWLEDGE_BASE_NAME]: input.kbName
+              [WellKnownTags.knowledge.KNOWLEDGE_SOURCE_TYPE]: this.type
             };
             if (!directory.startsWith(adk.environment.agent.directory)) {
               throw new Error("Directory path must be within the agent's directory");
@@ -43630,8 +43722,8 @@ var init_source_directory = __esm({
                 },
                 metadata: {
                   hash,
-                  sourceId: this.id,
-                  sourceType: this.type,
+                  dsId: this.id,
+                  dsType: this.type,
                   relPath: local.rel,
                   [WellKnownMetadata.knowledge.TITLE]: title
                 }
@@ -43668,10 +43760,10 @@ var init_source_directory = __esm({
 
 // src/primitives/data-sources/index.ts
 function isDirectorySource(source) {
-  return source.type === "directory";
+  return source.type === "document";
 }
 function isWebsiteSource(source) {
-  return source.type === "website";
+  return source.type === "web-page";
 }
 function isTableSource(source) {
   return source.type === "table";
@@ -44404,12 +44496,13 @@ var init_knowledge_indexing = __esm({
       description: "Built-in workflow to re-index all data sources in a knowledge base",
       input: z13.object({
         kbName: z13.string(),
+        kbId: z13.string(),
         force: z13.boolean().optional().describe("Force re-indexing even if files haven't changed").default(false)
       }),
       timeout: "180m",
       output: SyncOutput,
       handler: async ({ input, step: step2 }) => {
-        const { kbName } = input;
+        const { kbName, kbId } = input;
         const kb = adk.project.knowledge.find((x) => x.name === kbName);
         if (!kb) {
           throw new Error(`Knowledge base '${kbName}' not found`);
@@ -44425,7 +44518,8 @@ var init_knowledge_indexing = __esm({
                 statuses: ["in_progress", "listening", "pending", "paused"],
                 input: {
                   kbName,
-                  sourceId: source.id,
+                  kbId,
+                  dsId: source.id,
                   force: input.force || false
                 }
               }).then((x) => x.id)
@@ -44454,6 +44548,7 @@ var init_knowledge = __esm({
     init_define_BUILD();
     init_define_PACKAGE_VERSIONS();
     init_knowledge_indexing();
+    init_runtime2();
     ((Typings8) => {
       Typings8.Primitive = "knowledge";
     })(Typings5 || (Typings5 = {}));
@@ -45901,7 +45996,9 @@ var init_tracked_state = __esm({
       /** Workflow-specific state (persists across workflow executions) */
       workflowState: "workflowState",
       /** Workflow cached steps executions */
-      workflowSteps: "workflowSteps"
+      workflowSteps: "workflowSteps",
+      /** Data source metadata for dashboard visibility */
+      dsData: "dsData"
     };
     TrackedState2 = class _TrackedState {
       type;