cap-copilot-sdk 0.1.1 → 0.2.3

package/src/index.js

@@ -1,41 +1,30 @@
 "use strict";
-/**
- * @btp-copilot/cap-plugin — CDS plugin entry point.
- *
- * On CAP startup this plugin:
- *   1. Reads the CDS compiled model (entity schemas, types, relationships)
- *   2. Reads the project folder structure (srv handlers, db schema files,
- *      Fiori XML views, manifests, i18n labels, annotations)
- *   3. Iterates every running CDS service's srv.entities for runtime info
- *   4. POSTs all collected documents to the BTP Copilot backend so that the
- *      chatbot can answer context-aware questions about this app
- *
- * Minimal configuration in your app's package.json:
- * {
- *   "cds": {
- *     "requires": {
- *       "btp-copilot": {
- *         "kind": "btp-copilot",
- *         "backendUrl": "https://your-copilot.cfapps.eu10.hana.ondemand.com",
- *         "appId": "my-sales-app",
- *         "appName": "My Sales Application",
- *         "serviceUrl": "/odata/v4/SalesService",
- *         "token": "<bearer-token-or-read-from-env>"
- *       }
- *     },
- *     "plugins": ["@btp-copilot/cap-plugin"]
- *   }
- * }
- *
- * Or programmatically (in a custom server.js):
- *   const btpCopilot = require('@btp-copilot/cap-plugin');
- *   btpCopilot({ backendUrl: '...', appId: '...' });
- */
 
 const path = require("path");
 const { extract, extractFromServices } = require("./SchemaExtractor");
 const { scan } = require("./ProjectScanner");
-const { register } = require("./Registrar");
+const { register, registerAllServiceTools } = require("./Registrar");
+const { sendEntityData } = require("./DataSender");
+
+function _detectAppBaseUrl() {
+  const vcap = process.env.VCAP_APPLICATION;
+  if (vcap) {
+    try {
+      const parsed = JSON.parse(vcap);
+      const uris = parsed.application_uris ?? parsed.uris ?? [];
+      if (uris.length > 0) return `https://${uris[0]}`;
+    } catch (err) {
+      console.warn("Failed to parse VCAP_APPLICATION:", err.message);
+    }
+  }
+
+  const k8sUrl = process.env.APPLICATION_URL ?? process.env.APP_URL;
+  if (k8sUrl) return k8sUrl;
+
+  // ── Local development ──────────────────────────────────────────────────────
+  const port = process.env.PORT ?? 4004;
+  return `http://localhost:${port}`;
+}
 
 module.exports = (options = {}) => {
   const cds = require("@sap/cds");
@@ -49,26 +38,28 @@ module.exports = (options = {}) => {
   );
 
   const {
-    backendUrl = process.env.BTP_COPILOT_URL,
+    backendUrl = process.env.BTP_COPILOT_URL ?? "http://localhost:8000",
     appId = process.env.BTP_COPILOT_APP_ID,
     appName,
     serviceUrl,
+    serviceUrls,
+    appBaseUrl = process.env.BTP_COPILOT_APP_BASE_URL ?? _detectAppBaseUrl(),
     token = process.env.BTP_COPILOT_TOKEN,
     includeProjectFiles = true,
     includeHandlers = true,
     includeViews = true,
+    // verboseLogging: when true, CAP logs print the FULL JSON of every record
+    // pushed to the chatbot (instead of a 300-char truncated sample).
+    // Enable temporarily when debugging why the chatbot has wrong/missing data.
+    // WARNING: can produce very large console output for entities with wide rows.
+    verboseLogging = process.env.BTP_COPILOT_VERBOSE === "true" ? true : false,
   } = cfg;
 
   if (!backendUrl || !appId) {
-    log.warn(
-      "Missing required config (backendUrl, appId). " +
-        "Set them in cds.requires.btp-copilot or via BTP_COPILOT_URL / BTP_COPILOT_APP_ID env vars. " +
-        "Skipping registration.",
-    );
+    log.warn("Missing required config (backendUrl, appId).");
     return;
   }
 
-  // Validate appId format (backend requires ^[a-zA-Z0-9_-]+$)
   if (!/^[a-zA-Z0-9_-]+$/.test(appId)) {
     log.warn(
       `appId "${appId}" contains invalid characters. Use only letters, digits, underscores and hyphens.`,
@@ -81,12 +72,10 @@ module.exports = (options = {}) => {
       log.info(`Collecting context for app "${appId}"…`);
       const allDocs = [];
 
-      // ── 1. CDS model schema (entities, relationships, services) ─────────────
-      const schemaDocs = extract(cds, { serviceUrl });
+      const schemaDocs = extract(cds, { serviceUrl, serviceList });
       allDocs.push(...schemaDocs);
       log.info(`  + ${schemaDocs.length} schema documents`);
 
-      // ── 2. Runtime srv.entities for each running service ────────────────────
       const serviceList = Array.isArray(services)
         ? services
         : Object.values(services ?? {});
@@ -97,7 +86,6 @@ module.exports = (options = {}) => {
         log.info(`  + ${runtimeDocs.length} runtime service documents`);
       }
 
-      // ── 3. Project folder structure, src files, manifests ──────────────────
       if (includeProjectFiles) {
         const projectRoot = cds.env.root ?? process.cwd();
         const fileDocs = scan(projectRoot, { includeHandlers, includeViews });
@@ -111,14 +99,19 @@ module.exports = (options = {}) => {
       }
 
       log.info(
-        `Registering ${Math.min(allDocs.length, 50)} documents (of ${allDocs.length} total) for app "${appId}"…`,
+        `Registering ${allDocs.length} schema/project document(s) for app "${appId}"…`,
       );
 
+      // ── Schema + project docs: backend may skip if unchanged ────────────────
       const ok = await register({
         backendUrl,
         appId,
         appName: appName ?? appId,
         serviceUrl,
+        // Persist base URL in main app registry so the backend can resolve live
+        // OData queries even after it restarts (service-tool registrations are
+        // in-memory and lost on restart; the app registry persists).
+        appBaseUrl,
         documents: allDocs,
         token,
       });
@@ -132,8 +125,260 @@ module.exports = (options = {}) => {
           `✘ Registration failed for "${appId}". Check backendUrl (${backendUrl}) and token.`,
         );
       }
+
+
+      const countDoc = await _snapshotEntityCounts(serviceList, cds);
+      const sampleDocs = await _snapshotEntitySamples(serviceList, cds);
+      const dataDocs = [];
+      if (countDoc) dataDocs.push(countDoc);
+      dataDocs.push(...sampleDocs);
+
+      if (dataDocs.length > 0) {
+        log.info(
+          `Pushing ${dataDocs.length} data snapshot document(s) (bypasses schema cache)…`,
+        );
+        await register({
+          backendUrl,
+          appId,
+          appName: appName ?? appId,
+          documents: dataDocs,
+          token,
+          replace: false,
+        });
+        log.info(`  + ${dataDocs.length} data snapshot document(s) indexed.`);
+      }
+
+      // ── OData service tool registration ─────────────────────────────────────
+      // Register each running service as a separate tool so the backend maps
+      // entity names to the correct OData path unambiguously.
+      // When no explicit serviceUrls config is given, auto-register ALL running
+      // services (each with its own entity list, never merged).
+      if (appBaseUrl) {
+        const liveQueryUrls = serviceUrls ?? (serviceUrl ? [serviceUrl] : []);
+
+        // Build { serviceUrl, entities, entityFields } per service.
+        // entityFields is a map of entityName → [fieldName, ...] so the backend
+        // agent can build OData $filter clauses without parsing RAG text.
+        const _buildSvcEntry = (srv) => {
+          const svcUrl = srv.path ?? srv.options?.path ?? "";
+          const entityMap = srv.entities ?? {};
+          const entities = Object.keys(entityMap).filter((n) => !n.startsWith("_"));
+          const entityFields = {};
+          for (const [eName, eDef] of Object.entries(entityMap)) {
+            if (eName.startsWith("_") || !eDef) continue;
+            entityFields[eName] = Object.keys(eDef.elements ?? {}).filter(
+              (k) => !k.startsWith("_"),
+            );
+          }
+          return { serviceUrl: svcUrl, entities, entityFields };
+        };
+
+        // Build per-service registrations: use explicit list when configured,
+        // otherwise auto-discover every service that has entities.
+        const toRegister =
+          liveQueryUrls.length > 0
+            ? // Explicit list: match running services by path
+              serviceList
+                .filter((srv) => {
+                  const srvPath = srv.path ?? srv.options?.path ?? "";
+                  return liveQueryUrls.some(
+                    (u) =>
+                      u === srvPath ||
+                      srvPath.endsWith(u) ||
+                      u.endsWith(srvPath),
+                  );
+                })
+                .map(_buildSvcEntry)
+                .filter((s) => s.serviceUrl)
+            : // Auto-discover: every service with a path and at least one entity
+              serviceList
+                .map(_buildSvcEntry)
+                .filter((s) => s.serviceUrl && s.entities.length > 0);
+
+        if (toRegister.length > 0) {
+          log.info(
+            `Auto-discovered ${toRegister.length} OData service(s) for live-query tool registration.`,
+          );
+          for (const { serviceUrl: sUrl, entities, entityFields } of toRegister) {
+            await registerAllServiceTools({
+              backendUrl,
+              appId,
+              appName: appName ?? appId,
+              serviceUrls: [sUrl],
+              entities,
+              // Pass per-service entity→fields map so the backend agent can build
+              // OData $filter clauses without parsing RAG text
+              entityFieldsMap: { [sUrl]: entityFields ?? {} },
+              token,
+              appBaseUrl,
+            });
+          }
+          log.info(
+            `  + OData live-query tools registered for ${toRegister.length} service(s)`,
+          );
+        }
+      } else {
+        log.info(
+          "  ℹ Live OData query tool not registered — appBaseUrl could not be determined." +
+            " Set BTP_COPILOT_APP_BASE_URL or deploy to CF/Kyma for auto-detection.",
+        );
+      }
+
+      const dataCfg = { backendUrl, appId, appName: appName ?? appId, token, verboseLogging };
+      _registerLiveDataHooks(serviceList, dataCfg, log);
     } catch (err) {
       log.error("Registration error —", err.message);
     }
   });
 };
+
+const PAGE_SIZE = 50;
+
+const MAX_PAGES = 10;
+
+async function _snapshotEntitySamples(services, cds) {
+  const { SELECT } = cds;
+  const docs = [];
+
+  for (const srv of services) {
+    if (!srv?.entities) continue;
+    for (const [name, entity] of Object.entries(srv.entities)) {
+      if (!entity || name.startsWith("_")) continue;
+      try {
+        const hasModifiedAt =
+          entity.elements && "modifiedAt" in entity.elements;
+
+        let totalCount = null;
+        try {
+          const cr = await cds.run(
+            SELECT.one.from(entity).columns("count(*) as n"),
+          );
+          totalCount = Number(cr?.n ?? 0);
+        } catch {
+        }
+
+        let page = 0;
+
+        while (page < MAX_PAGES) {
+          const query = hasModifiedAt
+            ? SELECT.from(entity)
+                .orderBy("modifiedAt desc")
+                .limit(PAGE_SIZE, page * PAGE_SIZE)
+            : SELECT.from(entity).limit(PAGE_SIZE, page * PAGE_SIZE);
+
+          const rows = await cds.run(query);
+          if (!rows || rows.length === 0) break;
+
+          const fromRec = page * PAGE_SIZE + 1;
+          const toRec = page * PAGE_SIZE + rows.length;
+          const coverageNote =
+            totalCount !== null
+              ? totalCount <= toRec
+                ? `COMPLETE dataset — all ${totalCount} record(s) are shown below`
+                : `PARTIAL snapshot — showing records ${fromRec}–${toRec} of ${totalCount} total. ` +
+                  `Records ${toRec + 1}–${totalCount} are NOT in this document. ` +
+                  `Do NOT say there are no records for an ID just because it is absent here; ` +
+                  `use the live OData query tool or ask the user to navigate to that record.`
+              : `Records ${fromRec}–${toRec} (total count unknown)`;
+
+          const docTitle =
+            page === 0
+              ? `Entity data: ${name}`
+              : `Entity data: ${name} — page ${page + 1}`;
+
+          const content = [
+            `Entity: ${name}`,
+            `Coverage: ${coverageNote}`,
+            `Fetched at: ${new Date().toISOString()}`,
+            "",
+            "Records:",
+            JSON.stringify(rows, null, 2),
+          ].join("\n");
+
+          docs.push({ title: docTitle, content });
+
+          if (rows.length < PAGE_SIZE) break;
+          page++;
+        }
+      } catch (err) {
+        const reason = err?.message ?? "read failed";
+        docs.push({
+          title: `Entity data: ${name}`,
+          content: [
+            `Entity: ${name}`,
+            `Status: UNAVAILABLE`,
+            `Reason: ${reason}`,
+            "Note: This entity's read handler may depend on an external service",
+            "that is currently unreachable or has changed its API.",
+          ].join("\n"),
+        });
+      }
+    }
+  }
+
+  return docs;
+}
+
+function _registerLiveDataHooks(services, cfg, log) {
+  for (const srv of services) {
+    if (!srv?.entities || typeof srv.after !== "function") continue;
+    for (const [name] of Object.entries(srv.entities)) {
+      if (name.startsWith("_")) continue;
+      try {
+        srv.after("READ", name, async (data) => {
+          if (!data) return;
+          const records = Array.isArray(data) ? data : [data];
+          if (records.length === 0) return;
+          // $count filtering is handled inside sendEntityData
+          sendEntityData(cfg, name, records, `Entity data: ${name}`).catch(
+            () => {},
+          );
+        });
+
+        srv.after(["CREATE", "UPDATE"], name, async (data) => {
+          if (!data) return;
+          const records = Array.isArray(data) ? data : [data];
+          if (records.length === 0) return;
+          sendEntityData(
+            cfg,
+            name,
+            records,
+            `Entity data: ${name} — mutations`,
+          ).catch(() => {});
+        });
+      } catch (err) {
+        log.warn(
+          `Could not register live-data hook for ${name}: ${err.message}`,
+        );
+      }
+    }
+  }
+}
+
+async function _snapshotEntityCounts(services, cds) {
+  const { SELECT } = cds;
+  const lines = ["Entity record counts (at application startup):"];
+  let hasAny = false;
+
+  for (const srv of services) {
+    if (!srv?.entities) continue;
+    for (const [name, entity] of Object.entries(srv.entities)) {
+      if (!entity || name.startsWith("_")) continue;
+      try {
+        const result = await cds.run(
+          SELECT.one.from(entity).columns("count(*) as n"),
+        );
+        const n = result?.n ?? 0;
+        lines.push(`  ${name}: ${Number(n).toLocaleString()} record(s)`);
+        hasAny = true;
+      } catch (err) {
+        const reason = err?.message ?? "read failed";
+        lines.push(`  ${name}: UNAVAILABLE — ${reason}`);
+        hasAny = true;
+      }
+    }
+  }
+
+  if (!hasAny) return null;
+  return { title: "Live entity record counts", content: lines.join("\n") };
+}