cap-copilot-sdk 0.1.0 → 0.2.2

package/package.json

@@ -1,11 +1,21 @@
 {
   "name": "cap-copilot-sdk",
-  "version": "0.1.0",
+  "version": "0.2.2",
   "description": "CDS plugin: auto-extracts entity schemas, relationships, project structure and registers with BTP Copilot backend",
   "main": "cds-plugin.js",
-  "files": ["src", "cds-plugin.js"],
+  "exports": {
+    ".": "./cds-plugin.js",
+    "./cds-plugin": "./cds-plugin.js"
+  },
+  "files": ["src", "cds-plugin.js", "postinstall.js"],
+  "scripts": {
+    "postinstall": "node postinstall.js"
+  },
   "keywords": ["sap", "cap", "cds", "btp", "ai", "chatbot", "plugin"],
   "license": "MIT",
+  "cds": {
+    "plugin": true
+  },
   "peerDependencies": {
     "@sap/cds": ">=7.0.0"
   }

package/postinstall.js

@@ -0,0 +1,81 @@
+/**
+ * postinstall.js — auto-configure consuming CAP app's package.json
+ *
+ * Runs automatically after `npm install cap-copilot-sdk`.
+ * Adds only the app-specific keys (appId, appName) to cds.requires["btp-copilot"].
+ *
+ * URL defaults (backendUrl, iframeUrl) are intentionally kept inside the SDK —
+ * they are not written to the consuming app's package.json so developers never
+ * need to see or manage them.  Override via env vars when deploying to production:
+ *   BTP_COPILOT_URL        → backend URL
+ *   BTP_COPILOT_IFRAME_URL → frontend chatbot UI URL
+ */
+"use strict";
+const fs = require("fs");
+const path = require("path");
+
+// Set BTP_COPILOT_SKIP_POSTINSTALL=1 to opt out of auto-configuration.
+// Useful in CI/CD pipelines where package.json is read-only or managed by tooling.
+if (process.env.BTP_COPILOT_SKIP_POSTINSTALL === "1") process.exit(0);
+
+// INIT_CWD = directory from which `npm install` was run (the consuming app root)
+const appRoot = process.env.INIT_CWD;
+
+if (!appRoot || appRoot === __dirname || appRoot === path.resolve(__dirname, "../..")) {
+  process.exit(0);
+}
+
+const pkgPath = path.join(appRoot, "package.json");
+if (!fs.existsSync(pkgPath)) process.exit(0);
+
+let pkg;
+try {
+  pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
+} catch {
+  process.exit(0);
+}
+
+const hasCds = pkg.dependencies?.["@sap/cds"] || pkg.devDependencies?.["@sap/cds"];
+if (!hasCds) process.exit(0);
+
+const cdsSection = pkg.cds ?? {};
+const requires = cdsSection.requires ?? {};
+const existing = requires["btp-copilot"] ?? {};
+
+// Only the app-specific keys belong in package.json.
+// URL defaults are owned by the SDK (cds-plugin.js) — not exposed to the user.
+const safeId = (pkg.name ?? "cap-app").replace(/[^a-zA-Z0-9_-]/g, "-");
+const appDefaults = {
+  appId: existing.appId ?? safeId,
+  appName: existing.appName ?? pkg.description ?? pkg.name ?? "CAP App",
+};
+
+const changed =
+  existing.appId === undefined ||
+  existing.appName === undefined ||
+  existing.backendUrl !== undefined ||
+  existing.iframeUrl !== undefined;
+
+if (!changed) {
+  process.exit(0);
+}
+
+requires["btp-copilot"] = { ...existing, ...appDefaults };
+
+// Remove URL keys if a previous version of postinstall wrote them — they belong in the SDK
+delete requires["btp-copilot"].backendUrl;
+delete requires["btp-copilot"].iframeUrl;
+
+cdsSection.requires = requires;
+pkg.cds = cdsSection;
+
+try {
+  fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n", "utf8");
+  console.log(`\x1b[36m[cap-copilot-sdk]\x1b[0m \u2714 Configured cds.requires["btp-copilot"] in package.json`);
+  console.log(`\x1b[36m[cap-copilot-sdk]\x1b[0m   appId="${appDefaults.appId}" | appName="${appDefaults.appName}"`);
+  console.log(`\x1b[36m[cap-copilot-sdk]\x1b[0m   To use custom backend/frontend URLs set env vars:`);
+  console.log(`\x1b[36m[cap-copilot-sdk]\x1b[0m     BTP_COPILOT_URL=https://your-backend.cfapps.eu10.hana.ondemand.com`);
+  console.log(`\x1b[36m[cap-copilot-sdk]\x1b[0m     BTP_COPILOT_IFRAME_URL=https://your-frontend.cfapps.eu10.hana.ondemand.com`);
+} catch {
+  // Non-fatal
+}

package/src/DataSender.js

@@ -0,0 +1,469 @@
+"use strict";
+const { register } = require("./Registrar");
+
+const DEBOUNCE_MS = 2000;
+const MAX_RECORDS = 200;
+// Truncate individual document content to prevent extremely large payloads.
+// Content exceeding this is cut with a [truncated] marker so the backend still
+// receives a valid document rather than an oversized JSON payload.
+const MAX_DOC_CONTENT_BYTES = 512_000; // 512 KB per document
+
+// Global rate limiter: enforce a minimum gap between consecutive flushes
+// (across ALL apps in this process) so the backend never receives more than
+// ~5 bulk pushes per second even under heavy parallel load.
+const MIN_FLUSH_GAP_MS = 200;
+let _lastFlushAt = 0;
+
+function _sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+// Per-app debounce queue: appId -> { cfg, docs: Map<title,content>, timer, flushing }
+const _queues = new Map();
+
+function _getQueue(cfg) {
+  if (!_queues.has(cfg.appId)) {
+    _queues.set(cfg.appId, {
+      cfg,
+      docs: new Map(),
+      timer: null,
+      flushing: false,
+    });
+  }
+  return _queues.get(cfg.appId);
+}
+
+async function _flush(appId) {
+  const q = _queues.get(appId);
+  if (!q || q.docs.size === 0 || q.flushing) return;
+
+  // Snapshot and clear immediately so new writes arriving during the HTTP
+  // round-trip are queued into a fresh batch, not lost.
+  const documents = Array.from(q.docs.entries()).map(([title, content]) => ({
+    title,
+    content: Buffer.byteLength(content) > MAX_DOC_CONTENT_BYTES
+      ? content.slice(0, MAX_DOC_CONTENT_BYTES) + "\n[truncated — content exceeded 512 KB]"
+      : content,
+  }));
+  q.docs.clear();
+  q.timer = null;
+  q.flushing = true;
+
+  // Global rate limit: wait if we flushed too recently
+  const gap = _lastFlushAt + MIN_FLUSH_GAP_MS - Date.now();
+  if (gap > 0) await _sleep(gap);
+  _lastFlushAt = Date.now();
+
+  console.log(
+    `[btp-copilot] Flushing ${documents.length} queued document(s) to backend…`,
+  );
+
+  const { cfg } = q;
+  try {
+    await register({
+      backendUrl: cfg.backendUrl,
+      appId: cfg.appId,
+      appName: cfg.appName,
+      token: cfg.token,
+      documents,
+      replace: false,
+    });
+  } finally {
+    q.flushing = false;
+    // Re-schedule if new docs arrived while the HTTP request was in flight
+    if (q.docs.size > 0 && !q.timer) {
+      q.timer = setTimeout(() => _flush(appId), DEBOUNCE_MS);
+    }
+  }
+}
+
+async function sendEntityData(cfg, entityName, data, label) {
+  const raw = Array.isArray(data) ? data : data != null ? [data] : [];
+
+  // Filter out $count-only responses. CAP fires after("READ") for
+  // GET /Entity/$count requests too. Depending on the CAP version the handler
+  // receives either a plain number (22), or a single-key object ({"$count":22}).
+  // Neither is real record data — drop them to avoid corrupting the index.
+  const records = raw.filter(
+    (r) =>
+      r != null &&
+      typeof r === "object" &&
+      !("$count" in r && Object.keys(r).length === 1) &&
+      !("@odata.count" in r && Object.keys(r).length === 1),
+  );
+
+  if (records.length === 0) return false;
+
+  if (!cfg.backendUrl || !cfg.appId) {
+    console.warn(
+      "[btp-copilot] sendData: backendUrl or appId not configured — skipping push. " +
+        'Ensure cds.requires["btp-copilot"].backendUrl and .appId are set.',
+    );
+    return false;
+  }
+
+  const capped =
+    records.length > MAX_RECORDS ? records.slice(0, MAX_RECORDS) : records;
+  if (capped.length < records.length) {
+    console.log(
+      `[btp-copilot] sendData("${entityName}") — ${records.length} records, truncated to ${MAX_RECORDS}`,
+    );
+  }
+
+  console.log(
+    `[btp-copilot] sendData("${entityName}") — ${capped.length} record(s) being pushed to chatbot`,
+  );
+  if (capped.length > 0) {
+    if (cfg.verboseLogging) {
+      // Full JSON for every record — set verboseLogging:true or BTP_COPILOT_VERBOSE=true
+      console.log(
+        `[btp-copilot]   full payload (${entityName}):\n${JSON.stringify(capped, null, 2)}`,
+      );
+    } else {
+      const sample = JSON.stringify(capped[0]);
+      console.log(
+        `[btp-copilot]   sample record[0]: ${
+          sample.length > 300 ? sample.slice(0, 300) + "..." : sample
+        }`,
+      );
+    }
+  }
+
+
+  const shortEntityName = entityName.split(".").pop();
+  const title = label ?? `${shortEntityName} entity data`;
+  const content = [
+    `Entity: ${shortEntityName}`,
+    `Record count: ${capped.length}`,
+    `Pushed at: ${new Date().toISOString()}`,
+    "",
+    "Records:",
+    JSON.stringify(capped, null, 2),
+  ].join("\n");
+
+  const q = _getQueue(cfg);
+  q.docs.set(title, content);
+
+  if (q.timer) clearTimeout(q.timer);
+  q.timer = setTimeout(() => _flush(cfg.appId), DEBOUNCE_MS);
+
+  return true;
+}
+
+function watchEntity(srv, entity, cfg, options = {}) {
+  const { groupByKey, labelPrefix, once = false } = options;
+  const entityName = entity.name ?? String(entity);
+  let _readSent = false;
+
+  function _push(rows) {
+    setImmediate(async () => {
+      try {
+        if (groupByKey) {
+          const groups = {};
+          for (const row of rows) {
+            const id = row[groupByKey];
+            if (id == null) continue;
+            (groups[id] = groups[id] || []).push(row);
+          }
+          await Promise.all(
+            Object.entries(groups).map(([id, groupRows]) =>
+              sendEntityData(
+                cfg,
+                entityName,
+                groupRows,
+                `${labelPrefix ?? entityName} ${id}`,
+              ),
+            ),
+          );
+        } else {
+          await sendEntityData(cfg, entityName, rows);
+        }
+      } catch (e) {
+        console.warn(
+          `[btp-copilot] watchEntity(${entityName}) failed:`,
+          e.message,
+        );
+      }
+    });
+  }
+
+  // ── Layer 1a: READ hook — push data as the user views it ──────────────────
+  srv.after("READ", entity, (results) => {
+    if (!results) return;
+    if (once && _readSent) return;
+    const rows = Array.isArray(results) ? results : [results];
+    if (rows.length === 0) return;
+    if (once) _readSent = true;
+    _push(rows);
+  });
+
+  // ── Layer 1b: Mutation hooks — push immediately when data is written ───────
+  // This is critical: records created via API, imports, migrations, or batch
+  // jobs are NEVER seen by the READ hook. Without this, the chatbot is blind
+  // to any data the user creates but hasn't navigated back to view.
+  if (!once) {
+    srv.after(["CREATE", "UPDATE"], entity, (result) => {
+      if (!result) return;
+      const rows = Array.isArray(result) ? result : [result];
+      if (rows.length === 0) return;
+      _push(rows);
+    });
+  }
+}
+
+/**
+ * Background paginated sync of all entities across all services.
+ *
+ * TWO MODES:
+ *
+ * multiTenant=false (default, single-user / dev):
+ *   Pushes raw records per page to the vector store. The AI can answer
+ *   questions about any record without a live OData call.
+ *
+ * multiTenant=true  (recommended for production apps with many users):
+ *   Pushes ONLY per-entity aggregate stats (total count + field list).
+ *   Raw record data is NEVER stored in the shared vector store — it is
+ *   fetched at query time via OData using each user's own JWT token.
+ *   This prevents User A's records from appearing in User B's answers.
+ *
+ * @param {object}    cds         - @sap/cds instance
+ * @param {object[]}  services    - array of CDS service instances
+ * @param {object}    cfg         - { backendUrl, appId, token, appName }
+ * @param {object}    [options]
+ * @param {number}    [options.pageSize=100]       rows per document page
+ * @param {string}    [options.modifiedSince]      ISO timestamp — only sync newer records
+ * @param {boolean}   [options.multiTenant=false]  true = push stats only, not raw records
+ * @returns {Promise<number>} total rows synced (0 in multiTenant mode)
+ */
+async function syncAllEntities(cds, services, cfg, options = {}) {
+  const { pageSize = 100, modifiedSince = null, multiTenant = false } = options;
+
+  // ── multiTenant mode: push per-entity stats only ─────────────────────────
+  // Each entity gets a single summary document: entity name, total count, and
+  // the fields that exist. The AI knows the shape of the data and how many
+  // records there are, but raw records are never stored in the shared vector
+  // store. Per-user, per-record data comes live via OData at chat time.
+  if (multiTenant) {
+    let db;
+    try { db = await cds.connect.to("db"); } catch { db = null; }
+
+    const svcList = Array.isArray(services) ? services : Object.values(services ?? {});
+    for (const svc of svcList) {
+      if (!svc.entities) continue;
+      for (const [entityName, entity] of Object.entries(svc.entities)) {
+        try {
+          const fields = entity.elements
+            ? Object.keys(entity.elements).filter((k) => !k.startsWith("_")).join(", ")
+            : "unknown";
+          let totalCount = "unknown";
+          if (db) {
+            try {
+              const { SELECT } = cds;
+              const r = await db.run(SELECT.one.from(entity).columns("count(*) as n"));
+              totalCount = r?.n ?? 0;
+            } catch { /* entity not queryable */ }
+          }
+          const content = [
+            `Entity: ${entityName}`,
+            `Total records in database: ${totalCount}`,
+            `Available fields: ${fields}`,
+            `Note: Live per-user data is fetched via OData at query time using the user's token.`,
+          ].join("\n");
+          const q = _getQueue(cfg);
+          q.docs.set(`${entityName} entity summary`, content);
+          if (q.timer) clearTimeout(q.timer);
+          q.timer = setTimeout(() => _flush(cfg.appId), DEBOUNCE_MS);
+        } catch { /* skip unqueryable entities */ }
+      }
+    }
+    return 0;
+  }
+
+
+  let db;
+  try {
+    db = await cds.connect.to("db");
+  } catch {
+    console.warn(
+      "[btp-copilot] syncAllEntities: could not connect to db — skipping startup sync.",
+    );
+    return 0;
+  }
+
+  // cds.SELECT may not be directly on the module depending on @sap/cds version;
+  // try cds.SELECT first, then cds.ql.SELECT (both are the same object).
+  const SELECT = cds.SELECT ?? cds.ql?.SELECT;
+  if (!SELECT) {
+    console.warn(
+      "[btp-copilot] syncAllEntities: cds.SELECT not available — skipping background sync.",
+    );
+    return 0;
+  }
+
+  const svcList = Array.isArray(services)
+    ? services
+    : Object.values(services ?? {});
+  let totalSynced = 0;
+
+  for (const svc of svcList) {
+    if (!svc.entities) continue;
+    for (const [entityName, entity] of Object.entries(svc.entities)) {
+      try {
+        const hasModifiedAt =
+          entity.elements && "modifiedAt" in entity.elements;
+
+        // Incremental: skip entities with no changes since last sync.
+        // Use CDS tagged-template syntax for parameterized queries — never string interpolation.
+        if (modifiedSince && hasModifiedAt) {
+          const changed = await db.run(
+            SELECT.from(entity).columns("count(*) as n").where`modifiedAt > ${modifiedSince}`,
+          );
+          if (!changed?.[0]?.n) continue;
+        }
+
+        // Count total records for the stats document
+        let totalCount = "unknown";
+        try {
+          const r = await db.run(SELECT.from(entity).columns("count(*) as n"));
+          totalCount = r?.[0]?.n ?? 0;
+        } catch { /* entity may not support COUNT (e.g. view, external) */ }
+
+        const fields = entity.elements
+          ? Object.keys(entity.elements).filter((k) => !k.startsWith("_")).join(", ")
+          : "unknown";
+
+        // Avoid SELECT * which picks up virtual elements (e.g. IsActiveEntity
+        // on draft-enabled entities) that have no DB column and cause SQL errors.
+        // Association/composition elements also have no column (they use FK cols).
+        const persistedCols = entity.elements
+          ? Object.entries(entity.elements)
+              .filter(([, el]) => !el.virtual && !el.target)
+              .map(([name]) => name)
+          : [];
+
+        // Paginate through ALL records — no artificial cap.
+        // Each page is stored as a separate titled document so the vector
+        // store can retrieve any record regardless of how large the table is.
+        let offset = 0;
+        let pageNum = 0;
+
+        while (true) {
+          const baseQ =
+            persistedCols.length > 0
+              ? SELECT.from(entity).columns(persistedCols)
+              : SELECT.from(entity);
+          const pageQ = hasModifiedAt
+            ? baseQ.orderBy("modifiedAt desc").limit(pageSize, offset)
+            : baseQ.limit(pageSize, offset);
+          const page = await db.run(pageQ);
+
+          if (!page || page.length === 0) break;
+
+          pageNum++;
+          totalSynced += page.length;
+
+          const docTitle =
+            pageNum === 1
+              ? `${entityName} entity data`
+              : `${entityName} entity data (page ${pageNum})`;
+
+          const content = [
+            `Entity: ${entityName}`,
+            `Total records in database: ${totalCount}`,
+            `Available fields: ${fields}`,
+            `Page: ${pageNum} (records ${offset + 1}–${offset + page.length} of ${totalCount})`,
+            "",
+            `Records:\n${JSON.stringify(page, null, 2)}`,
+          ].join("\n");
+
+          const q = _getQueue(cfg);
+          q.docs.set(docTitle, content);
+          if (q.timer) clearTimeout(q.timer);
+          q.timer = setTimeout(() => _flush(cfg.appId), DEBOUNCE_MS);
+
+          offset += page.length;
+          if (page.length < pageSize) break;
+        }
+
+        // If entity was empty, still push a summary doc so the AI knows it exists
+        if (pageNum === 0) {
+          const content = [
+            `Entity: ${entityName}`,
+            `Total records in database: ${totalCount}`,
+            `Available fields: ${fields}`,
+            "",
+            "No records found.",
+          ].join("\n");
+          const q = _getQueue(cfg);
+          q.docs.set(`${entityName} entity data`, content);
+          if (q.timer) clearTimeout(q.timer);
+          q.timer = setTimeout(() => _flush(cfg.appId), DEBOUNCE_MS);
+        }
+      } catch (e) {
+        // Log skipped entities so admins can diagnose issues (external services,
+        // views, or entities with unsupported field types are expected to fail here).
+        console.warn(
+          `[btp-copilot] syncAllEntities: skipping "${entityName}" — ${e.message}`,
+        );
+      }
+    }
+  }
+  return totalSynced;
+}
+
+
+/**
+ * Push an entity-level error as a context document to the chatbot backend.
+ *
+ * Call this whenever a CDS handler fails to read an entity (e.g. a broken
+ * remote service expand). The chatbot receives a document explaining that the
+ * entity is currently unavailable and why, so it can give the user a
+ * meaningful answer instead of an unexpected 500 / empty result.
+ *
+ * @param {object} cfg        - { backendUrl, appId, token }
+ * @param {string} entityName - The entity that failed (e.g. "Farms")
+ * @param {string} reason     - The error message from the caught exception
+ * @param {string} [hint]     - Optional developer hint (e.g. "check FarmService expand config")
+ */
+async function sendEntityError(cfg, entityName, reason, hint) {
+  if (!cfg.backendUrl || !cfg.appId) return;
+
+  const lines = [
+    `Entity: ${entityName}`,
+    `Status: UNAVAILABLE`,
+    `Reason: ${reason ?? "unknown error"}`,
+    `Recorded at: ${new Date().toISOString()}`,
+  ];
+  if (hint) lines.push(`Hint: ${hint}`);
+  lines.push(
+    "",
+    "The chatbot cannot fetch live data for this entity right now.",
+    "It will use any previously cached data where available.",
+  );
+
+  const title = `Entity error: ${entityName}`;
+  const content = lines.join("\n");
+
+  const q = _getQueue(cfg);
+  q.docs.set(title, content);
+  if (q.timer) clearTimeout(q.timer);
+  q.timer = setTimeout(() => _flush(cfg.appId), DEBOUNCE_MS);
+}
+
+/**
+ * Flush all pending debounce queues immediately.
+ * Call this on graceful shutdown to ensure no queued data is lost.
+ */
+async function flushAll() {
+  const pending = [..._queues.keys()];
+  await Promise.all(
+    pending.map(async (appId) => {
+      const q = _queues.get(appId);
+      if (!q) return;
+      if (q.timer) { clearTimeout(q.timer); q.timer = null; }
+      await _flush(appId);
+    }),
+  );
+}
+
+module.exports = { sendEntityData, sendEntityError, watchEntity, syncAllEntities, flushAll };

package/src/ProjectScanner.js

@@ -19,8 +19,8 @@
 const fs = require("fs");
 const path = require("path");
 
-const MAX_LINES = 300; // max content lines per document
-const MAX_HANDLER_LINES = 150; // handlers are trimmed more aggressively
+const MAX_LINES = 400;
+const MAX_HANDLER_LINES = 250;
 const SKIP_DIRS = new Set([
   "node_modules",
   ".git",
@@ -121,8 +121,8 @@ function scan(projectRoot, opts = {}) {
       const xmlFiles = _findFiles(appDir, [".xml"], 4).filter(
         (f) => !f.includes("node_modules"),
       );
-      for (const filePath of xmlFiles.slice(0, 10)) {
-        // cap at 10 XML files
+      for (const filePath of xmlFiles.slice(0, 20)) {
+        // cap at 20 XML files
         const rel = path.relative(projectRoot, filePath).replace(/\\/g, "/");
         const content = _readTrimmed(filePath, MAX_LINES);
         if (content) {
@@ -131,7 +131,6 @@ function scan(projectRoot, opts = {}) {
       }
     }
 
-    // i18n / properties files (list of labels)
     const i18nFiles = _findFilesByName(appDir, "i18n.properties", 3).concat(
       _findFilesByName(appDir, "i18n_en.properties", 3),
     );
@@ -142,6 +141,19 @@ function scan(projectRoot, opts = {}) {
         docs.push({ title: `UI labels (i18n): ${rel}`, content });
       }
     }
+
+    if (includeHandlers) {
+      const controllerFiles = _findFiles(appDir, [".js", ".ts", ".mjs"], 4).filter(
+        (f) => !f.includes("node_modules") && !f.includes("test"),
+      );
+      for (const filePath of controllerFiles.slice(0, 15)) {
+        const rel = path.relative(projectRoot, filePath).replace(/\\/g, "/");
+        const content = _readTrimmed(filePath, MAX_HANDLER_LINES);
+        if (content) {
+          docs.push({ title: `UI controller: ${rel}`, content });
+        }
+      }
+    }
   }
 
   return docs;