@talkpilot/core-db 1.3.51 → 1.3.52

package/src/talkpilot/auditLogs/__tests__/auditLogs.spec.ts

@@ -0,0 +1,92 @@
+import {
+  insertAuditLog,
+  getAuditLogsCollection,
+  findAuditLogs,
+  countAuditLogs,
+} from "../auditLogs.getters";
+import { AuditLogEntry } from "../auditLogs.types";
+
+describe("db.auditLogs", () => {
+  describe("insertAuditLog", () => {
+    it("should write an entry that can be read back with all fields intact", async () => {
+      const entry: AuditLogEntry = {
+        clientId: "deletedClient1",
+        adminEmail: "admin@callai.co.il",
+        action: "delete",
+        timestamp: new Date("2026-07-15T10:00:00.000Z"),
+        env: "prod",
+      };
+
+      await insertAuditLog(entry);
+
+      const stored = await getAuditLogsCollection().findOne({
+        clientId: "deletedClient1",
+      });
+      expect(stored).toMatchObject(entry);
+    });
+  });
+
+  describe("findAuditLogs", () => {
+    const seed = async () => {
+      await insertAuditLog({
+        clientId: "clientA",
+        adminEmail: "admin@callai.co.il",
+        action: "delete",
+        timestamp: new Date("2026-01-01T00:00:00.000Z"),
+      });
+      await insertAuditLog({
+        clientId: "clientB",
+        adminEmail: "admin@callai.co.il",
+        action: "delete",
+        timestamp: new Date("2026-02-01T00:00:00.000Z"),
+      });
+      await insertAuditLog({
+        clientId: "clientC",
+        adminEmail: "admin@callai.co.il",
+        action: "delete",
+        timestamp: new Date("2026-03-01T00:00:00.000Z"),
+      });
+    };
+
+    it("should return a page of entries, newest first", async () => {
+      await seed();
+
+      const page = await findAuditLogs({ limit: 2 });
+
+      expect(page).toHaveLength(2);
+      expect(page.map((e) => e.clientId)).toEqual(["clientC", "clientB"]);
+    });
+
+    it("should skip entries for later pages", async () => {
+      await seed();
+
+      const page = await findAuditLogs({ skip: 2, limit: 2 });
+
+      expect(page).toHaveLength(1);
+      expect(page[0].clientId).toBe("clientA");
+    });
+  });
+
+  describe("countAuditLogs", () => {
+    it("should return the total number of entries regardless of pagination", async () => {
+      await insertAuditLog({
+        clientId: "clientA",
+        adminEmail: "admin@callai.co.il",
+        action: "delete",
+        timestamp: new Date(),
+      });
+      await insertAuditLog({
+        clientId: "clientB",
+        adminEmail: "admin@callai.co.il",
+        action: "delete",
+        timestamp: new Date(),
+      });
+
+      expect(await countAuditLogs()).toBe(2);
+    });
+
+    it("should return 0 when there are no entries", async () => {
+      expect(await countAuditLogs()).toBe(0);
+    });
+  });
+});

package/src/talkpilot/auditLogs/auditLogs.getters.ts

@@ -0,0 +1,26 @@
+import { Collection } from "mongodb";
+import { getDb } from "../index";
+import { applyQueryOptions, QueryOptions } from "../utils/query.utils";
+import { AuditLogDoc, AuditLogEntry } from "./auditLogs.types";
+
+export const getAuditLogsCollection = (): Collection<AuditLogEntry> => {
+  return getDb().collection<AuditLogEntry>("auditLogs");
+};
+
+export const insertAuditLog = async (entry: AuditLogEntry): Promise<void> => {
+  await getAuditLogsCollection().insertOne(entry);
+};
+
+export const findAuditLogs = async (
+  options: Pick<QueryOptions, "skip"> & Required<Pick<QueryOptions, "limit">>,
+): Promise<AuditLogDoc[]> => {
+  const cursor = getAuditLogsCollection().find({});
+  return applyQueryOptions(cursor, {
+    sort: { timestamp: -1 },
+    ...options,
+  }).toArray();
+};
+
+export const countAuditLogs = async (): Promise<number> => {
+  return getAuditLogsCollection().countDocuments({});
+};

package/src/talkpilot/auditLogs/auditLogs.types.ts

@@ -0,0 +1,13 @@
+import { WithId } from "mongodb";
+
+export type AuditLogEntry = {
+  clientId: string;
+  adminEmail: string;
+  action: "delete";
+  timestamp: Date;
+  env?: string;
+  allSucceeded?: boolean;
+  failedCollections?: string[];
+};
+
+export type AuditLogDoc = WithId<AuditLogEntry>;

package/src/talkpilot/auditLogs/index.ts

@@ -0,0 +1,2 @@
+export * from "./auditLogs.types";
+export * from "./auditLogs.getters";

package/src/talkpilot/clients/__tests__/clients.getters.spec.ts

@@ -0,0 +1,33 @@
+import { getClientsCollection, getAllClientCredits } from "../clients.getters";
+import { createClient } from "../../../test-utils/factories";
+
+describe("db.clients", () => {
+  describe("getAllClientCredits", () => {
+    it("should return a Map of clientId to credits across all clients", async () => {
+      await getClientsCollection().insertMany([
+        createClient({ clientId: "creditsClientA", credits: 100 }),
+        createClient({ clientId: "creditsClientB", credits: 250 }),
+      ]);
+
+      const result = await getAllClientCredits();
+
+      expect(result).toBeInstanceOf(Map);
+      expect(result.get("creditsClientA")).toBe(100);
+      expect(result.get("creditsClientB")).toBe(250);
+    });
+
+    it("should default missing credits to 0", async () => {
+      const client = createClient({ clientId: "noCreditsClient" });
+      delete (client as { credits?: number }).credits;
+      await getClientsCollection().insertOne(client);
+
+      const result = await getAllClientCredits();
+      expect(result.get("noCreditsClient")).toBe(0);
+    });
+
+    it("should return an empty Map when there are no clients", async () => {
+      const result = await getAllClientCredits();
+      expect(result.size).toBe(0);
+    });
+  });
+});

package/src/talkpilot/clients/__tests__/clients.lifecycle.spec.ts

@@ -0,0 +1,166 @@
+import {
+  clientExistsInAnyCollection,
+  deleteClientDataAcrossDomains,
+} from "../clients.lifecycle";
+import { getPhoneNumbersCollection } from "../../phone_numbers/phone_numbers.getter";
+import {
+  createMuniIssue,
+  generateMuniIssueId,
+  getAuditLogsCollection,
+  getScansCollection,
+} from "../../../index";
+import {
+  createPhoneNumber,
+  createMuniIssueInput,
+  createScanFixture,
+} from "../../../test-utils/factories";
+
+describe("db.clients.lifecycle", () => {
+  describe("clientExistsInAnyCollection", () => {
+    it("should return true when the client exists only in phone_numbers (client_id field)", async () => {
+      const clientId = "existsPhoneOnly";
+      await getPhoneNumbersCollection().insertOne(
+        createPhoneNumber({ client_id: clientId }),
+      );
+
+      expect(await clientExistsInAnyCollection(clientId)).toBe(true);
+    });
+
+    it("should return true when the client exists only in muniIssues (municipal DB)", async () => {
+      const clientId = "existsMuniOnly";
+      await createMuniIssue(
+        createMuniIssueInput({ clientId }),
+        generateMuniIssueId(),
+      );
+
+      expect(await clientExistsInAnyCollection(clientId)).toBe(true);
+    });
+
+    it("should return false when the client does not exist anywhere", async () => {
+      expect(await clientExistsInAnyCollection("noSuchClientAtAll")).toBe(
+        false,
+      );
+    });
+
+    it("should not consider websitalk data (scans/websiteUrls) at all", async () => {
+      const clientId = "onlyInWebsitalk";
+      await getScansCollection().insertOne(createScanFixture({ clientId }));
+
+      expect(await clientExistsInAnyCollection(clientId)).toBe(false);
+    });
+
+    it("should not be fooled by unrelated collections lacking a clientId/client_id field", async () => {
+      await global.db
+        .collection("someUnrelatedCollection")
+        .insertOne({ someField: "someValue" });
+
+      expect(await clientExistsInAnyCollection("noSuchClientAtAll")).toBe(
+        false,
+      );
+    });
+  });
+
+  describe("deleteClientDataAcrossDomains", () => {
+    it("should return deleted:false with a reason and delete nothing when the client does not exist", async () => {
+      await getPhoneNumbersCollection().insertOne(
+        createPhoneNumber({ client_id: "unrelatedClient" }),
+      );
+
+      const summary = await deleteClientDataAcrossDomains(
+        "missingClient",
+        "admin@callai.co.il",
+      );
+
+      expect(summary.deleted).toBe(false);
+      if (!summary.deleted) {
+        expect(summary.reason).toBeTruthy();
+      }
+      expect(
+        await getPhoneNumbersCollection().findOne({
+          client_id: "unrelatedClient",
+        }),
+      ).not.toBeNull();
+    });
+
+    it("should not write an audit log entry when the client does not exist", async () => {
+      await deleteClientDataAcrossDomains(
+        "missingClientNoAudit",
+        "admin@callai.co.il",
+      );
+
+      expect(
+        await getAuditLogsCollection().findOne({
+          clientId: "missingClientNoAudit",
+        }),
+      ).toBeNull();
+    });
+
+    it("should delete the client's data across talkpilot and municipal domains", async () => {
+      const clientId = "deleteMeEverywhere";
+      await getPhoneNumbersCollection().insertOne(
+        createPhoneNumber({ client_id: clientId }),
+      );
+      await createMuniIssue(
+        createMuniIssueInput({ clientId }),
+        generateMuniIssueId(),
+      );
+
+      const adminEmail = "admin@callai.co.il";
+      const summary = await deleteClientDataAcrossDomains(
+        clientId,
+        adminEmail,
+      );
+
+      expect(summary.deleted).toBe(true);
+      if (summary.deleted) {
+        expect(summary.allSucceeded).toBe(true);
+        const byName = (name: string) =>
+          summary.results.find((r) => r.collectionName === name);
+        expect(byName("phone_numbers")?.deletedCount).toBe(1);
+        expect(byName("muniIssues")?.deletedCount).toBe(1);
+      }
+
+      expect(await clientExistsInAnyCollection(clientId)).toBe(false);
+
+      // The audit entry itself must survive the client's own deletion.
+      const auditEntry = await getAuditLogsCollection().findOne({ clientId });
+      expect(auditEntry).toMatchObject({
+        clientId,
+        adminEmail,
+        action: "delete",
+      });
+      expect(auditEntry?.timestamp).toBeInstanceOf(Date);
+    });
+
+    it("should not touch websitalk data (scans/websiteUrls) — callers must call the external website-talk deletion API separately", async () => {
+      const clientId = "hasWebsitalkData";
+      await getPhoneNumbersCollection().insertOne(
+        createPhoneNumber({ client_id: clientId }),
+      );
+      await getScansCollection().insertOne(createScanFixture({ clientId }));
+
+      await deleteClientDataAcrossDomains(clientId, "admin@callai.co.il");
+
+      expect(
+        await getScansCollection().findOne({ clientId }),
+      ).not.toBeNull();
+    });
+
+    it("should not delete unrelated clients' data", async () => {
+      const targetClient = "deleteTarget";
+      const otherClient = "keepMeClient";
+      await getPhoneNumbersCollection().insertOne(
+        createPhoneNumber({ client_id: targetClient }),
+      );
+      await getPhoneNumbersCollection().insertOne(
+        createPhoneNumber({ client_id: otherClient }),
+      );
+
+      await deleteClientDataAcrossDomains(targetClient, "admin@callai.co.il");
+
+      expect(
+        await getPhoneNumbersCollection().findOne({ client_id: otherClient }),
+      ).not.toBeNull();
+    });
+  });
+});

package/src/talkpilot/clients/clients.constants.ts

@@ -0,0 +1,2 @@
+/** Collections to always skip during client deletion — e.g. auditLogs, which must survive it. */
+export const DELETE_DENYLIST: string[] = ["auditLogs"];

package/src/talkpilot/clients/clients.getters.ts

@@ -14,3 +14,10 @@ export const getClientByAPIKey = async (
   });
   return client ? client : null;
 };
+
+export const getAllClientCredits = async (): Promise<Map<string, number>> => {
+  const docs = await getClientsCollection()
+    .find({}, { projection: { clientId: 1, credits: 1 } })
+    .toArray();
+  return new Map(docs.map((c) => [c.clientId, c.credits ?? 0]));
+};

package/src/talkpilot/clients/clients.lifecycle.ts

@@ -0,0 +1,116 @@
+import { Db } from "mongodb";
+import { getDb, insertAuditLog } from "../index";
+import { getDb as getMunicipalDataDb } from "../../municipal";
+import { DELETE_DENYLIST } from "./clients.constants";
+import {
+  CollectionMatch,
+  CollectionDeleteResult,
+  DeleteClientDataSummary,
+} from "./clients.types";
+
+// Skips websitalk on purpose — see the warning on `deleteClientDataAcrossDomains` below.
+const findClientMatchInDb = async (
+  db: Db,
+  clientId: string,
+): Promise<CollectionMatch[]> => {
+  const collections = await db.listCollections().toArray();
+  return Promise.all(
+    collections
+      .filter((c) => !DELETE_DENYLIST.includes(c.name))
+      .map(async (c) => {
+        const doc = await db
+          .collection(c.name)
+          .findOne({ $or: [{ clientId }, { client_id: clientId }] });
+        return { collectionName: c.name, matched: !!doc };
+      }),
+  );
+};
+
+export const clientExistsInAnyCollection = async (
+  clientId: string,
+): Promise<boolean> => {
+  const [talkpilotResults, municipalResults] = await Promise.all([
+    findClientMatchInDb(getDb(), clientId),
+    findClientMatchInDb(getMunicipalDataDb(), clientId),
+  ]);
+  return [...talkpilotResults, ...municipalResults].some((r) => r.matched);
+};
+
+const deleteClientFromDb = async (
+  db: Db,
+  clientId: string,
+): Promise<CollectionDeleteResult[]> => {
+  const collections = await db.listCollections().toArray();
+  return Promise.all(
+    collections
+      .filter((c) => !DELETE_DENYLIST.includes(c.name))
+      .map(async (c) => {
+        try {
+          const result = await db
+            .collection(c.name)
+            .deleteMany({ $or: [{ clientId }, { client_id: clientId }] });
+          return {
+            collectionName: c.name,
+            deletedCount: result.deletedCount,
+            success: true,
+          };
+        } catch (error) {
+          return {
+            collectionName: c.name,
+            deletedCount: 0,
+            success: false,
+            error: error instanceof Error ? error.message : String(error),
+          };
+        }
+      }),
+  );
+};
+
+/**
+ * Deletes a client's data across talkpilot/municipal collections and logs the action.
+ *
+ * ⚠️ Does NOT delete website-talk data — `scans`/`websiteUrls` are the lookup key an
+ * external API needs to clean up that domain. Callers must invoke that API separately.
+ */
+export const deleteClientDataAcrossDomains = async (
+  clientId: string,
+  adminEmail: string,
+): Promise<DeleteClientDataSummary> => {
+  const exists = await clientExistsInAnyCollection(clientId);
+  if (!exists) {
+    return {
+      clientId,
+      deleted: false,
+      reason: "Client does not exist in any collection",
+    };
+  }
+
+  const [talkpilotResults, municipalResults] = await Promise.all([
+    deleteClientFromDb(getDb(), clientId),
+    deleteClientFromDb(getMunicipalDataDb(), clientId),
+  ]);
+  const results = [...talkpilotResults, ...municipalResults];
+  const allSucceeded = results.every((r) => r.success);
+
+  await insertAuditLog({
+    clientId,
+    adminEmail,
+    action: "delete",
+    timestamp: new Date(),
+    allSucceeded,
+    ...(allSucceeded
+      ? {}
+      : {
+          failedCollections: results
+            .filter((r) => !r.success)
+            .map((r) => r.collectionName),
+        }),
+  });
+
+  return {
+    clientId,
+    deleted: true,
+    results,
+    allSucceeded,
+  };
+};

package/src/talkpilot/clients/clients.types.ts

@@ -16,3 +16,22 @@ export type Client = {
 };
 
 export type ClientDoc = Client & { _id: ObjectId };
+
+export type CollectionMatch = { collectionName: string; matched: boolean };
+
+export type CollectionDeleteResult = {
+  collectionName: string;
+  deletedCount: number;
+  success: boolean;
+  error?: string;
+};
+
+/** Discriminated union: `deleted` tells you which shape you got without guessing. */
+export type DeleteClientDataSummary =
+  | { clientId: string; deleted: false; reason: string }
+  | {
+      clientId: string;
+      deleted: true;
+      results: CollectionDeleteResult[];
+      allSucceeded: boolean;
+    };

package/src/talkpilot/clients/index.ts

@@ -1,4 +1,6 @@
+export * from "./clients.constants";
 export * from "./clients.getters";
 export * from "./clients.types";
 export * from "./clients.quota.getters";
 export * from "./clients.quota.types";
+export * from "./clients.lifecycle";

package/src/talkpilot/flows/__tests__/flows.schema.spec.ts

@@ -96,29 +96,4 @@ describe("flowMongoSchema", () => {
       toolItem.properties.backgroundContinuationInstructions.bsonType,
     ).toBe("string");
   });
-
-  it("defines internalTools as an optional map of tool configs", () => {
-    const schema: any = flowMongoSchema;
-
-    expect(schema.properties).toHaveProperty("internalTools");
-    expect(schema.required).not.toContain("internalTools");
-    expect(schema.properties.internalTools.bsonType).toEqual([
-      "object",
-      "null",
-    ]);
-
-    const toolConfig = schema.properties.internalTools.additionalProperties;
-    expect(toolConfig.required).toEqual(["enabled"]);
-    expect(toolConfig.properties.enabled.bsonType).toBe("bool");
-    expect(toolConfig.properties.maxCalls.bsonType).toEqual(["number", "null"]);
-
-    const param = toolConfig.properties.params.additionalProperties;
-    expect(param.properties.value.bsonType).toEqual([
-      "string",
-      "number",
-      "bool",
-      "array",
-      "null",
-    ]);
-  });
 });

package/src/talkpilot/flows/flows.getter.ts

@@ -1,6 +1,6 @@
-import type { Collection, ObjectId, OptionalId, WithId } from "mongodb";
+import type { Collection, ObjectId, WithId } from "mongodb";
 import { getDb } from "../index";
-import type { Flow, FlowUpdateParams } from "./flows.types";
+import type { Flow } from "./flows.types";
 
 export const getFlowsCollection = (): Collection<Flow> => {
   return getDb().collection<Flow>("flows");
@@ -12,23 +12,3 @@ export const findFlowById = (
 ): Promise<WithId<Flow> | null> => {
   return getFlowsCollection().findOne({ _id: flowId, clientId });
 };
-
-export const createFlow = async (flow: Flow): Promise<WithId<Flow>> => {
-  const { insertedId } = await getFlowsCollection().insertOne(
-    flow as OptionalId<Flow>,
-  );
-  return { ...flow, _id: insertedId };
-};
-
-/** Returns false when no flow matches the id for this client. */
-export const updateFlowById = async (
-  flowId: ObjectId,
-  clientId: string,
-  update: FlowUpdateParams,
-): Promise<boolean> => {
-  const { matchedCount } = await getFlowsCollection().updateOne(
-    { _id: flowId, clientId },
-    { $set: { ...update, updatedAt: new Date() } },
-  );
-  return matchedCount > 0;
-};

package/src/talkpilot/flows/flows.schema.ts

@@ -150,31 +150,6 @@ export const flowMongoSchema = {
         additionalProperties: false,
       },
     },
-    internalTools: {
-      bsonType: ["object", "null"],
-      additionalProperties: {
-        bsonType: "object",
-        required: ["enabled"],
-        properties: {
-          enabled: { bsonType: "bool" },
-          maxCalls: { bsonType: ["number", "null"] },
-          params: {
-            bsonType: ["object", "null"],
-            additionalProperties: {
-              bsonType: "object",
-              properties: {
-                description: { bsonType: ["string", "null"] },
-                value: {
-                  bsonType: ["string", "number", "bool", "array", "null"],
-                },
-              },
-              additionalProperties: false,
-            },
-          },
-        },
-        additionalProperties: false,
-      },
-    },
   },
   additionalProperties: false,
 } as const;

package/src/talkpilot/flows/flows.types.ts

@@ -152,21 +152,6 @@ export type ConversationSettings = {
   liveTimeUpdates?: boolean;
 };
 
-/* ----------  Internal tools ---------- */
-
-export type InternalToolParamConfig = {
-  description?: string;
-  /** Pinned value. When set, the agent is not asked for this parameter. */
-  value?: string | number | boolean | string[];
-};
-
-export type InternalToolConfig = {
-  enabled: boolean;
-  /** Max invocations per call. Unlimited when omitted. */
-  maxCalls?: number;
-  params?: Record<string, InternalToolParamConfig>;
-};
-
 /* ----------  The Flow object ---------- */
 
 export type Flow = {
@@ -198,14 +183,7 @@ export type Flow = {
   // AI provider configuration
   aiProvider?: any;
 
-  /** Internal tool toggles and per-parameter overrides, keyed by tool name. */
-  internalTools?: Record<string, InternalToolConfig>;
-
   // Timestamps
   createdAt?: Date;
   updatedAt?: Date;
 };
-
-export type FlowUpdateParams = Partial<
-  Omit<Flow, "_id" | "clientId" | "createdAt">
->;

package/src/talkpilot/flows/index.ts

@@ -1,7 +1,2 @@
-export {
-  getFlowsCollection,
-  findFlowById,
-  createFlow,
-  updateFlowById,
-} from "./flows.getter";
+export { getFlowsCollection, findFlowById } from "./flows.getter";
 export type * from "./flows.types";

package/src/talkpilot/index.ts

@@ -2,6 +2,7 @@ import { Db, ObjectId as MongoObjectId } from "mongodb";
 
 export * from "./flows";
 export * from "./contextNotes";
+export * from "./auditLogs";
 export * from "./products";
 export * from "./results";
 export * from "./sessions";