@talkpilot/core-db 1.3.44 → 1.3.45

package/src/talkpilot/calls/calls.types.ts

@@ -45,6 +45,7 @@ export type Call = {
   conferenceRole?: ConferenceRole;
   redirectedCall?: boolean;
   toolExecutions?: ToolExecution[];
+  transcriptionComparison?: Record<string, TranscriptionSegment[]>;
 };
 
 export type ToolExecution = {

package/src/talkpilot/clientsConfig/__tests__/clientsConfig.getters.spec.ts

@@ -5,7 +5,15 @@ import {
   updateActiveScanId,
   clearActiveScanId,
   updateProductConfig,
+  getActiveScanIds,
+  addActiveScanId,
+  removeActiveScanId,
+  getPrimaryScanId,
+  setPrimaryScanId,
+  clearPrimaryScanId,
 } from "../clientsConfig.getters";
+import { MAX_ACTIVE_SCANS } from "../clientsConfig.constants";
+import { createClientConfig } from "../../../test-utils/factories/talkpilot/clientsConfig";
 
 describe("updateProductConfig", () => {
   beforeEach(async () => {
@@ -103,3 +111,313 @@ describe("activeScanId", () => {
     expect(await getActiveScanId(clientId)).toBeUndefined();
   });
 });
+
+describe("activeScanIds (CP-1628)", () => {
+  beforeEach(async () => {
+    await getClientsConfigCollection().deleteMany({});
+  });
+
+  afterEach(async () => {
+    await getClientsConfigCollection().deleteMany({});
+  });
+
+  it("returns an empty list when nothing is set", async () => {
+    const clientId = "client-list-1";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: { companyName: "X", timezone: "UTC", language: "he" },
+        },
+      }),
+    );
+
+    expect(await getActiveScanIds(clientId)).toEqual([]);
+  });
+
+  it("falls back to wrapping the legacy activeScanId when activeScanIds is absent", async () => {
+    const clientId = "client-list-2";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanId: "scan-legacy",
+          },
+        },
+      }),
+    );
+
+    expect(await getActiveScanIds(clientId)).toEqual(["scan-legacy"]);
+  });
+
+  it("adds a scan id and returns it from getActiveScanIds", async () => {
+    const clientId = "client-list-3";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: { companyName: "X", timezone: "UTC", language: "he" },
+        },
+      }),
+    );
+
+    await addActiveScanId(clientId, "scan-a");
+    await addActiveScanId(clientId, "scan-b");
+
+    expect(await getActiveScanIds(clientId)).toEqual(["scan-a", "scan-b"]);
+  });
+
+  it("does not duplicate an already-present scan id, and does not count it against the cap", async () => {
+    const clientId = "client-list-4";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanIds: ["scan-a", "scan-b", "scan-c"],
+          },
+        },
+      }),
+    );
+
+    await expect(addActiveScanId(clientId, "scan-a")).resolves.toBeUndefined();
+    expect(await getActiveScanIds(clientId)).toEqual([
+      "scan-a",
+      "scan-b",
+      "scan-c",
+    ]);
+  });
+
+  it(`throws when adding a new scan id beyond MAX_ACTIVE_SCANS (${MAX_ACTIVE_SCANS})`, async () => {
+    const clientId = "client-list-5";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanIds: ["scan-a", "scan-b", "scan-c"],
+          },
+        },
+      }),
+    );
+
+    await expect(addActiveScanId(clientId, "scan-d")).rejects.toThrow(
+      `Client ${clientId} already has the maximum of ${MAX_ACTIVE_SCANS} active scans`,
+    );
+    expect(await getActiveScanIds(clientId)).toEqual([
+      "scan-a",
+      "scan-b",
+      "scan-c",
+    ]);
+  });
+
+  it("removes a scan id without affecting the others", async () => {
+    const clientId = "client-list-6";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanIds: ["scan-a", "scan-b"],
+          },
+        },
+      }),
+    );
+
+    await removeActiveScanId(clientId, "scan-a");
+    expect(await getActiveScanIds(clientId)).toEqual(["scan-b"]);
+  });
+
+  it("removing a scan id that isn't present is a harmless no-op", async () => {
+    const clientId = "client-list-7";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanIds: ["scan-a"],
+          },
+        },
+      }),
+    );
+
+    await expect(
+      removeActiveScanId(clientId, "scan-x"),
+    ).resolves.toBeUndefined();
+    expect(await getActiveScanIds(clientId)).toEqual(["scan-a"]);
+  });
+
+  it("throws when removing from a client that doesn't exist", async () => {
+    await expect(
+      removeActiveScanId("no-such-client", "scan-a"),
+    ).rejects.toThrow("No client config found for clientId: no-such-client");
+  });
+
+  it("migrates a legacy-only activeScanId into activeScanIds instead of dropping it when adding a new one", async () => {
+    const clientId = "client-list-8";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanId: "scan-legacy",
+          },
+        },
+      }),
+    );
+
+    await addActiveScanId(clientId, "scan-new");
+
+    expect(await getActiveScanIds(clientId)).toEqual(["scan-legacy", "scan-new"]);
+    const doc = await getClientsConfigCollection().findOne({ clientId });
+    expect(doc?.products?.websiteTalk?.activeScanId).toBeUndefined();
+  });
+
+  it("throws when adding to a client that doesn't exist", async () => {
+    await expect(
+      addActiveScanId("no-such-client", "scan-a"),
+    ).rejects.toThrow("No client config found for clientId: no-such-client");
+  });
+
+  it("unsets a legacy-only activeScanId when it is the one being removed", async () => {
+    const clientId = "client-list-9";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanId: "scan-legacy",
+          },
+        },
+      }),
+    );
+
+    await removeActiveScanId(clientId, "scan-legacy");
+
+    expect(await getActiveScanIds(clientId)).toEqual([]);
+    const doc = await getClientsConfigCollection().findOne({ clientId });
+    expect(doc?.products?.websiteTalk?.activeScanId).toBeUndefined();
+  });
+});
+
+describe("primaryScanId (CP-1628 follow-up)", () => {
+  beforeEach(async () => {
+    await getClientsConfigCollection().deleteMany({});
+  });
+
+  afterEach(async () => {
+    await getClientsConfigCollection().deleteMany({});
+  });
+
+  it("returns undefined when nothing is set", async () => {
+    const clientId = "client-primary-1";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: { companyName: "X", timezone: "UTC", language: "he" },
+        },
+      }),
+    );
+
+    expect(await getPrimaryScanId(clientId)).toBeUndefined();
+  });
+
+  it("sets and returns the primary scan id, independent of activeScanIds", async () => {
+    const clientId = "client-primary-2";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            activeScanIds: ["scan-a", "scan-b"],
+          },
+        },
+      }),
+    );
+
+    await setPrimaryScanId(clientId, "scan-c");
+
+    expect(await getPrimaryScanId(clientId)).toBe("scan-c");
+    expect(await getActiveScanIds(clientId)).toEqual(["scan-a", "scan-b"]);
+  });
+
+  it("replaces the previous primary scan id when set again", async () => {
+    const clientId = "client-primary-3";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            primaryScanId: "scan-a",
+          },
+        },
+      }),
+    );
+
+    await setPrimaryScanId(clientId, "scan-b");
+
+    expect(await getPrimaryScanId(clientId)).toBe("scan-b");
+  });
+
+  it("clears the primary scan id", async () => {
+    const clientId = "client-primary-4";
+    await getClientsConfigCollection().insertOne(
+      createClientConfig({
+        clientId,
+        products: {
+          websiteTalk: {
+            companyName: "X",
+            timezone: "UTC",
+            language: "he",
+            primaryScanId: "scan-a",
+          },
+        },
+      }),
+    );
+
+    await clearPrimaryScanId(clientId);
+
+    expect(await getPrimaryScanId(clientId)).toBeUndefined();
+  });
+
+  it("throws when setting the primary scan for a client that doesn't exist", async () => {
+    await expect(
+      setPrimaryScanId("no-such-client", "scan-a"),
+    ).rejects.toThrow("No client config found for clientId: no-such-client");
+  });
+
+  it("throws when clearing the primary scan for a client that doesn't exist", async () => {
+    await expect(
+      clearPrimaryScanId("no-such-client"),
+    ).rejects.toThrow("No client config found for clientId: no-such-client");
+  });
+});

package/src/talkpilot/clientsConfig/clientsConfig.constants.ts

@@ -0,0 +1,2 @@
+/** Maximum number of scans a client may have active at once. */
+export const MAX_ACTIVE_SCANS = 3;

package/src/talkpilot/clientsConfig/clientsConfig.getters.ts

@@ -6,8 +6,9 @@ import {
   KNOWN_PRODUCT_KEYS,
   KnownProductKey,
 } from "../index";
-import { Collection } from "mongodb";
+import { Collection, UpdateFilter } from "mongodb";
 import { resetClientQuotaSettings } from "../clients/clients.quota.getters";
+import { MAX_ACTIVE_SCANS } from "./clientsConfig.constants";
 
 export const getClientsConfigCollection = (): Collection<ClientConfigDoc> => {
   return getDb().collection<ClientConfigDoc>("clientsConfig");
@@ -17,7 +18,9 @@ export const getClientConfig = (clientId: string) => {
   return getClientsConfigCollection().findOne({ clientId });
 };
 
-export const getClientDisplayName = async (clientId: string): Promise<string> => {
+export const getClientDisplayName = async (
+  clientId: string,
+): Promise<string> => {
   const doc = await getClientsConfigCollection().findOne(
     { clientId },
     { projection: { "quota.clientDisplayName": 1, _id: 0 } },
@@ -71,6 +74,24 @@ export const getActiveScanId = async (
   return cfg?.products?.websiteTalk?.activeScanId;
 };
 
+/**
+ * Returns the client's active scan ids as a list.
+ * Falls back to wrapping the legacy singular `activeScanId` field when
+ * `activeScanIds` is absent — lets clients on the old field keep working
+ * without a data migration.
+ */
+export const getActiveScanIds = async (clientId: string): Promise<string[]> => {
+  const cfg = await getClientConfig(clientId);
+  const list = cfg?.products?.websiteTalk?.activeScanIds;
+  if (list && list.length > 0) return list;
+  const legacy = cfg?.products?.websiteTalk?.activeScanId;
+  return legacy ? [legacy] : [];
+};
+
+/**
+ * @deprecated Use `addActiveScanId` instead — supports multiple simultaneously
+ * active scans. This function overwrites the entire single-value field.
+ */
 export const updateActiveScanId = async (
   clientId: string,
   scanId: string,
@@ -78,6 +99,38 @@ export const updateActiveScanId = async (
   await updateProductConfig(clientId, "websiteTalk", { activeScanId: scanId });
 };
 
+/**
+ * Adds a scan id to the client's active list. Writes the full list back
+ * (rather than a bare array append) so a client still on the legacy
+ * singular field gets migrated instead of silently losing it.
+ */
+export const addActiveScanId = async (
+  clientId: string,
+  scanId: string,
+): Promise<void> => {
+  const current = await getActiveScanIds(clientId);
+  if (!current.includes(scanId) && current.length >= MAX_ACTIVE_SCANS) {
+    throw new Error(
+      `Client ${clientId} already has the maximum of ${MAX_ACTIVE_SCANS} active scans`,
+    );
+  }
+  const next = current.includes(scanId) ? current : [...current, scanId];
+  const result = await getClientsConfigCollection().updateOne(
+    { clientId },
+    {
+      $set: { "products.websiteTalk.activeScanIds": next },
+      $unset: { "products.websiteTalk.activeScanId": "" },
+    },
+  );
+  if (result.matchedCount === 0) {
+    throw new Error(`No client config found for clientId: ${clientId}`);
+  }
+};
+
+/**
+ * @deprecated Use `removeActiveScanId` instead — removes a single scan from
+ * the active list without clearing the others.
+ */
 export const clearActiveScanId = async (clientId: string): Promise<void> => {
   const result = await getClientsConfigCollection().updateOne(
     { clientId },
@@ -88,6 +141,67 @@ export const clearActiveScanId = async (clientId: string): Promise<void> => {
   }
 };
 
+/**
+ * Removes a single scan id from the active list. Also clears the legacy
+ * singular field when it matches, so a not-yet-migrated client's removal
+ * actually takes effect instead of silently no-op'ing.
+ */
+export const removeActiveScanId = async (
+  clientId: string,
+  scanId: string,
+): Promise<void> => {
+  const cfg = await getClientConfig(clientId);
+  if (!cfg) {
+    throw new Error(`No client config found for clientId: ${clientId}`);
+  }
+  const updateOps: UpdateFilter<ClientConfigDoc> = {
+    $pull: { "products.websiteTalk.activeScanIds": scanId },
+  };
+  if (cfg.products?.websiteTalk?.activeScanId === scanId) {
+    updateOps.$unset = { "products.websiteTalk.activeScanId": "" };
+  }
+  await getClientsConfigCollection().updateOne({ clientId }, updateOps);
+};
+
+/** Returns the scan whose content seeds the phone call's opening description, if one is set. */
+export const getPrimaryScanId = async (
+  clientId: string,
+): Promise<string | undefined> => {
+  const cfg = await getClientConfig(clientId);
+  return cfg?.products?.websiteTalk?.primaryScanId;
+};
+
+/**
+ * Sets the primary scan, replacing whatever was set before — exactly one
+ * primary scan at a time. Throws if the client doesn't exist.
+ */
+export const setPrimaryScanId = async (
+  clientId: string,
+  scanId: string,
+): Promise<void> => {
+  const result = await getClientsConfigCollection().updateOne(
+    { clientId },
+    { $set: { "products.websiteTalk.primaryScanId": scanId } },
+  );
+  if (result.matchedCount === 0) {
+    throw new Error(`No client config found for clientId: ${clientId}`);
+  }
+};
+
+/**
+ * Clears the primary scan. Callers that need the opening description fall
+ * back to the first active scan. Throws if the client doesn't exist.
+ */
+export const clearPrimaryScanId = async (clientId: string): Promise<void> => {
+  const result = await getClientsConfigCollection().updateOne(
+    { clientId },
+    { $unset: { "products.websiteTalk.primaryScanId": "" } },
+  );
+  if (result.matchedCount === 0) {
+    throw new Error(`No client config found for clientId: ${clientId}`);
+  }
+};
+
 export async function listClientIdsWithConfiguredQuota(): Promise<string[]> {
   const docs = await getClientsConfigCollection()
     .find({ "quota.totalQuota": { $exists: true, $ne: null } })

package/src/talkpilot/clientsConfig/clientsConfig.types.ts

@@ -64,6 +64,10 @@ export type WebsiteTalkProduct = {
   timezone: string;
   language: string;
   activeScanId?: string;
+  /** Up to MAX_ACTIVE_SCANS scan ids simultaneously serving RAG for phone calls. */
+  activeScanIds?: string[];
+  /** The single scan whose content seeds the phone call's opening description. */
+  primaryScanId?: string;
 };
 
 // ---------------------------------------------------------------------------

package/src/talkpilot/clientsConfig/index.ts

@@ -1,2 +1,3 @@
 export * from "./clientsConfig.types";
 export * from "./clientsConfig.getters";
+export * from "./clientsConfig.constants";

package/src/test-utils/factories/index.ts

@@ -13,5 +13,6 @@ export * from "./municipal/muniIssues";
 export * from "./municipal/streets";
 export * from "./municipal/tickets";
 export * from "./websitalk/scans";
+export * from "./websitalk/scanResources";
 export * from "./websitalk/websiteUrls";
 export * from "./websitalk/knowledgeDocuments";

package/src/test-utils/factories/websitalk/scanResources.ts

@@ -0,0 +1,26 @@
+import { Factory } from "fishery";
+import { faker } from "@faker-js/faker";
+import { ObjectId } from "mongodb";
+import {
+  URL_TASK_STATUSES,
+  type ScanResource,
+} from "../../../websitalk/scanResources";
+
+export const scanResourceFactory = Factory.define<ScanResource>(() => ({
+  urlHashId: faker.string.uuid(),
+  clientId: faker.string.uuid(),
+  scanId: new ObjectId().toString(),
+  websiteUrlId: new ObjectId().toString(),
+  url: faker.internet.url(),
+  status: URL_TASK_STATUSES.URL_PENDING,
+  depth: 0,
+  retryCount: 0,
+  createdAt: new Date(),
+  updatedAt: new Date(),
+}));
+
+export function createScanResourceFixture(
+  overrides?: Partial<ScanResource>,
+): ScanResource {
+  return scanResourceFactory.build(overrides);
+}

package/src/test-utils/factories/websitalk/scans.ts

@@ -17,6 +17,11 @@ export const scanFactory = Factory.define<Scan>(() => ({
   },
   createdAt: new Date(),
   updatedAt: new Date(),
+  startedAt: new Date(),
+  completedAt: null,
+  pagesCompleted: 0,
+  pagesFailedPermanent: 0,
+  pagesDuplicate: 0,
 }));
 
 export function createScanFixture(overrides?: Partial<Scan>): Scan {

package/src/websitalk/index.ts

@@ -1,6 +1,7 @@
 import { Db, ObjectId as MongoObjectId } from "mongodb";
 
 export * from "./scans";
+export * from "./scanResources";
 export * from "./websiteUrls";
 export * from "./knowledgeDocuments";
 export { websitalkMongodbClient } from "./mongodb-client";

package/src/websitalk/knowledgeDocuments/__tests__/knowledgeDocuments.spec.ts

@@ -40,21 +40,6 @@ describe("db.websitalk.knowledgeDocuments", () => {
       expect(saved?.createdAt).toBeInstanceOf(Date);
       expect(saved?.updatedAt).toBeInstanceOf(Date);
     });
-
-    it("should create a knowledge document with a preset _id", async () => {
-      const presetId = new (await import("mongodb")).ObjectId();
-      const id = await createKnowledgeDocument({
-        _id: presetId,
-        clientId,
-        filename: "preset.pdf",
-        contentType: KNOWLEDGE_CONTENT_TYPES.PDF,
-        fileType: KNOWLEDGE_FILE_TYPES.PDF,
-        storagePath: `websitalk/knowledge_files/test/${presetId.toString()}/original.pdf`,
-        byteSize: 512,
-      });
-
-      expect(id.toString()).toBe(presetId.toString());
-    });
   });
 
   describe("getKnowledgeDocumentById", () => {
@@ -160,15 +145,11 @@ describe("db.websitalk.knowledgeDocuments", () => {
         status: KNOWLEDGE_DOCUMENT_STATUSES.READY,
         progressPercent: 100,
         pageCount: 12,
-        byteSize: 2048,
-        storagePath: "websitalk/knowledge_files/test/doc-updated.pdf",
       });
 
       expect(updated?.status).toBe(KNOWLEDGE_DOCUMENT_STATUSES.READY);
       expect(updated?.progressPercent).toBe(100);
       expect(updated?.pageCount).toBe(12);
-      expect(updated?.byteSize).toBe(2048);
-      expect(updated?.storagePath).toBe("websitalk/knowledge_files/test/doc-updated.pdf");
       expect(updated?.updatedAt.getTime()).toBeGreaterThanOrEqual(
         before!.updatedAt.getTime(),
       );

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.types.ts

@@ -1,4 +1,4 @@
-import { ObjectId as MongoObjectId } from "mongodb";
+import { WithId, ObjectId as MongoObjectId } from "mongodb";
 
 import type {
   KNOWLEDGE_CONTENT_TYPES,
@@ -34,7 +34,7 @@ export type KnowledgeDocument = {
   updatedAt: Date;
 };
 
-export type KnowledgeDocumentDoc = KnowledgeDocument & { _id: MongoObjectId };
+export type KnowledgeDocumentDoc = WithId<KnowledgeDocument>;
 
 export type CreateKnowledgeDocumentInput = Omit<
   KnowledgeDocument,