@talkpilot/core-db 1.3.17 → 1.3.19

package/src/talkpilot/clientsConfig/__tests__/clientsConfig.getters.spec.ts

@@ -1,6 +1,9 @@
 import { describe, it, expect, beforeEach, afterEach } from "@jest/globals";
 import {
   getClientsConfigCollection,
+  getActiveScanId,
+  updateActiveScanId,
+  clearActiveScanId,
   updateProductConfig,
 } from "../clientsConfig.getters";
 
@@ -23,8 +26,6 @@ describe("updateProductConfig", () => {
         websiteTalk: {
           language: "English",
           companyName: "OldCorp",
-          defaultBaseUrl: "https://old-url.com",
-          logoUrl: "",
           timezone: "UTC",
         },
       },
@@ -38,9 +39,7 @@ describe("updateProductConfig", () => {
     expect(updatedDoc).toBeDefined();
     expect(updatedDoc?.products.websiteTalk?.language).toBe("Hebrew");
     expect(updatedDoc?.products.websiteTalk?.companyName).toBe("TestCorp");
-    expect(updatedDoc?.products.websiteTalk?.defaultBaseUrl).toBe(
-      "https://old-url.com",
-    );
+    expect(updatedDoc?.products.websiteTalk?.timezone).toBe("UTC");
   });
 
   it("should throw an error if no client config is found", async () => {
@@ -51,3 +50,56 @@ describe("updateProductConfig", () => {
     ).rejects.toThrow("No client config found for clientId: invalid-client");
   });
 });
+
+describe("activeScanId", () => {
+  beforeEach(async () => {
+    await getClientsConfigCollection().deleteMany({});
+  });
+
+  afterEach(async () => {
+    await getClientsConfigCollection().deleteMany({});
+  });
+
+  it("should return undefined when activeScanId is not set", async () => {
+    const clientId = "client-active";
+    await getClientsConfigCollection().insertOne({
+      clientId,
+      products: {
+        websiteTalk: { companyName: "X", timezone: "UTC", language: "he" },
+      },
+    } as any);
+
+    expect(await getActiveScanId(clientId)).toBeUndefined();
+  });
+
+  it("should persist activeScanId and return it from getActiveScanId", async () => {
+    const clientId = "client-active-2";
+    await getClientsConfigCollection().insertOne({
+      clientId,
+      products: {
+        websiteTalk: { companyName: "X", timezone: "UTC", language: "he" },
+      },
+    } as any);
+
+    await updateActiveScanId(clientId, "scan-abc");
+    expect(await getActiveScanId(clientId)).toBe("scan-abc");
+  });
+
+  it("should clear activeScanId and return undefined from getActiveScanId", async () => {
+    const clientId = "client-active-3";
+    await getClientsConfigCollection().insertOne({
+      clientId,
+      products: {
+        websiteTalk: {
+          companyName: "X",
+          timezone: "UTC",
+          language: "he",
+          activeScanId: "scan-old",
+        },
+      },
+    } as any);
+
+    await clearActiveScanId(clientId);
+    expect(await getActiveScanId(clientId)).toBeUndefined();
+  });
+});

package/src/talkpilot/clientsConfig/__tests__/clientsConfig.spec.ts

@@ -31,11 +31,9 @@ describe("db.clientsConfig", () => {
         clientId,
         products: {
           websiteTalk: {
-            logoUrl: "https://cdn.example.com/branding/acme/logo.png",
             companyName: "Acme Ltd",
             timezone: "Asia/Jerusalem",
             language: "he",
-            defaultBaseUrl: "https://acme.example",
           },
         },
       });
@@ -48,11 +46,9 @@ describe("db.clientsConfig", () => {
         clientId,
         products: {
           websiteTalk: {
-            logoUrl: "https://cdn.example.com/branding/acme/logo.png",
             companyName: "Acme Ltd",
             timezone: "Asia/Jerusalem",
             language: "he",
-            defaultBaseUrl: "https://acme.example",
           },
         },
       });

package/src/talkpilot/clientsConfig/clientsConfig.getters.ts

@@ -53,3 +53,27 @@ export async function updateProductConfig<K extends KnownProductKey>(
     throw new Error(`No client config found for clientId: ${clientId}`);
   }
 }
+
+export const getActiveScanId = async (
+  clientId: string,
+): Promise<string | undefined> => {
+  const cfg = await getClientConfig(clientId);
+  return cfg?.products?.websiteTalk?.activeScanId;
+};
+
+export const updateActiveScanId = async (
+  clientId: string,
+  scanId: string,
+): Promise<void> => {
+  await updateProductConfig(clientId, "websiteTalk", { activeScanId: scanId });
+};
+
+export const clearActiveScanId = async (clientId: string): Promise<void> => {
+  const result = await getClientsConfigCollection().updateOne(
+    { clientId },
+    { $unset: { "products.websiteTalk.activeScanId": "" } },
+  );
+  if (result.matchedCount === 0) {
+    throw new Error(`No client config found for clientId: ${clientId}`);
+  }
+};

package/src/talkpilot/clientsConfig/clientsConfig.types.ts

@@ -58,11 +58,11 @@ export type WebsiteTalkScheduler = {
 
 /** Product config for the websiteTalk */
 export type WebsiteTalkProduct = {
-  logoUrl: string;
   companyName: string;
   timezone: string;
   language: string;
-  defaultBaseUrl: string;
+  activeScanId?: string;
+  flowId?: string;
 };
 
 // ---------------------------------------------------------------------------

package/src/talkpilot/flows/__tests__/flows.schema.spec.ts

@@ -79,6 +79,9 @@ describe("flowMongoSchema", () => {
     expect(toolItem.properties).toHaveProperty("runInBackground");
     expect(toolItem.properties.runInBackground.bsonType).toBe("bool");
 
+    expect(toolItem.properties).toHaveProperty("useRedisTranscription");
+    expect(toolItem.properties.useRedisTranscription.bsonType).toBe("bool");
+
     expect(toolItem.properties).toHaveProperty("backgroundToolType");
     expect(toolItem.properties.backgroundToolType.bsonType).toBe("string");
     expect(toolItem.properties.backgroundToolType.enum).toEqual([

package/src/talkpilot/flows/flows.schema.ts

@@ -138,6 +138,7 @@ export const flowMongoSchema = {
           parameters: { bsonType: "object" },
           bodyFormat: { bsonType: "string", enum: ["json", "form-data"] },
           preventAudioDuringTool: { bsonType: "bool" },
+          useRedisTranscription: { bsonType: "bool" },
           sendUserAuthToken: { bsonType: "bool" },
           runInBackground: { bsonType: "bool" },
           backgroundToolType: {

package/src/talkpilot/flows/flows.types.ts

@@ -71,6 +71,8 @@ export type FlowTool = {
   parameters: Record<string, unknown>;
   bodyFormat?: "json" | "form-data";
   preventAudioDuringTool?: boolean;
+  /** When true, prefer cached transcription from Redis (default: false). */
+  useRedisTranscription?: boolean;
   sendUserAuthToken?: boolean;
   runInBackground?: boolean;
   backgroundToolType?: "backgroundToolAlways" | "backgroundToolOnce";

package/src/talkpilot/phone_numbers/phone_numbers.getter.ts

@@ -156,3 +156,16 @@ export const findClientByPhoneNumber = async (
   if (!client) throw new Error("Failed to get client");
   return client;
 };
+
+export const bindClientPhoneToFlow = async (
+  clientId: string,
+  flowId: string,
+): Promise<void> => {
+  const result = await getPhoneNumbersCollection().updateOne(
+    { client_id: clientId, is_primary: true },
+    { $set: { flow_id: new ObjectId(flowId), updatedAt: new Date() } },
+  );
+  if (result.matchedCount === 0) {
+    throw new Error(`No primary phone number found for client: ${clientId}`);
+  }
+};

package/src/test-utils/factories/index.ts

@@ -13,3 +13,4 @@ export * from "./municipal/muniIssues";
 export * from "./municipal/streets";
 export * from "./municipal/tickets";
 export * from "./websitalk/scans";
+export * from "./websitalk/websiteUrls";

package/src/test-utils/factories/websitalk/scans.ts

@@ -1,10 +1,11 @@
 import { Factory } from "fishery";
 import { faker } from "@faker-js/faker";
+import { ObjectId } from "mongodb";
 import type { Scan } from "../../../websitalk";
 
 export const scanFactory = Factory.define<Scan>(() => ({
   clientId: faker.string.uuid(),
-  baseUrl: faker.internet.url(),
+  websiteUrlId: new ObjectId().toString(),
   status: "CHECKING",
   activeTasksCount: 0,
   pagesScanned: 0,

package/src/test-utils/factories/websitalk/websiteUrls.ts

@@ -0,0 +1,17 @@
+import { Factory } from "fishery";
+import { faker } from "@faker-js/faker";
+import type { WebsiteUrl } from "../../../websitalk/websiteUrls";
+
+export const websiteUrlFactory = Factory.define<WebsiteUrl>(() => ({
+  clientId: faker.string.uuid(),
+  name: faker.company.name(),
+  baseUrl: faker.internet.url(),
+  createdAt: new Date(),
+  updatedAt: new Date(),
+}));
+
+export function createWebsiteUrlFixture(
+  overrides?: Partial<WebsiteUrl>,
+): WebsiteUrl {
+  return websiteUrlFactory.build(overrides);
+}

package/src/websitalk/index.ts

@@ -1,6 +1,7 @@
 import { Db, ObjectId as MongoObjectId } from "mongodb";
 
 export * from "./scans";
+export * from "./websiteUrls";
 export { websitalkMongodbClient } from "./mongodb-client";
 
 let db: Db;

package/src/websitalk/scans/__tests__/scans.spec.ts

@@ -6,7 +6,12 @@ import {
   countScans,
   createScanDoc,
   updateScanDoc,
-  getActiveScanStatus,
+  findInProgressScansByClient,
+  getScansByWebsiteUrl,
+  getLatestScanByWebsiteUrl,
+  getInProgressScanByWebsiteUrl,
+  getLatestCompletedScanByWebsiteUrl,
+  deleteScansByWebsiteUrl,
 } from "../scans.getters";
 import { createScanFixture } from "../../../test-utils/factories";
 import { ObjectId } from "mongodb";
@@ -37,7 +42,7 @@ describe("db.websitalk.scans", () => {
       expect(result.length).toBe(1);
       expect(result[0]).toMatchObject({
         clientId,
-        baseUrl: scanInside.baseUrl,
+        websiteUrlId: scanInside.websiteUrlId,
         status: "COMPLETED",
       });
       expect(result[0]).toHaveProperty("siteChrome");
@@ -83,12 +88,12 @@ describe("db.websitalk.scans", () => {
         clientId,
         new Date("2023-05-01"),
         new Date("2023-05-31"),
-        ["clientId", "baseUrl", "status"],
+        ["clientId", "websiteUrlId", "status"],
       );
 
       expect(result).toMatchObject({
         clientId,
-        baseUrl: scan.baseUrl,
+        websiteUrlId: scan.websiteUrlId,
         status: "COMPLETED",
       });
       expect(result).toHaveProperty("_id");
@@ -110,7 +115,7 @@ describe("db.websitalk.scans", () => {
       const count = await countScans({ clientId });
 
       expect(result.length).toBe(1);
-      expect(result[0].baseUrl).toBe(completed.baseUrl);
+      expect(result[0].websiteUrlId).toBe(completed.websiteUrlId);
       expect(count).toBe(2);
     });
   });
@@ -118,15 +123,15 @@ describe("db.websitalk.scans", () => {
   describe("createScanDoc()", () => {
     it("should insert a scan with default values for the rest of the fields", async () => {
       const clientId = "createClient";
-      const baseUrl = "https://example.com";
+      const websiteUrlId = new ObjectId().toString();
 
-      const { insertedId } = await createScanDoc({ clientId, baseUrl });
+      const { insertedId } = await createScanDoc({ clientId, websiteUrlId });
 
       const result = await getScansCollection().findOne({ _id: insertedId });
 
       expect(result).toMatchObject({
         clientId,
-        baseUrl,
+        websiteUrlId,
         status: "CHECKING",
         activeTasksCount: 0,
         pagesScanned: 0,
@@ -166,53 +171,143 @@ describe("db.websitalk.scans", () => {
     });
   });
 
-  describe("getActiveScanStatus()", () => {
-    it("should return isActive: true with the status when the latest scan is still in progress", async () => {
-      const clientId = "activeClient";
+  describe("in-progress scan helpers", () => {
+    const websiteUrlId = "website-123";
 
+    it("should return website scan history newest first", async () => {
       await getScansCollection().insertMany([
         createScanFixture({
-          clientId,
+          websiteUrlId,
           status: "COMPLETED",
           createdAt: new Date("2023-05-01"),
         }),
         createScanFixture({
-          clientId,
-          status: "SCANNING",
+          websiteUrlId,
+          status: "COMPLETED",
           createdAt: new Date("2023-05-15"),
         }),
       ]);
 
-      const result = await getActiveScanStatus(clientId);
+      const scans = await getScansByWebsiteUrl(websiteUrlId);
+      expect(scans).toHaveLength(2);
+      expect(scans[0].createdAt.getTime()).toBeGreaterThan(
+        scans[1].createdAt.getTime(),
+      );
+    });
+
+    it("should return the newest scan for a website", async () => {
+      await getScansCollection().insertMany([
+        createScanFixture({
+          websiteUrlId,
+          status: "COMPLETED",
+          createdAt: new Date("2023-05-01"),
+        }),
+        createScanFixture({
+          websiteUrlId,
+          status: "FAILED",
+          createdAt: new Date("2023-05-15"),
+        }),
+      ]);
 
-      expect(result).toEqual({ isActive: true, status: "SCANNING" });
+      const latest = await getLatestScanByWebsiteUrl(websiteUrlId);
+      expect(latest?.status).toBe("FAILED");
     });
 
-    it("should return isActive: false when the latest scan finished (COMPLETED / FAILED)", async () => {
-      const clientId = "finishedClient";
+    it("should return in-progress scan and return null when only completed exists", async () => {
+      await getScansCollection().insertMany([
+        createScanFixture({
+          websiteUrlId,
+          status: "COMPLETED",
+          createdAt: new Date("2023-05-01"),
+        }),
+        createScanFixture({
+          websiteUrlId,
+          status: "SCANNING",
+          createdAt: new Date("2023-05-15"),
+        }),
+      ]);
 
+      const inProgressScan = await getInProgressScanByWebsiteUrl(websiteUrlId);
+      expect(inProgressScan?.status).toBe("SCANNING");
+
+      await getScansCollection().deleteMany({ websiteUrlId });
+      await getScansCollection().insertOne(
+        createScanFixture({
+          websiteUrlId,
+          status: "COMPLETED",
+          createdAt: new Date("2023-05-20"),
+        }),
+      );
+
+      expect(await getInProgressScanByWebsiteUrl(websiteUrlId)).toBeNull();
+    });
+
+    it("should return all in-progress scans across websites for a client", async () => {
+      const clientId = "multiScanClient";
       await getScansCollection().insertMany([
         createScanFixture({
           clientId,
+          websiteUrlId: "website-a",
           status: "SCANNING",
-          createdAt: new Date("2023-05-01"),
         }),
         createScanFixture({
           clientId,
-          status: "FAILED",
-          createdAt: new Date("2023-05-15"),
+          websiteUrlId: "website-b",
+          status: "PROCESSING",
+        }),
+        createScanFixture({
+          clientId,
+          websiteUrlId: "website-c",
+          status: "COMPLETED",
         }),
       ]);
 
-      const result = await getActiveScanStatus(clientId);
+      const scans = await findInProgressScansByClient(clientId);
+      expect(scans).toHaveLength(2);
+      expect(scans.map((s) => s.websiteUrlId).sort()).toEqual([
+        "website-a",
+        "website-b",
+      ]);
+    });
+  });
 
-      expect(result).toEqual({ isActive: false });
+  describe("website delete helpers", () => {
+    it("should return the newest completed scan only", async () => {
+      const websiteUrlId = new ObjectId().toString();
+      await getScansCollection().insertMany([
+        createScanFixture({
+          websiteUrlId,
+          status: "FAILED",
+          createdAt: new Date("2023-05-25"),
+        }),
+        createScanFixture({
+          websiteUrlId,
+          status: "COMPLETED",
+          createdAt: new Date("2023-05-20"),
+        }),
+        createScanFixture({
+          websiteUrlId,
+          status: "COMPLETED",
+          createdAt: new Date("2023-05-22"),
+        }),
+      ]);
+
+      const latest = await getLatestCompletedScanByWebsiteUrl(websiteUrlId);
+      expect(latest?.status).toBe("COMPLETED");
+      expect(latest?.createdAt).toEqual(new Date("2023-05-22"));
     });
 
-    it("should return isActive: false when the client has no scans", async () => {
-      const result = await getActiveScanStatus("noScansClient");
+    it("should delete all scans for the website and return count", async () => {
+      const websiteUrlId = new ObjectId().toString();
+      await getScansCollection().insertMany([
+        createScanFixture({ websiteUrlId }),
+        createScanFixture({ websiteUrlId }),
+        createScanFixture({ websiteUrlId: new ObjectId().toString() }),
+      ]);
 
-      expect(result).toEqual({ isActive: false });
+      const deleted = await deleteScansByWebsiteUrl(websiteUrlId);
+      expect(deleted).toBe(2);
+      expect(await countScans({ websiteUrlId })).toBe(0);
     });
   });
 });

package/src/websitalk/scans/index.ts

@@ -1,2 +1,4 @@
+export * from "./scans.constants";
 export * from "./scans.getters";
 export * from "./scans.types";
+export * from "./scans.utils";

package/src/websitalk/scans/scans.constants.ts

@@ -0,0 +1,15 @@
+export const SCAN_STATUSES = {
+  CHECKING: "CHECKING",
+  PREPARING: "PREPARING",
+  SCANNING: "SCANNING",
+  PROCESSING: "PROCESSING",
+  COMPLETED: "COMPLETED",
+  FAILED: "FAILED",
+  OVERWRITTEN: "OVERWRITTEN",
+} as const;
+
+export const FINISHED_SCAN_STATUSES = [
+  SCAN_STATUSES.COMPLETED,
+  SCAN_STATUSES.FAILED,
+  SCAN_STATUSES.OVERWRITTEN,
+] as const;

package/src/websitalk/scans/scans.getters.ts

@@ -4,12 +4,9 @@ import {
   applyQueryOptions,
   QueryOptions,
 } from "../../talkpilot/utils/query.utils";
-import type {
-  ActiveScanStatus,
-  Scan,
-  ScanDoc,
-  ScanStatus,
-} from "./scans.types";
+import type { Scan, ScanDoc } from "./scans.types";
+import { FINISHED_SCAN_STATUSES, SCAN_STATUSES } from "./scans.constants";
+import { isInProgressScan } from "./scans.utils";
 
 export const getScansCollection = (): Collection<Scan> => {
   return getDb().collection<Scan>("scans");
@@ -57,11 +54,12 @@ export const getScanFieldsByClientAndDateRange = <K extends keyof Scan>(
 };
 
 export const createScanDoc = (
-  scan: Pick<Scan, "clientId" | "baseUrl" | "notifyPhoneNumber">,
+  scan: Pick<Scan, "clientId" | "websiteUrlId"> &
+    Partial<Pick<Scan, "notifyPhoneNumber">>,
 ) => {
   return getScansCollection().insertOne({
     ...scan,
-    status: "CHECKING",
+    status: SCAN_STATUSES.CHECKING,
     activeTasksCount: 0,
     pagesScanned: 0,
     pagesSkipped: 0,
@@ -72,21 +70,48 @@ export const createScanDoc = (
   });
 };
 
-const FINISHED_SCAN_STATUSES: ScanStatus[] = ["COMPLETED", "FAILED"];
+export const getScanById = async (id: string): Promise<ScanDoc | null> => {
+  return getScansCollection().findOne({ _id: new ObjectId(id) });
+};
 
-export const getActiveScanStatus = async (
-  clientId: string,
-): Promise<ActiveScanStatus> => {
-  const latestScan = await getScansCollection().findOne(
-    { clientId },
+export const getScansByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc[]> => {
+  return getScansCollection()
+    .find({ websiteUrlId })
+    .sort({ createdAt: -1 })
+    .toArray();
+};
+
+export const getLatestScanByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc | null> => {
+  return getScansCollection().findOne(
+    { websiteUrlId },
     { sort: { createdAt: -1 } },
   );
+};
 
-  if (!latestScan || FINISHED_SCAN_STATUSES.includes(latestScan.status)) {
-    return { isActive: false };
+export const getInProgressScanByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc | null> => {
+  const latestScan = await getLatestScanByWebsiteUrl(websiteUrlId);
+  if (!latestScan || !isInProgressScan(latestScan)) {
+    return null;
   }
+  return latestScan;
+};
 
-  return { isActive: true, status: latestScan.status };
+export const findInProgressScansByClient = async (
+  clientId: string,
+): Promise<ScanDoc[]> => {
+  return getScansCollection()
+    .find({
+      clientId,
+      status: { $nin: FINISHED_SCAN_STATUSES },
+    })
+    .sort({ createdAt: -1 })
+    .toArray();
 };
 
 export const updateScanDoc = async (
@@ -111,3 +136,25 @@ export const findScansByQuery = async (
 export const countScans = async (query: Filter<Scan>): Promise<number> => {
   return getScansCollection().countDocuments(query);
 };
+
+export const hasScansForWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<boolean> => {
+  return (await countScans({ websiteUrlId })) > 0;
+};
+
+export const getLatestCompletedScanByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc | null> => {
+  return getScansCollection().findOne(
+    { websiteUrlId, status: SCAN_STATUSES.COMPLETED },
+    { sort: { createdAt: -1 } },
+  );
+};
+
+export const deleteScansByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<number> => {
+  const result = await getScansCollection().deleteMany({ websiteUrlId });
+  return result.deletedCount;
+};

package/src/websitalk/scans/scans.types.ts

@@ -1,16 +1,8 @@
 import { WithId } from "mongodb";
 
-export type ScanStatus =
-  | "CHECKING"
-  | "PREPARING"
-  | "SCANNING"
-  | "PROCESSING"
-  | "COMPLETED"
-  | "FAILED";
+import type { SCAN_STATUSES } from "./scans.constants";
 
-export type ActiveScanStatus =
-  | { isActive: true; status: ScanStatus }
-  | { isActive: false };
+export type ScanStatus = (typeof SCAN_STATUSES)[keyof typeof SCAN_STATUSES];
 
 export type SiteChromeEntry = {
   url: string;
@@ -38,7 +30,7 @@ export type SkippedEntry = {
 
 export type Scan = {
   clientId: string;
-  baseUrl: string;
+  websiteUrlId: string;
   notifyPhoneNumber?: string | null;
   status: ScanStatus;
   activeTasksCount: number;

package/src/websitalk/scans/scans.utils.ts

@@ -0,0 +1,5 @@
+import type { Scan, ScanStatus } from "./scans.types";
+import { FINISHED_SCAN_STATUSES } from "./scans.constants";
+
+export const isInProgressScan = (scan: Scan): boolean =>
+  !(FINISHED_SCAN_STATUSES as readonly ScanStatus[]).includes(scan.status);