@talkpilot/core-db 1.3.39 → 1.3.40

package/src/websitalk/scans/scans.getters.ts

@@ -1,160 +1,166 @@
-import { Collection, Filter, ObjectId } from "mongodb";
-import { getDb } from "../index";
-import {
-  applyQueryOptions,
-  QueryOptions,
-} from "../../talkpilot/utils/query.utils";
-import type { Scan, ScanDoc } from "./scans.types";
-import { FINISHED_SCAN_STATUSES, SCAN_STATUSES } from "./scans.constants";
-import { isInProgressScan } from "./scans.utils";
-
-export const getScansCollection = (): Collection<Scan> => {
-  return getDb().collection<Scan>("scans");
-};
-
-export const getScansByClientAndDateRange = (
-  clientId: string,
-  startDate: Date,
-  endDate: Date,
-): Promise<ScanDoc[]> => {
-  return getScansCollection()
-    .find({
-      clientId,
-      createdAt: {
-        $gte: startDate,
-        $lte: endDate,
-      },
-    })
-    .toArray();
-};
-
-export const getScanFieldsByClientAndDateRange = <K extends keyof Scan>(
-  clientId: string,
-  startDate: Date,
-  endDate: Date,
-  fields: K[],
-): Promise<Pick<ScanDoc, "_id" | K>[]> => {
-  const projection = fields.reduce<Record<string, 1>>((acc, field) => {
-    acc[field as string] = 1;
-    return acc;
-  }, {});
-
-  return getScansCollection()
-    .find(
-      {
-        clientId,
-        createdAt: {
-          $gte: startDate,
-          $lte: endDate,
-        },
-      },
-      { projection },
-    )
-    .toArray() as Promise<Pick<ScanDoc, "_id" | K>[]>;
-};
-
-export const createScanDoc = (
-  scan: Pick<Scan, "clientId" | "websiteUrlId"> &
-    Partial<Pick<Scan, "notifyPhoneNumber">>,
-) => {
-  return getScansCollection().insertOne({
-    ...scan,
-    status: SCAN_STATUSES.CHECKING,
-    activeTasksCount: 0,
-    pagesScanned: 0,
-    pagesSkipped: 0,
-    skipped: [],
-    siteChrome: {} as Scan["siteChrome"],
-    createdAt: new Date(),
-    updatedAt: new Date(),
-  });
-};
-
-export const getScanById = async (id: string): Promise<ScanDoc | null> => {
-  return getScansCollection().findOne({ _id: new ObjectId(id) });
-};
-
-export const getScansByWebsiteUrl = async (
-  websiteUrlId: string,
-): Promise<ScanDoc[]> => {
-  return getScansCollection()
-    .find({ websiteUrlId })
-    .sort({ createdAt: -1 })
-    .toArray();
-};
-
-export const getLatestScanByWebsiteUrl = async (
-  websiteUrlId: string,
-): Promise<ScanDoc | null> => {
-  return getScansCollection().findOne(
-    { websiteUrlId },
-    { sort: { createdAt: -1 } },
-  );
-};
-
-export const getInProgressScanByWebsiteUrl = async (
-  websiteUrlId: string,
-): Promise<ScanDoc | null> => {
-  const latestScan = await getLatestScanByWebsiteUrl(websiteUrlId);
-  if (!latestScan || !isInProgressScan(latestScan)) {
-    return null;
-  }
-  return latestScan;
-};
-
-export const findInProgressScansByClient = async (
-  clientId: string,
-): Promise<ScanDoc[]> => {
-  return getScansCollection()
-    .find({
-      clientId,
-      status: { $nin: FINISHED_SCAN_STATUSES },
-    })
-    .sort({ createdAt: -1 })
-    .toArray();
-};
-
-export const updateScanDoc = async (
-  scanId: ObjectId,
-  updates: Partial<Omit<Scan, "clientId" | "createdAt">>,
-): Promise<ScanDoc | null> => {
-  return await getScansCollection().findOneAndUpdate(
-    { _id: scanId },
-    { $set: { ...updates, updatedAt: new Date() } },
-    { returnDocument: "after" },
-  );
-};
-
-export const findScansByQuery = async (
-  query: Filter<Scan>,
-  options?: QueryOptions,
-): Promise<ScanDoc[]> => {
-  const cursor = getScansCollection().find(query);
-  return await applyQueryOptions(cursor, options).toArray();
-};
-
-export const countScans = async (query: Filter<Scan>): Promise<number> => {
-  return getScansCollection().countDocuments(query);
-};
-
-export const hasScansForWebsiteUrl = async (
-  websiteUrlId: string,
-): Promise<boolean> => {
-  return (await countScans({ websiteUrlId })) > 0;
-};
-
-export const getLatestCompletedScanByWebsiteUrl = async (
-  websiteUrlId: string,
-): Promise<ScanDoc | null> => {
-  return getScansCollection().findOne(
-    { websiteUrlId, status: SCAN_STATUSES.COMPLETED },
-    { sort: { createdAt: -1 } },
-  );
-};
-
-export const deleteScansByWebsiteUrl = async (
-  websiteUrlId: string,
-): Promise<number> => {
-  const result = await getScansCollection().deleteMany({ websiteUrlId });
-  return result.deletedCount;
-};
+import { Collection, Filter, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import {
+  applyQueryOptions,
+  QueryOptions,
+} from "../../talkpilot/utils/query.utils";
+import type { Scan, ScanDoc } from "./scans.types";
+import { FINISHED_SCAN_STATUSES, SCAN_STATUSES } from "./scans.constants";
+import { isInProgressScan } from "./scans.utils";
+
+export const getScansCollection = (): Collection<Scan> => {
+  return getDb().collection<Scan>("scans");
+};
+
+export const getScansByClientAndDateRange = (
+  clientId: string,
+  startDate: Date,
+  endDate: Date,
+): Promise<ScanDoc[]> => {
+  return getScansCollection()
+    .find({
+      clientId,
+      createdAt: {
+        $gte: startDate,
+        $lte: endDate,
+      },
+    })
+    .toArray();
+};
+
+export const getScanFieldsByClientAndDateRange = <K extends keyof Scan>(
+  clientId: string,
+  startDate: Date,
+  endDate: Date,
+  fields: K[],
+): Promise<Pick<ScanDoc, "_id" | K>[]> => {
+  const projection = fields.reduce<Record<string, 1>>((acc, field) => {
+    acc[field as string] = 1;
+    return acc;
+  }, {});
+
+  return getScansCollection()
+    .find(
+      {
+        clientId,
+        createdAt: {
+          $gte: startDate,
+          $lte: endDate,
+        },
+      },
+      { projection },
+    )
+    .toArray() as Promise<Pick<ScanDoc, "_id" | K>[]>;
+};
+
+export const createScanDoc = (
+  scan: Pick<Scan, "clientId" | "websiteUrlId"> &
+    Partial<Pick<Scan, "notifyPhoneNumber">>,
+) => {
+  const now = new Date();
+  return getScansCollection().insertOne({
+    ...scan,
+    status: SCAN_STATUSES.CHECKING,
+    activeTasksCount: 0,
+    pagesScanned: 0,
+    pagesSkipped: 0,
+    skipped: [],
+    siteChrome: {} as Scan["siteChrome"],
+    createdAt: now,
+    updatedAt: now,
+    startedAt: now,
+    completedAt: null,
+    pagesCompleted: 0,
+    pagesFailedPermanent: 0,
+    pagesDuplicate: 0,
+  });
+};
+
+export const getScanById = async (id: string): Promise<ScanDoc | null> => {
+  return getScansCollection().findOne({ _id: new ObjectId(id) });
+};
+
+export const getScansByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc[]> => {
+  return getScansCollection()
+    .find({ websiteUrlId })
+    .sort({ createdAt: -1 })
+    .toArray();
+};
+
+export const getLatestScanByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc | null> => {
+  return getScansCollection().findOne(
+    { websiteUrlId },
+    { sort: { createdAt: -1 } },
+  );
+};
+
+export const getInProgressScanByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc | null> => {
+  const latestScan = await getLatestScanByWebsiteUrl(websiteUrlId);
+  if (!latestScan || !isInProgressScan(latestScan)) {
+    return null;
+  }
+  return latestScan;
+};
+
+export const findInProgressScansByClient = async (
+  clientId: string,
+): Promise<ScanDoc[]> => {
+  return getScansCollection()
+    .find({
+      clientId,
+      status: { $nin: FINISHED_SCAN_STATUSES },
+    })
+    .sort({ createdAt: -1 })
+    .toArray();
+};
+
+export const updateScanDoc = async (
+  scanId: ObjectId,
+  updates: Partial<Omit<Scan, "clientId" | "createdAt">>,
+): Promise<ScanDoc | null> => {
+  return await getScansCollection().findOneAndUpdate(
+    { _id: scanId },
+    { $set: { ...updates, updatedAt: new Date() } },
+    { returnDocument: "after" },
+  );
+};
+
+export const findScansByQuery = async (
+  query: Filter<Scan>,
+  options?: QueryOptions,
+): Promise<ScanDoc[]> => {
+  const cursor = getScansCollection().find(query);
+  return await applyQueryOptions(cursor, options).toArray();
+};
+
+export const countScans = async (query: Filter<Scan>): Promise<number> => {
+  return getScansCollection().countDocuments(query);
+};
+
+export const hasScansForWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<boolean> => {
+  return (await countScans({ websiteUrlId })) > 0;
+};
+
+export const getLatestCompletedScanByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<ScanDoc | null> => {
+  return getScansCollection().findOne(
+    { websiteUrlId, status: SCAN_STATUSES.COMPLETED },
+    { sort: { createdAt: -1 } },
+  );
+};
+
+export const deleteScansByWebsiteUrl = async (
+  websiteUrlId: string,
+): Promise<number> => {
+  const result = await getScansCollection().deleteMany({ websiteUrlId });
+  return result.deletedCount;
+};

package/src/websitalk/scans/scans.types.ts

@@ -1,45 +1,54 @@
-import { WithId } from "mongodb";
-
-import type { SCAN_STATUSES } from "./scans.constants";
-
-export type ScanStatus = (typeof SCAN_STATUSES)[keyof typeof SCAN_STATUSES];
-
-export type SiteChromeEntry = {
-  url: string;
-  urlCanonical?: string | null;
-  title?: string | null;
-  h1?: string | null;
-  language?: string | null;
-  sourceType?: string | null;
-  status?: string | null;
-  context?: string | null;
-  contentHash?: string | null;
-  errorMessage?: string | null;
-  breadcrumbs?: unknown;
-  description?: string | null;
-  sourceUrl?: string | null;
-  fetchedAt?: Date | null;
-  createdAt?: Date;
-  updatedAt?: Date;
-};
-
-export type SkippedEntry = {
-  path: string;
-  reason: string;
-};
-
-export type Scan = {
-  clientId: string;
-  websiteUrlId: string;
-  notifyPhoneNumber?: string | null;
-  status: ScanStatus;
-  activeTasksCount: number;
-  pagesScanned: number;
-  pagesSkipped: number;
-  skipped: SkippedEntry[];
-  siteChrome: SiteChromeEntry;
-  createdAt: Date;
-  updatedAt: Date;
-};
-
-export type ScanDoc = WithId<Scan>;
+import { WithId } from "mongodb";
+
+import type { SCAN_STATUSES } from "./scans.constants";
+
+export type ScanStatus = (typeof SCAN_STATUSES)[keyof typeof SCAN_STATUSES];
+
+export type SiteChromeEntry = {
+  url: string;
+  urlCanonical?: string | null;
+  title?: string | null;
+  h1?: string | null;
+  language?: string | null;
+  sourceType?: string | null;
+  status?: string | null;
+  context?: string | null;
+  contentHash?: string | null;
+  errorMessage?: string | null;
+  breadcrumbs?: unknown;
+  description?: string | null;
+  sourceUrl?: string | null;
+  fetchedAt?: Date | null;
+  createdAt?: Date;
+  updatedAt?: Date;
+};
+
+export type SkippedEntry = {
+  path: string;
+  reason: string;
+};
+
+export type Scan = {
+  clientId: string;
+  websiteUrlId: string;
+  notifyPhoneNumber?: string | null;
+  status: ScanStatus;
+  activeTasksCount: number;
+  pagesScanned: number;
+  pagesSkipped: number;
+  skipped: SkippedEntry[];
+  siteChrome: SiteChromeEntry;
+  createdAt: Date;
+  updatedAt: Date;
+
+  // Terminal-only snapshot, written once at finalize (Brain, outside core-db).
+  // Never read for live progress — an active scan's KPIs must come from
+  // getScanKpiCounts, not these fields, which are not kept in sync mid-scan.
+  startedAt?: Date;
+  completedAt?: Date | null;
+  pagesCompleted?: number;
+  pagesFailedPermanent?: number;
+  pagesDuplicate?: number;
+};
+
+export type ScanDoc = WithId<Scan>;

package/src/websitalk/scans/scans.utils.ts

@@ -1,5 +1,5 @@
-import type { Scan, ScanStatus } from "./scans.types";
-import { FINISHED_SCAN_STATUSES } from "./scans.constants";
-
-export const isInProgressScan = (scan: Scan): boolean =>
-  !(FINISHED_SCAN_STATUSES as readonly ScanStatus[]).includes(scan.status);
+import type { Scan, ScanStatus } from "./scans.types";
+import { FINISHED_SCAN_STATUSES } from "./scans.constants";
+
+export const isInProgressScan = (scan: Scan): boolean =>
+  !(FINISHED_SCAN_STATUSES as readonly ScanStatus[]).includes(scan.status);

package/src/websitalk/websiteUrls/__tests__/websiteUrls.spec.ts

@@ -1,77 +1,77 @@
-import {
-  createWebsiteUrl,
-  getWebsiteUrlsByClient,
-  getWebsiteUrlById,
-  getWebsiteUrlsCollection,
-  updateWebsiteUrl,
-  deleteWebsiteUrl,
-} from "../websiteUrls.getters";
-import { createWebsiteUrlFixture } from "../../../test-utils/factories/websitalk/websiteUrls";
-
-describe("db.websitalk.websiteUrls", () => {
-  const clientId = "test-client";
-
-  describe("createWebsiteUrl", () => {
-    it("should create a website URL configuration", async () => {
-      const id = await createWebsiteUrl({
-        clientId,
-        name: "Primary",
-        baseUrl: "https://example.com",
-      });
-
-      const saved = await getWebsiteUrlById(id.toString());
-      expect(saved).toMatchObject({
-        clientId,
-        name: "Primary",
-        baseUrl: "https://example.com",
-      });
-      expect(saved?.createdAt).toBeInstanceOf(Date);
-      expect(saved?.updatedAt).toBeInstanceOf(Date);
-    });
-  });
-
-  describe("getWebsiteUrlsByClient", () => {
-    it("should return all configurations for a client", async () => {
-      await getWebsiteUrlsCollection().insertMany([
-        createWebsiteUrlFixture({ clientId, name: "A" }),
-        createWebsiteUrlFixture({ clientId, name: "B" }),
-        createWebsiteUrlFixture({ clientId: "other-client", name: "Other" }),
-      ]);
-
-      const results = await getWebsiteUrlsByClient(clientId);
-      expect(results).toHaveLength(2);
-      expect(results.map((doc) => doc.name)).toEqual(["A", "B"]);
-    });
-  });
-
-  describe("updateWebsiteUrl", () => {
-    it("should update name and baseUrl", async () => {
-      const id = await createWebsiteUrl({
-        clientId,
-        name: "Old",
-        baseUrl: "https://old.example",
-      });
-
-      const updated = await updateWebsiteUrl(id.toString(), {
-        name: "New",
-        baseUrl: "https://new.example",
-      });
-
-      expect(updated?.name).toBe("New");
-      expect(updated?.baseUrl).toBe("https://new.example");
-    });
-  });
-
-  describe("deleteWebsiteUrl", () => {
-    it("should delete the website and return true", async () => {
-      const id = await createWebsiteUrl({
-        clientId: "delete-client",
-        name: "Gone",
-        baseUrl: "https://gone.example",
-      });
-
-      expect(await deleteWebsiteUrl(id.toString())).toBe(true);
-      expect(await getWebsiteUrlById(id.toString())).toBeNull();
-    });
-  });
-});
+import {
+  createWebsiteUrl,
+  getWebsiteUrlsByClient,
+  getWebsiteUrlById,
+  getWebsiteUrlsCollection,
+  updateWebsiteUrl,
+  deleteWebsiteUrl,
+} from "../websiteUrls.getters";
+import { createWebsiteUrlFixture } from "../../../test-utils/factories/websitalk/websiteUrls";
+
+describe("db.websitalk.websiteUrls", () => {
+  const clientId = "test-client";
+
+  describe("createWebsiteUrl", () => {
+    it("should create a website URL configuration", async () => {
+      const id = await createWebsiteUrl({
+        clientId,
+        name: "Primary",
+        baseUrl: "https://example.com",
+      });
+
+      const saved = await getWebsiteUrlById(id.toString());
+      expect(saved).toMatchObject({
+        clientId,
+        name: "Primary",
+        baseUrl: "https://example.com",
+      });
+      expect(saved?.createdAt).toBeInstanceOf(Date);
+      expect(saved?.updatedAt).toBeInstanceOf(Date);
+    });
+  });
+
+  describe("getWebsiteUrlsByClient", () => {
+    it("should return all configurations for a client", async () => {
+      await getWebsiteUrlsCollection().insertMany([
+        createWebsiteUrlFixture({ clientId, name: "A" }),
+        createWebsiteUrlFixture({ clientId, name: "B" }),
+        createWebsiteUrlFixture({ clientId: "other-client", name: "Other" }),
+      ]);
+
+      const results = await getWebsiteUrlsByClient(clientId);
+      expect(results).toHaveLength(2);
+      expect(results.map((doc) => doc.name)).toEqual(["A", "B"]);
+    });
+  });
+
+  describe("updateWebsiteUrl", () => {
+    it("should update name and baseUrl", async () => {
+      const id = await createWebsiteUrl({
+        clientId,
+        name: "Old",
+        baseUrl: "https://old.example",
+      });
+
+      const updated = await updateWebsiteUrl(id.toString(), {
+        name: "New",
+        baseUrl: "https://new.example",
+      });
+
+      expect(updated?.name).toBe("New");
+      expect(updated?.baseUrl).toBe("https://new.example");
+    });
+  });
+
+  describe("deleteWebsiteUrl", () => {
+    it("should delete the website and return true", async () => {
+      const id = await createWebsiteUrl({
+        clientId: "delete-client",
+        name: "Gone",
+        baseUrl: "https://gone.example",
+      });
+
+      expect(await deleteWebsiteUrl(id.toString())).toBe(true);
+      expect(await getWebsiteUrlById(id.toString())).toBeNull();
+    });
+  });
+});

package/src/websitalk/websiteUrls/index.ts

@@ -1,2 +1,2 @@
-export * from "./websiteUrls.types";
-export * from "./websiteUrls.getters";
+export * from "./websiteUrls.types";
+export * from "./websiteUrls.getters";