@talkpilot/core-db 1.1.15 → 1.1.16

package/src/test-utils/factories/municipal/muniIssues.ts

@@ -0,0 +1,32 @@
+import { Factory } from "fishery";
+import { faker } from "@faker-js/faker";
+import type { CreateMuniIssueInput } from "../../../municipal";
+
+/**
+ * Test factory for `createMuniIssue` input.
+ * `jira.key` uses `sequence` so each build gets a unique key (the real key
+ * comes from Jira in production).
+ */
+export const muniIssueInputFactory = Factory.define<CreateMuniIssueInput>(
+  ({ sequence }) => ({
+    callSid: `CA${faker.string.alphanumeric(10)}`,
+    clientId: faker.string.uuid(),
+    cityName: faker.location.city(),
+    issueContent: {
+      issuerName: faker.person.fullName(),
+      issuerRole: faker.person.jobTitle(),
+      issueDescription: faker.lorem.sentence(),
+      desiredBehavior: faker.lorem.sentence(),
+    },
+    jira: {
+      key: `CP-${sequence}`,
+      syncStatus: "created",
+    },
+  }),
+);
+
+export function createMuniIssueInput(
+  overrides?: Partial<CreateMuniIssueInput>,
+): CreateMuniIssueInput {
+  return muniIssueInputFactory.build(overrides);
+}

package/src/test-utils/factories/talkpilot/contextNotes.ts

@@ -0,0 +1,40 @@
+import { Factory } from "fishery";
+import { ObjectId } from "mongodb";
+import { faker } from "@faker-js/faker";
+import type { ContextNote, ContextNoteEntry } from "../../../talkpilot";
+
+export const contextNoteEntryFactory = Factory.define<ContextNoteEntry>(() => ({
+  _id: new ObjectId(),
+  title: faker.lorem.words(3),
+  content: faker.lorem.sentences(2),
+  activeFrom: new Date(),
+  expiresAt: null,
+  createdAt: new Date(),
+  updatedAt: new Date(),
+}));
+
+export const contextNoteFactory = Factory.define<
+  ContextNote & { _id: ObjectId }
+>(() => ({
+  _id: new ObjectId(),
+  clientId: faker.string.uuid(),
+  product: "Municipal",
+  notes: [],
+}));
+
+export function createContextNoteEntry(
+  overrides?: Partial<ContextNoteEntry>,
+): ContextNoteEntry {
+  return {
+    ...contextNoteEntryFactory.build(),
+    ...overrides,
+  } as ContextNoteEntry;
+}
+
+export function createContextNote(
+  overrides?: Partial<ContextNote>,
+): ContextNote & { _id: ObjectId } {
+  return { ...contextNoteFactory.build(), ...overrides } as ContextNote & {
+    _id: ObjectId;
+  };
+}

package/src/test-utils/factories/websitalk/scans.ts

@@ -0,0 +1,23 @@
+import { Factory } from "fishery";
+import { faker } from "@faker-js/faker";
+import type { Scan } from "../../../websitalk";
+
+export const scanFactory = Factory.define<Scan>(() => ({
+  clientId: faker.string.uuid(),
+  baseUrl: faker.internet.url(),
+  status: "CHECKING",
+  activeTasksCount: 0,
+  pagesScanned: 0,
+  pagesSkipped: 0,
+  skipped: [],
+  siteChrome: {
+    url: faker.internet.url(),
+    sourceType: "site_chrome",
+  },
+  createdAt: new Date(),
+  updatedAt: new Date(),
+}));
+
+export function createScanFixture(overrides?: Partial<Scan>): Scan {
+  return scanFactory.build(overrides);
+}

package/src/utils/date.utils.ts

@@ -0,0 +1,126 @@
+/**
+ * Shared timezone-aware date helpers for statistics aggregations.
+ */
+
+/** Milliseconds in a single day. */
+export const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** Maps a short weekday name (as produced by Intl) to its index (Sun = 0). */
+export const WEEKDAY: Record<string, number> = {
+  Sun: 0,
+  Mon: 1,
+  Tue: 2,
+  Wed: 3,
+  Thu: 4,
+  Fri: 5,
+  Sat: 6,
+};
+
+/** Parses a `YYYY-MM-DD` string into `[year, month, day]` numbers. */
+export const parseYmd = (dateStr: string): [number, number, number] => {
+  const [y, m, d] = dateStr.split("-").map(Number);
+  return [y, m, d];
+};
+
+/** Builds a zero-padded `YYYY-MM-DD` string from numeric parts. */
+export const ymdToStr = (y: number, m: number, d: number): string =>
+  `${y}-${String(m).padStart(2, "0")}-${String(d).padStart(2, "0")}`;
+
+/** Parses a `YYYY-MM-DD` string as midnight UTC. */
+export const toUtcDate = (dateStr: string): Date =>
+  new Date(`${dateStr}T00:00:00.000Z`);
+
+/** Formats a date as a `YYYY-MM-DD` string in UTC. */
+export const formatDate = (date: Date): string =>
+  date.toISOString().slice(0, 10);
+
+/** Adds `days` to a `YYYY-MM-DD` string, returning the resulting `YYYY-MM-DD`. */
+export const addDaysToDateStr = (dateStr: string, days: number): string =>
+  formatDate(new Date(toUtcDate(dateStr).getTime() + days * DAY_MS));
+
+/** Next civil calendar day (proleptic Gregorian). Avoids DST 24h steps that can stall iteration. */
+export const incrementYmd = (
+  y: number,
+  m: number,
+  d: number,
+): [number, number, number] => {
+  const next = new Date(Date.UTC(y, m - 1, d + 1));
+  return [next.getUTCFullYear(), next.getUTCMonth() + 1, next.getUTCDate()];
+};
+
+/**
+ * Formats a date as the calendar day in the given timezone.
+ * `en-CA` yields ISO-8601 `YYYY-MM-DD`, which is lexicographically sortable
+ * (string compare == chronological compare) and matches MongoDB's `%Y-%m-%d`.
+ */
+export const formatYmdInTz = (date: Date, timezone: string): string =>
+  new Intl.DateTimeFormat("en-CA", { timeZone: timezone }).format(date);
+
+/**
+ * Returns the calendar day in the given timezone as numeric parts.
+ * Reads parts by `type` (not string order), so `en-US` is fine here.
+ */
+export const getYmdInTz = (date: Date, timezone: string) => {
+  const parts = Object.fromEntries(
+    new Intl.DateTimeFormat("en-US", {
+      timeZone: timezone,
+      year: "numeric",
+      month: "numeric",
+      day: "numeric",
+    })
+      .formatToParts(date)
+      .map((p) => [p.type, Number(p.value)]),
+  );
+  return { y: parts.year, m: parts.month, d: parts.day };
+};
+
+/** Weekday index (Sun = 0) of a date in the given timezone. */
+export const getWeekdayInTz = (date: Date, timezone: string): number =>
+  WEEKDAY[
+    new Intl.DateTimeFormat("en-US", {
+      timeZone: timezone,
+      weekday: "short",
+    }).format(date)
+  ] ?? 0;
+
+/**
+ * Returns the given `YYYY-MM-DD` day at noon UTC — a stable anchor inside the
+ * calendar day, far from midnight so timezone/DST shifts can't cross a date boundary.
+ */
+export const dateAtNoonUtc = (dateStr: string): Date => {
+  const [y, m, d] = parseYmd(dateStr);
+  return new Date(Date.UTC(y, m - 1, d, 12, 0, 0));
+};
+
+/**
+ * UTC instant at the start of the given `YYYY-MM-DD` calendar day in `timezone`.
+ * Binary-searches the boundary so it stays correct across DST transitions.
+ */
+export const startOfCalendarDayInTz = (
+  dateStr: string,
+  timezone: string,
+): Date => {
+  const anchor = dateAtNoonUtc(dateStr).getTime();
+  let low = anchor - 2 * DAY_MS;
+  let high = anchor + DAY_MS;
+
+  while (high - low > 1 /* ms */) {
+    const mid = Math.floor((low + high) / 2);
+    if (formatYmdInTz(new Date(mid), timezone) >= dateStr) high = mid;
+    else low = mid;
+  }
+
+  return new Date(high);
+};
+
+/** UTC instant at the last millisecond of the given calendar day in `timezone`. */
+export const endOfCalendarDayInTz = (
+  dateStr: string,
+  timezone: string,
+): Date => {
+  const [y, m, d] = parseYmd(dateStr);
+  const nextDay = new Date(Date.UTC(y, m - 1, d + 1))
+    .toISOString()
+    .slice(0, 10);
+  return new Date(startOfCalendarDayInTz(nextDay, timezone).getTime() - 1);
+};

package/src/utils/shared.types.ts

@@ -0,0 +1,4 @@
+export interface CallLengthThresholds {
+  shortThreshold: number;
+  mediumThreshold: number;
+}

package/src/websitalk/index.ts

@@ -0,0 +1,15 @@
+import { Db, ObjectId as MongoObjectId } from "mongodb";
+
+export * from "./scans";
+export { websitalkMongodbClient } from "./mongodb-client";
+
+let db: Db;
+export const setDb = (d: Db) => {
+  db = d;
+};
+
+export const getDb = (): Db => {
+  if (!db) throw new Error("Website Talk DB not initialised");
+  return db;
+};
+export const ObjectId = MongoObjectId;

package/src/websitalk/mongodb-client.ts

@@ -0,0 +1,61 @@
+import { MongoClient, Db } from "mongodb";
+import { setDb } from "./index";
+import { validateConfig, validateMongoUri } from "../utils/validation";
+
+class WebsitalkMongoDBClient {
+  private client: MongoClient | null = null;
+  private db: Db | null = null;
+  private readonly defaultDbName = "website-talk";
+
+  async connect(uri?: string, dbName?: string): Promise<void> {
+    if (this.client) {
+      return;
+    }
+
+    const mongodbUri = uri || process.env.MONGO_URI || process.env.MONGODB_URI;
+    validateConfig("MONGO_URI", mongodbUri);
+    validateMongoUri(mongodbUri!);
+
+    try {
+      this.client = new MongoClient(mongodbUri!);
+      await this.client.connect();
+      const targetDbName =
+        dbName || process.env.WEBSITALK_DB_NAME || this.defaultDbName;
+      this.db = this.client.db(targetDbName);
+      setDb(this.db);
+      console.info(`[core-db] Website Talk MongoDB connected: ${targetDbName}`);
+    } catch (error) {
+      console.error("[core-db] Website Talk connection failed", error);
+      throw error;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    if (this.client) {
+      try {
+        await this.client.close();
+        this.client = null;
+        this.db = null;
+        console.info("Website Talk MongoDB disconnected successfully");
+      } catch (error) {
+        console.error("[core-db] Website Talk disconnection failed", error);
+        throw error;
+      }
+    }
+  }
+
+  getDb(): Db {
+    if (!this.db) {
+      throw new Error(
+        "Website Talk database not initialized. Call connect() first.",
+      );
+    }
+    return this.db;
+  }
+
+  isConnected(): boolean {
+    return this.client !== null && this.client !== undefined;
+  }
+}
+
+export const websitalkMongodbClient = new WebsitalkMongoDBClient();

package/src/websitalk/scans/__tests__/scans.spec.ts

@@ -0,0 +1,218 @@
+import {
+  getScansCollection,
+  getScansByClientAndDateRange,
+  getScanFieldsByClientAndDateRange,
+  findScansByQuery,
+  countScans,
+  createScanDoc,
+  updateScanDoc,
+  getActiveScanStatus,
+} from "../scans.getters";
+import { createScanFixture } from "../../../test-utils/factories";
+import { ObjectId } from "mongodb";
+
+describe("db.websitalk.scans", () => {
+  describe("getScansByClientAndDateRange()", () => {
+    it("should return full scan documents within the date range", async () => {
+      const clientId = "dateRangeClient";
+      const startDate = new Date("2023-05-01");
+      const endDate = new Date("2023-05-31");
+
+      const scanInside = createScanFixture({ clientId, status: "COMPLETED" });
+      const scanBefore = createScanFixture({ clientId });
+      const scanAfter = createScanFixture({ clientId });
+
+      await getScansCollection().insertMany([
+        { ...scanInside, createdAt: new Date("2023-05-15") },
+        { ...scanBefore, createdAt: new Date("2023-04-30") },
+        { ...scanAfter, createdAt: new Date("2023-06-01") },
+      ]);
+
+      const result = await getScansByClientAndDateRange(
+        clientId,
+        startDate,
+        endDate,
+      );
+
+      expect(result.length).toBe(1);
+      expect(result[0]).toMatchObject({
+        clientId,
+        baseUrl: scanInside.baseUrl,
+        status: "COMPLETED",
+      });
+      expect(result[0]).toHaveProperty("siteChrome");
+      expect(result[0]).toHaveProperty("skipped");
+    });
+
+    it("should respect the date range boundaries (inclusive)", async () => {
+      const clientId = "boundaryClient";
+      const onStart = createScanFixture({ clientId });
+      const onEnd = createScanFixture({ clientId });
+
+      await getScansCollection().insertMany([
+        { ...onStart, createdAt: new Date("2023-05-01") },
+        { ...onEnd, createdAt: new Date("2023-05-31") },
+      ]);
+
+      const result = await getScansByClientAndDateRange(
+        clientId,
+        new Date("2023-05-01"),
+        new Date("2023-05-31"),
+      );
+
+      expect(result.length).toBe(2);
+    });
+  });
+
+  describe("getScanFieldsByClientAndDateRange()", () => {
+    it("should return only the requested fields (plus _id)", async () => {
+      const clientId = "projectionClient";
+      const scan = createScanFixture({
+        clientId,
+        status: "COMPLETED",
+        siteChrome: { url: "https://example.com", sourceType: "site_chrome" },
+        skipped: [{ path: "/broken", reason: "HTTP 500" }],
+      });
+
+      await getScansCollection().insertOne({
+        ...scan,
+        createdAt: new Date("2023-05-15"),
+      });
+
+      const [result] = await getScanFieldsByClientAndDateRange(
+        clientId,
+        new Date("2023-05-01"),
+        new Date("2023-05-31"),
+        ["clientId", "baseUrl", "status"],
+      );
+
+      expect(result).toMatchObject({
+        clientId,
+        baseUrl: scan.baseUrl,
+        status: "COMPLETED",
+      });
+      expect(result).toHaveProperty("_id");
+      expect(result).not.toHaveProperty("siteChrome");
+      expect(result).not.toHaveProperty("skipped");
+      expect(result).not.toHaveProperty("createdAt");
+    });
+  });
+
+  describe("findScansByQuery() / countScans()", () => {
+    it("should find scans matching a raw filter and count them", async () => {
+      const clientId = "queryClient";
+      const completed = createScanFixture({ clientId, status: "COMPLETED" });
+      const scanning = createScanFixture({ clientId, status: "SCANNING" });
+
+      await getScansCollection().insertMany([completed, scanning]);
+
+      const result = await findScansByQuery({ clientId, status: "COMPLETED" });
+      const count = await countScans({ clientId });
+
+      expect(result.length).toBe(1);
+      expect(result[0].baseUrl).toBe(completed.baseUrl);
+      expect(count).toBe(2);
+    });
+  });
+
+  describe("createScanDoc()", () => {
+    it("should insert a scan with default values for the rest of the fields", async () => {
+      const clientId = "createClient";
+      const baseUrl = "https://example.com";
+
+      const { insertedId } = await createScanDoc({ clientId, baseUrl });
+
+      const result = await getScansCollection().findOne({ _id: insertedId });
+
+      expect(result).toMatchObject({
+        clientId,
+        baseUrl,
+        status: "CHECKING",
+        activeTasksCount: 0,
+        pagesScanned: 0,
+        pagesSkipped: 0,
+        skipped: [],
+        siteChrome: {},
+      });
+      expect(result?.createdAt).toBeInstanceOf(Date);
+      expect(result?.updatedAt).toBeInstanceOf(Date);
+    });
+  });
+
+  describe("updateScanDoc()", () => {
+    it("should update the given fields and bump updatedAt", async () => {
+      const scan = createScanFixture({ status: "SCANNING", pagesScanned: 0 });
+      const { insertedId } = await getScansCollection().insertOne(scan);
+
+      const result = await updateScanDoc(insertedId, {
+        status: "COMPLETED",
+        pagesScanned: 10,
+      });
+
+      expect(result).toMatchObject({
+        _id: insertedId,
+        status: "COMPLETED",
+        pagesScanned: 10,
+      });
+      expect(result?.updatedAt.getTime()).toBeGreaterThan(
+        scan.updatedAt.getTime(),
+      );
+    });
+
+    it("should return null when the scan does not exist", async () => {
+      const result = await updateScanDoc(new ObjectId(), { status: "FAILED" });
+
+      expect(result).toBeNull();
+    });
+  });
+
+  describe("getActiveScanStatus()", () => {
+    it("should return isActive: true with the status when the latest scan is still in progress", async () => {
+      const clientId = "activeClient";
+
+      await getScansCollection().insertMany([
+        createScanFixture({
+          clientId,
+          status: "COMPLETED",
+          createdAt: new Date("2023-05-01"),
+        }),
+        createScanFixture({
+          clientId,
+          status: "SCANNING",
+          createdAt: new Date("2023-05-15"),
+        }),
+      ]);
+
+      const result = await getActiveScanStatus(clientId);
+
+      expect(result).toEqual({ isActive: true, status: "SCANNING" });
+    });
+
+    it("should return isActive: false when the latest scan finished (COMPLETED / FAILED)", async () => {
+      const clientId = "finishedClient";
+
+      await getScansCollection().insertMany([
+        createScanFixture({
+          clientId,
+          status: "SCANNING",
+          createdAt: new Date("2023-05-01"),
+        }),
+        createScanFixture({
+          clientId,
+          status: "FAILED",
+          createdAt: new Date("2023-05-15"),
+        }),
+      ]);
+
+      const result = await getActiveScanStatus(clientId);
+
+      expect(result).toEqual({ isActive: false });
+    });
+
+    it("should return isActive: false when the client has no scans", async () => {
+      const result = await getActiveScanStatus("noScansClient");
+
+      expect(result).toEqual({ isActive: false });
+    });
+  });
+});

package/src/websitalk/scans/index.ts

@@ -0,0 +1,2 @@
+export * from "./scans.getters";
+export * from "./scans.types";

package/src/websitalk/scans/scans.getters.ts

@@ -0,0 +1,113 @@
+import { Collection, Filter, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import {
+  applyQueryOptions,
+  QueryOptions,
+} from "../../talkpilot/utils/query.utils";
+import type {
+  ActiveScanStatus,
+  Scan,
+  ScanDoc,
+  ScanStatus,
+} from "./scans.types";
+
+export const getScansCollection = (): Collection<Scan> => {
+  return getDb().collection<Scan>("scans");
+};
+
+export const getScansByClientAndDateRange = (
+  clientId: string,
+  startDate: Date,
+  endDate: Date,
+): Promise<ScanDoc[]> => {
+  return getScansCollection()
+    .find({
+      clientId,
+      createdAt: {
+        $gte: startDate,
+        $lte: endDate,
+      },
+    })
+    .toArray();
+};
+
+export const getScanFieldsByClientAndDateRange = <K extends keyof Scan>(
+  clientId: string,
+  startDate: Date,
+  endDate: Date,
+  fields: K[],
+): Promise<Pick<ScanDoc, "_id" | K>[]> => {
+  const projection = fields.reduce<Record<string, 1>>((acc, field) => {
+    acc[field as string] = 1;
+    return acc;
+  }, {});
+
+  return getScansCollection()
+    .find(
+      {
+        clientId,
+        createdAt: {
+          $gte: startDate,
+          $lte: endDate,
+        },
+      },
+      { projection },
+    )
+    .toArray() as Promise<Pick<ScanDoc, "_id" | K>[]>;
+};
+
+export const createScanDoc = (
+  scan: Pick<Scan, "clientId" | "baseUrl" | "notifyPhoneNumber">,
+) => {
+  return getScansCollection().insertOne({
+    ...scan,
+    status: "CHECKING",
+    activeTasksCount: 0,
+    pagesScanned: 0,
+    pagesSkipped: 0,
+    skipped: [],
+    siteChrome: {} as Scan["siteChrome"],
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  });
+};
+
+const FINISHED_SCAN_STATUSES: ScanStatus[] = ["COMPLETED", "FAILED"];
+
+export const getActiveScanStatus = async (
+  clientId: string,
+): Promise<ActiveScanStatus> => {
+  const latestScan = await getScansCollection().findOne(
+    { clientId },
+    { sort: { createdAt: -1 } },
+  );
+
+  if (!latestScan || FINISHED_SCAN_STATUSES.includes(latestScan.status)) {
+    return { isActive: false };
+  }
+
+  return { isActive: true, status: latestScan.status };
+};
+
+export const updateScanDoc = async (
+  scanId: ObjectId,
+  updates: Partial<Omit<Scan, "clientId" | "createdAt">>,
+): Promise<ScanDoc | null> => {
+  return await getScansCollection().findOneAndUpdate(
+    { _id: scanId },
+    { $set: { ...updates, updatedAt: new Date() } },
+    { returnDocument: "after" },
+  );
+};
+
+export const findScansByQuery = async (
+  query: Filter<Scan>,
+  options?: QueryOptions,
+): Promise<ScanDoc[]> => {
+  const cursor = getScansCollection().find(query);
+  return await applyQueryOptions(cursor, options).toArray();
+};
+
+export const countScans = async (query: Filter<Scan>): Promise<number> => {
+  return getScansCollection().countDocuments(query);
+};

package/src/websitalk/scans/scans.types.ts

@@ -0,0 +1,53 @@
+import { WithId } from "mongodb";
+
+export type ScanStatus =
+  | "CHECKING"
+  | "PREPARING"
+  | "SCANNING"
+  | "PROCESSING"
+  | "COMPLETED"
+  | "FAILED";
+
+export type ActiveScanStatus =
+  | { isActive: true; status: ScanStatus }
+  | { isActive: false };
+
+export type SiteChromeEntry = {
+  url: string;
+  urlCanonical?: string | null;
+  title?: string | null;
+  h1?: string | null;
+  language?: string | null;
+  sourceType?: string | null;
+  status?: string | null;
+  context?: string | null;
+  contentHash?: string | null;
+  errorMessage?: string | null;
+  breadcrumbs?: unknown;
+  description?: string | null;
+  sourceUrl?: string | null;
+  fetchedAt?: Date | null;
+  createdAt?: Date;
+  updatedAt?: Date;
+};
+
+export type SkippedEntry = {
+  path: string;
+  reason: string;
+};
+
+export type Scan = {
+  clientId: string;
+  baseUrl: string;
+  notifyPhoneNumber?: string | null;
+  status: ScanStatus;
+  activeTasksCount: number;
+  pagesScanned: number;
+  pagesSkipped: number;
+  skipped: SkippedEntry[];
+  siteChrome: SiteChromeEntry;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+export type ScanDoc = WithId<Scan>;