@talkpilot/core-db 1.3.39 → 1.3.41

package/src/utils/date.utils.ts

@@ -1,126 +1,126 @@
-/**
- * Shared timezone-aware date helpers for statistics aggregations.
- */
-
-/** Milliseconds in a single day. */
-export const DAY_MS = 24 * 60 * 60 * 1000;
-
-/** Maps a short weekday name (as produced by Intl) to its index (Sun = 0). */
-export const WEEKDAY: Record<string, number> = {
-  Sun: 0,
-  Mon: 1,
-  Tue: 2,
-  Wed: 3,
-  Thu: 4,
-  Fri: 5,
-  Sat: 6,
-};
-
-/** Parses a `YYYY-MM-DD` string into `[year, month, day]` numbers. */
-export const parseYmd = (dateStr: string): [number, number, number] => {
-  const [y, m, d] = dateStr.split("-").map(Number);
-  return [y, m, d];
-};
-
-/** Builds a zero-padded `YYYY-MM-DD` string from numeric parts. */
-export const ymdToStr = (y: number, m: number, d: number): string =>
-  `${y}-${String(m).padStart(2, "0")}-${String(d).padStart(2, "0")}`;
-
-/** Parses a `YYYY-MM-DD` string as midnight UTC. */
-export const toUtcDate = (dateStr: string): Date =>
-  new Date(`${dateStr}T00:00:00.000Z`);
-
-/** Formats a date as a `YYYY-MM-DD` string in UTC. */
-export const formatDate = (date: Date): string =>
-  date.toISOString().slice(0, 10);
-
-/** Adds `days` to a `YYYY-MM-DD` string, returning the resulting `YYYY-MM-DD`. */
-export const addDaysToDateStr = (dateStr: string, days: number): string =>
-  formatDate(new Date(toUtcDate(dateStr).getTime() + days * DAY_MS));
-
-/** Next civil calendar day (proleptic Gregorian). Avoids DST 24h steps that can stall iteration. */
-export const incrementYmd = (
-  y: number,
-  m: number,
-  d: number,
-): [number, number, number] => {
-  const next = new Date(Date.UTC(y, m - 1, d + 1));
-  return [next.getUTCFullYear(), next.getUTCMonth() + 1, next.getUTCDate()];
-};
-
-/**
- * Formats a date as the calendar day in the given timezone.
- * `en-CA` yields ISO-8601 `YYYY-MM-DD`, which is lexicographically sortable
- * (string compare == chronological compare) and matches MongoDB's `%Y-%m-%d`.
- */
-export const formatYmdInTz = (date: Date, timezone: string): string =>
-  new Intl.DateTimeFormat("en-CA", { timeZone: timezone }).format(date);
-
-/**
- * Returns the calendar day in the given timezone as numeric parts.
- * Reads parts by `type` (not string order), so `en-US` is fine here.
- */
-export const getYmdInTz = (date: Date, timezone: string) => {
-  const parts = Object.fromEntries(
-    new Intl.DateTimeFormat("en-US", {
-      timeZone: timezone,
-      year: "numeric",
-      month: "numeric",
-      day: "numeric",
-    })
-      .formatToParts(date)
-      .map((p) => [p.type, Number(p.value)]),
-  );
-  return { y: parts.year, m: parts.month, d: parts.day };
-};
-
-/** Weekday index (Sun = 0) of a date in the given timezone. */
-export const getWeekdayInTz = (date: Date, timezone: string): number =>
-  WEEKDAY[
-    new Intl.DateTimeFormat("en-US", {
-      timeZone: timezone,
-      weekday: "short",
-    }).format(date)
-  ] ?? 0;
-
-/**
- * Returns the given `YYYY-MM-DD` day at noon UTC — a stable anchor inside the
- * calendar day, far from midnight so timezone/DST shifts can't cross a date boundary.
- */
-export const dateAtNoonUtc = (dateStr: string): Date => {
-  const [y, m, d] = parseYmd(dateStr);
-  return new Date(Date.UTC(y, m - 1, d, 12, 0, 0));
-};
-
-/**
- * UTC instant at the start of the given `YYYY-MM-DD` calendar day in `timezone`.
- * Binary-searches the boundary so it stays correct across DST transitions.
- */
-export const startOfCalendarDayInTz = (
-  dateStr: string,
-  timezone: string,
-): Date => {
-  const anchor = dateAtNoonUtc(dateStr).getTime();
-  let low = anchor - 2 * DAY_MS;
-  let high = anchor + DAY_MS;
-
-  while (high - low > 1 /* ms */) {
-    const mid = Math.floor((low + high) / 2);
-    if (formatYmdInTz(new Date(mid), timezone) >= dateStr) high = mid;
-    else low = mid;
-  }
-
-  return new Date(high);
-};
-
-/** UTC instant at the last millisecond of the given calendar day in `timezone`. */
-export const endOfCalendarDayInTz = (
-  dateStr: string,
-  timezone: string,
-): Date => {
-  const [y, m, d] = parseYmd(dateStr);
-  const nextDay = new Date(Date.UTC(y, m - 1, d + 1))
-    .toISOString()
-    .slice(0, 10);
-  return new Date(startOfCalendarDayInTz(nextDay, timezone).getTime() - 1);
-};
+/**
+ * Shared timezone-aware date helpers for statistics aggregations.
+ */
+
+/** Milliseconds in a single day. */
+export const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** Maps a short weekday name (as produced by Intl) to its index (Sun = 0). */
+export const WEEKDAY: Record<string, number> = {
+  Sun: 0,
+  Mon: 1,
+  Tue: 2,
+  Wed: 3,
+  Thu: 4,
+  Fri: 5,
+  Sat: 6,
+};
+
+/** Parses a `YYYY-MM-DD` string into `[year, month, day]` numbers. */
+export const parseYmd = (dateStr: string): [number, number, number] => {
+  const [y, m, d] = dateStr.split("-").map(Number);
+  return [y, m, d];
+};
+
+/** Builds a zero-padded `YYYY-MM-DD` string from numeric parts. */
+export const ymdToStr = (y: number, m: number, d: number): string =>
+  `${y}-${String(m).padStart(2, "0")}-${String(d).padStart(2, "0")}`;
+
+/** Parses a `YYYY-MM-DD` string as midnight UTC. */
+export const toUtcDate = (dateStr: string): Date =>
+  new Date(`${dateStr}T00:00:00.000Z`);
+
+/** Formats a date as a `YYYY-MM-DD` string in UTC. */
+export const formatDate = (date: Date): string =>
+  date.toISOString().slice(0, 10);
+
+/** Adds `days` to a `YYYY-MM-DD` string, returning the resulting `YYYY-MM-DD`. */
+export const addDaysToDateStr = (dateStr: string, days: number): string =>
+  formatDate(new Date(toUtcDate(dateStr).getTime() + days * DAY_MS));
+
+/** Next civil calendar day (proleptic Gregorian). Avoids DST 24h steps that can stall iteration. */
+export const incrementYmd = (
+  y: number,
+  m: number,
+  d: number,
+): [number, number, number] => {
+  const next = new Date(Date.UTC(y, m - 1, d + 1));
+  return [next.getUTCFullYear(), next.getUTCMonth() + 1, next.getUTCDate()];
+};
+
+/**
+ * Formats a date as the calendar day in the given timezone.
+ * `en-CA` yields ISO-8601 `YYYY-MM-DD`, which is lexicographically sortable
+ * (string compare == chronological compare) and matches MongoDB's `%Y-%m-%d`.
+ */
+export const formatYmdInTz = (date: Date, timezone: string): string =>
+  new Intl.DateTimeFormat("en-CA", { timeZone: timezone }).format(date);
+
+/**
+ * Returns the calendar day in the given timezone as numeric parts.
+ * Reads parts by `type` (not string order), so `en-US` is fine here.
+ */
+export const getYmdInTz = (date: Date, timezone: string) => {
+  const parts = Object.fromEntries(
+    new Intl.DateTimeFormat("en-US", {
+      timeZone: timezone,
+      year: "numeric",
+      month: "numeric",
+      day: "numeric",
+    })
+      .formatToParts(date)
+      .map((p) => [p.type, Number(p.value)]),
+  );
+  return { y: parts.year, m: parts.month, d: parts.day };
+};
+
+/** Weekday index (Sun = 0) of a date in the given timezone. */
+export const getWeekdayInTz = (date: Date, timezone: string): number =>
+  WEEKDAY[
+    new Intl.DateTimeFormat("en-US", {
+      timeZone: timezone,
+      weekday: "short",
+    }).format(date)
+  ] ?? 0;
+
+/**
+ * Returns the given `YYYY-MM-DD` day at noon UTC — a stable anchor inside the
+ * calendar day, far from midnight so timezone/DST shifts can't cross a date boundary.
+ */
+export const dateAtNoonUtc = (dateStr: string): Date => {
+  const [y, m, d] = parseYmd(dateStr);
+  return new Date(Date.UTC(y, m - 1, d, 12, 0, 0));
+};
+
+/**
+ * UTC instant at the start of the given `YYYY-MM-DD` calendar day in `timezone`.
+ * Binary-searches the boundary so it stays correct across DST transitions.
+ */
+export const startOfCalendarDayInTz = (
+  dateStr: string,
+  timezone: string,
+): Date => {
+  const anchor = dateAtNoonUtc(dateStr).getTime();
+  let low = anchor - 2 * DAY_MS;
+  let high = anchor + DAY_MS;
+
+  while (high - low > 1 /* ms */) {
+    const mid = Math.floor((low + high) / 2);
+    if (formatYmdInTz(new Date(mid), timezone) >= dateStr) high = mid;
+    else low = mid;
+  }
+
+  return new Date(high);
+};
+
+/** UTC instant at the last millisecond of the given calendar day in `timezone`. */
+export const endOfCalendarDayInTz = (
+  dateStr: string,
+  timezone: string,
+): Date => {
+  const [y, m, d] = parseYmd(dateStr);
+  const nextDay = new Date(Date.UTC(y, m - 1, d + 1))
+    .toISOString()
+    .slice(0, 10);
+  return new Date(startOfCalendarDayInTz(nextDay, timezone).getTime() - 1);
+};

package/src/utils/shared.types.ts

@@ -1,4 +1,4 @@
-export interface CallLengthThresholds {
-  shortThreshold: number;
-  mediumThreshold: number;
-}
+export interface CallLengthThresholds {
+  shortThreshold: number;
+  mediumThreshold: number;
+}

package/src/utils/validation.ts

@@ -1,23 +1,23 @@
-/**
- * Simple validation utilities for the core-db package.
- * Designed to be lightweight and only run when needed.
- */
-
-export const validateConfig = (configName: string, value?: string): string => {
-  if (!value || value.trim() === "") {
-    throw new Error(
-      `[core-db] Configuration Error: '${configName}' is missing. ` +
-        `Please provide it via parameters or set the corresponding environment variable.`,
-    );
-  }
-  return value;
-};
-
-export const validateMongoUri = (uri: string): void => {
-  if (!uri.startsWith("mongodb://") && !uri.startsWith("mongodb+srv://")) {
-    throw new Error(
-      `[core-db] Connection Error: The provided MongoDB URI is invalid. ` +
-        `It must start with 'mongodb://' or 'mongodb+srv://'. Received: '${uri}'`,
-    );
-  }
-};
+/**
+ * Simple validation utilities for the core-db package.
+ * Designed to be lightweight and only run when needed.
+ */
+
+export const validateConfig = (configName: string, value?: string): string => {
+  if (!value || value.trim() === "") {
+    throw new Error(
+      `[core-db] Configuration Error: '${configName}' is missing. ` +
+        `Please provide it via parameters or set the corresponding environment variable.`,
+    );
+  }
+  return value;
+};
+
+export const validateMongoUri = (uri: string): void => {
+  if (!uri.startsWith("mongodb://") && !uri.startsWith("mongodb+srv://")) {
+    throw new Error(
+      `[core-db] Connection Error: The provided MongoDB URI is invalid. ` +
+        `It must start with 'mongodb://' or 'mongodb+srv://'. Received: '${uri}'`,
+    );
+  }
+};

package/src/websitalk/index.ts

@@ -1,16 +1,17 @@
-import { Db, ObjectId as MongoObjectId } from "mongodb";
-
-export * from "./scans";
-export * from "./websiteUrls";
-export { websitalkMongodbClient } from "./mongodb-client";
-
-let db: Db;
-export const setDb = (d: Db) => {
-  db = d;
-};
-
-export const getDb = (): Db => {
-  if (!db) throw new Error("Website Talk DB not initialised");
-  return db;
-};
-export const ObjectId = MongoObjectId;
+import { Db, ObjectId as MongoObjectId } from "mongodb";
+
+export * from "./scans";
+export * from "./websiteUrls";
+export * from "./knowledgeDocuments";
+export { websitalkMongodbClient } from "./mongodb-client";
+
+let db: Db;
+export const setDb = (d: Db) => {
+  db = d;
+};
+
+export const getDb = (): Db => {
+  if (!db) throw new Error("Website Talk DB not initialised");
+  return db;
+};
+export const ObjectId = MongoObjectId;

package/src/websitalk/knowledgeDocuments/__tests__/knowledgeDocuments.spec.ts

@@ -0,0 +1,123 @@
+import {
+  KNOWLEDGE_DOCUMENT_STATUSES,
+  KNOWLEDGE_FILE_TYPES,
+} from "../knowledgeDocuments.constants";
+import {
+  createKnowledgeDocument,
+  deleteKnowledgeDocument,
+  getKnowledgeDocumentById,
+  getKnowledgeDocumentsCollection,
+  listKnowledgeDocumentsByClientId,
+  listReadyKnowledgeDocumentIdsByClientId,
+  updateKnowledgeDocument,
+} from "../knowledgeDocuments.getters";
+import { createKnowledgeDocumentFixture } from "../../../test-utils/factories/websitalk/knowledgeDocuments";
+
+describe("db.websitalk.knowledgeDocuments", () => {
+  const clientId = "test-client";
+
+  describe("createKnowledgeDocument", () => {
+    it("should create a knowledge document", async () => {
+      const id = await createKnowledgeDocument({
+        clientId,
+        filename: "faq.pdf",
+        contentType: "application/pdf",
+        fileType: KNOWLEDGE_FILE_TYPES.PDF,
+        storagePath: "knowledge/test/faq.pdf",
+        byteSize: 2048,
+      });
+
+      const saved = await getKnowledgeDocumentById(id.toString());
+      expect(saved).toMatchObject({
+        clientId,
+        filename: "faq.pdf",
+        status: KNOWLEDGE_DOCUMENT_STATUSES.UPLOADING,
+        progressPercent: 0,
+        fileType: KNOWLEDGE_FILE_TYPES.PDF,
+      });
+      expect(saved?.createdAt).toBeInstanceOf(Date);
+      expect(saved?.updatedAt).toBeInstanceOf(Date);
+    });
+  });
+
+  describe("listKnowledgeDocumentsByClientId", () => {
+    it("should return documents for a client only", async () => {
+      await getKnowledgeDocumentsCollection().insertMany([
+        createKnowledgeDocumentFixture({ clientId, filename: "a.pdf" }),
+        createKnowledgeDocumentFixture({ clientId, filename: "b.pdf" }),
+        createKnowledgeDocumentFixture({
+          clientId: "other-client",
+          filename: "c.pdf",
+        }),
+      ]);
+
+      const results = await listKnowledgeDocumentsByClientId(clientId);
+      expect(results).toHaveLength(2);
+      expect(results.map((doc) => doc.filename).sort()).toEqual([
+        "a.pdf",
+        "b.pdf",
+      ]);
+    });
+  });
+
+  describe("listReadyKnowledgeDocumentIdsByClientId", () => {
+    it("should return only READY document ids", async () => {
+      await getKnowledgeDocumentsCollection().insertMany([
+        createKnowledgeDocumentFixture({
+          clientId,
+          status: KNOWLEDGE_DOCUMENT_STATUSES.READY,
+        }),
+        createKnowledgeDocumentFixture({
+          clientId,
+          status: KNOWLEDGE_DOCUMENT_STATUSES.PROCESSING,
+        }),
+        createKnowledgeDocumentFixture({
+          clientId: "other",
+          status: KNOWLEDGE_DOCUMENT_STATUSES.READY,
+        }),
+      ]);
+
+      const ids = await listReadyKnowledgeDocumentIdsByClientId(clientId);
+      expect(ids).toHaveLength(1);
+    });
+  });
+
+  describe("updateKnowledgeDocument", () => {
+    it("should update status and progress", async () => {
+      const id = await createKnowledgeDocument({
+        clientId,
+        filename: "doc.pdf",
+        contentType: "application/pdf",
+        fileType: KNOWLEDGE_FILE_TYPES.PDF,
+        storagePath: "knowledge/test/doc.pdf",
+        byteSize: 100,
+      });
+
+      const updated = await updateKnowledgeDocument(id.toString(), {
+        status: KNOWLEDGE_DOCUMENT_STATUSES.READY,
+        progressPercent: 100,
+        pageCount: 12,
+      });
+
+      expect(updated?.status).toBe(KNOWLEDGE_DOCUMENT_STATUSES.READY);
+      expect(updated?.progressPercent).toBe(100);
+      expect(updated?.pageCount).toBe(12);
+    });
+  });
+
+  describe("deleteKnowledgeDocument", () => {
+    it("should delete and return true", async () => {
+      const id = await createKnowledgeDocument({
+        clientId: "delete-client",
+        filename: "gone.pdf",
+        contentType: "application/pdf",
+        fileType: KNOWLEDGE_FILE_TYPES.PDF,
+        storagePath: "knowledge/delete/gone.pdf",
+        byteSize: 10,
+      });
+
+      expect(await deleteKnowledgeDocument(id.toString())).toBe(true);
+      expect(await getKnowledgeDocumentById(id.toString())).toBeNull();
+    });
+  });
+});

package/src/websitalk/knowledgeDocuments/index.ts

@@ -0,0 +1,3 @@
+export * from "./knowledgeDocuments.constants";
+export * from "./knowledgeDocuments.types";
+export * from "./knowledgeDocuments.getters";

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.constants.ts

@@ -0,0 +1,19 @@
+export const KNOWLEDGE_DOCUMENTS_COLLECTION = "knowledgeDocuments";
+
+export const KNOWLEDGE_DOCUMENT_STATUSES = {
+  UPLOADING: "UPLOADING",
+  EXTRACTING: "EXTRACTING",
+  PROCESSING: "PROCESSING",
+  READY: "READY",
+  FAILED: "FAILED",
+} as const;
+
+export type KnowledgeDocumentStatus =
+  (typeof KNOWLEDGE_DOCUMENT_STATUSES)[keyof typeof KNOWLEDGE_DOCUMENT_STATUSES];
+
+export const KNOWLEDGE_FILE_TYPES = {
+  PDF: "pdf",
+} as const;
+
+export type KnowledgeFileType =
+  (typeof KNOWLEDGE_FILE_TYPES)[keyof typeof KNOWLEDGE_FILE_TYPES];

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.getters.ts

@@ -0,0 +1,85 @@
+import { Collection, ObjectId as MongoObjectId } from "mongodb";
+import { getDb, ObjectId } from "../index";
+import {
+  KNOWLEDGE_DOCUMENTS_COLLECTION,
+  KNOWLEDGE_DOCUMENT_STATUSES,
+} from "./knowledgeDocuments.constants";
+import type {
+  CreateKnowledgeDocumentInput,
+  KnowledgeDocument,
+  KnowledgeDocumentDoc,
+  UpdateKnowledgeDocumentInput,
+} from "./knowledgeDocuments.types";
+
+export const getKnowledgeDocumentsCollection = (): Collection<KnowledgeDocument> => {
+  return getDb().collection<KnowledgeDocument>(KNOWLEDGE_DOCUMENTS_COLLECTION);
+};
+
+export const createKnowledgeDocument = async (
+  input: CreateKnowledgeDocumentInput,
+): Promise<MongoObjectId> => {
+  const now = new Date();
+  const { insertedId } = await getKnowledgeDocumentsCollection().insertOne({
+    clientId: input.clientId,
+    filename: input.filename,
+    contentType: input.contentType,
+    fileType: input.fileType,
+    storagePath: input.storagePath,
+    markdownStoragePath: input.markdownStoragePath,
+    byteSize: input.byteSize,
+    pageCount: input.pageCount,
+    charCount: input.charCount,
+    contentHash: input.contentHash,
+    status: input.status ?? KNOWLEDGE_DOCUMENT_STATUSES.UPLOADING,
+    errorMessage: input.errorMessage,
+    progressPercent: input.progressPercent ?? 0,
+    createdAt: now,
+    updatedAt: now,
+  });
+  return insertedId;
+};
+
+export const getKnowledgeDocumentById = async (
+  id: string,
+): Promise<KnowledgeDocumentDoc | null> => {
+  return getKnowledgeDocumentsCollection().findOne({ _id: new ObjectId(id) });
+};
+
+export const listKnowledgeDocumentsByClientId = async (
+  clientId: string,
+): Promise<KnowledgeDocumentDoc[]> => {
+  return getKnowledgeDocumentsCollection()
+    .find({ clientId })
+    .sort({ createdAt: -1 })
+    .toArray();
+};
+
+export const listReadyKnowledgeDocumentIdsByClientId = async (
+  clientId: string,
+): Promise<string[]> => {
+  const docs = await getKnowledgeDocumentsCollection()
+    .find(
+      { clientId, status: KNOWLEDGE_DOCUMENT_STATUSES.READY },
+      { projection: { _id: 1 } },
+    )
+    .toArray();
+  return docs.map((doc) => doc._id.toString());
+};
+
+export const updateKnowledgeDocument = async (
+  id: string,
+  updates: UpdateKnowledgeDocumentInput,
+): Promise<KnowledgeDocumentDoc | null> => {
+  return getKnowledgeDocumentsCollection().findOneAndUpdate(
+    { _id: new ObjectId(id) },
+    { $set: { ...updates, updatedAt: new Date() } },
+    { returnDocument: "after" },
+  );
+};
+
+export const deleteKnowledgeDocument = async (id: string): Promise<boolean> => {
+  const result = await getKnowledgeDocumentsCollection().deleteOne({
+    _id: new ObjectId(id),
+  });
+  return result.deletedCount === 1;
+};

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.types.ts

@@ -0,0 +1,37 @@
+import { ObjectId } from "mongodb";
+import type {
+  KnowledgeDocumentStatus,
+  KnowledgeFileType,
+} from "./knowledgeDocuments.constants";
+
+export type KnowledgeDocument = {
+  clientId: string;
+  filename: string;
+  contentType: string;
+  fileType: KnowledgeFileType;
+  storagePath: string;
+  markdownStoragePath?: string;
+  byteSize: number;
+  pageCount?: number;
+  charCount?: number;
+  contentHash?: string;
+  status: KnowledgeDocumentStatus;
+  errorMessage?: string;
+  progressPercent: number;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+export type KnowledgeDocumentDoc = KnowledgeDocument & { _id: ObjectId };
+
+export type CreateKnowledgeDocumentInput = Omit<
+  KnowledgeDocument,
+  "createdAt" | "updatedAt" | "progressPercent" | "status"
+> & {
+  status?: KnowledgeDocumentStatus;
+  progressPercent?: number;
+};
+
+export type UpdateKnowledgeDocumentInput = Partial<
+  Omit<KnowledgeDocument, "clientId" | "createdAt" | "updatedAt">
+>;