@talkpilot/core-db 1.3.40 → 1.3.41

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.constants.ts

@@ -0,0 +1,19 @@
+export const KNOWLEDGE_DOCUMENTS_COLLECTION = "knowledgeDocuments";
+
+export const KNOWLEDGE_DOCUMENT_STATUSES = {
+  UPLOADING: "UPLOADING",
+  EXTRACTING: "EXTRACTING",
+  PROCESSING: "PROCESSING",
+  READY: "READY",
+  FAILED: "FAILED",
+} as const;
+
+export type KnowledgeDocumentStatus =
+  (typeof KNOWLEDGE_DOCUMENT_STATUSES)[keyof typeof KNOWLEDGE_DOCUMENT_STATUSES];
+
+export const KNOWLEDGE_FILE_TYPES = {
+  PDF: "pdf",
+} as const;
+
+export type KnowledgeFileType =
+  (typeof KNOWLEDGE_FILE_TYPES)[keyof typeof KNOWLEDGE_FILE_TYPES];

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.getters.ts

@@ -0,0 +1,85 @@
+import { Collection, ObjectId as MongoObjectId } from "mongodb";
+import { getDb, ObjectId } from "../index";
+import {
+  KNOWLEDGE_DOCUMENTS_COLLECTION,
+  KNOWLEDGE_DOCUMENT_STATUSES,
+} from "./knowledgeDocuments.constants";
+import type {
+  CreateKnowledgeDocumentInput,
+  KnowledgeDocument,
+  KnowledgeDocumentDoc,
+  UpdateKnowledgeDocumentInput,
+} from "./knowledgeDocuments.types";
+
+export const getKnowledgeDocumentsCollection = (): Collection<KnowledgeDocument> => {
+  return getDb().collection<KnowledgeDocument>(KNOWLEDGE_DOCUMENTS_COLLECTION);
+};
+
+export const createKnowledgeDocument = async (
+  input: CreateKnowledgeDocumentInput,
+): Promise<MongoObjectId> => {
+  const now = new Date();
+  const { insertedId } = await getKnowledgeDocumentsCollection().insertOne({
+    clientId: input.clientId,
+    filename: input.filename,
+    contentType: input.contentType,
+    fileType: input.fileType,
+    storagePath: input.storagePath,
+    markdownStoragePath: input.markdownStoragePath,
+    byteSize: input.byteSize,
+    pageCount: input.pageCount,
+    charCount: input.charCount,
+    contentHash: input.contentHash,
+    status: input.status ?? KNOWLEDGE_DOCUMENT_STATUSES.UPLOADING,
+    errorMessage: input.errorMessage,
+    progressPercent: input.progressPercent ?? 0,
+    createdAt: now,
+    updatedAt: now,
+  });
+  return insertedId;
+};
+
+export const getKnowledgeDocumentById = async (
+  id: string,
+): Promise<KnowledgeDocumentDoc | null> => {
+  return getKnowledgeDocumentsCollection().findOne({ _id: new ObjectId(id) });
+};
+
+export const listKnowledgeDocumentsByClientId = async (
+  clientId: string,
+): Promise<KnowledgeDocumentDoc[]> => {
+  return getKnowledgeDocumentsCollection()
+    .find({ clientId })
+    .sort({ createdAt: -1 })
+    .toArray();
+};
+
+export const listReadyKnowledgeDocumentIdsByClientId = async (
+  clientId: string,
+): Promise<string[]> => {
+  const docs = await getKnowledgeDocumentsCollection()
+    .find(
+      { clientId, status: KNOWLEDGE_DOCUMENT_STATUSES.READY },
+      { projection: { _id: 1 } },
+    )
+    .toArray();
+  return docs.map((doc) => doc._id.toString());
+};
+
+export const updateKnowledgeDocument = async (
+  id: string,
+  updates: UpdateKnowledgeDocumentInput,
+): Promise<KnowledgeDocumentDoc | null> => {
+  return getKnowledgeDocumentsCollection().findOneAndUpdate(
+    { _id: new ObjectId(id) },
+    { $set: { ...updates, updatedAt: new Date() } },
+    { returnDocument: "after" },
+  );
+};
+
+export const deleteKnowledgeDocument = async (id: string): Promise<boolean> => {
+  const result = await getKnowledgeDocumentsCollection().deleteOne({
+    _id: new ObjectId(id),
+  });
+  return result.deletedCount === 1;
+};

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.types.ts

@@ -0,0 +1,37 @@
+import { ObjectId } from "mongodb";
+import type {
+  KnowledgeDocumentStatus,
+  KnowledgeFileType,
+} from "./knowledgeDocuments.constants";
+
+export type KnowledgeDocument = {
+  clientId: string;
+  filename: string;
+  contentType: string;
+  fileType: KnowledgeFileType;
+  storagePath: string;
+  markdownStoragePath?: string;
+  byteSize: number;
+  pageCount?: number;
+  charCount?: number;
+  contentHash?: string;
+  status: KnowledgeDocumentStatus;
+  errorMessage?: string;
+  progressPercent: number;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+export type KnowledgeDocumentDoc = KnowledgeDocument & { _id: ObjectId };
+
+export type CreateKnowledgeDocumentInput = Omit<
+  KnowledgeDocument,
+  "createdAt" | "updatedAt" | "progressPercent" | "status"
+> & {
+  status?: KnowledgeDocumentStatus;
+  progressPercent?: number;
+};
+
+export type UpdateKnowledgeDocumentInput = Partial<
+  Omit<KnowledgeDocument, "clientId" | "createdAt" | "updatedAt">
+>;

package/src/websitalk/scans/scans.getters.ts

@@ -57,7 +57,6 @@ export const createScanDoc = (
   scan: Pick<Scan, "clientId" | "websiteUrlId"> &
     Partial<Pick<Scan, "notifyPhoneNumber">>,
 ) => {
-  const now = new Date();
   return getScansCollection().insertOne({
     ...scan,
     status: SCAN_STATUSES.CHECKING,
@@ -66,13 +65,8 @@ export const createScanDoc = (
     pagesSkipped: 0,
     skipped: [],
     siteChrome: {} as Scan["siteChrome"],
-    createdAt: now,
-    updatedAt: now,
-    startedAt: now,
-    completedAt: null,
-    pagesCompleted: 0,
-    pagesFailedPermanent: 0,
-    pagesDuplicate: 0,
+    createdAt: new Date(),
+    updatedAt: new Date(),
   });
 };
 

package/src/websitalk/scans/scans.types.ts

@@ -40,15 +40,6 @@ export type Scan = {
   siteChrome: SiteChromeEntry;
   createdAt: Date;
   updatedAt: Date;
-
-  // Terminal-only snapshot, written once at finalize (Brain, outside core-db).
-  // Never read for live progress — an active scan's KPIs must come from
-  // getScanKpiCounts, not these fields, which are not kept in sync mid-scan.
-  startedAt?: Date;
-  completedAt?: Date | null;
-  pagesCompleted?: number;
-  pagesFailedPermanent?: number;
-  pagesDuplicate?: number;
 };
 
 export type ScanDoc = WithId<Scan>;

package/src/talkpilot/clientsConfig/clientsConfig.constants.ts

@@ -1,2 +0,0 @@
-/** Maximum number of scans a client may have active at once. */
-export const MAX_ACTIVE_SCANS = 3;

package/src/test-utils/factories/websitalk/scanResources.ts

@@ -1,26 +0,0 @@
-import { Factory } from "fishery";
-import { faker } from "@faker-js/faker";
-import { ObjectId } from "mongodb";
-import {
-  URL_TASK_STATUSES,
-  type ScanResource,
-} from "../../../websitalk/scanResources";
-
-export const scanResourceFactory = Factory.define<ScanResource>(() => ({
-  urlHashId: faker.string.uuid(),
-  clientId: faker.string.uuid(),
-  scanId: new ObjectId().toString(),
-  websiteUrlId: new ObjectId().toString(),
-  url: faker.internet.url(),
-  status: URL_TASK_STATUSES.URL_PENDING,
-  depth: 0,
-  retryCount: 0,
-  createdAt: new Date(),
-  updatedAt: new Date(),
-}));
-
-export function createScanResourceFixture(
-  overrides?: Partial<ScanResource>,
-): ScanResource {
-  return scanResourceFactory.build(overrides);
-}

package/src/websitalk/scanResources/__tests__/scanResources.spec.ts

@@ -1,136 +0,0 @@
-import { ObjectId } from "mongodb";
-import {
-  getScanResourcesCollection,
-  getScanKpiCounts,
-  getScanPages,
-  getScanResourceById,
-  getScanResourceCountsByScanIds,
-} from "../scanResources.getters";
-import { boardColumn } from "../scanResources.utils";
-import { createScanResourceFixture } from "../../../test-utils/factories";
-
-describe("db.websitalk.scanResources", () => {
-  describe("boardColumn()", () => {
-    it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => {
-      expect(boardColumn("URL_FAILED")).toBe("FAILED");
-    });
-
-    it("folds URL_PERMANENT_FAILURE into FAILED", () => {
-      expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED");
-    });
-
-    it("folds duplicate content into COMPLETED", () => {
-      expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED");
-    });
-  });
-
-  describe("getScanKpiCounts()", () => {
-    it("sums to the scan's total task count and omits zero-count statuses", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
-      ]);
-
-      const counts = await getScanKpiCounts(scanId);
-
-      expect(counts.URL_COMPLETED).toBe(2);
-      expect(counts.URL_PERMANENT_FAILURE).toBe(1);
-      expect(counts.URL_PENDING).toBeUndefined();
-      const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
-      expect(total).toBe(3);
-    });
-  });
-
-  describe("getScanPages()", () => {
-    it("never returns context or stageLogs, and paginates within one column", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany(
-        Array.from({ length: 3 }, () =>
-          createScanResourceFixture({
-            scanId,
-            status: "URL_COMPLETED",
-            context: "heavy markdown content",
-            stageLogs: [{ from: null, to: "URL_COMPLETED", occurredAt: new Date(), details: {} }],
-          }),
-        ),
-      );
-
-      const page1 = await getScanPages({ scanId, column: "COMPLETED", limit: 2 });
-      expect(page1.items).toHaveLength(2);
-      expect(page1.nextCursor).not.toBeNull();
-      page1.items.forEach((item) => {
-        expect(item).not.toHaveProperty("context");
-        expect(item).not.toHaveProperty("stageLogs");
-      });
-
-      const page2 = await getScanPages({
-        scanId,
-        column: "COMPLETED",
-        limit: 2,
-        cursor: page1.nextCursor,
-      });
-      expect(page2.items).toHaveLength(1);
-      expect(page2.nextCursor).toBeNull();
-    });
-
-    it("folds URL_FAILED into the FAILED column query", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }),
-        createScanResourceFixture({ scanId, status: "URL_FAILED" }),
-        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
-      ]);
-
-      const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" });
-      expect(ready.items).toHaveLength(1);
-
-      const failed = await getScanPages({ scanId, column: "FAILED" });
-      expect(failed.items).toHaveLength(2);
-    });
-  });
-
-  describe("getScanResourceCountsByScanIds()", () => {
-    it("returns per-scan totals and omits scans with no resources", async () => {
-      const scanA = new ObjectId().toString();
-      const scanB = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }),
-      ]);
-
-      const counts = await getScanResourceCountsByScanIds([scanA, scanB, "missing"]);
-      expect(counts[scanA]).toBe(2);
-      expect(counts[scanB]).toBe(1);
-      expect(counts.missing).toBeUndefined();
-    });
-  });
-
-  describe("getScanResourceById()", () => {
-    it("returns the full document including stageLogs in insertion order", async () => {
-      const scanId = new ObjectId().toString();
-      const stageLogs = [
-        { from: null, to: "URL_PENDING" as const, occurredAt: new Date("2024-01-01"), details: {} },
-        { from: "URL_PENDING" as const, to: "URL_SCANNING" as const, occurredAt: new Date("2024-01-02"), details: {} },
-      ];
-      const fixture = createScanResourceFixture({ scanId, stageLogs, context: "full content" });
-      const { insertedId } = await getScanResourcesCollection().insertOne(fixture);
-
-      const result = await getScanResourceById(scanId, insertedId.toString());
-
-      expect(result?.context).toBe("full content");
-      expect(result?.stageLogs).toEqual(stageLogs);
-    });
-
-    it("returns null when the scan does not match", async () => {
-      const fixture = createScanResourceFixture();
-      const { insertedId } = await getScanResourcesCollection().insertOne(fixture);
-
-      const result = await getScanResourceById(new ObjectId().toString(), insertedId.toString());
-
-      expect(result).toBeNull();
-    });
-  });
-});

package/src/websitalk/scanResources/index.ts

@@ -1,4 +0,0 @@
-export * from "./scanResources.constants";
-export * from "./scanResources.getters";
-export * from "./scanResources.types";
-export * from "./scanResources.utils";

package/src/websitalk/scanResources/scanResources.constants.ts

@@ -1,47 +0,0 @@
-export const SCAN_RESOURCES_COLLECTION = "scanResources";
-
-export const URL_TASK_STATUSES = {
-  URL_PENDING: "URL_PENDING",
-  URL_SCANNING: "URL_SCANNING",
-  URL_READY_TO_PROCESS: "URL_READY_TO_PROCESS",
-  URL_PENDING_CHUNKS: "URL_PENDING_CHUNKS",
-  URL_CHUNKING: "URL_CHUNKING",
-  URL_EMBEDDING: "URL_EMBEDDING",
-  URL_COMPLETED: "URL_COMPLETED",
-  URL_FAILED: "URL_FAILED",
-  URL_DUPLICATE_CONTENT: "URL_DUPLICATE_CONTENT",
-  URL_PERMANENT_FAILURE: "URL_PERMANENT_FAILURE",
-} as const;
-
-export const BOARD_COLUMN_STATUSES = {
-  PENDING: [URL_TASK_STATUSES.URL_PENDING],
-  SCANNING: [URL_TASK_STATUSES.URL_SCANNING],
-  READY_TO_PROCESS: [URL_TASK_STATUSES.URL_READY_TO_PROCESS],
-  CHUNKING: [URL_TASK_STATUSES.URL_CHUNKING, URL_TASK_STATUSES.URL_PENDING_CHUNKS],
-  PROCESSING: [URL_TASK_STATUSES.URL_EMBEDDING],
-  COMPLETED: [
-    URL_TASK_STATUSES.URL_COMPLETED,
-    URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
-  ],
-  FAILED: [
-    URL_TASK_STATUSES.URL_PERMANENT_FAILURE,
-    URL_TASK_STATUSES.URL_FAILED,
-  ],
-} as const;
-
-export const SCAN_RESOURCE_BOARD_PROJECTION = {
-  url: 1,
-  title: 1,
-  status: 1,
-  statusEnteredAt: 1,
-  retryCount: 1,
-  processRetryCount: 1,
-  globalResurrections: 1,
-  noRetry: 1,
-  errorMessage: 1,
-  updatedAt: 1,
-  sourceType: 1,
-} as const;
-
-export const DEFAULT_PAGE_LIMIT = 4;
-export const MAX_PAGE_LIMIT = 50;

package/src/websitalk/scanResources/scanResources.getters.ts

@@ -1,94 +0,0 @@
-import { Collection, Filter, ObjectId } from "mongodb";
-import { getDb } from "../index";
-import {
-  DEFAULT_PAGE_LIMIT,
-  MAX_PAGE_LIMIT,
-  SCAN_RESOURCES_COLLECTION,
-  SCAN_RESOURCE_BOARD_PROJECTION,
-} from "./scanResources.constants";
-import type {
-  PagedResult,
-  ScanPagesQuery,
-  ScanResource,
-  ScanResourceBoardProjection,
-  ScanResourceDoc,
-  UrlTaskStatus,
-} from "./scanResources.types";
-import { decodeCursor, encodeCursor, statusesForColumn } from "./scanResources.utils";
-
-export const getScanResourcesCollection = (): Collection<ScanResource> =>
-  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
-
-export const getScanKpiCounts = async (
-  scanId: string,
-): Promise<Partial<Record<UrlTaskStatus, number>>> => {
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: UrlTaskStatus; count: number }>([
-      { $match: { scanId } },
-      { $group: { _id: "$status", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
-    acc[row._id] = row.count;
-    return acc;
-  }, {});
-};
-
-export const getScanResourceCountsByScanIds = async (
-  scanIds: string[],
-): Promise<Record<string, number>> => {
-  if (scanIds.length === 0) return {};
-
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: string; count: number }>([
-      { $match: { scanId: { $in: scanIds } } },
-      { $group: { _id: "$scanId", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
-};
-
-export const getScanPages = async (
-  query: ScanPagesQuery,
-): Promise<PagedResult<ScanResourceBoardProjection>> => {
-  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
-
-  const filter: Filter<ScanResource> = {
-    scanId: query.scanId,
-    status: { $in: [...statusesForColumn(query.column)] },
-  };
-  if (query.cursor) {
-    filter._id = { $gt: decodeCursor(query.cursor) };
-  }
-  if (query.search) {
-    filter.$or = [
-      { url: { $regex: query.search, $options: "i" } },
-      { title: { $regex: query.search, $options: "i" } },
-    ];
-  }
-  if (query.since) {
-    filter.updatedAt = { $gte: query.since };
-  }
-
-  const docs = await getScanResourcesCollection()
-    .find(filter, {
-      projection: SCAN_RESOURCE_BOARD_PROJECTION,
-      sort: { _id: 1 },
-      limit: limit + 1,
-    })
-    .toArray();
-
-  const hasMore = docs.length > limit;
-  const items = (hasMore ? docs.slice(0, limit) : docs) as ScanResourceBoardProjection[];
-  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
-
-  return { items, nextCursor };
-};
-
-export const getScanResourceById = async (
-  scanId: string,
-  pageId: string,
-): Promise<ScanResourceDoc | null> =>
-  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });

package/src/websitalk/scanResources/scanResources.types.ts

@@ -1,90 +0,0 @@
-import { WithId } from "mongodb";
-import type {
-  BOARD_COLUMN_STATUSES,
-  URL_TASK_STATUSES,
-} from "./scanResources.constants";
-
-export type UrlTaskStatus =
-  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
-
-export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
-
-export type StageLogEntry = {
-  from: UrlTaskStatus | null;
-  to: UrlTaskStatus;
-  occurredAt: Date;
-  details: Record<string, unknown>;
-};
-
-export type ScanResource = {
-  urlHashId: string;
-  clientId: string;
-  scanId: string;
-  websiteUrlId: string;
-  url: string;
-  urlCanonical?: string | null;
-  title?: string | null;
-  h1?: string | null;
-  language?: string | null;
-  sourceType?: string | null;
-  context?: string | null;
-  contentHash?: string | null;
-  duplicateOfTaskId?: string | null;
-  breadcrumbs?: unknown;
-  description?: string | null;
-  sourceUrl?: string | null;
-  errorMessage?: string | null;
-
-  status: UrlTaskStatus;
-  depth: number;
-  retryCount: number;
-
-  processRetryCount?: number;
-  globalResurrections?: number;
-  noRetry?: boolean;
-  reviveAt?: Date | null;
-  statusEnteredAt?: Date;
-
-  httpStatus?: number;
-  finalUrl?: string;
-  batchDone?: number;
-  batchTotal?: number;
-
-  stageLogs?: StageLogEntry[];
-
-  createdAt: Date;
-  updatedAt: Date;
-  fetchedAt?: Date | null;
-};
-
-export type ScanResourceDoc = WithId<ScanResource>;
-
-export type ScanResourceBoardProjection = Pick<
-  ScanResourceDoc,
-  | "_id"
-  | "url"
-  | "title"
-  | "status"
-  | "statusEnteredAt"
-  | "retryCount"
-  | "processRetryCount"
-  | "globalResurrections"
-  | "noRetry"
-  | "errorMessage"
-  | "updatedAt"
-  | "sourceType"
->;
-
-export type ScanPagesQuery = {
-  scanId: string;
-  column: BoardColumn;
-  search?: string;
-  cursor?: string | null;
-  since?: Date;
-  limit?: number;
-};
-
-export type PagedResult<T> = {
-  items: T[];
-  nextCursor: string | null;
-};

package/src/websitalk/scanResources/scanResources.utils.ts

@@ -1,28 +0,0 @@
-import { ObjectId } from "mongodb";
-import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
-import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
-
-const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
-  BOARD_COLUMN_STATUSES,
-).reduce(
-  (acc, [column, statuses]) => {
-    for (const status of statuses) {
-      acc[status] = column as BoardColumn;
-    }
-    return acc;
-  },
-  {} as Record<UrlTaskStatus, BoardColumn>,
-);
-
-export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
-  STATUS_TO_COLUMN[status];
-
-export const statusesForColumn = (
-  column: BoardColumn,
-): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
-
-export const encodeCursor = (id: ObjectId): string =>
-  Buffer.from(id.toHexString()).toString("base64");
-
-export const decodeCursor = (cursor: string): ObjectId =>
-  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));