@talkpilot/core-db 1.3.54 → 1.3.55

package/src/websitalk/scanResources/__tests__/scanResources.spec.ts

@@ -1,212 +1,248 @@
-import { ObjectId } from "mongodb";
-import {
-  getScanResourcesCollection,
-  getScanKpiCounts,
-  getScanPages,
-  getScanResourceById,
-  getScanResourceCountsByScanIds,
-  deleteScanResourcesByScanId,
-} from "../scanResources.getters";
-import { boardColumn } from "../scanResources.utils";
-import { createScanResourceFixture } from "../../../test-utils/factories";
-
-describe("db.websitalk.scanResources", () => {
-  describe("boardColumn()", () => {
-    it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => {
-      expect(boardColumn("URL_FAILED")).toBe("FAILED");
-    });
-
-    it("folds URL_PERMANENT_FAILURE into FAILED", () => {
-      expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED");
-    });
-
-    it("folds duplicate content into COMPLETED", () => {
-      expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED");
-    });
-  });
-
-  describe("getScanKpiCounts()", () => {
-    it("sums to the scan's total task count and omits zero-count statuses", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
-      ]);
-
-      const counts = await getScanKpiCounts(scanId);
-
-      expect(counts.URL_COMPLETED).toBe(2);
-      expect(counts.URL_PERMANENT_FAILURE).toBe(1);
-      expect(counts.URL_PENDING).toBeUndefined();
-      const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
-      expect(total).toBe(3);
-    });
-  });
-
-  describe("getScanPages()", () => {
-    it("never returns context or stageLogs, and paginates within one column", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany(
-        Array.from({ length: 3 }, () =>
-          createScanResourceFixture({
-            scanId,
-            status: "URL_COMPLETED",
-            context: "heavy markdown content",
-            stageLogs: [
-              {
-                from: null,
-                to: "URL_COMPLETED",
-                occurredAt: new Date(),
-                details: {},
-              },
-            ],
-          }),
-        ),
-      );
-
-      const page1 = await getScanPages({
-        scanId,
-        column: "COMPLETED",
-        limit: 2,
-      });
-      expect(page1.items).toHaveLength(2);
-      expect(page1.nextCursor).not.toBeNull();
-      page1.items.forEach((item) => {
-        expect(item).not.toHaveProperty("context");
-        expect(item).not.toHaveProperty("stageLogs");
-      });
-
-      const page2 = await getScanPages({
-        scanId,
-        column: "COMPLETED",
-        limit: 2,
-        cursor: page1.nextCursor,
-      });
-      expect(page2.items).toHaveLength(1);
-      expect(page2.nextCursor).toBeNull();
-    });
-
-    it("folds URL_FAILED into the FAILED column query", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }),
-        createScanResourceFixture({ scanId, status: "URL_FAILED" }),
-        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
-      ]);
-
-      const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" });
-      expect(ready.items).toHaveLength(1);
-
-      const failed = await getScanPages({ scanId, column: "FAILED" });
-      expect(failed.items).toHaveLength(2);
-    });
-
-    it("finds a URL containing regex-special characters when searched verbatim", async () => {
-      const scanId = new ObjectId().toString();
-      const url =
-        "https://www.industry.org.il/index.php?dir=site&page=expert&op=item&cs=3022";
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED", url }),
-        createScanResourceFixture({
-          scanId,
-          status: "URL_COMPLETED",
-          url: "https://example.com/other",
-        }),
-      ]);
-
-      const result = await getScanPages({
-        scanId,
-        column: "COMPLETED",
-        search: url,
-      });
-
-      expect(result.items).toHaveLength(1);
-      expect(result.items[0].url).toBe(url);
-    });
-  });
-
-  describe("getScanResourceCountsByScanIds()", () => {
-    it("returns per-scan totals and omits scans with no resources", async () => {
-      const scanA = new ObjectId().toString();
-      const scanB = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }),
-      ]);
-
-      const counts = await getScanResourceCountsByScanIds([
-        scanA,
-        scanB,
-        "missing",
-      ]);
-      expect(counts[scanA]).toBe(2);
-      expect(counts[scanB]).toBe(1);
-      expect(counts.missing).toBeUndefined();
-    });
-  });
-
-  describe("getScanResourceById()", () => {
-    it("returns the full document including stageLogs in insertion order", async () => {
-      const scanId = new ObjectId().toString();
-      const stageLogs = [
-        {
-          from: null,
-          to: "URL_PENDING" as const,
-          occurredAt: new Date("2024-01-01"),
-          details: {},
-        },
-        {
-          from: "URL_PENDING" as const,
-          to: "URL_SCANNING" as const,
-          occurredAt: new Date("2024-01-02"),
-          details: {},
-        },
-      ];
-      const fixture = createScanResourceFixture({
-        scanId,
-        stageLogs,
-        context: "full content",
-      });
-      const { insertedId } =
-        await getScanResourcesCollection().insertOne(fixture);
-
-      const result = await getScanResourceById(scanId, insertedId.toString());
-
-      expect(result?.context).toBe("full content");
-      expect(result?.stageLogs).toEqual(stageLogs);
-    });
-
-    it("returns null when the scan does not match", async () => {
-      const fixture = createScanResourceFixture();
-      const { insertedId } =
-        await getScanResourcesCollection().insertOne(fixture);
-
-      const result = await getScanResourceById(
-        new ObjectId().toString(),
-        insertedId.toString(),
-      );
-
-      expect(result).toBeNull();
-    });
-  });
-
-  describe("deleteScanResourcesByScanId()", () => {
-    it("should delete all resources for the scan and return the count", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId }),
-        createScanResourceFixture({ scanId }),
-        createScanResourceFixture({ scanId: new ObjectId().toString() }),
-      ]);
-
-      const deleted = await deleteScanResourcesByScanId(scanId);
-
-      expect(deleted).toBe(2);
-      expect(
-        await getScanResourcesCollection().countDocuments({ scanId }),
-      ).toBe(0);
-    });
-  });
-});
+import { ObjectId } from "mongodb";
+import {
+  getScanResourcesCollection,
+  getScanKpiCounts,
+  getScanPages,
+  getScanResourceById,
+  getScanResourceCountsByScanIds,
+  listPageSummariesByScanIds,
+} from "../scanResources.getters";
+import { URL_TASK_STATUSES } from "../scanResources.constants";
+import { boardColumn } from "../scanResources.utils";
+import { createScanResourceFixture } from "../../../test-utils/factories";
+
+describe("db.websitalk.scanResources", () => {
+  describe("boardColumn()", () => {
+    it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => {
+      expect(boardColumn("URL_FAILED")).toBe("FAILED");
+    });
+
+    it("folds URL_PERMANENT_FAILURE into FAILED", () => {
+      expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED");
+    });
+
+    it("folds duplicate content into COMPLETED", () => {
+      expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED");
+    });
+  });
+
+  describe("getScanKpiCounts()", () => {
+    it("sums to the scan's total task count and omits zero-count statuses", async () => {
+      const scanId = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
+      ]);
+
+      const counts = await getScanKpiCounts(scanId);
+
+      expect(counts.URL_COMPLETED).toBe(2);
+      expect(counts.URL_PERMANENT_FAILURE).toBe(1);
+      expect(counts.URL_PENDING).toBeUndefined();
+      const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
+      expect(total).toBe(3);
+    });
+  });
+
+  describe("getScanPages()", () => {
+    it("never returns context or stageLogs, and paginates within one column", async () => {
+      const scanId = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany(
+        Array.from({ length: 3 }, () =>
+          createScanResourceFixture({
+            scanId,
+            status: "URL_COMPLETED",
+            context: "heavy markdown content",
+            stageLogs: [
+              {
+                from: null,
+                to: "URL_COMPLETED",
+                occurredAt: new Date(),
+                details: {},
+              },
+            ],
+          }),
+        ),
+      );
+
+      const page1 = await getScanPages({
+        scanId,
+        column: "COMPLETED",
+        limit: 2,
+      });
+      expect(page1.items).toHaveLength(2);
+      expect(page1.nextCursor).not.toBeNull();
+      page1.items.forEach((item) => {
+        expect(item).not.toHaveProperty("context");
+        expect(item).not.toHaveProperty("stageLogs");
+      });
+
+      const page2 = await getScanPages({
+        scanId,
+        column: "COMPLETED",
+        limit: 2,
+        cursor: page1.nextCursor,
+      });
+      expect(page2.items).toHaveLength(1);
+      expect(page2.nextCursor).toBeNull();
+    });
+
+    it("folds URL_FAILED into the FAILED column query", async () => {
+      const scanId = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }),
+        createScanResourceFixture({ scanId, status: "URL_FAILED" }),
+        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
+      ]);
+
+      const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" });
+      expect(ready.items).toHaveLength(1);
+
+      const failed = await getScanPages({ scanId, column: "FAILED" });
+      expect(failed.items).toHaveLength(2);
+    });
+
+    it("finds a URL containing regex-special characters when searched verbatim", async () => {
+      const scanId = new ObjectId().toString();
+      const url =
+        "https://www.industry.org.il/index.php?dir=site&page=expert&op=item&cs=3022";
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId, status: "URL_COMPLETED", url }),
+        createScanResourceFixture({
+          scanId,
+          status: "URL_COMPLETED",
+          url: "https://example.com/other",
+        }),
+      ]);
+
+      const result = await getScanPages({
+        scanId,
+        column: "COMPLETED",
+        search: url,
+      });
+
+      expect(result.items).toHaveLength(1);
+      expect(result.items[0].url).toBe(url);
+    });
+  });
+
+  describe("getScanResourceCountsByScanIds()", () => {
+    it("returns per-scan totals and omits scans with no resources", async () => {
+      const scanA = new ObjectId().toString();
+      const scanB = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }),
+      ]);
+
+      const counts = await getScanResourceCountsByScanIds([
+        scanA,
+        scanB,
+        "missing",
+      ]);
+      expect(counts[scanA]).toBe(2);
+      expect(counts[scanB]).toBe(1);
+      expect(counts.missing).toBeUndefined();
+    });
+  });
+
+  describe("listPageSummariesByScanIds()", () => {
+    it("returns completed pages for the given scans", async () => {
+      const scanA = new ObjectId().toString();
+      const scanB = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({
+          scanId: scanA,
+          status: URL_TASK_STATUSES.URL_COMPLETED,
+          url: "https://a.example/page",
+          title: "A",
+          context: "summary a",
+          summaryTokens: 12,
+          unsummarized: false,
+        }),
+        createScanResourceFixture({
+          scanId: scanA,
+          status: URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
+          context: "dup",
+        }),
+        createScanResourceFixture({
+          scanId: scanB,
+          status: URL_TASK_STATUSES.URL_COMPLETED,
+          url: "https://b.example/page",
+          context: "summary b",
+        }),
+        createScanResourceFixture({
+          scanId: new ObjectId().toString(),
+          status: URL_TASK_STATUSES.URL_COMPLETED,
+          context: "other scan",
+        }),
+      ]);
+
+      const pages = await listPageSummariesByScanIds([scanA, scanB]);
+      expect(pages).toHaveLength(2);
+      expect(pages.map((page) => page.scanId).sort()).toEqual(
+        [scanA, scanB].sort(),
+      );
+      const pageA = pages.find((page) => page.scanId === scanA);
+      expect(pageA).toMatchObject({
+        url: "https://a.example/page",
+        title: "A",
+        context: "summary a",
+        summaryTokens: 12,
+        unsummarized: false,
+        status: URL_TASK_STATUSES.URL_COMPLETED,
+      });
+    });
+
+    it("returns an empty array when scanIds is empty", async () => {
+      expect(await listPageSummariesByScanIds([])).toEqual([]);
+    });
+  });
+
+  describe("getScanResourceById()", () => {
+    it("returns the full document including stageLogs in insertion order", async () => {
+      const scanId = new ObjectId().toString();
+      const stageLogs = [
+        {
+          from: null,
+          to: "URL_PENDING" as const,
+          occurredAt: new Date("2024-01-01"),
+          details: {},
+        },
+        {
+          from: "URL_PENDING" as const,
+          to: "URL_SCANNING" as const,
+          occurredAt: new Date("2024-01-02"),
+          details: {},
+        },
+      ];
+      const fixture = createScanResourceFixture({
+        scanId,
+        stageLogs,
+        context: "full content",
+      });
+      const { insertedId } =
+        await getScanResourcesCollection().insertOne(fixture);
+
+      const result = await getScanResourceById(scanId, insertedId.toString());
+
+      expect(result?.context).toBe("full content");
+      expect(result?.stageLogs).toEqual(stageLogs);
+    });
+
+    it("returns null when the scan does not match", async () => {
+      const fixture = createScanResourceFixture();
+      const { insertedId } =
+        await getScanResourcesCollection().insertOne(fixture);
+
+      const result = await getScanResourceById(
+        new ObjectId().toString(),
+        insertedId.toString(),
+      );
+
+      expect(result).toBeNull();
+    });
+  });
+});

package/src/websitalk/scanResources/index.ts

@@ -1,4 +1,4 @@
-export * from "./scanResources.constants";
-export * from "./scanResources.getters";
-export * from "./scanResources.types";
-export * from "./scanResources.utils";
+export * from "./scanResources.constants";
+export * from "./scanResources.getters";
+export * from "./scanResources.types";
+export * from "./scanResources.utils";

package/src/websitalk/scanResources/scanResources.constants.ts

@@ -1,47 +1,60 @@
-export const SCAN_RESOURCES_COLLECTION = "scanResources";
-
-export const URL_TASK_STATUSES = {
-  URL_PENDING: "URL_PENDING",
-  URL_SCANNING: "URL_SCANNING",
-  URL_READY_TO_PROCESS: "URL_READY_TO_PROCESS",
-  URL_PENDING_CHUNKS: "URL_PENDING_CHUNKS",
-  URL_CHUNKING: "URL_CHUNKING",
-  URL_EMBEDDING: "URL_EMBEDDING",
-  URL_COMPLETED: "URL_COMPLETED",
-  URL_FAILED: "URL_FAILED",
-  URL_DUPLICATE_CONTENT: "URL_DUPLICATE_CONTENT",
-  URL_PERMANENT_FAILURE: "URL_PERMANENT_FAILURE",
-} as const;
-
-export const BOARD_COLUMN_STATUSES = {
-  PENDING: [URL_TASK_STATUSES.URL_PENDING],
-  SCANNING: [URL_TASK_STATUSES.URL_SCANNING],
-  READY_TO_PROCESS: [URL_TASK_STATUSES.URL_READY_TO_PROCESS],
-  CHUNKING: [URL_TASK_STATUSES.URL_CHUNKING, URL_TASK_STATUSES.URL_PENDING_CHUNKS],
-  PROCESSING: [URL_TASK_STATUSES.URL_EMBEDDING],
-  COMPLETED: [
-    URL_TASK_STATUSES.URL_COMPLETED,
-    URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
-  ],
-  FAILED: [
-    URL_TASK_STATUSES.URL_PERMANENT_FAILURE,
-    URL_TASK_STATUSES.URL_FAILED,
-  ],
-} as const;
-
-export const SCAN_RESOURCE_BOARD_PROJECTION = {
-  url: 1,
-  title: 1,
-  status: 1,
-  statusEnteredAt: 1,
-  retryCount: 1,
-  processRetryCount: 1,
-  globalResurrections: 1,
-  noRetry: 1,
-  errorMessage: 1,
-  updatedAt: 1,
-  sourceType: 1,
-} as const;
-
-export const DEFAULT_PAGE_LIMIT = 4;
-export const MAX_PAGE_LIMIT = 50;
+export const SCAN_RESOURCES_COLLECTION = "scanResources";
+
+export const URL_TASK_STATUSES = {
+  URL_PENDING: "URL_PENDING",
+  URL_SCANNING: "URL_SCANNING",
+  URL_READY_TO_PROCESS: "URL_READY_TO_PROCESS",
+  URL_PENDING_CHUNKS: "URL_PENDING_CHUNKS",
+  URL_CHUNKING: "URL_CHUNKING",
+  URL_EMBEDDING: "URL_EMBEDDING",
+  URL_COMPLETED: "URL_COMPLETED",
+  URL_FAILED: "URL_FAILED",
+  URL_DUPLICATE_CONTENT: "URL_DUPLICATE_CONTENT",
+  URL_PERMANENT_FAILURE: "URL_PERMANENT_FAILURE",
+} as const;
+
+export const BOARD_COLUMN_STATUSES = {
+  PENDING: [URL_TASK_STATUSES.URL_PENDING],
+  SCANNING: [URL_TASK_STATUSES.URL_SCANNING],
+  READY_TO_PROCESS: [URL_TASK_STATUSES.URL_READY_TO_PROCESS],
+  CHUNKING: [
+    URL_TASK_STATUSES.URL_CHUNKING,
+    URL_TASK_STATUSES.URL_PENDING_CHUNKS,
+  ],
+  PROCESSING: [URL_TASK_STATUSES.URL_EMBEDDING],
+  COMPLETED: [
+    URL_TASK_STATUSES.URL_COMPLETED,
+    URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
+  ],
+  FAILED: [
+    URL_TASK_STATUSES.URL_PERMANENT_FAILURE,
+    URL_TASK_STATUSES.URL_FAILED,
+  ],
+} as const;
+
+export const SCAN_RESOURCE_BOARD_PROJECTION = {
+  url: 1,
+  title: 1,
+  status: 1,
+  statusEnteredAt: 1,
+  retryCount: 1,
+  processRetryCount: 1,
+  globalResurrections: 1,
+  noRetry: 1,
+  errorMessage: 1,
+  updatedAt: 1,
+  sourceType: 1,
+} as const;
+
+export const SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION = {
+  scanId: 1,
+  url: 1,
+  title: 1,
+  context: 1,
+  summaryTokens: 1,
+  unsummarized: 1,
+  status: 1,
+} as const;
+
+export const DEFAULT_PAGE_LIMIT = 4;
+export const MAX_PAGE_LIMIT = 50;