@talkpilot/core-db 1.3.60 → 1.3.61

package/src/websitalk/scanResources/scanResources.getters.ts

@@ -1,128 +1,121 @@
-import { Collection, Filter, ObjectId } from "mongodb";
-import { getDb } from "../index";
-import {
-  DEFAULT_PAGE_LIMIT,
-  MAX_PAGE_LIMIT,
-  SCAN_RESOURCES_COLLECTION,
-  SCAN_RESOURCE_BOARD_PROJECTION,
-  SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION,
-  URL_TASK_STATUSES,
-} from "./scanResources.constants";
-import type {
-  PagedResult,
-  ScanPagesQuery,
-  ScanResource,
-  ScanResourceBoardProjection,
-  ScanResourceDoc,
-  ScanResourcePageSummary,
-  UrlTaskStatus,
-} from "./scanResources.types";
-import {
-  decodeCursor,
-  encodeCursor,
-  escapeRegex,
-  statusesForColumn,
-} from "./scanResources.utils";
-
-export const getScanResourcesCollection = (): Collection<ScanResource> =>
-  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
-
-export const getScanKpiCounts = async (
-  scanId: string,
-): Promise<Partial<Record<UrlTaskStatus, number>>> => {
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: UrlTaskStatus; count: number }>([
-      { $match: { scanId } },
-      { $group: { _id: "$status", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
-    acc[row._id] = row.count;
-    return acc;
-  }, {});
-};
-
-export const getScanResourceCountsByScanIds = async (
-  scanIds: string[],
-): Promise<Record<string, number>> => {
-  if (scanIds.length === 0) return {};
-
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: string; count: number }>([
-      { $match: { scanId: { $in: scanIds } } },
-      { $group: { _id: "$scanId", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
-};
-
-export const listPageSummariesByScanIds = async (
-  scanIds: string[],
-): Promise<ScanResourcePageSummary[]> => {
-  if (scanIds.length === 0) return [];
-
-  return getScanResourcesCollection()
-    .find(
-      {
-        scanId: { $in: scanIds },
-        status: URL_TASK_STATUSES.URL_COMPLETED,
-      },
-      { projection: SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION },
-    )
-    .toArray() as Promise<ScanResourcePageSummary[]>;
-};
-
-export const getScanPages = async (
-  query: ScanPagesQuery,
-): Promise<PagedResult<ScanResourceBoardProjection>> => {
-  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
-
-  const filter: Filter<ScanResource> = {
-    scanId: query.scanId,
-    status: { $in: [...statusesForColumn(query.column)] },
-  };
-  if (query.cursor) {
-    filter._id = { $gt: decodeCursor(query.cursor) };
-  }
-  if (query.search) {
-    const searchPattern = escapeRegex(query.search);
-    filter.$or = [
-      { url: { $regex: searchPattern, $options: "i" } },
-      { title: { $regex: searchPattern, $options: "i" } },
-    ];
-  }
-  if (query.since) {
-    filter.updatedAt = { $gte: query.since };
-  }
-
-  const docs = await getScanResourcesCollection()
-    .find(filter, {
-      projection: SCAN_RESOURCE_BOARD_PROJECTION,
-      sort: { _id: 1 },
-      limit: limit + 1,
-    })
-    .toArray();
-
-  const hasMore = docs.length > limit;
-  const items = (
-    hasMore ? docs.slice(0, limit) : docs
-  ) as ScanResourceBoardProjection[];
-  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
-
-  return { items, nextCursor };
-};
-
-export const getScanResourceById = async (
-  scanId: string,
-  pageId: string,
-): Promise<ScanResourceDoc | null> =>
-  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });
-
-export const deleteScanResourcesByScanId = async (
-  scanId: string,
-): Promise<number> => {
-  const result = await getScanResourcesCollection().deleteMany({ scanId });
-  return result.deletedCount;
-};
+import { Collection, Filter, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import {
+  DEFAULT_PAGE_LIMIT,
+  MAX_PAGE_LIMIT,
+  SCAN_RESOURCES_COLLECTION,
+  SCAN_RESOURCE_BOARD_PROJECTION,
+  SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION,
+  URL_TASK_STATUSES,
+} from "./scanResources.constants";
+import type {
+  PagedResult,
+  ScanPagesQuery,
+  ScanResource,
+  ScanResourceBoardProjection,
+  ScanResourceDoc,
+  ScanResourcePageSummary,
+  UrlTaskStatus,
+} from "./scanResources.types";
+import {
+  decodeCursor,
+  encodeCursor,
+  escapeRegex,
+  statusesForColumn,
+} from "./scanResources.utils";
+
+export const getScanResourcesCollection = (): Collection<ScanResource> =>
+  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
+
+export const getScanKpiCounts = async (
+  scanId: string,
+): Promise<Partial<Record<UrlTaskStatus, number>>> => {
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: UrlTaskStatus; count: number }>([
+      { $match: { scanId } },
+      { $group: { _id: "$status", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
+    acc[row._id] = row.count;
+    return acc;
+  }, {});
+};
+
+export const getScanResourceCountsByScanIds = async (
+  scanIds: string[],
+): Promise<Record<string, number>> => {
+  if (scanIds.length === 0) return {};
+
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: string; count: number }>([
+      { $match: { scanId: { $in: scanIds } } },
+      { $group: { _id: "$scanId", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
+};
+
+export const listPageSummariesByScanIds = async (
+  scanIds: string[],
+): Promise<ScanResourcePageSummary[]> => {
+  if (scanIds.length === 0) return [];
+
+  return getScanResourcesCollection()
+    .find(
+      {
+        scanId: { $in: scanIds },
+        status: URL_TASK_STATUSES.URL_COMPLETED,
+      },
+      { projection: SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION },
+    )
+    .toArray() as Promise<ScanResourcePageSummary[]>;
+};
+
+export const getScanPages = async (
+  query: ScanPagesQuery,
+): Promise<PagedResult<ScanResourceBoardProjection>> => {
+  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
+
+  const filter: Filter<ScanResource> = {
+    scanId: query.scanId,
+    status: { $in: [...statusesForColumn(query.column)] },
+  };
+  if (query.cursor) {
+    filter._id = { $gt: decodeCursor(query.cursor) };
+  }
+  if (query.search) {
+    const searchPattern = escapeRegex(query.search);
+    filter.$or = [
+      { url: { $regex: searchPattern, $options: "i" } },
+      { title: { $regex: searchPattern, $options: "i" } },
+    ];
+  }
+  if (query.since) {
+    filter.updatedAt = { $gte: query.since };
+  }
+
+  const docs = await getScanResourcesCollection()
+    .find(filter, {
+      projection: SCAN_RESOURCE_BOARD_PROJECTION,
+      sort: { _id: 1 },
+      limit: limit + 1,
+    })
+    .toArray();
+
+  const hasMore = docs.length > limit;
+  const items = (
+    hasMore ? docs.slice(0, limit) : docs
+  ) as ScanResourceBoardProjection[];
+  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
+
+  return { items, nextCursor };
+};
+
+export const getScanResourceById = async (
+  scanId: string,
+  pageId: string,
+): Promise<ScanResourceDoc | null> =>
+  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });

package/src/websitalk/scanResources/scanResources.types.ts

@@ -1,105 +1,105 @@
-import { WithId } from "mongodb";
-import type {
-  BOARD_COLUMN_STATUSES,
-  URL_TASK_STATUSES,
-} from "./scanResources.constants";
-
-export type UrlTaskStatus =
-  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
-
-export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
-
-export type StageLogEntry = {
-  from: UrlTaskStatus | null;
-  to: UrlTaskStatus;
-  occurredAt: Date;
-  details: Record<string, unknown>;
-};
-
-export type ScanResource = {
-  urlHashId: string;
-  clientId: string;
-  scanId: string;
-  websiteUrlId: string;
-  url: string;
-  urlCanonical?: string | null;
-  title?: string | null;
-  h1?: string | null;
-  language?: string | null;
-  sourceType?: string | null;
-  context?: string | null;
-  rawContext?: string | null;
-  summaryTokens?: number | null;
-  unsummarized?: boolean | null;
-  contentHash?: string | null;
-  duplicateOfTaskId?: string | null;
-  breadcrumbs?: unknown;
-  description?: string | null;
-  sourceUrl?: string | null;
-  errorMessage?: string | null;
-
-  status: UrlTaskStatus;
-  depth: number;
-  retryCount: number;
-
-  processRetryCount?: number;
-  globalResurrections?: number;
-  noRetry?: boolean;
-  reviveAt?: Date | null;
-  statusEnteredAt?: Date;
-
-  httpStatus?: number;
-  finalUrl?: string;
-  batchDone?: number;
-  batchTotal?: number;
-
-  stageLogs?: StageLogEntry[];
-
-  createdAt: Date;
-  updatedAt: Date;
-  fetchedAt?: Date | null;
-};
-
-export type ScanResourceDoc = WithId<ScanResource>;
-
-export type ScanResourceBoardProjection = Pick<
-  ScanResourceDoc,
-  | "_id"
-  | "url"
-  | "title"
-  | "status"
-  | "statusEnteredAt"
-  | "retryCount"
-  | "processRetryCount"
-  | "globalResurrections"
-  | "noRetry"
-  | "errorMessage"
-  | "updatedAt"
-  | "sourceType"
->;
-
-export type ScanResourcePageSummary = Pick<
-  ScanResourceDoc,
-  | "_id"
-  | "scanId"
-  | "url"
-  | "title"
-  | "context"
-  | "summaryTokens"
-  | "unsummarized"
-  | "status"
->;
-
-export type ScanPagesQuery = {
-  scanId: string;
-  column: BoardColumn;
-  search?: string;
-  cursor?: string | null;
-  since?: Date;
-  limit?: number;
-};
-
-export type PagedResult<T> = {
-  items: T[];
-  nextCursor: string | null;
-};
+import { WithId } from "mongodb";
+import type {
+  BOARD_COLUMN_STATUSES,
+  URL_TASK_STATUSES,
+} from "./scanResources.constants";
+
+export type UrlTaskStatus =
+  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
+
+export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
+
+export type StageLogEntry = {
+  from: UrlTaskStatus | null;
+  to: UrlTaskStatus;
+  occurredAt: Date;
+  details: Record<string, unknown>;
+};
+
+export type ScanResource = {
+  urlHashId: string;
+  clientId: string;
+  scanId: string;
+  websiteUrlId: string;
+  url: string;
+  urlCanonical?: string | null;
+  title?: string | null;
+  h1?: string | null;
+  language?: string | null;
+  sourceType?: string | null;
+  context?: string | null;
+  rawContext?: string | null;
+  summaryTokens?: number | null;
+  unsummarized?: boolean | null;
+  contentHash?: string | null;
+  duplicateOfTaskId?: string | null;
+  breadcrumbs?: unknown;
+  description?: string | null;
+  sourceUrl?: string | null;
+  errorMessage?: string | null;
+
+  status: UrlTaskStatus;
+  depth: number;
+  retryCount: number;
+
+  processRetryCount?: number;
+  globalResurrections?: number;
+  noRetry?: boolean;
+  reviveAt?: Date | null;
+  statusEnteredAt?: Date;
+
+  httpStatus?: number;
+  finalUrl?: string;
+  batchDone?: number;
+  batchTotal?: number;
+
+  stageLogs?: StageLogEntry[];
+
+  createdAt: Date;
+  updatedAt: Date;
+  fetchedAt?: Date | null;
+};
+
+export type ScanResourceDoc = WithId<ScanResource>;
+
+export type ScanResourceBoardProjection = Pick<
+  ScanResourceDoc,
+  | "_id"
+  | "url"
+  | "title"
+  | "status"
+  | "statusEnteredAt"
+  | "retryCount"
+  | "processRetryCount"
+  | "globalResurrections"
+  | "noRetry"
+  | "errorMessage"
+  | "updatedAt"
+  | "sourceType"
+>;
+
+export type ScanResourcePageSummary = Pick<
+  ScanResourceDoc,
+  | "_id"
+  | "scanId"
+  | "url"
+  | "title"
+  | "context"
+  | "summaryTokens"
+  | "unsummarized"
+  | "status"
+>;
+
+export type ScanPagesQuery = {
+  scanId: string;
+  column: BoardColumn;
+  search?: string;
+  cursor?: string | null;
+  since?: Date;
+  limit?: number;
+};
+
+export type PagedResult<T> = {
+  items: T[];
+  nextCursor: string | null;
+};

package/src/websitalk/scanResources/scanResources.utils.ts

@@ -1,31 +1,31 @@
-import { ObjectId } from "mongodb";
-import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
-import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
-
-const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
-  BOARD_COLUMN_STATUSES,
-).reduce(
-  (acc, [column, statuses]) => {
-    for (const status of statuses) {
-      acc[status] = column as BoardColumn;
-    }
-    return acc;
-  },
-  {} as Record<UrlTaskStatus, BoardColumn>,
-);
-
-export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
-  STATUS_TO_COLUMN[status];
-
-export const statusesForColumn = (
-  column: BoardColumn,
-): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
-
-export const encodeCursor = (id: ObjectId): string =>
-  Buffer.from(id.toHexString()).toString("base64");
-
-export const decodeCursor = (cursor: string): ObjectId =>
-  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
-
-export const escapeRegex = (text: string): string =>
-  text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+import { ObjectId } from "mongodb";
+import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
+import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
+
+const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
+  BOARD_COLUMN_STATUSES,
+).reduce(
+  (acc, [column, statuses]) => {
+    for (const status of statuses) {
+      acc[status] = column as BoardColumn;
+    }
+    return acc;
+  },
+  {} as Record<UrlTaskStatus, BoardColumn>,
+);
+
+export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
+  STATUS_TO_COLUMN[status];
+
+export const statusesForColumn = (
+  column: BoardColumn,
+): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
+
+export const encodeCursor = (id: ObjectId): string =>
+  Buffer.from(id.toHexString()).toString("base64");
+
+export const decodeCursor = (cursor: string): ObjectId =>
+  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
+
+export const escapeRegex = (text: string): string =>
+  text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");

package/src/websitalk/scans/__tests__/scans.spec.ts

@@ -12,9 +12,7 @@ import {
   getInProgressScanByWebsiteUrl,
   getLatestCompletedScanByWebsiteUrl,
   deleteScansByWebsiteUrl,
-  deleteScanById,
 } from "../scans.getters";
-import { FINISHED_SCAN_STATUSES, SCAN_STATUSES } from "../scans.constants";
 import { createScanFixture } from "../../../test-utils/factories";
 import { ObjectId } from "mongodb";
 
@@ -140,33 +138,12 @@ describe("db.websitalk.scans", () => {
         pagesSkipped: 0,
         skipped: [],
         siteChrome: {},
-        pausedAt: null,
       });
       expect(result?.createdAt).toBeInstanceOf(Date);
       expect(result?.updatedAt).toBeInstanceOf(Date);
     });
   });
 
-  describe("pausedAt", () => {
-    it("is set on pause and cleared back to null on resume, via updateScanDoc", async () => {
-      const scan = createScanFixture({ status: "SCANNING", pausedAt: null });
-      const { insertedId } = await getScansCollection().insertOne(scan);
-
-      const pausedAt = new Date();
-      const paused = await updateScanDoc(insertedId, {
-        status: "PAUSED",
-        pausedAt,
-      });
-      expect(paused?.pausedAt).toEqual(pausedAt);
-
-      const resumed = await updateScanDoc(insertedId, {
-        status: "SCANNING",
-        pausedAt: null,
-      });
-      expect(resumed?.pausedAt).toBeNull();
-    });
-  });
-
   describe("updateScanDoc()", () => {
     it("should update the given fields and bump updatedAt", async () => {
       const scan = createScanFixture({ status: "SCANNING", pagesScanned: 0 });
@@ -333,30 +310,4 @@ describe("db.websitalk.scans", () => {
       expect(await countScans({ websiteUrlId })).toBe(0);
     });
   });
-
-  describe("deleteScanById()", () => {
-    it("should delete the scan and return true", async () => {
-      const { insertedId } =
-        await getScansCollection().insertOne(createScanFixture());
-
-      expect(await deleteScanById(insertedId)).toBe(true);
-      expect(
-        await getScansCollection().findOne({ _id: insertedId }),
-      ).toBeNull();
-    });
-
-    it("should return false when the scan does not exist", async () => {
-      expect(await deleteScanById(new ObjectId())).toBe(false);
-    });
-  });
-
-  describe("FINISHED_SCAN_STATUSES", () => {
-    it("should include CANCELED — canceling is final, not resumable", () => {
-      expect(FINISHED_SCAN_STATUSES).toContain(SCAN_STATUSES.CANCELED);
-    });
-
-    it("should NOT include PAUSED — pausing is reversible", () => {
-      expect(FINISHED_SCAN_STATUSES).not.toContain(SCAN_STATUSES.PAUSED);
-    });
-  });
 });

package/src/websitalk/scans/index.ts

@@ -1,4 +1,4 @@
-export * from "./scans.constants";
-export * from "./scans.getters";
-export * from "./scans.types";
-export * from "./scans.utils";
+export * from "./scans.constants";
+export * from "./scans.getters";
+export * from "./scans.types";
+export * from "./scans.utils";

package/src/websitalk/scans/scans.constants.ts

@@ -1,18 +1,15 @@
-export const SCAN_STATUSES = {
-  CHECKING: "CHECKING",
-  PREPARING: "PREPARING",
-  SCANNING: "SCANNING",
-  PROCESSING: "PROCESSING",
-  COMPLETED: "COMPLETED",
-  FAILED: "FAILED",
-  OVERWRITTEN: "OVERWRITTEN",
-  PAUSED: "PAUSED",
-  CANCELED: "CANCELED",
-} as const;
-
-export const FINISHED_SCAN_STATUSES = [
-  SCAN_STATUSES.COMPLETED,
-  SCAN_STATUSES.FAILED,
-  SCAN_STATUSES.OVERWRITTEN,
-  SCAN_STATUSES.CANCELED,
-] as const;
+export const SCAN_STATUSES = {
+  CHECKING: "CHECKING",
+  PREPARING: "PREPARING",
+  SCANNING: "SCANNING",
+  PROCESSING: "PROCESSING",
+  COMPLETED: "COMPLETED",
+  FAILED: "FAILED",
+  OVERWRITTEN: "OVERWRITTEN",
+} as const;
+
+export const FINISHED_SCAN_STATUSES = [
+  SCAN_STATUSES.COMPLETED,
+  SCAN_STATUSES.FAILED,
+  SCAN_STATUSES.OVERWRITTEN,
+] as const;