@talkpilot/core-db 1.3.48 → 1.3.50

package/src/websitalk/scanResources/scanResources.constants.ts

@@ -1,47 +1,47 @@
-export const SCAN_RESOURCES_COLLECTION = "scanResources";
-
-export const URL_TASK_STATUSES = {
-  URL_PENDING: "URL_PENDING",
-  URL_SCANNING: "URL_SCANNING",
-  URL_READY_TO_PROCESS: "URL_READY_TO_PROCESS",
-  URL_PENDING_CHUNKS: "URL_PENDING_CHUNKS",
-  URL_CHUNKING: "URL_CHUNKING",
-  URL_EMBEDDING: "URL_EMBEDDING",
-  URL_COMPLETED: "URL_COMPLETED",
-  URL_FAILED: "URL_FAILED",
-  URL_DUPLICATE_CONTENT: "URL_DUPLICATE_CONTENT",
-  URL_PERMANENT_FAILURE: "URL_PERMANENT_FAILURE",
-} as const;
-
-export const BOARD_COLUMN_STATUSES = {
-  PENDING: [URL_TASK_STATUSES.URL_PENDING],
-  SCANNING: [URL_TASK_STATUSES.URL_SCANNING],
-  READY_TO_PROCESS: [URL_TASK_STATUSES.URL_READY_TO_PROCESS],
-  CHUNKING: [URL_TASK_STATUSES.URL_CHUNKING, URL_TASK_STATUSES.URL_PENDING_CHUNKS],
-  PROCESSING: [URL_TASK_STATUSES.URL_EMBEDDING],
-  COMPLETED: [
-    URL_TASK_STATUSES.URL_COMPLETED,
-    URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
-  ],
-  FAILED: [
-    URL_TASK_STATUSES.URL_PERMANENT_FAILURE,
-    URL_TASK_STATUSES.URL_FAILED,
-  ],
-} as const;
-
-export const SCAN_RESOURCE_BOARD_PROJECTION = {
-  url: 1,
-  title: 1,
-  status: 1,
-  statusEnteredAt: 1,
-  retryCount: 1,
-  processRetryCount: 1,
-  globalResurrections: 1,
-  noRetry: 1,
-  errorMessage: 1,
-  updatedAt: 1,
-  sourceType: 1,
-} as const;
-
-export const DEFAULT_PAGE_LIMIT = 4;
-export const MAX_PAGE_LIMIT = 50;
+export const SCAN_RESOURCES_COLLECTION = "scanResources";
+
+export const URL_TASK_STATUSES = {
+  URL_PENDING: "URL_PENDING",
+  URL_SCANNING: "URL_SCANNING",
+  URL_READY_TO_PROCESS: "URL_READY_TO_PROCESS",
+  URL_PENDING_CHUNKS: "URL_PENDING_CHUNKS",
+  URL_CHUNKING: "URL_CHUNKING",
+  URL_EMBEDDING: "URL_EMBEDDING",
+  URL_COMPLETED: "URL_COMPLETED",
+  URL_FAILED: "URL_FAILED",
+  URL_DUPLICATE_CONTENT: "URL_DUPLICATE_CONTENT",
+  URL_PERMANENT_FAILURE: "URL_PERMANENT_FAILURE",
+} as const;
+
+export const BOARD_COLUMN_STATUSES = {
+  PENDING: [URL_TASK_STATUSES.URL_PENDING],
+  SCANNING: [URL_TASK_STATUSES.URL_SCANNING],
+  READY_TO_PROCESS: [URL_TASK_STATUSES.URL_READY_TO_PROCESS],
+  CHUNKING: [URL_TASK_STATUSES.URL_CHUNKING, URL_TASK_STATUSES.URL_PENDING_CHUNKS],
+  PROCESSING: [URL_TASK_STATUSES.URL_EMBEDDING],
+  COMPLETED: [
+    URL_TASK_STATUSES.URL_COMPLETED,
+    URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
+  ],
+  FAILED: [
+    URL_TASK_STATUSES.URL_PERMANENT_FAILURE,
+    URL_TASK_STATUSES.URL_FAILED,
+  ],
+} as const;
+
+export const SCAN_RESOURCE_BOARD_PROJECTION = {
+  url: 1,
+  title: 1,
+  status: 1,
+  statusEnteredAt: 1,
+  retryCount: 1,
+  processRetryCount: 1,
+  globalResurrections: 1,
+  noRetry: 1,
+  errorMessage: 1,
+  updatedAt: 1,
+  sourceType: 1,
+} as const;
+
+export const DEFAULT_PAGE_LIMIT = 4;
+export const MAX_PAGE_LIMIT = 50;

package/src/websitalk/scanResources/scanResources.getters.ts

@@ -1,102 +1,102 @@
-import { Collection, Filter, ObjectId } from "mongodb";
-import { getDb } from "../index";
-import {
-  DEFAULT_PAGE_LIMIT,
-  MAX_PAGE_LIMIT,
-  SCAN_RESOURCES_COLLECTION,
-  SCAN_RESOURCE_BOARD_PROJECTION,
-} from "./scanResources.constants";
-import type {
-  PagedResult,
-  ScanPagesQuery,
-  ScanResource,
-  ScanResourceBoardProjection,
-  ScanResourceDoc,
-  UrlTaskStatus,
-} from "./scanResources.types";
-import {
-  decodeCursor,
-  encodeCursor,
-  escapeRegex,
-  statusesForColumn,
-} from "./scanResources.utils";
-
-export const getScanResourcesCollection = (): Collection<ScanResource> =>
-  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
-
-export const getScanKpiCounts = async (
-  scanId: string,
-): Promise<Partial<Record<UrlTaskStatus, number>>> => {
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: UrlTaskStatus; count: number }>([
-      { $match: { scanId } },
-      { $group: { _id: "$status", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
-    acc[row._id] = row.count;
-    return acc;
-  }, {});
-};
-
-export const getScanResourceCountsByScanIds = async (
-  scanIds: string[],
-): Promise<Record<string, number>> => {
-  if (scanIds.length === 0) return {};
-
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: string; count: number }>([
-      { $match: { scanId: { $in: scanIds } } },
-      { $group: { _id: "$scanId", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
-};
-
-export const getScanPages = async (
-  query: ScanPagesQuery,
-): Promise<PagedResult<ScanResourceBoardProjection>> => {
-  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
-
-  const filter: Filter<ScanResource> = {
-    scanId: query.scanId,
-    status: { $in: [...statusesForColumn(query.column)] },
-  };
-  if (query.cursor) {
-    filter._id = { $gt: decodeCursor(query.cursor) };
-  }
-  if (query.search) {
-    const searchPattern = escapeRegex(query.search);
-    filter.$or = [
-      { url: { $regex: searchPattern, $options: "i" } },
-      { title: { $regex: searchPattern, $options: "i" } },
-    ];
-  }
-  if (query.since) {
-    filter.updatedAt = { $gte: query.since };
-  }
-
-  const docs = await getScanResourcesCollection()
-    .find(filter, {
-      projection: SCAN_RESOURCE_BOARD_PROJECTION,
-      sort: { _id: 1 },
-      limit: limit + 1,
-    })
-    .toArray();
-
-  const hasMore = docs.length > limit;
-  const items = (
-    hasMore ? docs.slice(0, limit) : docs
-  ) as ScanResourceBoardProjection[];
-  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
-
-  return { items, nextCursor };
-};
-
-export const getScanResourceById = async (
-  scanId: string,
-  pageId: string,
-): Promise<ScanResourceDoc | null> =>
-  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });
+import { Collection, Filter, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import {
+  DEFAULT_PAGE_LIMIT,
+  MAX_PAGE_LIMIT,
+  SCAN_RESOURCES_COLLECTION,
+  SCAN_RESOURCE_BOARD_PROJECTION,
+} from "./scanResources.constants";
+import type {
+  PagedResult,
+  ScanPagesQuery,
+  ScanResource,
+  ScanResourceBoardProjection,
+  ScanResourceDoc,
+  UrlTaskStatus,
+} from "./scanResources.types";
+import {
+  decodeCursor,
+  encodeCursor,
+  escapeRegex,
+  statusesForColumn,
+} from "./scanResources.utils";
+
+export const getScanResourcesCollection = (): Collection<ScanResource> =>
+  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
+
+export const getScanKpiCounts = async (
+  scanId: string,
+): Promise<Partial<Record<UrlTaskStatus, number>>> => {
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: UrlTaskStatus; count: number }>([
+      { $match: { scanId } },
+      { $group: { _id: "$status", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
+    acc[row._id] = row.count;
+    return acc;
+  }, {});
+};
+
+export const getScanResourceCountsByScanIds = async (
+  scanIds: string[],
+): Promise<Record<string, number>> => {
+  if (scanIds.length === 0) return {};
+
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: string; count: number }>([
+      { $match: { scanId: { $in: scanIds } } },
+      { $group: { _id: "$scanId", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
+};
+
+export const getScanPages = async (
+  query: ScanPagesQuery,
+): Promise<PagedResult<ScanResourceBoardProjection>> => {
+  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
+
+  const filter: Filter<ScanResource> = {
+    scanId: query.scanId,
+    status: { $in: [...statusesForColumn(query.column)] },
+  };
+  if (query.cursor) {
+    filter._id = { $gt: decodeCursor(query.cursor) };
+  }
+  if (query.search) {
+    const searchPattern = escapeRegex(query.search);
+    filter.$or = [
+      { url: { $regex: searchPattern, $options: "i" } },
+      { title: { $regex: searchPattern, $options: "i" } },
+    ];
+  }
+  if (query.since) {
+    filter.updatedAt = { $gte: query.since };
+  }
+
+  const docs = await getScanResourcesCollection()
+    .find(filter, {
+      projection: SCAN_RESOURCE_BOARD_PROJECTION,
+      sort: { _id: 1 },
+      limit: limit + 1,
+    })
+    .toArray();
+
+  const hasMore = docs.length > limit;
+  const items = (
+    hasMore ? docs.slice(0, limit) : docs
+  ) as ScanResourceBoardProjection[];
+  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
+
+  return { items, nextCursor };
+};
+
+export const getScanResourceById = async (
+  scanId: string,
+  pageId: string,
+): Promise<ScanResourceDoc | null> =>
+  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });

package/src/websitalk/scanResources/scanResources.types.ts

@@ -1,90 +1,90 @@
-import { WithId } from "mongodb";
-import type {
-  BOARD_COLUMN_STATUSES,
-  URL_TASK_STATUSES,
-} from "./scanResources.constants";
-
-export type UrlTaskStatus =
-  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
-
-export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
-
-export type StageLogEntry = {
-  from: UrlTaskStatus | null;
-  to: UrlTaskStatus;
-  occurredAt: Date;
-  details: Record<string, unknown>;
-};
-
-export type ScanResource = {
-  urlHashId: string;
-  clientId: string;
-  scanId: string;
-  websiteUrlId: string;
-  url: string;
-  urlCanonical?: string | null;
-  title?: string | null;
-  h1?: string | null;
-  language?: string | null;
-  sourceType?: string | null;
-  context?: string | null;
-  contentHash?: string | null;
-  duplicateOfTaskId?: string | null;
-  breadcrumbs?: unknown;
-  description?: string | null;
-  sourceUrl?: string | null;
-  errorMessage?: string | null;
-
-  status: UrlTaskStatus;
-  depth: number;
-  retryCount: number;
-
-  processRetryCount?: number;
-  globalResurrections?: number;
-  noRetry?: boolean;
-  reviveAt?: Date | null;
-  statusEnteredAt?: Date;
-
-  httpStatus?: number;
-  finalUrl?: string;
-  batchDone?: number;
-  batchTotal?: number;
-
-  stageLogs?: StageLogEntry[];
-
-  createdAt: Date;
-  updatedAt: Date;
-  fetchedAt?: Date | null;
-};
-
-export type ScanResourceDoc = WithId<ScanResource>;
-
-export type ScanResourceBoardProjection = Pick<
-  ScanResourceDoc,
-  | "_id"
-  | "url"
-  | "title"
-  | "status"
-  | "statusEnteredAt"
-  | "retryCount"
-  | "processRetryCount"
-  | "globalResurrections"
-  | "noRetry"
-  | "errorMessage"
-  | "updatedAt"
-  | "sourceType"
->;
-
-export type ScanPagesQuery = {
-  scanId: string;
-  column: BoardColumn;
-  search?: string;
-  cursor?: string | null;
-  since?: Date;
-  limit?: number;
-};
-
-export type PagedResult<T> = {
-  items: T[];
-  nextCursor: string | null;
-};
+import { WithId } from "mongodb";
+import type {
+  BOARD_COLUMN_STATUSES,
+  URL_TASK_STATUSES,
+} from "./scanResources.constants";
+
+export type UrlTaskStatus =
+  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
+
+export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
+
+export type StageLogEntry = {
+  from: UrlTaskStatus | null;
+  to: UrlTaskStatus;
+  occurredAt: Date;
+  details: Record<string, unknown>;
+};
+
+export type ScanResource = {
+  urlHashId: string;
+  clientId: string;
+  scanId: string;
+  websiteUrlId: string;
+  url: string;
+  urlCanonical?: string | null;
+  title?: string | null;
+  h1?: string | null;
+  language?: string | null;
+  sourceType?: string | null;
+  context?: string | null;
+  contentHash?: string | null;
+  duplicateOfTaskId?: string | null;
+  breadcrumbs?: unknown;
+  description?: string | null;
+  sourceUrl?: string | null;
+  errorMessage?: string | null;
+
+  status: UrlTaskStatus;
+  depth: number;
+  retryCount: number;
+
+  processRetryCount?: number;
+  globalResurrections?: number;
+  noRetry?: boolean;
+  reviveAt?: Date | null;
+  statusEnteredAt?: Date;
+
+  httpStatus?: number;
+  finalUrl?: string;
+  batchDone?: number;
+  batchTotal?: number;
+
+  stageLogs?: StageLogEntry[];
+
+  createdAt: Date;
+  updatedAt: Date;
+  fetchedAt?: Date | null;
+};
+
+export type ScanResourceDoc = WithId<ScanResource>;
+
+export type ScanResourceBoardProjection = Pick<
+  ScanResourceDoc,
+  | "_id"
+  | "url"
+  | "title"
+  | "status"
+  | "statusEnteredAt"
+  | "retryCount"
+  | "processRetryCount"
+  | "globalResurrections"
+  | "noRetry"
+  | "errorMessage"
+  | "updatedAt"
+  | "sourceType"
+>;
+
+export type ScanPagesQuery = {
+  scanId: string;
+  column: BoardColumn;
+  search?: string;
+  cursor?: string | null;
+  since?: Date;
+  limit?: number;
+};
+
+export type PagedResult<T> = {
+  items: T[];
+  nextCursor: string | null;
+};

package/src/websitalk/scanResources/scanResources.utils.ts

@@ -1,31 +1,31 @@
-import { ObjectId } from "mongodb";
-import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
-import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
-
-const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
-  BOARD_COLUMN_STATUSES,
-).reduce(
-  (acc, [column, statuses]) => {
-    for (const status of statuses) {
-      acc[status] = column as BoardColumn;
-    }
-    return acc;
-  },
-  {} as Record<UrlTaskStatus, BoardColumn>,
-);
-
-export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
-  STATUS_TO_COLUMN[status];
-
-export const statusesForColumn = (
-  column: BoardColumn,
-): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
-
-export const encodeCursor = (id: ObjectId): string =>
-  Buffer.from(id.toHexString()).toString("base64");
-
-export const decodeCursor = (cursor: string): ObjectId =>
-  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
-
-export const escapeRegex = (text: string): string =>
-  text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+import { ObjectId } from "mongodb";
+import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
+import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
+
+const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
+  BOARD_COLUMN_STATUSES,
+).reduce(
+  (acc, [column, statuses]) => {
+    for (const status of statuses) {
+      acc[status] = column as BoardColumn;
+    }
+    return acc;
+  },
+  {} as Record<UrlTaskStatus, BoardColumn>,
+);
+
+export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
+  STATUS_TO_COLUMN[status];
+
+export const statusesForColumn = (
+  column: BoardColumn,
+): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
+
+export const encodeCursor = (id: ObjectId): string =>
+  Buffer.from(id.toHexString()).toString("base64");
+
+export const decodeCursor = (cursor: string): ObjectId =>
+  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
+
+export const escapeRegex = (text: string): string =>
+  text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");