@talkpilot/core-db 1.3.39 → 1.3.40

package/src/websitalk/scanResources/scanResources.getters.ts

@@ -0,0 +1,94 @@
+import { Collection, Filter, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import {
+  DEFAULT_PAGE_LIMIT,
+  MAX_PAGE_LIMIT,
+  SCAN_RESOURCES_COLLECTION,
+  SCAN_RESOURCE_BOARD_PROJECTION,
+} from "./scanResources.constants";
+import type {
+  PagedResult,
+  ScanPagesQuery,
+  ScanResource,
+  ScanResourceBoardProjection,
+  ScanResourceDoc,
+  UrlTaskStatus,
+} from "./scanResources.types";
+import { decodeCursor, encodeCursor, statusesForColumn } from "./scanResources.utils";
+
+export const getScanResourcesCollection = (): Collection<ScanResource> =>
+  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
+
+export const getScanKpiCounts = async (
+  scanId: string,
+): Promise<Partial<Record<UrlTaskStatus, number>>> => {
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: UrlTaskStatus; count: number }>([
+      { $match: { scanId } },
+      { $group: { _id: "$status", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
+    acc[row._id] = row.count;
+    return acc;
+  }, {});
+};
+
+export const getScanResourceCountsByScanIds = async (
+  scanIds: string[],
+): Promise<Record<string, number>> => {
+  if (scanIds.length === 0) return {};
+
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: string; count: number }>([
+      { $match: { scanId: { $in: scanIds } } },
+      { $group: { _id: "$scanId", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
+};
+
+export const getScanPages = async (
+  query: ScanPagesQuery,
+): Promise<PagedResult<ScanResourceBoardProjection>> => {
+  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
+
+  const filter: Filter<ScanResource> = {
+    scanId: query.scanId,
+    status: { $in: [...statusesForColumn(query.column)] },
+  };
+  if (query.cursor) {
+    filter._id = { $gt: decodeCursor(query.cursor) };
+  }
+  if (query.search) {
+    filter.$or = [
+      { url: { $regex: query.search, $options: "i" } },
+      { title: { $regex: query.search, $options: "i" } },
+    ];
+  }
+  if (query.since) {
+    filter.updatedAt = { $gte: query.since };
+  }
+
+  const docs = await getScanResourcesCollection()
+    .find(filter, {
+      projection: SCAN_RESOURCE_BOARD_PROJECTION,
+      sort: { _id: 1 },
+      limit: limit + 1,
+    })
+    .toArray();
+
+  const hasMore = docs.length > limit;
+  const items = (hasMore ? docs.slice(0, limit) : docs) as ScanResourceBoardProjection[];
+  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
+
+  return { items, nextCursor };
+};
+
+export const getScanResourceById = async (
+  scanId: string,
+  pageId: string,
+): Promise<ScanResourceDoc | null> =>
+  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });

package/src/websitalk/scanResources/scanResources.types.ts

@@ -0,0 +1,90 @@
+import { WithId } from "mongodb";
+import type {
+  BOARD_COLUMN_STATUSES,
+  URL_TASK_STATUSES,
+} from "./scanResources.constants";
+
+export type UrlTaskStatus =
+  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
+
+export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
+
+export type StageLogEntry = {
+  from: UrlTaskStatus | null;
+  to: UrlTaskStatus;
+  occurredAt: Date;
+  details: Record<string, unknown>;
+};
+
+export type ScanResource = {
+  urlHashId: string;
+  clientId: string;
+  scanId: string;
+  websiteUrlId: string;
+  url: string;
+  urlCanonical?: string | null;
+  title?: string | null;
+  h1?: string | null;
+  language?: string | null;
+  sourceType?: string | null;
+  context?: string | null;
+  contentHash?: string | null;
+  duplicateOfTaskId?: string | null;
+  breadcrumbs?: unknown;
+  description?: string | null;
+  sourceUrl?: string | null;
+  errorMessage?: string | null;
+
+  status: UrlTaskStatus;
+  depth: number;
+  retryCount: number;
+
+  processRetryCount?: number;
+  globalResurrections?: number;
+  noRetry?: boolean;
+  reviveAt?: Date | null;
+  statusEnteredAt?: Date;
+
+  httpStatus?: number;
+  finalUrl?: string;
+  batchDone?: number;
+  batchTotal?: number;
+
+  stageLogs?: StageLogEntry[];
+
+  createdAt: Date;
+  updatedAt: Date;
+  fetchedAt?: Date | null;
+};
+
+export type ScanResourceDoc = WithId<ScanResource>;
+
+export type ScanResourceBoardProjection = Pick<
+  ScanResourceDoc,
+  | "_id"
+  | "url"
+  | "title"
+  | "status"
+  | "statusEnteredAt"
+  | "retryCount"
+  | "processRetryCount"
+  | "globalResurrections"
+  | "noRetry"
+  | "errorMessage"
+  | "updatedAt"
+  | "sourceType"
+>;
+
+export type ScanPagesQuery = {
+  scanId: string;
+  column: BoardColumn;
+  search?: string;
+  cursor?: string | null;
+  since?: Date;
+  limit?: number;
+};
+
+export type PagedResult<T> = {
+  items: T[];
+  nextCursor: string | null;
+};

package/src/websitalk/scanResources/scanResources.utils.ts

@@ -0,0 +1,28 @@
+import { ObjectId } from "mongodb";
+import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
+import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
+
+const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
+  BOARD_COLUMN_STATUSES,
+).reduce(
+  (acc, [column, statuses]) => {
+    for (const status of statuses) {
+      acc[status] = column as BoardColumn;
+    }
+    return acc;
+  },
+  {} as Record<UrlTaskStatus, BoardColumn>,
+);
+
+export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
+  STATUS_TO_COLUMN[status];
+
+export const statusesForColumn = (
+  column: BoardColumn,
+): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
+
+export const encodeCursor = (id: ObjectId): string =>
+  Buffer.from(id.toHexString()).toString("base64");
+
+export const decodeCursor = (cursor: string): ObjectId =>
+  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));