@talkpilot/core-db 1.3.54 → 1.3.55

package/src/websitalk/scanResources/scanResources.getters.ts

@@ -1,109 +1,121 @@
-import { Collection, Filter, ObjectId } from "mongodb";
-import { getDb } from "../index";
-import {
-  DEFAULT_PAGE_LIMIT,
-  MAX_PAGE_LIMIT,
-  SCAN_RESOURCES_COLLECTION,
-  SCAN_RESOURCE_BOARD_PROJECTION,
-} from "./scanResources.constants";
-import type {
-  PagedResult,
-  ScanPagesQuery,
-  ScanResource,
-  ScanResourceBoardProjection,
-  ScanResourceDoc,
-  UrlTaskStatus,
-} from "./scanResources.types";
-import {
-  decodeCursor,
-  encodeCursor,
-  escapeRegex,
-  statusesForColumn,
-} from "./scanResources.utils";
-
-export const getScanResourcesCollection = (): Collection<ScanResource> =>
-  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
-
-export const getScanKpiCounts = async (
-  scanId: string,
-): Promise<Partial<Record<UrlTaskStatus, number>>> => {
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: UrlTaskStatus; count: number }>([
-      { $match: { scanId } },
-      { $group: { _id: "$status", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
-    acc[row._id] = row.count;
-    return acc;
-  }, {});
-};
-
-export const getScanResourceCountsByScanIds = async (
-  scanIds: string[],
-): Promise<Record<string, number>> => {
-  if (scanIds.length === 0) return {};
-
-  const rows = await getScanResourcesCollection()
-    .aggregate<{ _id: string; count: number }>([
-      { $match: { scanId: { $in: scanIds } } },
-      { $group: { _id: "$scanId", count: { $sum: 1 } } },
-    ])
-    .toArray();
-
-  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
-};
-
-export const getScanPages = async (
-  query: ScanPagesQuery,
-): Promise<PagedResult<ScanResourceBoardProjection>> => {
-  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
-
-  const filter: Filter<ScanResource> = {
-    scanId: query.scanId,
-    status: { $in: [...statusesForColumn(query.column)] },
-  };
-  if (query.cursor) {
-    filter._id = { $gt: decodeCursor(query.cursor) };
-  }
-  if (query.search) {
-    const searchPattern = escapeRegex(query.search);
-    filter.$or = [
-      { url: { $regex: searchPattern, $options: "i" } },
-      { title: { $regex: searchPattern, $options: "i" } },
-    ];
-  }
-  if (query.since) {
-    filter.updatedAt = { $gte: query.since };
-  }
-
-  const docs = await getScanResourcesCollection()
-    .find(filter, {
-      projection: SCAN_RESOURCE_BOARD_PROJECTION,
-      sort: { _id: 1 },
-      limit: limit + 1,
-    })
-    .toArray();
-
-  const hasMore = docs.length > limit;
-  const items = (
-    hasMore ? docs.slice(0, limit) : docs
-  ) as ScanResourceBoardProjection[];
-  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
-
-  return { items, nextCursor };
-};
-
-export const getScanResourceById = async (
-  scanId: string,
-  pageId: string,
-): Promise<ScanResourceDoc | null> =>
-  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });
-
-export const deleteScanResourcesByScanId = async (
-  scanId: string,
-): Promise<number> => {
-  const result = await getScanResourcesCollection().deleteMany({ scanId });
-  return result.deletedCount;
-};
+import { Collection, Filter, ObjectId } from "mongodb";
+import { getDb } from "../index";
+import {
+  DEFAULT_PAGE_LIMIT,
+  MAX_PAGE_LIMIT,
+  SCAN_RESOURCES_COLLECTION,
+  SCAN_RESOURCE_BOARD_PROJECTION,
+  SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION,
+  URL_TASK_STATUSES,
+} from "./scanResources.constants";
+import type {
+  PagedResult,
+  ScanPagesQuery,
+  ScanResource,
+  ScanResourceBoardProjection,
+  ScanResourceDoc,
+  ScanResourcePageSummary,
+  UrlTaskStatus,
+} from "./scanResources.types";
+import {
+  decodeCursor,
+  encodeCursor,
+  escapeRegex,
+  statusesForColumn,
+} from "./scanResources.utils";
+
+export const getScanResourcesCollection = (): Collection<ScanResource> =>
+  getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
+
+export const getScanKpiCounts = async (
+  scanId: string,
+): Promise<Partial<Record<UrlTaskStatus, number>>> => {
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: UrlTaskStatus; count: number }>([
+      { $match: { scanId } },
+      { $group: { _id: "$status", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
+    acc[row._id] = row.count;
+    return acc;
+  }, {});
+};
+
+export const getScanResourceCountsByScanIds = async (
+  scanIds: string[],
+): Promise<Record<string, number>> => {
+  if (scanIds.length === 0) return {};
+
+  const rows = await getScanResourcesCollection()
+    .aggregate<{ _id: string; count: number }>([
+      { $match: { scanId: { $in: scanIds } } },
+      { $group: { _id: "$scanId", count: { $sum: 1 } } },
+    ])
+    .toArray();
+
+  return Object.fromEntries(rows.map((row) => [row._id, row.count]));
+};
+
+export const listPageSummariesByScanIds = async (
+  scanIds: string[],
+): Promise<ScanResourcePageSummary[]> => {
+  if (scanIds.length === 0) return [];
+
+  return getScanResourcesCollection()
+    .find(
+      {
+        scanId: { $in: scanIds },
+        status: URL_TASK_STATUSES.URL_COMPLETED,
+      },
+      { projection: SCAN_RESOURCE_PAGE_SUMMARY_PROJECTION },
+    )
+    .toArray() as Promise<ScanResourcePageSummary[]>;
+};
+
+export const getScanPages = async (
+  query: ScanPagesQuery,
+): Promise<PagedResult<ScanResourceBoardProjection>> => {
+  const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
+
+  const filter: Filter<ScanResource> = {
+    scanId: query.scanId,
+    status: { $in: [...statusesForColumn(query.column)] },
+  };
+  if (query.cursor) {
+    filter._id = { $gt: decodeCursor(query.cursor) };
+  }
+  if (query.search) {
+    const searchPattern = escapeRegex(query.search);
+    filter.$or = [
+      { url: { $regex: searchPattern, $options: "i" } },
+      { title: { $regex: searchPattern, $options: "i" } },
+    ];
+  }
+  if (query.since) {
+    filter.updatedAt = { $gte: query.since };
+  }
+
+  const docs = await getScanResourcesCollection()
+    .find(filter, {
+      projection: SCAN_RESOURCE_BOARD_PROJECTION,
+      sort: { _id: 1 },
+      limit: limit + 1,
+    })
+    .toArray();
+
+  const hasMore = docs.length > limit;
+  const items = (
+    hasMore ? docs.slice(0, limit) : docs
+  ) as ScanResourceBoardProjection[];
+  const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
+
+  return { items, nextCursor };
+};
+
+export const getScanResourceById = async (
+  scanId: string,
+  pageId: string,
+): Promise<ScanResourceDoc | null> =>
+  getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });

package/src/websitalk/scanResources/scanResources.types.ts

@@ -1,90 +1,105 @@
-import { WithId } from "mongodb";
-import type {
-  BOARD_COLUMN_STATUSES,
-  URL_TASK_STATUSES,
-} from "./scanResources.constants";
-
-export type UrlTaskStatus =
-  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
-
-export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
-
-export type StageLogEntry = {
-  from: UrlTaskStatus | null;
-  to: UrlTaskStatus;
-  occurredAt: Date;
-  details: Record<string, unknown>;
-};
-
-export type ScanResource = {
-  urlHashId: string;
-  clientId: string;
-  scanId: string;
-  websiteUrlId: string;
-  url: string;
-  urlCanonical?: string | null;
-  title?: string | null;
-  h1?: string | null;
-  language?: string | null;
-  sourceType?: string | null;
-  context?: string | null;
-  contentHash?: string | null;
-  duplicateOfTaskId?: string | null;
-  breadcrumbs?: unknown;
-  description?: string | null;
-  sourceUrl?: string | null;
-  errorMessage?: string | null;
-
-  status: UrlTaskStatus;
-  depth: number;
-  retryCount: number;
-
-  processRetryCount?: number;
-  globalResurrections?: number;
-  noRetry?: boolean;
-  reviveAt?: Date | null;
-  statusEnteredAt?: Date;
-
-  httpStatus?: number;
-  finalUrl?: string;
-  batchDone?: number;
-  batchTotal?: number;
-
-  stageLogs?: StageLogEntry[];
-
-  createdAt: Date;
-  updatedAt: Date;
-  fetchedAt?: Date | null;
-};
-
-export type ScanResourceDoc = WithId<ScanResource>;
-
-export type ScanResourceBoardProjection = Pick<
-  ScanResourceDoc,
-  | "_id"
-  | "url"
-  | "title"
-  | "status"
-  | "statusEnteredAt"
-  | "retryCount"
-  | "processRetryCount"
-  | "globalResurrections"
-  | "noRetry"
-  | "errorMessage"
-  | "updatedAt"
-  | "sourceType"
->;
-
-export type ScanPagesQuery = {
-  scanId: string;
-  column: BoardColumn;
-  search?: string;
-  cursor?: string | null;
-  since?: Date;
-  limit?: number;
-};
-
-export type PagedResult<T> = {
-  items: T[];
-  nextCursor: string | null;
-};
+import { WithId } from "mongodb";
+import type {
+  BOARD_COLUMN_STATUSES,
+  URL_TASK_STATUSES,
+} from "./scanResources.constants";
+
+export type UrlTaskStatus =
+  (typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
+
+export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
+
+export type StageLogEntry = {
+  from: UrlTaskStatus | null;
+  to: UrlTaskStatus;
+  occurredAt: Date;
+  details: Record<string, unknown>;
+};
+
+export type ScanResource = {
+  urlHashId: string;
+  clientId: string;
+  scanId: string;
+  websiteUrlId: string;
+  url: string;
+  urlCanonical?: string | null;
+  title?: string | null;
+  h1?: string | null;
+  language?: string | null;
+  sourceType?: string | null;
+  context?: string | null;
+  rawContext?: string | null;
+  summaryTokens?: number | null;
+  unsummarized?: boolean | null;
+  contentHash?: string | null;
+  duplicateOfTaskId?: string | null;
+  breadcrumbs?: unknown;
+  description?: string | null;
+  sourceUrl?: string | null;
+  errorMessage?: string | null;
+
+  status: UrlTaskStatus;
+  depth: number;
+  retryCount: number;
+
+  processRetryCount?: number;
+  globalResurrections?: number;
+  noRetry?: boolean;
+  reviveAt?: Date | null;
+  statusEnteredAt?: Date;
+
+  httpStatus?: number;
+  finalUrl?: string;
+  batchDone?: number;
+  batchTotal?: number;
+
+  stageLogs?: StageLogEntry[];
+
+  createdAt: Date;
+  updatedAt: Date;
+  fetchedAt?: Date | null;
+};
+
+export type ScanResourceDoc = WithId<ScanResource>;
+
+export type ScanResourceBoardProjection = Pick<
+  ScanResourceDoc,
+  | "_id"
+  | "url"
+  | "title"
+  | "status"
+  | "statusEnteredAt"
+  | "retryCount"
+  | "processRetryCount"
+  | "globalResurrections"
+  | "noRetry"
+  | "errorMessage"
+  | "updatedAt"
+  | "sourceType"
+>;
+
+export type ScanResourcePageSummary = Pick<
+  ScanResourceDoc,
+  | "_id"
+  | "scanId"
+  | "url"
+  | "title"
+  | "context"
+  | "summaryTokens"
+  | "unsummarized"
+  | "status"
+>;
+
+export type ScanPagesQuery = {
+  scanId: string;
+  column: BoardColumn;
+  search?: string;
+  cursor?: string | null;
+  since?: Date;
+  limit?: number;
+};
+
+export type PagedResult<T> = {
+  items: T[];
+  nextCursor: string | null;
+};

package/src/websitalk/scanResources/scanResources.utils.ts

@@ -1,31 +1,31 @@
-import { ObjectId } from "mongodb";
-import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
-import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
-
-const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
-  BOARD_COLUMN_STATUSES,
-).reduce(
-  (acc, [column, statuses]) => {
-    for (const status of statuses) {
-      acc[status] = column as BoardColumn;
-    }
-    return acc;
-  },
-  {} as Record<UrlTaskStatus, BoardColumn>,
-);
-
-export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
-  STATUS_TO_COLUMN[status];
-
-export const statusesForColumn = (
-  column: BoardColumn,
-): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
-
-export const encodeCursor = (id: ObjectId): string =>
-  Buffer.from(id.toHexString()).toString("base64");
-
-export const decodeCursor = (cursor: string): ObjectId =>
-  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
-
-export const escapeRegex = (text: string): string =>
-  text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+import { ObjectId } from "mongodb";
+import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
+import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
+
+const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
+  BOARD_COLUMN_STATUSES,
+).reduce(
+  (acc, [column, statuses]) => {
+    for (const status of statuses) {
+      acc[status] = column as BoardColumn;
+    }
+    return acc;
+  },
+  {} as Record<UrlTaskStatus, BoardColumn>,
+);
+
+export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
+  STATUS_TO_COLUMN[status];
+
+export const statusesForColumn = (
+  column: BoardColumn,
+): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
+
+export const encodeCursor = (id: ObjectId): string =>
+  Buffer.from(id.toHexString()).toString("base64");
+
+export const decodeCursor = (cursor: string): ObjectId =>
+  new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
+
+export const escapeRegex = (text: string): string =>
+  text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");