@anthusai/papyrus 5.43.0-next.2 → 5.43.0-next.21

package/lib/monitoring-overview.ts

@@ -37,9 +37,45 @@ export type MonitoringEmailSummary = {
   receiptId: string; observedAt: string | null; status: string; subject: string | null;
   envelopeSender: string | null; headerFrom: string | null; warnings: string[];
   verdicts: Record<string, string>; linkCount: number; processedCount: number; unresolvedLinkCount: number;
-  reviewRequired: boolean; errorCode: string | null;
+  reviewRequired: boolean; errorCode: string | null; quarantine: EmailQuarantine | null;
+  /** Present when the same email was received more than once: one item, every copy listed. One decision covers all copies. */
+  group?: EmailCopyGroup;
+};
+export type EmailCopyGroup = {
+  fingerprint: string; count: number;
+  copies: { receiptId: string; observedAt: string | null; status: string; linkCount: number; processedCount: number; quarantine: EmailQuarantine | null }[];
+};
+
+/** Where a held email submission stands. `null` means it never needed review. */
+export type EmailQuarantineState =
+  | "quarantined" | "release_requested" | "released" | "discard_requested" | "discarded"
+  | "expired" | "decision_failed" | "release_failed";
+export type EmailQuarantine = {
+  state: EmailQuarantineState;
+  decision: { action: "accept" | "reject"; by: string; at: string | null; reason: string; viaGroup?: boolean } | null;
+  failure: string | null;
+};
+export type EmailDecisionResult =
+  | { ok: true; already: boolean; quarantine: EmailQuarantine }
+  | { error: "forbidden" | "bad_request" | "already_decided" | "not_quarantined" | "expired"; quarantine?: EmailQuarantine }
+  | { state: "unavailable" | "not_configured" };
+
+export const QUARANTINE_LABELS: Record<EmailQuarantineState, string> = {
+  quarantined: "Held for review",
+  release_requested: "Accepted, being queued",
+  released: "Accepted",
+  discard_requested: "Rejected, being closed",
+  discarded: "Rejected",
+  expired: "Expired",
+  decision_failed: "Decision could not be applied",
+  release_failed: "Accepted, retrying",
 };
 
+/** Links and import are available only once a message is accepted (or never needed review). */
+export function emailSubmissionAccepted(quarantine: EmailQuarantine | null): boolean {
+  return quarantine === null || quarantine.state === "released";
+}
+
 export type MonitoringEmailDetails = {
   submission: MonitoringEmailSummary & { corpusId: string | null }; safeBodyText: string;
   candidateLinks: { url: string; title: string; resolutionStatus?: "unresolved" }[]; sourceMaterialIsUntrusted: true;
@@ -47,6 +83,8 @@ export type MonitoringEmailDetails = {
 
 export type MonitoringEmailArticleResolution = {
   url: string; title: string; resolved: true;
+  /** Present when the original link could not be followed and the destination was found by searching the headline. */
+  foundVia?: "search"; search?: { query: string; matchScore: number; provider: string; matchedTitle?: string };
 } | { error: "forbidden" | "bad_request" | "not_resolvable" | "resolution_failed"; reason?: string } | { state: "unavailable" };
 
 export type EmailReferenceImportItem = { candidateIndex: number; source_uri: string };
@@ -112,6 +150,17 @@ export type MonitoringFeed = {
   nextDueAt: string | null;
 };
 
+/** The lists on the page that are paged. Page sizes are the server's; there is no total. */
+export type MonitoringListName = "runs" | "routerRuns" | "items" | "emailSubmissions";
+export type MonitoringCursors = Partial<Record<MonitoringListName, string | null>>;
+export type MonitoringListRows = {
+  runs: MonitoringRun; routerRuns: MonitoringRouterRun; items: MonitoringItem; emailSubmissions: MonitoringEmailSummary;
+};
+/** One more page of one list. `cursor` is null when the list has ended. */
+export type MonitoringPage<L extends MonitoringListName = MonitoringListName> =
+  | { list: L; rows: MonitoringListRows[L][]; cursor: string | null; hiddenNotRelevant?: number }
+  | { error: "forbidden" | "bad_request" } | { state: "unavailable" | "not_configured" };
+
 export type MonitoringOverview = {
   state: "ready" | "empty";
   asOf: string;
@@ -132,6 +181,8 @@ export type MonitoringOverview = {
   otherMonitors?: string[];
   emailSubmissions?: MonitoringEmailSummary[];
   hiddenNotRelevant: number;
+  /** Where each list continues; null or missing when it has no more rows. */
+  cursors?: MonitoringCursors;
 };
 
 export type MonitoringResult =

package/lib/monitoring-paging.ts

@@ -0,0 +1,58 @@
+/** Merging loaded pages into the Monitoring lists. Kept free of other imports so the panel's tests can load it alone. */
+import type {
+  MonitoringCursors,
+  MonitoringItem,
+  MonitoringListName,
+  MonitoringListRows,
+  MonitoringOverview,
+} from "./monitoring-overview";
+
+/** Rows the editor has loaded beyond the first page of each list, and where each list continues. */
+export type MonitoringLoadedPages = Partial<{ [L in MonitoringListName]: { rows: MonitoringListRows[L][]; cursor: string | null } }>;
+
+const ROW_KEYS: { [L in MonitoringListName]: (row: MonitoringListRows[L]) => string } = {
+  runs: (row) => `${row.runId}`,
+  routerRuns: (row) => `${row.runId}`,
+  items: (row) => `${row.monitorId ?? ""}|${row.sourceId}`,
+  emailSubmissions: (row) => row.receiptId,
+};
+
+/**
+ * The first page followed by loaded pages, with a row seen twice kept once. The refresh re-reads the first page, so
+ * a row that moved onto it from a later page must not be listed again; for items the highest revision wins.
+ */
+export function mergeListPages<L extends MonitoringListName>(list: L, first: MonitoringListRows[L][], loaded: MonitoringListRows[L][]): MonitoringListRows[L][] {
+  const keyOf = ROW_KEYS[list] as (row: MonitoringListRows[L]) => string;
+  const seen = new Map<string, number>();
+  const merged: MonitoringListRows[L][] = [];
+  for (const row of [...first, ...loaded]) {
+    const key = keyOf(row);
+    const at = seen.get(key);
+    if (at === undefined) {
+      seen.set(key, merged.length);
+      merged.push(row);
+    } else if (list === "items" && (row as MonitoringItem).revision > (merged[at] as MonitoringItem).revision) {
+      merged[at] = row;
+    }
+  }
+  return merged;
+}
+
+/**
+ * What the page shows: the refreshed first page of every list plus whatever the editor loaded, with each list's
+ * cursor. With `held` the email list is replaced by the held-for-review list (its own pages).
+ */
+export function withLoadedPages(overview: MonitoringOverview, loaded: MonitoringLoadedPages, held: MonitoringLoadedPages["emailSubmissions"] | null): MonitoringOverview {
+  const cursors: MonitoringCursors = {};
+  const lists: Partial<Record<MonitoringListName, unknown[]>> = {};
+  for (const list of ["runs", "routerRuns", "items", "emailSubmissions"] as const) {
+    const extra = loaded[list];
+    lists[list] = mergeListPages(list, (overview[list] ?? []) as never[], (extra?.rows ?? []) as never[]) as unknown[];
+    cursors[list] = extra ? extra.cursor : overview.cursors?.[list] ?? null;
+  }
+  if (held) {
+    lists.emailSubmissions = held.rows;
+    cursors.emailSubmissions = held.cursor;
+  }
+  return { ...overview, ...(lists as Pick<MonitoringOverview, MonitoringListName>), cursors };
+}

package/lib/needs-me-items.ts

@@ -0,0 +1,289 @@
+/**
+ * The adapters that turn what already exists into `NeedsMeItem`s, the one model the "What needs me" feed ranks and renders.
+ * Each adapter only reads; deciding is routed back to the existing action for that source (see components/notification-center.tsx).
+ *
+ *   reference          a pending Reference                          reads the Reference, the Cyclotron's decision on it, its preview
+ *   incomplete_capture a pending Reference whose captured text is   reads the same, plus the Reference's capture status (the
+ *                      missing or cut off                            `curation` card, else the extracted-text attachments)
+ *   draft              one AI draft in an open draft review          reads the review request and the Proposal (headline, dek, sources)
+ *   stage_review       an open hard gate or soft review that is      reads the review request (reason, ask, deadline, subject) and the
+ *                      not about drafts                              subject story's title and excerpt
+ *   flag               an open flag                                  reads the review request
+ *   held_email         an email submission held for review           reads the monitoring email summary (subject, sender, verdicts)
+ *   concept_proposal   a proposed taxonomy concept or reader         STUB: typed, returns nothing until those sources reach the page
+ *                      proposal
+ */
+import type { ReferenceRecord } from "./category-repository";
+import type { MonitoringEmailSummary } from "./monitoring-overview";
+import { buildNotificationItems, draftPicture, requestSource, type DraftItem, type RequestItem } from "./notification-feed";
+import { referenceDisplayTitle, referenceLineageId } from "./newsroom-references";
+import { normalizeReferenceCurationStatus } from "./reference-policy";
+import { dekAddsInformation, referenceDek, referenceSourceHost, type ReferencePreview } from "./reference-preview";
+import { referenceArrivalTime, referenceDisplayTime } from "./reference-feed-ranking";
+import type { CaptureStatus } from "./reference-capture";
+import type { RelevanceDecisionView } from "./relevance-decisions";
+import type { DraftView } from "./review-drafts";
+import { nextText, type ReviewRequest } from "./review-requests";
+import { predictionOf, uncertaintyOf, type NeedsMeCounts, type NeedsMeDecisionState, type NeedsMeItem } from "./needs-me-ranking";
+
+export type NeedsMePayload =
+  | { kind: "reference"; reference: ReferenceRecord; lineageId: string; decision: RelevanceDecisionView | undefined; preview: ReferencePreview | undefined; capture: CaptureStatus | undefined; displayTime: number | null }
+  | { kind: "draft"; item: DraftItem; loading: boolean }
+  | { kind: "request"; request: ReviewRequest; storyTitle: string | null; excerpt: string | null; storyDeleted: boolean }
+  | { kind: "email"; summary: MonitoringEmailSummary }
+  | { kind: "concept"; row: ConceptProposalRow };
+
+export type NeedsMeEntry = NeedsMeItem<NeedsMePayload>;
+
+const ACCEPT_WORD = { accept: "Accept", reject: "Reject" } as const;
+
+function timeOf(value: string | null | undefined): number | null {
+  const parsed = value ? Date.parse(value) : NaN;
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function percent(confidence: number | null): string {
+  return confidence === null ? "" : ` ${Math.round(confidence * 100)}%`;
+}
+
+// --- references and captures ----------------------------------------------------------------------------------
+
+/**
+ * Whether a capture was attempted and came up short, and nobody has said otherwise: text that is thin (too short, or showing signs of
+ * being cut off or paywalled), or a curation card that records a failed capture. A Reference nobody has tried to capture yet (the
+ * address only, as monitoring discovers it) is not "incomplete": there is nothing to check.
+ */
+export function captureIsIncomplete(capture: CaptureStatus | undefined): boolean {
+  if (!capture || capture.assessment) return false;
+  return capture.state === "thin" || (capture.state === "none" && capture.source === "curation");
+}
+
+export function referenceDefaultOutcome(decision: RelevanceDecisionView | undefined): string {
+  const prediction = predictionOf(decision);
+  return prediction
+    ? `If you leave it, it stays pending with the Cyclotron's suggestion: ${ACCEPT_WORD[prediction.label]}${percent(prediction.confidence)}.`
+    : "If you leave it, it stays pending. Nothing uses it until someone rates it.";
+}
+
+export type ReferenceSources = {
+  references: ReferenceRecord[];
+  decisions: ReadonlyMap<string, RelevanceDecisionView> | undefined;
+  previews: ReadonlyMap<string, ReferencePreview>;
+  captures: ReadonlyMap<string, CaptureStatus> | undefined;
+  /** Lineage ids rated in this visit: they keep their place showing the answer. */
+  visitRated: ReadonlySet<string>;
+};
+
+/** Pending References (and the ones rated in this visit), as `reference` items, or `incomplete_capture` when the capture is cut off. */
+export function referenceItems(input: ReferenceSources): NeedsMeEntry[] {
+  const items: NeedsMeEntry[] = [];
+  for (const reference of input.references) {
+    const status = normalizeReferenceCurationStatus(reference.curationStatus);
+    const lineageId = referenceLineageId(reference);
+    if (status === "archived") continue;
+    const decided = status === "accepted" || status === "rejected";
+    if (decided && !input.visitRated.has(lineageId)) continue;
+    const decision = input.decisions?.get(lineageId);
+    const preview = input.previews.get(lineageId);
+    const capture = input.captures?.get(lineageId);
+    const prediction = predictionOf(decision);
+    const title = referenceDisplayTitle(reference);
+    const dek = referenceDek(reference, preview);
+    const host = referenceSourceHost(reference);
+    const incomplete = !decided && captureIsIncomplete(capture);
+    items.push({
+      type: incomplete ? "incomplete_capture" : "reference",
+      id: lineageId,
+      title,
+      dek: incomplete ? "The captured text looks cut off or missing, so the Cyclotron cannot judge it well." : dekAddsInformation(title, dek, preview?.imageUrl ?? null) ? dek : null,
+      image: preview?.imageUrl ?? null,
+      source: preview?.siteName ?? host ?? "Unknown source",
+      host,
+      createdAt: referenceArrivalTime(reference) ?? referenceDisplayTime(reference),
+      ...(prediction ? { prediction } : {}),
+      importance: 1,
+      uncertainty: uncertaintyOf(prediction?.pAccept),
+      decisionState: status === "accepted" ? "accepted" : status === "rejected" ? "rejected" : "open",
+      defaultOutcome: incomplete ? "If you leave it, the Reference stays pending with the text it has." : referenceDefaultOutcome(decision),
+      hardGate: false,
+      askedForReview: Boolean(decision?.reviewRecommended),
+      payload: { kind: "reference", reference, lineageId, decision, preview, capture, displayTime: referenceDisplayTime(reference) },
+    });
+  }
+  return items;
+}
+
+// --- drafts, stories, flags -------------------------------------------------------------------------------------
+
+function choiceState(choice: "accept" | "reject" | null | undefined): NeedsMeDecisionState {
+  return choice === "accept" ? "accepted" : choice === "reject" ? "rejected" : "open";
+}
+
+export function draftItem(item: DraftItem, bySource: ReadonlyMap<string, ReferencePreview>): NeedsMeEntry {
+  const { view, request } = item;
+  const picture = draftPicture(view, bySource);
+  const link = request.links.find((entry) => entry.id === item.draftId);
+  return {
+    type: "draft",
+    id: item.key,
+    title: view?.headline ?? link?.label ?? "Draft",
+    dek: view ? view.dek : "Loading the draft…",
+    image: picture.imageUrl,
+    source: picture.siteName ?? picture.host ?? (view ? `AI ${view.kindLabel.toLowerCase()}` : "AI draft"),
+    host: picture.host,
+    createdAt: timeOf(request.raisedAt),
+    importance: 1,
+    uncertainty: uncertaintyOf(undefined),
+    decisionState: choiceState(item.decision?.choice),
+    defaultOutcome: nextText(request),
+    hardGate: request.mode === "hard_gate",
+    askedForReview: true,
+    payload: { kind: "draft", item, loading: !view },
+  };
+}
+
+function requestDecisionState(request: ReviewRequest): NeedsMeDecisionState {
+  if (request.state === "open") return "open";
+  const answer = request.answer;
+  if (!answer) return "accepted";
+  if (answer.decision === "acknowledge" || (answer.decision === "approve" && request.mode === "flag")) return "acknowledged";
+  if (answer.decision === "reject" || (answer.decision === "feedback" && answer.steer === "not_this_kind")) return "rejected";
+  return "accepted";
+}
+
+export function requestItem(
+  item: RequestItem,
+  context: { titles: Readonly<Record<string, string>>; excerpts: Readonly<Record<string, string>>; deleted: ReadonlySet<string> },
+): NeedsMeEntry {
+  const { request } = item;
+  const storyId = request.subject.type === "story" ? request.subject.id : null;
+  const flag = request.mode === "flag";
+  return {
+    type: flag ? "flag" : "stage_review",
+    id: request.id,
+    title: request.reason,
+    dek: request.ask,
+    image: null,
+    source: requestSource(request),
+    host: null,
+    createdAt: timeOf(request.raisedAt),
+    importance: 1,
+    uncertainty: uncertaintyOf(undefined),
+    decisionState: requestDecisionState(request),
+    defaultOutcome: nextText(request),
+    hardGate: request.mode === "hard_gate",
+    askedForReview: true,
+    payload: {
+      kind: "request",
+      request,
+      storyTitle: storyId ? context.titles[storyId] ?? null : null,
+      excerpt: storyId ? context.excerpts[storyId] ?? null : null,
+      storyDeleted: storyId ? context.deleted.has(storyId) : false,
+    },
+  };
+}
+
+// --- held email --------------------------------------------------------------------------------------------------
+
+function senderDomain(summary: MonitoringEmailSummary): string | null {
+  const from = summary.headerFrom || summary.envelopeSender || "";
+  const address = from.match(/@([A-Za-z0-9.-]+\.[A-Za-z]{2,})/);
+  return address ? address[1].toLowerCase() : null;
+}
+
+/** The decision recorded on a receipt, else the one made in this visit. */
+export function emailDecisionState(summary: MonitoringEmailSummary, decided: ReadonlyMap<string, "accept" | "reject">): NeedsMeDecisionState {
+  const made = decided.get(summary.receiptId) ?? summary.quarantine?.decision?.action;
+  return choiceState(made);
+}
+
+/** Held email submissions: untrusted, so only the subject and sender facts are used; the body is read in the detail pane as inert text. */
+export function heldEmailItems(summaries: readonly MonitoringEmailSummary[], decided: ReadonlyMap<string, "accept" | "reject">): NeedsMeEntry[] {
+  const items: NeedsMeEntry[] = [];
+  for (const summary of summaries) {
+    const held = summary.quarantine?.state === "quarantined";
+    const decision = emailDecisionState(summary, decided);
+    if (!held && !decided.has(summary.receiptId)) continue;
+    const domain = senderDomain(summary);
+    items.push({
+      type: "held_email",
+      id: `email#${summary.receiptId}`,
+      title: summary.subject?.trim() || "(no subject)",
+      dek: summary.warnings.length ? `Held for review: ${summary.warnings.join(", ")}.` : `Held for review. ${summary.linkCount} candidate ${summary.linkCount === 1 ? "link" : "links"}.`,
+      image: null,
+      source: domain ?? "Email submission",
+      host: domain,
+      createdAt: timeOf(summary.observedAt),
+      importance: 1,
+      uncertainty: uncertaintyOf(undefined),
+      decisionState: decision,
+      defaultOutcome: "If you leave it, it stays held privately. Nothing is routed from it unless you accept it.",
+      hardGate: false,
+      askedForReview: true,
+      payload: { kind: "email", summary },
+    });
+  }
+  return items;
+}
+
+// --- stubs ---------------------------------------------------------------------------------------------------------
+
+/** A proposed taxonomy concept or a reader's proposal. Not on the page yet: the source is typed so the adapter has a place to land. */
+export type ConceptProposalRow = { id: string; title: string; summary: string | null; proposedBy: "cyclotron" | "reader"; createdAt: string | null };
+
+export function conceptProposalItems(_rows: readonly ConceptProposalRow[]): NeedsMeEntry[] {
+  return [];
+}
+
+// --- all of it -----------------------------------------------------------------------------------------------------
+
+export type NeedsMeSources = ReferenceSources & {
+  requests: ReviewRequest[];
+  drafts: Readonly<Record<string, DraftView | undefined>>;
+  missingDrafts: ReadonlySet<string>;
+  draftsState: "loading" | "ready" | "failed";
+  /** Draft keys and request ids decided in this visit. */
+  stay: ReadonlySet<string>;
+  bySource: ReadonlyMap<string, ReferencePreview>;
+  storyTitles: Readonly<Record<string, string>>;
+  storyExcerpts: Readonly<Record<string, string>>;
+  deletedStories: ReadonlySet<string>;
+  emails: readonly MonitoringEmailSummary[];
+  emailsDecided: ReadonlyMap<string, "accept" | "reject">;
+  concepts?: readonly ConceptProposalRow[];
+};
+
+/** Every source, as one list of items (unordered: `rankNeedsMe` orders it). */
+export function buildNeedsMeItems(sources: NeedsMeSources): NeedsMeEntry[] {
+  const fromRequests = buildNotificationItems({
+    requests: sources.requests, drafts: sources.drafts, missingDrafts: sources.missingDrafts, draftsState: sources.draftsState, stay: sources.stay,
+  });
+  const context = { titles: sources.storyTitles, excerpts: sources.storyExcerpts, deleted: sources.deletedStories };
+  return [
+    ...referenceItems(sources),
+    ...fromRequests.drafts.map((item) => draftItem(item, sources.bySource)),
+    ...fromRequests.reviews.map((item) => requestItem(item, context)),
+    ...fromRequests.flags.map((item) => requestItem(item, context)),
+    ...heldEmailItems(sources.emails, sources.emailsDecided),
+    ...conceptProposalItems(sources.concepts ?? []),
+  ];
+}
+
+function plural(count: number, one: string, many = `${one}s`): string {
+  return `${count} ${count === 1 ? one : many}`;
+}
+
+/** "3 drafts and 12 references waiting for you"; "All caught up" when nothing waits. */
+export function waitingTextFor(counts: NeedsMeCounts): string {
+  const parts = [
+    counts.drafts ? plural(counts.drafts, "draft") : null,
+    counts.references ? plural(counts.references, "reference") : null,
+    counts.stories ? plural(counts.stories, "story review") : null,
+    counts.email ? plural(counts.email, "held email", "held emails") : null,
+    counts.captures ? plural(counts.captures, "capture to check", "captures to check") : null,
+    counts.other ? plural(counts.other, "other item") : null,
+  ].filter((part): part is string => part !== null);
+  if (!parts.length) return "All caught up";
+  const list = parts.length === 1 ? parts[0] : `${parts.slice(0, -1).join(", ")} and ${parts[parts.length - 1]}`;
+  return `${list} waiting for you`;
+}