@anthusai/papyrus 5.44.0 → 5.45.0

package/lib/reference-feed-ranking.ts

@@ -0,0 +1,191 @@
+/**
+ * How the References feed is ordered, and which cards belong in which tab.
+ *
+ * Plain words, as the "How this feed is ranked" popover says them:
+ *   1. The Cyclotron's prediction becomes a probability that you will Accept (P). A prediction of Accept at 80% is
+ *      P = 0.8; a prediction of Reject at 80% is P = 0.2; no prediction is P = 0.5.
+ *   2. Old items lose their say. The score is 0.5 + (P - 0.5) * w, where w = exp(-age / tau) is a freshness weight.
+ *      Age pulls an item toward "uncertain" (0.5), never to the bottom.
+ *   3. tau is learned from how old the References you accepted were when you accepted them
+ *      (tau = median age / ln 2). With fewer than 10 accepted examples nothing is guessed: no decay.
+ *   4. Highest score first. The strongest Reject predictions end up last. Ties go to the newest item.
+ *   5. Unrated items older than 3 tau move to "Earlier" at the bottom. Nothing is deleted.
+ *
+ * The same recipe ranks monitor stories (`learn_freshness` in src/papyrus_content/ideas/monitor_stories.py).
+ */
+import type { ReferenceRecord } from "./category-repository";
+import type { RelevanceDecisionView } from "./relevance-decisions";
+import { normalizeReferenceCurationStatus } from "./reference-policy";
+import { referenceLineageId } from "./newsroom-references";
+
+/** Fewer accepted examples than this and there is no freshness weight: a constant is never guessed. */
+export const MINIMUM_FRESHNESS_EXAMPLES = 10;
+/** An unrated item whose age passes this many tau has a weight below exp(-3), about 5%, and moves to Earlier. */
+export const EARLIER_AFTER_TAUS = 3;
+/** The probability that stands for "no prediction". */
+export const UNDECIDED_PROBABILITY = 0.5;
+/** A prediction that recorded a label but no confidence is a mild lean, not a coin toss and not a certainty. */
+export const UNRECORDED_CONFIDENCE = 0.6;
+
+const DAY_MILLISECONDS = 86_400_000;
+
+export type FeedFreshness = {
+  /** Null when there were too few accepted examples to learn from: no decay at all. */
+  tauDays: number | null;
+  medianAgeDays: number | null;
+  examples: number;
+};
+
+function timeOf(value: string | null | undefined): number | null {
+  if (!value) return null;
+  const parsed = Date.parse(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+/** When the item reached the newsroom. The same arrival the monitor stories use. */
+export function referenceArrivalTime(reference: ReferenceRecord): number | null {
+  return timeOf(reference.createdAt) ?? timeOf(reference.importedAt) ?? timeOf(reference.retrievedAt) ?? timeOf(reference.sourcePublishedAt);
+}
+
+/** The date a card shows: when the source says it was published, else when it arrived. */
+export function referenceDisplayTime(reference: ReferenceRecord): number | null {
+  return timeOf(reference.sourcePublishedAt) ?? timeOf(reference.sourceUpdatedAt) ?? referenceArrivalTime(reference);
+}
+
+/**
+ * tau from the age at acceptance of accepted References: median / ln 2, since the median age of an exponential
+ * exp(-age / tau) is tau * ln 2. `ignore` keeps the item an editor just rated from moving the very ranking they are looking at.
+ */
+export function learnFreshness(references: ReferenceRecord[], ignore: ReadonlySet<string> = new Set()): FeedFreshness {
+  const ages: number[] = [];
+  for (const reference of references) {
+    if (normalizeReferenceCurationStatus(reference.curationStatus) !== "accepted" || ignore.has(referenceLineageId(reference))) continue;
+    const arrived = referenceArrivalTime(reference);
+    const decided = timeOf(reference.curationStatusUpdatedAt);
+    if (arrived === null || decided === null) continue;
+    const age = (decided - arrived) / DAY_MILLISECONDS;
+    if (age > 0) ages.push(age);
+  }
+  if (ages.length < MINIMUM_FRESHNESS_EXAMPLES) return { tauDays: null, medianAgeDays: null, examples: ages.length };
+  ages.sort((left, right) => left - right);
+  const middle = Math.floor(ages.length / 2);
+  const median = ages.length % 2 ? ages[middle] : (ages[middle - 1] + ages[middle]) / 2;
+  return { tauDays: median / Math.LN2, medianAgeDays: median, examples: ages.length };
+}
+
+/** exp(-age / tau) at `now`; 1 without a learned tau or an unreadable arrival. */
+export function freshnessWeight(reference: ReferenceRecord, freshness: FeedFreshness, now: number): number {
+  const arrived = referenceArrivalTime(reference);
+  if (freshness.tauDays === null || arrived === null) return 1;
+  return Math.exp(-Math.max(now - arrived, 0) / DAY_MILLISECONDS / freshness.tauDays);
+}
+
+/** P(the editor accepts), from the Cyclotron's decision: its confidence is in the label it chose. */
+export function acceptProbability(decision: RelevanceDecisionView | undefined): number {
+  if (!decision) return UNDECIDED_PROBABILITY;
+  const confidence = decision.confidence ?? UNRECORDED_CONFIDENCE;
+  const accepts = decision.label === (decision.positiveLabel ?? "include");
+  return accepts ? confidence : 1 - confidence;
+}
+
+export type FeedItem = {
+  reference: ReferenceRecord;
+  lineageId: string;
+  decision: RelevanceDecisionView | undefined;
+  /** P(accept) before freshness. */
+  probability: number;
+  weight: number;
+  /** 0.5 + (P - 0.5) * weight. Highest is first. */
+  score: number;
+  /** True when this unrated item is older than the Earlier threshold. */
+  earlier: boolean;
+};
+
+export function scoreFeedItem(reference: ReferenceRecord, decision: RelevanceDecisionView | undefined, freshness: FeedFreshness, now: number): FeedItem {
+  const probability = acceptProbability(decision);
+  const weight = freshnessWeight(reference, freshness, now);
+  const arrived = referenceArrivalTime(reference);
+  const earlier = freshness.tauDays !== null && arrived !== null && (now - arrived) / DAY_MILLISECONDS > EARLIER_AFTER_TAUS * freshness.tauDays;
+  return {
+    reference,
+    lineageId: referenceLineageId(reference),
+    decision,
+    probability,
+    weight,
+    score: UNDECIDED_PROBABILITY + (probability - UNDECIDED_PROBABILITY) * weight,
+    earlier,
+  };
+}
+
+/** Highest score first; ties go to the newest item, then to a stable id order. */
+export function compareFeedItems(left: FeedItem, right: FeedItem): number {
+  if (right.score !== left.score) return right.score - left.score;
+  const newest = (referenceDisplayTime(right.reference) ?? 0) - (referenceDisplayTime(left.reference) ?? 0);
+  return newest || left.lineageId.localeCompare(right.lineageId);
+}
+
+export type ReferenceFeedTabKey = "feed" | "rated" | "earlier";
+
+export type ReferenceFeedPartition = {
+  freshness: FeedFreshness;
+  /** Pending items, plus the ones rated during this visit (they keep their place), best prediction first. */
+  feed: FeedItem[];
+  /** Everything an editor has answered, most recent answer first. */
+  rated: FeedItem[];
+  /** Unrated items past the freshness threshold, ranked like the feed. Still ratable. */
+  earlier: FeedItem[];
+};
+
+/**
+ * Splits canonical References into the three tabs.
+ * `visitRated` holds lineage ids rated since the page opened: they stay in Feed, in place, showing the answer.
+ */
+export function partitionReferenceFeed(
+  references: ReferenceRecord[],
+  decisions: ReadonlyMap<string, RelevanceDecisionView> | undefined,
+  options: { now: number; visitRated?: ReadonlySet<string>; tauDaysOverride?: number | null },
+): ReferenceFeedPartition {
+  const visitRated = options.visitRated ?? new Set<string>();
+  const learned = learnFreshness(references, visitRated);
+  const freshness: FeedFreshness = options.tauDaysOverride && options.tauDaysOverride > 0
+    ? { ...learned, tauDays: options.tauDaysOverride }
+    : learned;
+  const feed: FeedItem[] = [];
+  const rated: FeedItem[] = [];
+  const earlier: FeedItem[] = [];
+  for (const reference of references) {
+    const status = normalizeReferenceCurationStatus(reference.curationStatus);
+    const lineageId = referenceLineageId(reference);
+    const item = scoreFeedItem(reference, decisions?.get(lineageId), freshness, options.now);
+    if (status === "archived") continue;
+    if (status === "accepted" || status === "rejected") {
+      rated.push(item);
+      if (visitRated.has(lineageId)) (item.earlier ? earlier : feed).push(item);
+    } else if (item.earlier) earlier.push(item);
+    else feed.push(item);
+  }
+  feed.sort(compareFeedItems);
+  earlier.sort(compareFeedItems);
+  rated.sort((left, right) => (timeOf(right.reference.curationStatusUpdatedAt) ?? 0) - (timeOf(left.reference.curationStatusUpdatedAt) ?? 0)
+    || left.lineageId.localeCompare(right.lineageId));
+  return { freshness, feed, rated, earlier };
+}
+
+export type FeedGroupKey = "top" | "more" | "least";
+
+/**
+ * The feed's sections, from the ranked list (already best first). No fixed counts: Top predictions are the Accepts at least half as
+ * strong as the strongest one, Least likely are the predicted Rejects, and More for you is everything between (weaker Accepts and
+ * items with no prediction). A group with no members is absent.
+ */
+export function groupFeedItems(items: FeedItem[]): Array<{ key: FeedGroupKey; items: FeedItem[] }> {
+  const strongest = items.reduce((best, item) => Math.max(best, item.score - UNDECIDED_PROBABILITY), 0);
+  const groups: Record<FeedGroupKey, FeedItem[]> = { top: [], more: [], least: [] };
+  for (const item of items) {
+    const strength = item.score - UNDECIDED_PROBABILITY;
+    if (strength < 0) groups.least.push(item);
+    else if (strength > 0 && strength >= strongest / 2) groups.top.push(item);
+    else groups.more.push(item);
+  }
+  return (["top", "more", "least"] as const).filter((key) => groups[key].length).map((key) => ({ key, items: groups[key] }));
+}

package/lib/reference-preview.ts

@@ -0,0 +1,112 @@
+/**
+ * What a Reference card shows besides its title: the source page's preview image, a one-line dek and the site name.
+ *
+ * Storage: a `ReferenceAttachment` with role "preview" (sort key "002-preview"), no schema change. `sourceUri` is the
+ * page's og:image / twitter:image address and `metadata` holds `{ description, siteName, title, fetchedAt }`. The newsroom
+ * already loads every attachment with the References, so the feed needs no extra request. `papyrus references enrich-images`
+ * (src/papyrus_content/reference_images.py) writes them.
+ */
+import type { ReferenceAttachmentRecord, ReferenceRecord } from "./category-repository";
+
+export const REFERENCE_PREVIEW_ROLE = "preview";
+
+export type ReferencePreview = {
+  imageUrl: string | null;
+  description: string | null;
+  siteName: string | null;
+  title: string | null;
+};
+
+function record(value: unknown): Record<string, unknown> {
+  if (typeof value === "string") {
+    try { return record(JSON.parse(value)); } catch { return {}; }
+  }
+  return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+function text(value: unknown): string | null {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function httpAddress(value: unknown): string | null {
+  const candidate = text(value);
+  return candidate && /^https?:\/\//i.test(candidate) ? candidate : null;
+}
+
+/** The preview of each Reference lineage, from its "preview" attachment. A lineage with none has no entry. */
+export function referencePreviewsByLineage(attachments: ReferenceAttachmentRecord[] | undefined): Map<string, ReferencePreview> {
+  const previews = new Map<string, ReferencePreview>();
+  for (const attachment of attachments ?? []) {
+    if (attachment.role !== REFERENCE_PREVIEW_ROLE || !attachment.referenceLineageId) continue;
+    const metadata = record(attachment.metadata);
+    previews.set(attachment.referenceLineageId, {
+      imageUrl: httpAddress(attachment.sourceUri) ?? httpAddress(metadata.imageUrl),
+      description: text(metadata.description),
+      siteName: text(metadata.siteName),
+      title: text(metadata.title),
+    });
+  }
+  return previews;
+}
+
+/** The one-line dek: the source page's description, else a summary recorded on the Reference. */
+export function referenceDek(reference: ReferenceRecord, preview: ReferencePreview | undefined): string | null {
+  if (preview?.description) return preview.description;
+  const metadata = record(reference.metadata);
+  for (const key of ["subtitle", "summary", "abstract", "description"]) {
+    const value = text(metadata[key]);
+    if (value) return value;
+  }
+  return null;
+}
+
+/** The source's host without "www.", or null when the Reference has no web address. */
+export function referenceSourceHost(reference: ReferenceRecord): string | null {
+  const candidate = httpAddress(reference.sourceUri);
+  if (!candidate) return null;
+  try { return new URL(candidate).hostname.replace(/^www\./i, "") || null; } catch { return null; }
+}
+
+/** Short age for a card: "5m", "3h", "2d", "6w", "4mo". */
+export function formatFeedAge(time: number | null, now: number): string | null {
+  if (time === null) return null;
+  const seconds = Math.max(0, Math.round((now - time) / 1000));
+  if (seconds < 60) return "now";
+  const minutes = Math.round(seconds / 60);
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.round(minutes / 60);
+  if (hours < 24) return `${hours}h`;
+  const days = Math.round(hours / 24);
+  if (days < 14) return `${days}d`;
+  if (days < 60) return `${Math.round(days / 7)}w`;
+  return `${Math.round(days / 30)}mo`;
+}
+
+function words(text: string): string[] {
+  return text.toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N}]+/gu, " ").split(" ").filter((word) => word.length > 1);
+}
+
+/**
+ * Whether the dek tells the reader something the headline and picture do not. A dek that repeats the headline, or that is mostly
+ * the headline's own words, is dropped; so is the description of a GitHub repository, whose social-card picture already prints it.
+ */
+export function dekAddsInformation(title: string, dek: string | null, imageUrl: string | null): boolean {
+  if (!dek) return false;
+  let imageHost = "";
+  try { imageHost = imageUrl ? new URL(imageUrl).hostname : ""; } catch { /* no host */ }
+  if (/(^|\.)opengraph\.githubassets\.com$/i.test(imageHost)) return false;
+  const titleWords = new Set(words(title));
+  const dekWords = words(dek);
+  if (!dekWords.length) return false;
+  const shared = dekWords.filter((word) => titleWords.has(word)).length;
+  const covered = titleWords.size ? [...titleWords].filter((word) => dekWords.includes(word)).length / titleWords.size : 0;
+  return shared / dekWords.length < 0.6 && covered < 0.8;
+}
+
+/** Pictures that are a text card, not a photograph: a repository social card, or any picture whose ratio is almost exactly 2:1. Cropping cuts the words off. */
+export function isCardStyleImage(url: string | null, naturalRatio: number | null): boolean {
+  let host = "";
+  try { host = url ? new URL(url).hostname : ""; } catch { /* no host */ }
+  if (/(^|\.)(opengraph\.githubassets\.com|repository-images\.githubusercontent\.com|github\.com)$/i.test(host)) return true;
+  return naturalRatio !== null && Math.abs(naturalRatio - 2) < 0.04;
+}

package/package.json

@@ -140,12 +140,16 @@
     "./components/reference-cyclotron-priming": "./components/reference-cyclotron-priming.tsx",
     "./components/reference-cyclotron-status": "./components/reference-cyclotron-status.tsx",
     "./components/reference-cyclotron-suggestion": "./components/reference-cyclotron-suggestion.tsx",
+    "./components/reference-feed": "./components/reference-feed.tsx",
     "./components/reference-live-agreement": "./components/reference-live-agreement.tsx",
     "./components/reference-priming": "./components/reference-priming.tsx",
     "./components/reference-relevance-review": "./components/reference-relevance-review.tsx",
     "./components/reference-review-card": "./components/reference-review-card.tsx",
     "./components/reference-section-suggestion": "./components/reference-section-suggestion.tsx",
     "./components/reference-source-preview": "./components/reference-source-preview.tsx",
+    "./components/references-page-loading": "./components/references-page-loading.tsx",
+    "./components/references-prefetch": "./components/references-prefetch.ts",
+    "./components/review-rating-grid": "./components/review-rating-grid.tsx",
     "./components/settings-page": "./components/settings-page.tsx",
     "./components/signup-entries-panel": "./components/signup-entries-panel.tsx",
     "./components/staging-banner": "./components/staging-banner.tsx",
@@ -240,8 +244,10 @@
     "./lib/layout-scenarios": "./lib/layout-scenarios.ts",
     "./lib/layout-types": "./lib/layout-types.ts",
     "./lib/layout-warnings": "./lib/layout-warnings.ts",
+    "./lib/local-preview-feed-fixture": "./lib/local-preview-feed-fixture.ts",
     "./lib/local-preview-section-fixtures": "./lib/local-preview-section-fixtures.ts",
     "./lib/local-preview-seed": "./lib/local-preview-seed.ts",
+    "./lib/local-preview-synthetic-rejects": "./lib/local-preview-synthetic-rejects.ts",
     "./lib/local-preview-tenant": "./lib/local-preview-tenant.ts",
     "./lib/local-preview": "./lib/local-preview.ts",
     "./lib/markus-body": "./lib/markus-body.ts",
@@ -252,6 +258,7 @@
     "./lib/monitoring-client": "./lib/monitoring-client.ts",
     "./lib/monitoring-overview-fixture": "./lib/monitoring-overview-fixture.ts",
     "./lib/monitoring-overview": "./lib/monitoring-overview.ts",
+    "./lib/monitoring-paging": "./lib/monitoring-paging.ts",
     "./lib/news-desk-session": "./lib/news-desk-session.ts",
     "./lib/newsletter-api": "./lib/newsletter-api.ts",
     "./lib/newsletter-client": "./lib/newsletter-client.ts",
@@ -314,7 +321,10 @@
     "./lib/reader-storage-url": "./lib/reader-storage-url.ts",
     "./lib/reader-subject": "./lib/reader-subject.ts",
     "./lib/reference-display": "./lib/reference-display.ts",
+    "./lib/reference-feed-cache": "./lib/reference-feed-cache.ts",
+    "./lib/reference-feed-ranking": "./lib/reference-feed-ranking.ts",
     "./lib/reference-policy": "./lib/reference-policy.ts",
+    "./lib/reference-preview": "./lib/reference-preview.ts",
     "./lib/reference-review-details": "./lib/reference-review-details.ts",
     "./lib/reference-review-request": "./lib/reference-review-request.ts",
     "./lib/reference-source-preview": "./lib/reference-source-preview.ts",
@@ -470,7 +480,7 @@
       "optional": true
     }
   },
-  "version": "5.44.0",
+  "version": "5.45.0",
   "dependencies": {
     "@aws-amplify/adapter-nextjs": "^1.7.3",
     "@aws-crypto/sha256-js": "5.2.0",

package/routes.manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.44.0",
+  "version": "5.45.0",
   "routes": [
     {
       "file": "app/%5F_preview/[[...path]]/route.ts",