extract-cite 1.0.0 → 1.0.1

package/dist/llm/content-check.d.ts

@@ -0,0 +1,36 @@
+import { ContentCheck } from './types';
+export interface ContentInput {
+    /** The first `words` words of the content, for the prompt. */
+    text: string;
+    /** How many words that is. */
+    words: number;
+    /** One line per block of the page: selector, word count, opening words. */
+    outline: string;
+    /** The parsed page, to check the model's selectors against. */
+    document: any;
+}
+/** The first `max` words of `text`. */
+export declare function firstWords(text: string, max: number): string;
+/** HTML or text → plain text. */
+export declare function contentToText(content: string): string;
+/** `tag#id.class.class`, from parts that need no escaping. */
+export declare function selectorFor(el: any): string;
+/**
+ * The page's blocks as an indented list the model can pick selectors from:
+ * landmarks (`header`, `nav`, `aside`, `footer`, `main`, `article`, …) and
+ * elements with an id, class or role, with their word count and first words.
+ */
+export declare function pageOutline(document: any, maxLines?: number): string;
+/** What the model is shown for the content check. */
+export declare function prepareContent(html: string, options?: {
+    content?: string;
+    text?: string;
+    words?: number;
+}): ContentInput;
+/**
+ * Normalizes and checks the model's `contentCheck`: an unknown verdict becomes
+ * `"unknown"`, signals and examples not in the text are dropped, and selectors
+ * that are invalid or match nothing on the page are dropped.
+ * @param haystack the content and page text the model was shown
+ */
+export declare function parseContentCheck(reply: unknown, input: ContentInput, haystack: string): ContentCheck;

package/dist/llm/extract-cite-llm.d.ts

@@ -11,6 +11,8 @@ import { ExtractCiteLLMOptions, ExtractCiteLLMResult } from './types';
  * cite.formatted.apa?.text;  // full APA 7 entry
  * cite.needsReview;          // fields to check by hand
  * cite.citation.authors[0].qualifications; // bio facts, quoted from the page
+ * cite.contentCheck?.verdict; // "full", "paywalled", "truncated", …
+ * cite.contentCheck?.tips;    // header, nav, sidebar, footer blocks to cut
  * ```
  *
  * Throws when the page cannot be fetched or the model call fails; a missing

package/dist/llm/index.d.ts

@@ -4,4 +4,5 @@ export * from './extract-cite-llm';
 export * from './format-citation';
 export { preparePage, fetchPageHTML, clipText } from './page-text';
 export { parseCitationReply, normalizeDate } from './parse-reply';
-export { SYSTEM_PROMPT, buildUserPrompt, missingFields } from './prompt';
+export { SYSTEM_PROMPT, CONTENT_CHECK_PROMPT, systemPrompt, buildUserPrompt, missingFields } from './prompt';
+export { prepareContent, parseContentCheck, pageOutline, contentToText, firstWords } from './content-check';

package/dist/llm/prompt.d.ts

@@ -1,7 +1,15 @@
 import { ExtractCiteResult } from '../html-to-cite/extract-cite';
 import { CitationField } from './types';
 import { PageInput } from './page-text';
+import { ContentInput } from './content-check';
 export declare const SYSTEM_PROMPT: string;
+/**
+ * Appended to `SYSTEM_PROMPT` when the content check is on: `<content>` is the
+ * start of the extracted text, `<outline>` the page's blocks as selectors.
+ */
+export declare const CONTENT_CHECK_PROMPT: string;
+/** The system prompt, with the content check appended when it is on. */
+export declare function systemPrompt(checkContent?: boolean): string;
 /** Which candidate fields the regex pass filled, for the prompt. */
 export declare function describeCandidates(partial: ExtractCiteResult): Record<string, string | null>;
 /** The citation fields the regex pass cannot supply at all. */
@@ -11,4 +19,6 @@ export declare function buildUserPrompt(args: {
     partial: ExtractCiteResult;
     missing: CitationField[];
     page: PageInput;
+    /** The content check's input; leave out to skip the check. */
+    content?: ContentInput;
 }): string;

package/dist/llm/types.d.ts

@@ -55,11 +55,65 @@ export type CitationField = "authors" | "title" | "containerTitle" | "publisher"
 export declare const CITATION_FIELDS: CitationField[];
 /** A field (or one author's qualifications) a human should check. */
 export interface ReviewItem {
-    /** A citation field, or `qualifications:<author name>`. */
+    /** A citation field, `qualifications:<author name>`, or `content` when the content check found no full article. */
     field: string;
     confidence: number;
     reason: string;
 }
+/** What the content check concluded about the extracted text. */
+export type ContentVerdict = 
+/** The article body is all there. */
+"full"
+/** A subscribe or sign-in wall replaces or cuts the body. */
+ | "paywalled"
+/** The body stops early without a wall: a teaser, an abstract, "Read more". */
+ | "truncated"
+/** A bot check, consent wall, access-denied or error page instead of the article. */
+ | "blocked"
+/** A home page, index or listing, with no single article. */
+ | "not-article"
+/** The model gave no usable verdict. */
+ | "unknown";
+export declare const CONTENT_VERDICTS: ContentVerdict[];
+/** The kinds of clutter a readability tip can be about. */
+export type PageRegion = "header" | "nav" | "sidebar" | "footer" | "ads" | "related" | "newsletter" | "share" | "comments" | "cookie-banner" | "other";
+export declare const PAGE_REGIONS: PageRegion[];
+/** Clutter the extraction kept, and how to cut it. */
+export interface ReadabilityTip {
+    region: PageRegion;
+    /** CSS selector of the block to cut. Kept only when it matches an element on the page. */
+    selector?: string;
+    /** A short snippet of the clutter, kept only when it is in the text. */
+    example?: string;
+    /** How to cut it, in a sentence. */
+    tip: string;
+}
+/** Whether the extracted content is the full article, and how to make it cleaner. */
+export interface ContentCheck {
+    verdict: ContentVerdict;
+    /** True only for `"full"`. */
+    isFullContent: boolean;
+    /** 0–1. */
+    confidence: number;
+    /** Phrases from the text that show the verdict ("Subscribe to continue reading"); only ones that are in the text are kept. */
+    signals: string[];
+    /** One or two sentences for the reader. */
+    note: string;
+    /** CSS selector of the element that holds the article body. Kept only when it matches. */
+    contentSelector?: string;
+    /** Clutter the content still holds (header, nav, sidebar, footer, …), biggest first. */
+    tips: ReadabilityTip[];
+    /**
+     * The checked selectors in the shape of an `extract-webpage`
+     * `extract-selectors-per-domain.json` entry, ready to paste.
+     */
+    selectors: {
+        content: string[];
+        remove: string[];
+    };
+    /** Words of content the model was shown. */
+    wordsChecked: number;
+}
 export interface FormattedCitation {
     /** Plain text. */
     text: string;
@@ -82,6 +136,8 @@ export interface ExtractCiteLLMResult {
     origin: Partial<Record<CitationField, "regex" | "llm" | "both">>;
     /** One entry per requested style. */
     formatted: Partial<Record<CitationStyle, FormattedCitation>>;
+    /** Full article or paywall stub, plus tips for cutting clutter. Absent with `checkContent: false`. */
+    contentCheck?: ContentCheck;
     model: string;
     ms: number;
 }
@@ -104,6 +160,15 @@ export interface ExtractCiteLLMOptions {
     reviewThreshold?: number;
     /** Characters of page text sent to the model. Default 12000. */
     maxChars?: number;
+    /**
+     * The content extracted from the page (HTML or text), e.g. extract-webpage's
+     * `html`, for the content check. Default: the page's own visible text.
+     */
+    content?: string;
+    /** Check that the content is the full article and suggest what to cut. Default true. */
+    checkContent?: boolean;
+    /** Words of content the check is shown, from the start. Default 3000. */
+    contentWords?: number;
     /** Abort the model call after this many milliseconds. Default 60000. */
     timeoutMs?: number;
     /** ISO date printed as the access date. Default: today. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-cite",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "module": "./dist/extract-cite.es.js",
   "description": "Cite any webpage: regex/metadata extraction of author, date, title and source, plus an optional LLM pass (OpenRouter by default) that completes the APA parts, scores confidence per field, flags fields for review, extracts author qualifications, and formats APA, MLA, Chicago, Harvard, IEEE and BibTeX.",
   "author": "vtempest <grokthiscontact@gmail.com>",
@@ -58,4 +58,4 @@
     "author",
     "scraping"
   ]
-}
+}

package/src/llm/content-check.ts

@@ -0,0 +1,187 @@
+/**
+ * @fileoverview The content check that rides along with the citation call:
+ * the first few thousand words of the extracted content and an outline of the
+ * page's blocks go to the model, which says whether this is the full article or
+ * a paywall stub, and which header, nav, sidebar and footer blocks to cut. The
+ * reply is checked like the citation: quoted phrases must be in the text and
+ * selectors must match an element on the page, or they are dropped.
+ */
+import { parseHTML } from "linkedom";
+import { appearsIn, clamp01 } from "./parse-reply";
+import {
+  CONTENT_VERDICTS,
+  PAGE_REGIONS,
+  type ContentCheck,
+  type ContentVerdict,
+  type PageRegion,
+  type ReadabilityTip,
+} from "./types";
+
+type Json = Record<string, any>;
+
+const REMOVE = "script,style,noscript,svg,template,iframe,canvas";
+const LANDMARK = /^(header|nav|aside|footer|main|article|section|form)$/;
+/** An id or class usable in a selector without escaping. */
+const IDENT = /^[A-Za-z_-][\w-]*$/;
+
+export interface ContentInput {
+  /** The first `words` words of the content, for the prompt. */
+  text: string;
+  /** How many words that is. */
+  words: number;
+  /** One line per block of the page: selector, word count, opening words. */
+  outline: string;
+  /** The parsed page, to check the model's selectors against. */
+  document: any;
+}
+
+const countWords = (text: string) => (text.match(/\S+/g) || []).length;
+
+/** The first `max` words of `text`. */
+export function firstWords(text: string, max: number): string {
+  const words = text.match(/\S+/g) || [];
+  return words.length <= max ? words.join(" ") : `${words.slice(0, max).join(" ")} […]`;
+}
+
+/** A node's text with a space between elements, so "Home</a><a>News" does not run together. */
+function textOf(node: any): string {
+  if (node.nodeType === 3) return node.data || "";
+  if (node.nodeType !== 1 || node.matches?.(REMOVE)) return "";
+  return Array.from(node.childNodes || []).map(textOf).join(" ");
+}
+
+const squash = (text: string) => text.replace(/\s+/g, " ").trim();
+
+/** HTML or text → plain text. */
+export function contentToText(content: string): string {
+  if (!/<[a-z!/][^>]*>/i.test(content)) return squash(content);
+  const { document } = parseHTML(`<html><body>${content}</body></html>`);
+  return document.body ? squash(textOf(document.body)) : "";
+}
+
+/** `tag#id.class.class`, from parts that need no escaping. */
+export function selectorFor(el: any): string {
+  const tag = String(el.tagName || "").toLowerCase();
+  const id = el.getAttribute?.("id");
+  if (id && IDENT.test(id)) return `${tag}#${id}`;
+  const classes = String(el.getAttribute?.("class") || "")
+    .split(/\s+/)
+    .filter((c) => IDENT.test(c))
+    .slice(0, 2);
+  const role = el.getAttribute?.("role");
+  const roleAttr = role && IDENT.test(role) ? `[role="${role}"]` : "";
+  return `${tag}${classes.map((c) => `.${c}`).join("")}${roleAttr}`;
+}
+
+/**
+ * The page's blocks as an indented list the model can pick selectors from:
+ * landmarks (`header`, `nav`, `aside`, `footer`, `main`, `article`, …) and
+ * elements with an id, class or role, with their word count and first words.
+ */
+export function pageOutline(document: any, maxLines = 60): string {
+  const lines: string[] = [];
+  const walk = (parent: any, depth: number, level: number) => {
+    for (const el of Array.from(parent.children || []) as any[]) {
+      if (lines.length >= maxLines) return;
+      const tag = String(el.tagName || "").toLowerCase();
+      if (el.matches?.(REMOVE)) continue;
+      const text = squash(textOf(el));
+      const words = countWords(text);
+      if (words < 3) continue;
+      // A bare `div` says nothing and would match every div on the page.
+      const selector = selectorFor(el);
+      const named = LANDMARK.test(tag) || selector !== tag;
+      if (named) {
+        const opening = text.split(" ").slice(0, 8).join(" ");
+        lines.push(`${"  ".repeat(level)}${selector} (${words} words) "${opening}"`);
+      }
+      if (depth < 8 && level < 4) walk(el, depth + 1, named ? level + 1 : level);
+    }
+  };
+  if (document.body) walk(document.body, 0, 0);
+  return lines.join("\n");
+}
+
+/** What the model is shown for the content check. */
+export function prepareContent(
+  html: string,
+  options: { content?: string; text?: string; words?: number } = {}
+): ContentInput {
+  const { document } = parseHTML(html);
+  const source = options.content ?? options.text;
+  const full = source !== undefined ? contentToText(source) : document.body ? squash(textOf(document.body)) : "";
+  const text = firstWords(full, options.words ?? 3000);
+  return { text, words: Math.min(countWords(full), options.words ?? 3000), outline: pageOutline(document), document };
+}
+
+/** Bare tags that are safe to cut whole: every one of them is page chrome. */
+const CHROME_TAG = /^(header|nav|aside|footer)$/i;
+
+/** Is `selector` valid and does it match something on the page? */
+function matches(document: any, selector: string): boolean {
+  try {
+    return Boolean(document.querySelector(selector));
+  } catch {
+    return false;
+  }
+}
+
+/** A trimmed, tag-free string, capped. */
+function str(value: unknown, max: number): string | undefined {
+  if (typeof value !== "string") return undefined;
+  const clean = value.replace(/<[^>]*>/g, "").replace(/\s+/g, " ").trim();
+  if (!clean || /^(null|none|n\/a)$/i.test(clean)) return undefined;
+  return clean.slice(0, max);
+}
+
+/**
+ * Normalizes and checks the model's `contentCheck`: an unknown verdict becomes
+ * `"unknown"`, signals and examples not in the text are dropped, and selectors
+ * that are invalid or match nothing on the page are dropped.
+ * @param haystack the content and page text the model was shown
+ */
+export function parseContentCheck(reply: unknown, input: ContentInput, haystack: string): ContentCheck {
+  const r = (reply && typeof reply === "object" ? reply : {}) as Json;
+  const verdict: ContentVerdict = CONTENT_VERDICTS.includes(r.verdict) ? r.verdict : "unknown";
+  const selector = (value: unknown, bareOk: RegExp) => {
+    const s = str(value, 200);
+    if (!s || !matches(input.document, s)) return undefined;
+    // A bare tag like `div` or `section` would cut (or keep) far too much.
+    return /^[a-z][a-z0-9]*$/i.test(s) && !bareOk.test(s) ? undefined : s;
+  };
+
+  const signals = (Array.isArray(r.signals) ? r.signals : [])
+    .map((s: unknown) => str(s, 160))
+    .filter((s: string | undefined): s is string => Boolean(s) && appearsIn(s, haystack))
+    .slice(0, 5);
+
+  const tips: ReadabilityTip[] = [];
+  for (const raw of Array.isArray(r.tips) ? r.tips.slice(0, 12) : []) {
+    if (!raw || typeof raw !== "object") continue;
+    const tip = str(raw.tip, 300);
+    if (!tip) continue;
+    const example = str(raw.example, 160);
+    tips.push({
+      region: PAGE_REGIONS.includes(raw.region) ? (raw.region as PageRegion) : "other",
+      selector: selector(raw.selector, CHROME_TAG),
+      example: example && appearsIn(example, haystack) ? example : undefined,
+      tip,
+    });
+    if (tips.length === 8) break;
+  }
+
+  const contentSelector = selector(r.contentSelector, /^(main|article)$/i);
+  const remove = [...new Set(tips.map((t) => t.selector).filter(Boolean) as string[])];
+
+  return {
+    verdict,
+    isFullContent: verdict === "full",
+    confidence: verdict === "unknown" ? 0 : clamp01(r.confidence),
+    signals,
+    note: str(r.note, 400) || (verdict === "unknown" ? "The model did not check the content." : ""),
+    contentSelector,
+    tips,
+    selectors: { content: contentSelector ? [contentSelector] : [], remove },
+    wordsChecked: input.words,
+  };
+}

package/src/llm/extract-cite-llm.ts

@@ -1,15 +1,18 @@
 /**
  * @fileoverview `extractCiteLLM`: the cheap regex/metadata pass first, then one
  * model call that is handed what was found and what is still missing, completes
- * the APA parts, scores each one, and reads the authors' qualifications.
+ * the APA parts, scores each one, and reads the authors' qualifications. The
+ * same call checks that the extracted content is the full article, not a
+ * paywall stub, and names the header, nav, sidebar and footer blocks to cut.
  */
 import { parseHTML } from "linkedom";
 import { extractCite } from "../html-to-cite/extract-cite";
 import { callLLM } from "./call-llm";
+import { parseContentCheck, prepareContent } from "./content-check";
 import { formatCitations } from "./format-citation";
 import { fetchPageHTML, preparePage } from "./page-text";
 import { parseCitationReply } from "./parse-reply";
-import { buildUserPrompt, missingFields, SYSTEM_PROMPT } from "./prompt";
+import { buildUserPrompt, missingFields, systemPrompt } from "./prompt";
 import {
   CITATION_STYLES,
   type CitationField,
@@ -29,6 +32,8 @@ import {
  * cite.formatted.apa?.text;  // full APA 7 entry
  * cite.needsReview;          // fields to check by hand
  * cite.citation.authors[0].qualifications; // bio facts, quoted from the page
+ * cite.contentCheck?.verdict; // "full", "paywalled", "truncated", …
+ * cite.contentCheck?.tips;    // header, nav, sidebar, footer blocks to cut
  * ```
  *
  * Throws when the page cannot be fetched or the model call fails; a missing
@@ -36,7 +41,7 @@ import {
  */
 export async function extractCiteLLM(options: ExtractCiteLLMOptions): Promise<ExtractCiteLLMResult> {
   const started = Date.now();
-  const { url, reviewThreshold = 0.7, styles = [...CITATION_STYLES] } = options;
+  const { url, reviewThreshold = 0.7, styles = [...CITATION_STYLES], checkContent = true } = options;
 
   let html = options.html;
   if (!html && options.text === undefined) {
@@ -51,6 +56,9 @@ export async function extractCiteLLM(options: ExtractCiteLLMOptions): Promise<Ex
 
   // 2. One model call with the partial in hand.
   const page = preparePage(html, { text: options.text, maxChars: options.maxChars });
+  const content = checkContent
+    ? prepareContent(html, { content: options.content, text: options.text, words: options.contentWords })
+    : undefined;
   const { json, model } = await callLLM({
     apiKey: options.apiKey,
     model: options.model,
@@ -58,8 +66,8 @@ export async function extractCiteLLM(options: ExtractCiteLLMOptions): Promise<Ex
     headers: options.headers,
     timeoutMs: options.timeoutMs,
     fetch: options.fetch,
-    system: SYSTEM_PROMPT,
-    user: buildUserPrompt({ url, partial, missing, page }),
+    system: systemPrompt(checkContent),
+    user: buildUserPrompt({ url, partial, missing, page, content }),
   });
 
   // 3. Validate it against the page and score it.
@@ -71,6 +79,24 @@ export async function extractCiteLLM(options: ExtractCiteLLMOptions): Promise<Ex
     reviewThreshold
   );
 
+  // 4. The content check, held to the same standard: quotes must be in the
+  // text, selectors must match the page.
+  const contentCheck = content
+    ? parseContentCheck(
+        (json as { contentCheck?: unknown } | null)?.contentCheck,
+        content,
+        [content.text, page.text, page.pageTitle].join("\n")
+      )
+    : undefined;
+  if (contentCheck && !contentCheck.isFullContent && contentCheck.verdict !== "unknown") {
+    parsed.needsReview.push({
+      field: "content",
+      confidence: contentCheck.confidence,
+      reason: `${contentCheck.verdict}: ${contentCheck.note || contentCheck.signals[0] || "not the full article"}`,
+    });
+    parsed.needsReview.sort((a, b) => a.confidence - b.confidence);
+  }
+
   // `agree` means the regex pass and the model read the same value; anything
   // else the model supplied, because the regex pass had nothing or was overruled.
   const origin: ExtractCiteLLMResult["origin"] = {};
@@ -92,6 +118,7 @@ export async function extractCiteLLM(options: ExtractCiteLLMOptions): Promise<Ex
     missing,
     origin,
     formatted: formatCitations(parsed.citation, styles),
+    contentCheck,
     model,
     ms: Date.now() - started,
   };

package/src/llm/index.ts

@@ -4,4 +4,5 @@ export * from "./extract-cite-llm";
 export * from "./format-citation";
 export { preparePage, fetchPageHTML, clipText } from "./page-text";
 export { parseCitationReply, normalizeDate } from "./parse-reply";
-export { SYSTEM_PROMPT, buildUserPrompt, missingFields } from "./prompt";
+export { SYSTEM_PROMPT, CONTENT_CHECK_PROMPT, systemPrompt, buildUserPrompt, missingFields } from "./prompt";
+export { prepareContent, parseContentCheck, pageOutline, contentToText, firstWords } from "./content-check";

package/src/llm/prompt.ts

@@ -1,11 +1,13 @@
 /**
  * @fileoverview The prompt: the regex pass's partial citation goes in as
  * candidates to verify, the still-missing parts are named, and the page text
- * is fenced as untrusted data.
+ * is fenced as untrusted data. With the content check on, the same call also
+ * judges whether the extracted content is the full article and what to cut.
  */
 import type { ExtractCiteResult } from "../html-to-cite/extract-cite";
-import { CITATION_FIELDS, SOURCE_TYPES, type CitationField } from "./types";
+import { CITATION_FIELDS, CONTENT_VERDICTS, PAGE_REGIONS, SOURCE_TYPES, type CitationField } from "./types";
 import type { PageInput } from "./page-text";
+import type { ContentInput } from "./content-check";
 
 export const SYSTEM_PROMPT = `You build bibliographic citations from web pages for students and researchers. A wrong author or date is worse than a missing one, so never guess.
 
@@ -44,6 +46,37 @@ Reply with one JSON object and nothing else:
   }
 }`;
 
+/**
+ * Appended to `SYSTEM_PROMPT` when the content check is on: `<content>` is the
+ * start of the extracted text, `<outline>` the page's blocks as selectors.
+ */
+export const CONTENT_CHECK_PROMPT = `
+Also check the content. <content> is the start of the text extracted from this page as the article (or the page's own text when nothing was extracted), and <outline> lists the page's blocks as CSS selectors with their word counts and opening words. Readers will get this text as the article, so say whether it is the whole article and what clutter to cut.
+
+- verdict, one of: ${CONTENT_VERDICTS.filter((v) => v !== "unknown").join(", ")}.
+  "full": the article body is there, from its opening to its end, or it runs past the sample with no sign of stopping. A short article is still full.
+  "paywalled": a subscribe, sign-in or register wall replaces or cuts off the body, usually a teaser paragraph and then a prompt to pay or log in.
+  "truncated": the body stops early with no wall: a "Read more" or "Continue reading" link, a cut mid-sentence, or only an abstract or summary where the page promises more.
+  "blocked": a bot check, captcha, cookie or consent wall, access-denied or error page instead of the article.
+  "not-article": a home page, index, search results or listing, not one article.
+  Judge only by what the text shows, not by what you know about the site.
+- signals: up to 5 short phrases copied exactly from the text that show the verdict ("Subscribe to continue reading", "Already a subscriber? Sign in"). Empty when the verdict is full.
+- tips: the clutter in <content> that is not the article, so it can be cut for readability: the site header and top bar, navigation menus, sidebars, footers, ads, related-article lists, newsletter and subscribe boxes, share buttons, comments, cookie banners. For each: region (one of ${PAGE_REGIONS.join(", ")}), the selector from <outline> that holds it (copy it exactly; null if no outline line fits, never invent one), a short example copied exactly from the text, and a one-sentence tip on how to cut it. Biggest first, at most 8. None when the content is clean.
+- contentSelector: the selector from <outline> of the element that holds the article body, or null.
+- note: one or two plain sentences for the reader: is this the full article, and if not, what is missing.
+
+Add this key to the JSON object:
+  "contentCheck": {
+    "verdict": string, "confidence": number, "signals": string[], "note": string,
+    "contentSelector": string|null,
+    "tips": [ { "region": string, "selector": string|null, "example": string|null, "tip": string } ]
+  }`;
+
+/** The system prompt, with the content check appended when it is on. */
+export function systemPrompt(checkContent = true): string {
+  return checkContent ? SYSTEM_PROMPT + CONTENT_CHECK_PROMPT : SYSTEM_PROMPT;
+}
+
 /** Which candidate fields the regex pass filled, for the prompt. */
 export function describeCandidates(partial: ExtractCiteResult): Record<string, string | null> {
   return {
@@ -72,15 +105,17 @@ export function missingFields(partial: ExtractCiteResult): CitationField[] {
 }
 
 /** Stops page text from closing the fence it is wrapped in. */
-const unfence = (text: string) => text.replace(/<\/?(page|text|meta|json-ld|title)\b[^>]*>/gi, "");
+const unfence = (text: string) => text.replace(/<\/?(page|text|meta|json-ld|title|content|outline)\b[^>]*>/gi, "");
 
 export function buildUserPrompt(args: {
   url?: string;
   partial: ExtractCiteResult;
   missing: CitationField[];
   page: PageInput;
+  /** The content check's input; leave out to skip the check. */
+  content?: ContentInput;
 }): string {
-  const { url, partial, missing, page } = args;
+  const { url, partial, missing, page, content } = args;
   const parts = [
     `URL: ${url || "(none given)"}`,
     `Candidates from the regex pass: ${JSON.stringify(describeCandidates(partial))}`,
@@ -90,6 +125,13 @@ export function buildUserPrompt(args: {
   ];
   if (page.meta) parts.push(`<meta>\n${unfence(page.meta)}\n</meta>`);
   if (page.jsonLd) parts.push(`<json-ld>\n${unfence(page.jsonLd)}\n</json-ld>`);
-  parts.push(`<text>\n${unfence(page.text)}\n</text>`, "</page>");
+  parts.push(`<text>\n${unfence(page.text)}\n</text>`);
+  if (content) {
+    parts.push(
+      `<content words="${content.words}">\n${unfence(content.text)}\n</content>`,
+      `<outline>\n${unfence(content.outline) || "(no blocks with an id, class or role)"}\n</outline>`
+    );
+  }
+  parts.push("</page>");
   return parts.join("\n");
 }