@relvehq/agent-ready 0.4.1 → 0.4.3

package/README.md

@@ -18,7 +18,7 @@ they got before.
 On a real tool page: raw HTML 97,944 tokens, a generic converter 4,618, a
 refined file 1,695.
 
-**Start here:** the onboarding guide at <https://relvehq.com/agent-ready>
+**Start here:** the onboarding guide at <https://engine.relve.ai/search/agent-ready>
 (every stack, the WordPress plugin download, and how to test it).
 
 ## Requirements
@@ -135,11 +135,18 @@ choose what to serve: that would be cloaking.
    1. a **template** for the route (`"/blog/:slug"`), written from your data
    2. a **stored** file (`store.get(path)`), e.g. one written by `refine`
    3. **convert**: fetch the page's own HTML (public, no cookies) and turn its
-      main content into Markdown (Readability + turndown, GFM tables). Under 200
-      characters of real text is discarded.
+      main content into Markdown (Readability + turndown, GFM tables).
+      Readability is used only when it keeps most of `<main>`'s text, numbers
+      and tables; otherwise `<main>` is converted whole. Closed accordion
+      answers are kept, loading placeholders ("Loading…", skeletons) are
+      dropped, and FAQ answers the HTML lacks are filled in word for word from
+      the page's FAQPage JSON-LD, under the page's own FAQ heading (never added
+      to a page that shows no FAQ). Under 200 characters of real text is
+      discarded.
    4. nothing usable: an `Accept` request gets the normal HTML page, a `.md`
       request gets a 404
-4. **Enrich**: YAML front matter (title, canonical, source), facts from JSON-LD, token count.
+4. **Enrich**: YAML front matter (title, description, canonical, source), facts from JSON-LD, token count.
+   A doc that brings its own front matter keeps it; only a missing title or description is added.
 5. **Respond**:
 
 ```
@@ -367,6 +374,7 @@ if (d.markdown) {
 | `internalPrefix` | `/__md` | rewrite target |
 | `fetchTimeoutMs` | `8000` | page fetch for convert |
 | `minChars` | `200` | quality gate for convert |
+| `maxHtmlBytes` | `5242880` (5 MB) | largest page HTML convert reads; raise it for one very large page |
 
 ## Commands
 

package/SETUP.md

@@ -1,6 +1,6 @@
 # Setting up @relvehq/agent-ready
 
-A step-by-step guide for the developer who adds this package to a site. The onboarding page with the same steps, the WordPress plugin download and a test block: https://relvehq.com/agent-ready
+A step-by-step guide for the developer who adds this package to a site. The onboarding page with the same steps, the WordPress plugin download and a test block: https://engine.relve.ai/search/agent-ready
 
 ## The one command
 

package/dist/core/config.d.ts

@@ -107,6 +107,8 @@ export interface AgentReadyConfig {
     fetchTimeoutMs?: number;
     /** Quality gate: converted Markdown with less plain text than this is discarded. Default 200. */
     minChars?: number;
+    /** Largest page HTML the converter reads; bigger pages get no Markdown. Default 5 MB. */
+    maxHtmlBytes?: number;
 }
 export interface ResolvedAgentReadyConfig {
     readonly __resolved: true;
@@ -127,6 +129,7 @@ export interface ResolvedAgentReadyConfig {
     fetchOrigin: ((request: Request) => string) | null;
     fetchTimeoutMs: number;
     minChars: number;
+    maxHtmlBytes: number;
 }
 /** Never served as Markdown, whatever the venture's config says. */
 export declare const ALWAYS_EXCLUDE: readonly string[];
@@ -134,6 +137,8 @@ export declare const DEFAULT_CACHE_CONTROL = "public, s-maxage=300, stale-while-
 export declare const DEFAULT_NEGOTIATED_CACHE_CONTROL = "private, max-age=300";
 export declare const DEFAULT_INTERNAL_PREFIX = "/__md";
 export declare const DEFAULT_INTERNAL_HEADER = "x-agent-ready-internal";
+/** Default cap on the page HTML the converter reads (maxHtmlBytes). */
+export declare const DEFAULT_MAX_HTML_BYTES: number;
 /** Request header the middleware adapter sets on its rewrite: how Markdown was asked for. */
 export declare const VIA_HEADER = "x-agent-ready-via";
 export declare function defineAgentReady(config: AgentReadyConfig | ResolvedAgentReadyConfig): ResolvedAgentReadyConfig;

package/dist/core/config.js

@@ -7,7 +7,7 @@
 //
 // EDGE-SAFE. No Node APIs. Middleware bundles import this module.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VIA_HEADER = exports.DEFAULT_INTERNAL_HEADER = exports.DEFAULT_INTERNAL_PREFIX = exports.DEFAULT_NEGOTIATED_CACHE_CONTROL = exports.DEFAULT_CACHE_CONTROL = exports.ALWAYS_EXCLUDE = void 0;
+exports.VIA_HEADER = exports.DEFAULT_MAX_HTML_BYTES = exports.DEFAULT_INTERNAL_HEADER = exports.DEFAULT_INTERNAL_PREFIX = exports.DEFAULT_NEGOTIATED_CACHE_CONTROL = exports.DEFAULT_CACHE_CONTROL = exports.ALWAYS_EXCLUDE = void 0;
 exports.defineAgentReady = defineAgentReady;
 exports.isEnabled = isEnabled;
 exports.isExcluded = isExcluded;
@@ -30,6 +30,8 @@ exports.DEFAULT_CACHE_CONTROL = "public, s-maxage=300, stale-while-revalidate=86
 exports.DEFAULT_NEGOTIATED_CACHE_CONTROL = "private, max-age=300";
 exports.DEFAULT_INTERNAL_PREFIX = "/__md";
 exports.DEFAULT_INTERNAL_HEADER = "x-agent-ready-internal";
+/** Default cap on the page HTML the converter reads (maxHtmlBytes). */
+exports.DEFAULT_MAX_HTML_BYTES = 5 * 1024 * 1024;
 /** Request header the middleware adapter sets on its rewrite: how Markdown was asked for. */
 exports.VIA_HEADER = "x-agent-ready-via";
 function defineAgentReady(config) {
@@ -57,6 +59,7 @@ function defineAgentReady(config) {
         fetchOrigin: c.fetchOrigin ?? null,
         fetchTimeoutMs: c.fetchTimeoutMs ?? 8000,
         minChars: c.minChars ?? 200,
+        maxHtmlBytes: c.maxHtmlBytes && c.maxHtmlBytes > 0 ? c.maxHtmlBytes : exports.DEFAULT_MAX_HTML_BYTES,
     };
 }
 /** The master switch. A throwing `enabled` function counts as off. */

package/dist/core/convert.d.ts

@@ -19,4 +19,21 @@ export declare function htmlToMarkdown(html: string, opts: ConvertOptions): Conv
  * Scripts never run here, so do the swap ourselves before [hidden] is stripped.
  */
 export declare function unfoldNextStreaming(doc: El): void;
+/**
+ * Accordions and FAQ toggles: the question is a <button> wired to its panel by
+ * aria-controls, and a closed panel is [hidden] (or rendered empty). Show the
+ * panel and keep the question as text, before buttons and [hidden] are
+ * stripped. Tabs, popups and menus are left alone, and so is a panel that is a
+ * menu, listbox or dialog or holds navigation; a trigger whose panel isn't on
+ * the page is left too. Hidden elements whose class names an accordion or FAQ
+ * are shown as well.
+ */
+export declare function openDisclosures(doc: El): void;
+/**
+ * Loading placeholders: busy regions, progress bars, skeleton / shimmer /
+ * spinner / loader elements (small, no media), and blocks whose whole text is
+ * a loading message ("Loading reviews…"). Real text that only starts with
+ * "Loading" stays: the message needs an ellipsis, or to be the bare word.
+ */
+export declare function dropPlaceholders(doc: El): void;
 export {};

package/dist/core/convert.js

@@ -4,13 +4,17 @@
 //   1. Read metadata (title, description, lang, canonical, JSON-LD) from <head>.
 //   2. Unfold Next.js streamed Suspense content (<div hidden id="S:n">) into
 //      its placeholder, so App Router pages convert with their real content.
-//   3. Strip chrome: script/style/nav/footer/forms/dialogs/cookie banners.
-//   4. Pick the main content: Mozilla Readability, unless it returns too little
-//      compared with <main>/<article>/<body>, which are used instead.
-//   5. Absolutize links and images, flatten block-level links, give header-less
+//   3. Open accordions (keep the question, show the panel) and drop loading
+//      placeholders, before buttons and [hidden] go.
+//   4. Strip chrome: script/style/nav/footer/forms/dialogs/cookie banners.
+//   5. Pick the main content: Mozilla Readability, unless it returns too little
+//      of <main>/<article>/<body> (text, numbers or tables), which are used instead.
+//   6. Absolutize links and images, flatten block-level links, give header-less
 //      tables a header row so GFM can render them.
-//   6. Turndown + GFM (tables, strikethrough, task lists), then collapse whitespace.
-//   7. Quality gate: under `minChars` of plain text -> null (caller serves HTML).
+//   7. Turndown + GFM (tables, strikethrough, task lists), then collapse whitespace.
+//   8. FAQ answers the HTML lacks (closed accordion panels are often empty)
+//      are filled from the FAQPage JSON-LD, under the page's own FAQ heading.
+//   9. Quality gate: under `minChars` of plain text -> null (caller serves HTML).
 //
 // NODE ONLY (linkedom, turndown). Never import this from middleware.
 var __importDefault = (this && this.__importDefault) || function (mod) {
@@ -19,10 +23,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.htmlToMarkdown = htmlToMarkdown;
 exports.unfoldNextStreaming = unfoldNextStreaming;
+exports.openDisclosures = openDisclosures;
+exports.dropPlaceholders = dropPlaceholders;
 const linkedom_1 = require("linkedom");
 const readability_1 = require("@mozilla/readability");
 const turndown_1 = __importDefault(require("turndown"));
 const turndown_plugin_gfm_1 = require("turndown-plugin-gfm");
+const faq_1 = require("./faq");
 const JUNK = [
     "script",
     "style",
@@ -57,6 +64,19 @@ const JUNK = [
 ].join(",");
 const COOKIE_RE = /(^|[\s_-])(cookie|cookies|consent|gdpr|onetrust|cookiebot|cc-window|cc-banner|cmp)([\s_-]|$)/i;
 const BLOCK = "div,p,section,article,header,ul,ol,li,table,h1,h2,h3,h4,h5,h6,blockquote,pre,figure";
+// Accordions (changes/agent-ready-quality).
+const DISCLOSURE_CLASS_RE = /(^|[\s_-])(accordion|faqs?)([\s_-]|$)/i;
+const NOT_A_DISCLOSURE_ROLE = /^(tab|menuitem|combobox|option)$/i;
+const NOT_A_PANEL_ROLE = /^(menu|menubar|listbox|dialog|alertdialog|navigation|tree|grid)$/i;
+// Expander labels are controls, not content: their panel opens, the label goes.
+const CONTROL_LABEL_RE = /^(?:(?:show|read|view|see|load)\s+(?:more|less|all)|expand|collapse|toggle|open|close)\b/i;
+// Loading placeholders.
+const PLACEHOLDER_RE = /(^|[\s_-])(skeleton|shimmer|spinner|loader|loading)([\s_-]|$)|(^|\s)animate-(pulse|spin)(\s|$)/i;
+const LOADING_TEXT_RE = /^(?:loading|fetching|please wait)\b[^.!?]{0,40}(?:…|\.{3})$|^(?:loading|fetching|please wait)\.?$/i;
+const PLACEHOLDER_MAX_TEXT = 200;
+// Data-heavy pages: Readability must keep this share of <main>'s numbers and tables.
+const KEEP_DATA_SHARE = 0.8;
+const MIN_NUMBERS_TO_JUDGE = 10;
 let turndown = null;
 function getTurndown() {
     if (turndown)
@@ -82,6 +102,15 @@ function getTurndown() {
             return (firstCell ? "| " : " ") + text + " |";
         },
     });
+    // A <summary> that is only text is a question or a label: keep it bold. One
+    // holding a heading renders as that heading.
+    td.addRule("agentReadySummary", {
+        filter: (node) => node.nodeName === "SUMMARY" && !node.querySelector("h1, h2, h3, h4, h5, h6"),
+        replacement(content) {
+            const text = content.replace(/\s+/g, " ").trim();
+            return text ? `\n\n**${text}**\n\n` : "";
+        },
+    });
     td.remove(["script", "style", "noscript", "iframe", "button", "form", "svg"]);
     turndown = td;
     return td;
@@ -95,6 +124,8 @@ function htmlToMarkdown(html, opts) {
     const meta = readMeta(document, pageUrl);
     const firstH1 = textOf(document.querySelector("h1"));
     unfoldNextStreaming(document);
+    openDisclosures(document);
+    dropPlaceholders(document);
     stripChrome(document);
     const root = document.querySelector("main") || document.querySelector("article") || document.body || document.documentElement;
     const rootStrategy = document.querySelector("main")
@@ -112,8 +143,14 @@ function htmlToMarkdown(html, opts) {
         const article = new readability_1.Readability(rdoc, { charThreshold: minChars, keepClasses: false }).parse();
         const articleText = textLength(article?.textContent ?? "");
         // Readability is great on articles but can keep a fraction of a listing or
-        // profile page. Use it only when it kept most of what <main> holds.
-        if (article && article.content && articleText >= minChars && articleText >= rootText * 0.6) {
+        // profile page. Use it only when it kept most of what <main> holds: its
+        // text, and on data-heavy pages its numbers and tables (a tool page kept
+        // 65-87% of the text but 20-70% of the prices, traffic and scores).
+        if (article &&
+            article.content &&
+            articleText >= minChars &&
+            articleText >= rootText * 0.6 &&
+            keepsTheData(root, article.content, article.textContent ?? "")) {
             contentHtml = article.content;
             strategy = "readability";
             readabilityTitle = article.title || undefined;
@@ -131,6 +168,9 @@ function htmlToMarkdown(html, opts) {
     const heading = firstH1 || readabilityTitle || meta.title;
     if (heading && !/^#\s/m.test(markdown))
         markdown = `# ${oneLine(heading)}\n\n${markdown}`;
+    const faq = (0, faq_1.faqPairs)(meta.jsonLd ?? []);
+    if (faq.length)
+        markdown = (0, faq_1.fillFaqAnswers)(markdown, faq, (pair) => faqAnswerMarkdown(pair, pageUrl));
     const textLen = plainTextLength(markdown);
     if (!markdown || textLen < minChars)
         return null;
@@ -186,6 +226,96 @@ function unfoldNextStreaming(doc) {
         segment.parentNode.removeChild(segment);
     }
 }
+/**
+ * Accordions and FAQ toggles: the question is a <button> wired to its panel by
+ * aria-controls, and a closed panel is [hidden] (or rendered empty). Show the
+ * panel and keep the question as text, before buttons and [hidden] are
+ * stripped. Tabs, popups and menus are left alone, and so is a panel that is a
+ * menu, listbox or dialog or holds navigation; a trigger whose panel isn't on
+ * the page is left too. Hidden elements whose class names an accordion or FAQ
+ * are shown as well.
+ */
+function openDisclosures(doc) {
+    for (const trigger of Array.from(doc.querySelectorAll("[aria-controls][aria-expanded]"))) {
+        if (trigger.hasAttribute("aria-haspopup") || NOT_A_DISCLOSURE_ROLE.test(trigger.getAttribute("role") || ""))
+            continue;
+        const panels = String(trigger.getAttribute("aria-controls"))
+            .split(/\s+/)
+            .filter(Boolean)
+            .map((id) => byId(doc, id))
+            .filter(Boolean);
+        if (!panels.length)
+            continue;
+        const notAPanel = (p) => NOT_A_PANEL_ROLE.test(p.getAttribute("role") || "") ||
+            String(p.nodeName).toLowerCase() === "nav" ||
+            !!p.querySelector("nav, [role=navigation], [role=menu]");
+        if (panels.some(notAPanel))
+            continue;
+        panels.forEach(show);
+        keepTriggerText(doc, trigger);
+    }
+    for (const el of Array.from(doc.querySelectorAll("[class]"))) {
+        if (DISCLOSURE_CLASS_RE.test(el.getAttribute("class") || ""))
+            show(el);
+    }
+}
+function byId(doc, id) {
+    if (doc.getElementById)
+        return doc.getElementById(id);
+    return doc.querySelector(`[id="${id.replace(/["\\]/g, "\\$&")}"]`);
+}
+function show(el) {
+    el.removeAttribute("hidden");
+    if (String(el.getAttribute("aria-hidden")).toLowerCase() === "true")
+        el.removeAttribute("aria-hidden");
+    const style = el.getAttribute("style");
+    if (style)
+        el.setAttribute("style", style.replace(/(?:display\s*:\s*none|visibility\s*:\s*hidden)\s*;?/gi, ""));
+}
+/** A question <button> becomes its text: plain inside a heading, else a bold paragraph. */
+function keepTriggerText(doc, trigger) {
+    if (String(trigger.nodeName).toLowerCase() !== "button" || !trigger.parentNode)
+        return;
+    for (const icon of Array.from(trigger.querySelectorAll("svg, [aria-hidden=true]")))
+        remove(icon);
+    const text = oneLine(trigger.textContent || "");
+    if (!text || CONTROL_LABEL_RE.test(text))
+        return;
+    if (trigger.closest && trigger.closest("h1, h2, h3, h4, h5, h6, summary, dt")) {
+        trigger.parentNode.replaceChild(doc.createTextNode(text), trigger);
+        return;
+    }
+    const p = doc.createElement("p");
+    const strong = doc.createElement("strong");
+    strong.textContent = text;
+    p.appendChild(strong);
+    trigger.parentNode.replaceChild(p, trigger);
+}
+/**
+ * Loading placeholders: busy regions, progress bars, skeleton / shimmer /
+ * spinner / loader elements (small, no media), and blocks whose whole text is
+ * a loading message ("Loading reviews…"). Real text that only starts with
+ * "Loading" stays: the message needs an ellipsis, or to be the bare word.
+ */
+function dropPlaceholders(doc) {
+    const small = (el) => textLength(el.textContent || "") < PLACEHOLDER_MAX_TEXT && !el.querySelector("img, picture, video, table");
+    const skip = (el) => /^(html|body|main|img|picture|video|source|td|th)$/.test(String(el.nodeName).toLowerCase());
+    for (const el of Array.from(doc.querySelectorAll('[aria-busy="true"], [role="progressbar"]'))) {
+        if (!skip(el) && small(el))
+            remove(el);
+    }
+    for (const el of Array.from(doc.querySelectorAll("[id], [class]"))) {
+        if (skip(el))
+            continue;
+        const hay = `${el.getAttribute("id") || ""} ${el.getAttribute("class") || ""}`;
+        if (PLACEHOLDER_RE.test(hay) && small(el))
+            remove(el);
+    }
+    for (const el of Array.from(doc.querySelectorAll("p, div, span, li, section, small, em, strong"))) {
+        if (el.parentNode && LOADING_TEXT_RE.test(oneLine(el.textContent || "")))
+            remove(el);
+    }
+}
 function stripChrome(doc) {
     for (const el of Array.from(doc.querySelectorAll(JUNK)))
         remove(el);
@@ -289,6 +419,49 @@ function headerRows(doc) {
         }
     }
 }
+/**
+ * Readability kept the data: 80% of <main>'s distinct numbers (judged when it
+ * has 10 or more) and 80% of its tables.
+ */
+function keepsTheData(root, articleHtml, articleText) {
+    if (!root)
+        return true;
+    const rootNums = distinctNumbers(root.textContent || "");
+    if (rootNums.size >= MIN_NUMBERS_TO_JUDGE) {
+        const kept = distinctNumbers(articleText);
+        let n = 0;
+        for (const x of rootNums)
+            if (kept.has(x))
+                n++;
+        if (n < rootNums.size * KEEP_DATA_SHARE)
+            return false;
+    }
+    const rootTables = root.querySelectorAll("table").length;
+    const keptTables = (articleHtml.match(/<table\b/gi) ?? []).length;
+    return rootTables === 0 || keptTables >= rootTables * KEEP_DATA_SHARE;
+}
+/** Numbers of 2+ characters, thousands separators removed ("1,400" = "1400"). */
+function distinctNumbers(s) {
+    // Typed: under the site's tsconfig, `match() ?? []` is a union whose
+    // .filter callback gets `never` (the site build fails on it).
+    const nums = s.replace(/(\d),(?=\d{3})/g, "$1").match(/\d+(?:\.\d+)?/g) ?? [];
+    return new Set(nums.filter((n) => n.length > 1));
+}
+/** An FAQ answer from JSON-LD as Markdown: HTML through turndown, plain text as it is. */
+function faqAnswerMarkdown(pair, pageUrl) {
+    if ((0, faq_1.answerHasHtml)(pair)) {
+        const { document: adoc } = (0, linkedom_1.parseHTML)(`<!doctype html><html><head></head><body>${pair.answer}</body></html>`);
+        absolutize(adoc, pageUrl);
+        const md = tidy(getTurndown().turndown(adoc.body.innerHTML));
+        if (md)
+            return md;
+    }
+    // Plain answers can carry Markdown links to site paths: make them absolute.
+    return (0, faq_1.plainAnswer)(pair.answerText).replace(/\]\((\/[^)\s]*)\)/g, (m, path) => {
+        const abs = absolute(path, pageUrl);
+        return abs ? `](${abs})` : m;
+    });
+}
 function unwrap(el) {
     const parent = el.parentNode;
     if (!parent)

package/dist/core/enrich.d.ts

@@ -16,12 +16,21 @@ export declare function estimateTokens(text: string): number;
 export declare function hasFrontMatter(markdown: string): boolean;
 /**
  * Builds the final Markdown body. A doc that already carries front matter
- * (hand-written or stored docs usually do) is left exactly as it is, so the
- * package never stacks a second block on top of an author's.
+ * (hand-written or stored docs usually do) keeps its block, so the package
+ * never stacks a second one on top of an author's; only a `title` or
+ * `description` the block lacks is filled in from the doc's metadata.
  */
 export declare function enrich(doc: MarkdownDoc, meta: EnrichMeta): Enriched;
 /** A YAML front-matter block. Empty values are dropped; strings are JSON-quoted (valid YAML). */
 export declare function frontMatter(fields: Record<string, string | number | null | undefined>): string;
+/** Top-level keys of the doc's front matter; empty when it has none. */
+export declare function frontMatterKeys(markdown: string): Set<string>;
+/**
+ * Sets one key of the doc's front matter: replaces its value (with any
+ * indented continuation lines) or adds it after `title`, else first. Other
+ * lines are kept as they are. A doc without front matter is returned unchanged.
+ */
+export declare function setFrontMatterField(markdown: string, key: string, value: string): string;
 /**
  * A short "## Structured data" section from the page's JSON-LD, or null when
  * nothing useful is there. Only well-known, flat facts are listed; the raw

package/dist/core/enrich.js

@@ -7,6 +7,8 @@ exports.estimateTokens = estimateTokens;
 exports.hasFrontMatter = hasFrontMatter;
 exports.enrich = enrich;
 exports.frontMatter = frontMatter;
+exports.frontMatterKeys = frontMatterKeys;
+exports.setFrontMatterField = setFrontMatterField;
 exports.jsonLdFacts = jsonLdFacts;
 /** Rough token count: ~4 characters per token for English prose. */
 function estimateTokens(text) {
@@ -17,13 +19,20 @@ function hasFrontMatter(markdown) {
 }
 /**
  * Builds the final Markdown body. A doc that already carries front matter
- * (hand-written or stored docs usually do) is left exactly as it is, so the
- * package never stacks a second block on top of an author's.
+ * (hand-written or stored docs usually do) keeps its block, so the package
+ * never stacks a second one on top of an author's; only a `title` or
+ * `description` the block lacks is filled in from the doc's metadata.
  */
 function enrich(doc, meta) {
     const markdown = doc.markdown.trim();
     if (hasFrontMatter(markdown)) {
-        const body = markdown + "\n";
+        let md = markdown;
+        const keys = frontMatterKeys(md);
+        if (doc.title && !keys.has("title"))
+            md = setFrontMatterField(md, "title", doc.title);
+        if (doc.description && !keys.has("description"))
+            md = setFrontMatterField(md, "description", doc.description);
+        const body = md + "\n";
         return { body, tokens: estimateTokens(body) };
     }
     const fm = frontMatter({
@@ -50,6 +59,47 @@ function frontMatter(fields) {
     lines.push("---");
     return lines.join("\n");
 }
+const FRONT_MATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/;
+/** Top-level keys of the doc's front matter; empty when it has none. */
+function frontMatterKeys(markdown) {
+    const keys = new Set();
+    const m = FRONT_MATTER_RE.exec(markdown);
+    if (!m)
+        return keys;
+    for (const line of m[1].split(/\r?\n/)) {
+        const k = /^([A-Za-z_][\w-]*):/.exec(line);
+        if (k)
+            keys.add(k[1]);
+    }
+    return keys;
+}
+/**
+ * Sets one key of the doc's front matter: replaces its value (with any
+ * indented continuation lines) or adds it after `title`, else first. Other
+ * lines are kept as they are. A doc without front matter is returned unchanged.
+ */
+function setFrontMatterField(markdown, key, value) {
+    const m = FRONT_MATTER_RE.exec(markdown);
+    if (!m)
+        return markdown;
+    const lines = m[1].split(/\r?\n/);
+    const entry = `${key}: ${yamlScalar(value)}`;
+    const span = (start) => {
+        let end = start + 1;
+        while (end < lines.length && /^[ \t]/.test(lines[end]))
+            end++;
+        return end;
+    };
+    const at = lines.findIndex((l) => l.startsWith(`${key}:`));
+    if (at >= 0) {
+        lines.splice(at, span(at) - at, entry);
+    }
+    else {
+        const title = lines.findIndex((l) => l.startsWith("title:"));
+        lines.splice(title >= 0 ? span(title) : 0, 0, entry);
+    }
+    return `---\n${lines.join("\n")}\n---${m[2] ? "\n" : ""}${markdown.slice(m[0].length)}`;
+}
 function yamlScalar(value) {
     if (typeof value === "number")
         return String(value);

package/dist/core/faq.d.ts

@@ -0,0 +1,39 @@
+export interface FaqPair {
+    /** The question, on one line. */
+    question: string;
+    /** The answer as the JSON-LD gives it (may hold HTML). */
+    answer: string;
+    /** The answer as plain text: tags stripped, entities decoded, one line. */
+    answerText: string;
+}
+/** Every Question / acceptedAnswer pair of the page's FAQPage blocks, in page order, deduped by question. */
+export declare function faqPairs(jsonLd: unknown[]): FaqPair[];
+/** True when the answer holds HTML markup rather than plain text. */
+export declare function answerHasHtml(pair: FaqPair): boolean;
+/** Does the page's text (one line per block) show an FAQ heading? */
+export declare function pageShowsFaq(text: string): boolean;
+/** Does the Markdown have an FAQ heading? */
+export declare function hasFaqSection(markdown: string): boolean;
+/**
+ * Convert: write each answer the FAQ section lacks, word for word, under its
+ * question line; a question with no line is appended at the end of the
+ * section as a heading one level down. No FAQ heading: returned unchanged.
+ */
+export declare function fillFaqAnswers(markdown: string, pairs: FaqPair[], render: (pair: FaqPair) => string): string;
+/**
+ * Refine: the FAQ section holds exactly the page's questions and answers, in
+ * the page's order. Null when the document has no FAQ section.
+ */
+export declare function replaceFaqSection(markdown: string, pairs: FaqPair[]): string | null;
+/**
+ * Refine: add a "## FAQ" section with the page's questions and answers, before
+ * a closing "## About…" / "## Source…" section when the document has one,
+ * else at the end.
+ */
+export declare function addFaqSection(markdown: string, pairs: FaqPair[]): string;
+/** Pairs whose answer is not in the FAQ section word for word; null when there is no section. */
+export declare function missingFaqAnswers(markdown: string, pairs: FaqPair[]): FaqPair[] | null;
+/** A plain answer as a Markdown paragraph: a leading "#", ">", "- " or "1." can't start a block. */
+export declare function plainAnswer(text: string): string;
+/** HTML or text -> one line of plain text. */
+export declare function plainText(s: string): string;