@relvehq/agent-ready 0.4.2 → 0.4.3

package/dist/core/faq.js

@@ -0,0 +1,232 @@
+"use strict";
+// FAQ: the page's FAQPage JSON-LD answers, and the FAQ section of a Markdown
+// document (changes/agent-ready-quality).
+//
+//   convert  fills answers the HTML lacks (accordions that render closed
+//            panels empty) under the page's own FAQ heading
+//   refine   swaps the page's questions and answers into a rewrite, word for
+//            word, so a model never rewords them
+//
+// A document without an FAQ heading never gets one from convert: the page
+// shows no FAQ (owner, 2026-11-03). EDGE-SAFE (pure).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.faqPairs = faqPairs;
+exports.answerHasHtml = answerHasHtml;
+exports.pageShowsFaq = pageShowsFaq;
+exports.hasFaqSection = hasFaqSection;
+exports.fillFaqAnswers = fillFaqAnswers;
+exports.replaceFaqSection = replaceFaqSection;
+exports.addFaqSection = addFaqSection;
+exports.missingFaqAnswers = missingFaqAnswers;
+exports.plainAnswer = plainAnswer;
+exports.plainText = plainText;
+/** Every Question / acceptedAnswer pair of the page's FAQPage blocks, in page order, deduped by question. */
+function faqPairs(jsonLd) {
+    const pairs = [];
+    const seen = new Set();
+    const visit = (node) => {
+        if (Array.isArray(node))
+            return node.forEach(visit);
+        if (!node || typeof node !== "object")
+            return;
+        const o = node;
+        if (Array.isArray(o["@graph"]))
+            visit(o["@graph"]);
+        const types = [].concat(o["@type"] ?? []).map(String);
+        if (!types.includes("FAQPage"))
+            return;
+        for (const q of [].concat(o.mainEntity ?? [])) {
+            if (!q || typeof q !== "object")
+                continue;
+            const qo = q;
+            const question = plainText(String(qo.name ?? qo.text ?? ""));
+            const accepted = [].concat(qo.acceptedAnswer ?? [])[0];
+            const answer = accepted && typeof accepted === "object" ? String(accepted.text ?? "").trim() : "";
+            const answerText = plainText(answer);
+            if (!question || !answerText || seen.has(question.toLowerCase()))
+                continue;
+            seen.add(question.toLowerCase());
+            pairs.push({ question, answer, answerText });
+        }
+    };
+    jsonLd.forEach(visit);
+    return pairs;
+}
+/** True when the answer holds HTML markup rather than plain text. */
+function answerHasHtml(pair) {
+    return /<[a-z][^>]*>/i.test(pair.answer);
+}
+const FAQ_TITLE = /\b(?:faqs?|frequently asked questions?)\b/i;
+/** Does the page's text (one line per block) show an FAQ heading? */
+function pageShowsFaq(text) {
+    return text.split("\n").some((line) => line.trim().length <= 80 && FAQ_TITLE.test(line));
+}
+function findSection(lines) {
+    let fenced = false;
+    let found = null;
+    // Front matter is YAML: a "# comment" there is not a heading.
+    let start = 0;
+    if (lines[0] === "---") {
+        const close = lines.indexOf("---", 1);
+        if (close > 0)
+            start = close + 1;
+    }
+    for (let i = start; i < lines.length; i++) {
+        if (/^\s*(```|~~~)/.test(lines[i]))
+            fenced = !fenced;
+        if (fenced)
+            continue;
+        const h = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(lines[i]);
+        if (!h)
+            continue;
+        const level = h[1].length;
+        if (found) {
+            if (level <= found.level) {
+                found.end = i;
+                return found;
+            }
+            continue;
+        }
+        if (FAQ_TITLE.test(h[2]))
+            found = { heading: i, level, end: lines.length };
+    }
+    return found;
+}
+/** Does the Markdown have an FAQ heading? */
+function hasFaqSection(markdown) {
+    return findSection(splitLines(markdown)) !== null;
+}
+/**
+ * Convert: write each answer the FAQ section lacks, word for word, under its
+ * question line; a question with no line is appended at the end of the
+ * section as a heading one level down. No FAQ heading: returned unchanged.
+ */
+function fillFaqAnswers(markdown, pairs, render) {
+    const lines = splitLines(markdown);
+    const section = findSection(lines);
+    if (!section || !pairs.length)
+        return markdown;
+    const appended = [];
+    for (const pair of pairs) {
+        const body = lines.slice(section.heading + 1, section.end).join("\n");
+        if (contains(body, pair.answerText))
+            continue;
+        const at = lines.findIndex((l, i) => i > section.heading && i < section.end && sameQuestion(l, pair.question));
+        if (at === -1) {
+            appended.push(pair);
+            continue;
+        }
+        const add = ["", render(pair)];
+        lines.splice(at + 1, 0, ...add);
+        section.end += add.length;
+    }
+    if (appended.length) {
+        const sub = "#".repeat(Math.min(section.level + 1, 6));
+        const add = appended.flatMap((p) => ["", `${sub} ${p.question}`, "", render(p)]);
+        lines.splice(section.end, 0, ...add, "");
+    }
+    return tidy(lines.join("\n"));
+}
+/**
+ * Refine: the FAQ section holds exactly the page's questions and answers, in
+ * the page's order. Null when the document has no FAQ section.
+ */
+function replaceFaqSection(markdown, pairs) {
+    const lines = splitLines(markdown);
+    const section = findSection(lines);
+    if (!section)
+        return null;
+    const sub = "#".repeat(Math.min(section.level + 1, 6));
+    lines.splice(section.heading + 1, section.end - section.heading - 1, "", ...renderPairs(pairs, sub), "");
+    return tidy(lines.join("\n"));
+}
+/**
+ * Refine: add a "## FAQ" section with the page's questions and answers, before
+ * a closing "## About…" / "## Source…" section when the document has one,
+ * else at the end.
+ */
+function addFaqSection(markdown, pairs) {
+    const lines = splitLines(markdown);
+    let at = lines.length;
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const h = /^##\s+(.+)$/.exec(lines[i]);
+        if (!h)
+            continue;
+        if (/^(about|source)/i.test(h[1].trim()))
+            at = i;
+        break;
+    }
+    lines.splice(at, 0, "", "## FAQ", "", ...renderPairs(pairs, "###"), "");
+    return tidy(lines.join("\n"));
+}
+/** Pairs whose answer is not in the FAQ section word for word; null when there is no section. */
+function missingFaqAnswers(markdown, pairs) {
+    const lines = splitLines(markdown);
+    const section = findSection(lines);
+    if (!section)
+        return null;
+    const body = lines.slice(section.heading + 1, section.end).join("\n");
+    return pairs.filter((p) => !contains(body, p.answerText));
+}
+/** A plain answer as a Markdown paragraph: a leading "#", ">", "- " or "1." can't start a block. */
+function plainAnswer(text) {
+    return text.replace(/^(#{1,6}\s|>|[-+*]\s)/, "\\$1").replace(/^(\d+)\.(\s)/, "$1\\.$2");
+}
+function renderPairs(pairs, sub) {
+    return pairs.flatMap((p, i) => [...(i ? [""] : []), `${sub} ${p.question}`, "", plainAnswer(p.answerText)]);
+}
+// ── text ────────────────────────────────────────────────────────────────
+/** HTML or text -> one line of plain text. */
+function plainText(s) {
+    return decodeEntities(s.replace(/<br\s*\/?>|<\/(?:p|li|div|h[1-6])>/gi, " ").replace(/<[^>]+>/g, ""))
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function decodeEntities(s) {
+    return s.replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos|nbsp|#39);/gi, (m, e) => {
+        const k = e.toLowerCase();
+        if (k === "amp")
+            return "&";
+        if (k === "lt")
+            return "<";
+        if (k === "gt")
+            return ">";
+        if (k === "quot")
+            return '"';
+        if (k === "apos" || k === "#39")
+            return "'";
+        if (k === "nbsp")
+            return " ";
+        const code = k.startsWith("#x") ? parseInt(k.slice(2), 16) : parseInt(k.slice(1), 10);
+        return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
+    });
+}
+/** Markdown -> comparable text: link text kept, markup and all whitespace dropped. */
+function squash(md) {
+    return md
+        .normalize("NFKC")
+        .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1")
+        .replace(/^\s*(?:[-+*]|\d+\.)\s+/gm, "")
+        .replace(/\\([\\`*_{}[\]()#+\-.!|>])/g, "$1")
+        .replace(/[*_`#>|]/g, "")
+        .replace(/[‘’]/g, "'")
+        .replace(/[“”]/g, '"')
+        .replace(/[–—]/g, "-")
+        .replace(/\s+/g, "")
+        .toLowerCase();
+}
+function contains(body, answerText) {
+    const needle = squash(answerText);
+    return needle.length > 0 && squash(body).includes(needle);
+}
+function sameQuestion(line, question) {
+    const q = (s) => squash(s).replace(/[?.:!]+$/, "");
+    const l = q(line);
+    return l.length > 0 && l === q(question);
+}
+function splitLines(markdown) {
+    return markdown.replace(/\r\n?/g, "\n").split("\n");
+}
+function tidy(md) {
+    return md.replace(/\n{3,}/g, "\n\n").trim();
+}

package/dist/core/resolve.js

@@ -50,7 +50,9 @@ exports.resolveMarkdown = resolveMarkdown;
 exports.defaultFetchOrigin = defaultFetchOrigin;
 exports.matchPattern = matchPattern;
 const config_1 = require("./config");
-const MAX_HTML_BYTES = 5 * 1024 * 1024;
+// const MAX_HTML_BYTES = 5 * 1024 * 1024;
+// Now cfg.maxHtmlBytes (default 5 MB, core/config.ts), so a site with one very
+// large page can raise it.
 async function resolveMarkdown(cfg, input, deps = {}) {
     const { path, search, request } = input;
     const pageUrl = cfg.site + (path === "/" ? "/" : path) + search;
@@ -107,11 +109,11 @@ async function resolveMarkdown(cfg, input, deps = {}) {
     }
     const contentType = res.headers.get("content-type") ?? "";
     const declared = Number(res.headers.get("content-length") ?? "0");
-    if (declared > MAX_HTML_BYTES)
+    if (declared > cfg.maxHtmlBytes)
         return { ...miss, originStatus: res.status };
     const body = await res.text().catch(() => "");
     const html = { status: res.status, contentType: contentType || "text/html; charset=utf-8", body };
-    if (!res.ok || !/text\/html|application\/xhtml/i.test(contentType) || body.length > MAX_HTML_BYTES) {
+    if (!res.ok || !/text\/html|application\/xhtml/i.test(contentType) || body.length > cfg.maxHtmlBytes) {
         return { ...miss, html, originStatus: res.status };
     }
     const convert = deps.convert ?? (await Promise.resolve().then(() => __importStar(require("./convert")))).htmlToMarkdown;

package/dist/core/version.d.ts

@@ -1 +1 @@
-export declare const PACKAGE_VERSION = "0.4.2";
+export declare const PACKAGE_VERSION = "0.4.3";

package/dist/core/version.js

@@ -5,4 +5,4 @@
 // EDGE-SAFE.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PACKAGE_VERSION = void 0;
-exports.PACKAGE_VERSION = "0.4.2";
+exports.PACKAGE_VERSION = "0.4.3";

package/dist/refine/check.d.ts

@@ -1,6 +1,6 @@
 import type { PageInput } from "./types";
 import type { PromptSettings } from "./prompts";
-export type IssueKind = "number" | "url" | "frontmatter" | "hash" | "section" | "h1" | "html" | "placeholder" | "tokens" | "empty" | "plan";
+export type IssueKind = "number" | "url" | "frontmatter" | "hash" | "section" | "h1" | "html" | "placeholder" | "tokens" | "empty" | "plan" | "description" | "faq";
 export interface CheckIssue {
     kind: IssueKind;
     detail: string;

package/dist/refine/check.js

@@ -17,14 +17,20 @@
 //                prices of other products shown on the page; 2026-10-29)
 //   tokens       over the prompt's token budget
 //   empty        no real body
+//   description  no `description` in the front matter
+//   faq          the page shows an FAQ with FAQPage answers, and the document
+//                lacks the section or an answer word for word (refine swaps
+//                the page's in before checking; this guards other callers)
 //
-// Frontmatter values are not number-checked: they hold the hash, timestamps
-// and dates reformatted as YYYY-MM-DD by design.
+// Frontmatter values are not number-checked (they hold the hash, timestamps
+// and dates reformatted as YYYY-MM-DD by design), except the description,
+// which is prose: its numbers and URLs are checked like the body's.
 // EDGE-SAFE (pure). It cannot catch a reworded claim; people still read.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.checkDoc = checkDoc;
 exports.planNames = planNames;
 const enrich_1 = require("../core/enrich");
+const faq_1 = require("../core/faq");
 const URL_RE = /https?:\/\/[^\s<>()[\]|"'`]+/g;
 const NUM_RE = /\d+(?:[.,]\d+)*/g;
 const HTML_RE = /<\/?(?:a|b|br|div|em|h[1-6]|i|img|li|ol|p|script|span|strong|style|table|td|th|tr|ul)\b[^>]*>/i;
@@ -36,6 +42,7 @@ function checkDoc(doc, input, settings, opts = {}) {
     // Frontmatter.
     const fm = /^---\n([\s\S]*?)\n---\n?/.exec(md);
     const body = fm ? md.slice(fm[0].length) : md;
+    let description = "";
     if (!fm) {
         add("frontmatter", "no frontmatter block");
     }
@@ -51,6 +58,9 @@ function checkDoc(doc, input, settings, opts = {}) {
             add("frontmatter", `canonical ${meta.canonical} is not ${input.url}`);
         if (meta.source_hash !== input.sourceHash)
             add("hash", `source_hash ${meta.source_hash ?? "(none)"} does not match the input`);
+        description = meta.description ?? "";
+        if (!description)
+            add("description", "no description in the front matter");
     }
     const prose = body.replace(/^```[\s\S]*?^```/gm, "");
     if (prose.replace(/[#|\-\s>*_]/g, "").length < 40)
@@ -69,8 +79,32 @@ function checkDoc(doc, input, settings, opts = {}) {
         add("placeholder", `left in: ${left[0]}`);
     if (tokens > settings.tokenBudget)
         add("tokens", `${tokens} tokens, budget ${settings.tokenBudget}`);
+    // FAQ: the page's own answers, word for word.
+    const pairs = (0, faq_1.faqPairs)(input.jsonLd);
+    if (pairs.length) {
+        const missing = (0, faq_1.missingFaqAnswers)(md, pairs);
+        if (missing === null) {
+            if ((0, faq_1.pageShowsFaq)(input.text))
+                add("faq", "the page's FAQ section is missing");
+        }
+        else {
+            for (const p of missing.slice(0, 3))
+                add("faq", `the answer to "${p.question.slice(0, 80)}" is not the page's`);
+            if (missing.length > 3)
+                add("faq", `${missing.length - 3} more answers are not the page's`);
+        }
+    }
     // Facts: everything the input contains, as one haystack.
-    const haystack = [input.text, input.links.map((l) => `${l.url} ${l.text}`).join("\n"), JSON.stringify(input.jsonLd), input.url].join("\n");
+    const haystack = [
+        input.text,
+        input.links.map((l) => `${l.url} ${l.text}`).join("\n"),
+        JSON.stringify(input.jsonLd),
+        input.url,
+        input.description ?? "",
+    ].join("\n");
+    // What is fact-checked: the body, plus the description (prose, unlike the
+    // rest of the front matter).
+    const claims = `${body}\n${description}`;
     const known = new Set();
     const knownNoQuery = new Set();
     for (const u of haystack.match(URL_RE) ?? []) {
@@ -90,14 +124,14 @@ function checkDoc(doc, input, settings, opts = {}) {
             // not a path
         }
     }
-    for (const raw of body.match(URL_RE) ?? []) {
+    for (const raw of claims.match(URL_RE) ?? []) {
         const u = trimUrl(raw);
         if (!known.has(normUrl(u)) && !knownNoQuery.has(noQuery(u)))
             add("url", u);
     }
     const knownNums = new Set((haystack.match(NUM_RE) ?? []).map(canonNum));
     const seen = new Set();
-    for (const n of body.replace(URL_RE, " ").match(NUM_RE) ?? []) {
+    for (const n of claims.replace(URL_RE, " ").match(NUM_RE) ?? []) {
         const c = canonNum(n);
         if (knownNums.has(c) || seen.has(c))
             continue;

package/dist/refine/extract.js

@@ -9,9 +9,12 @@
 //   jsonLd     every JSON-LD block on the page (top-level arrays flattened)
 //   sourceHash SHA-256 of `text + "\n" + JSON.stringify(jsonLd)`: the page's
 //              fingerprint; a new hash means the page changed
+//   description the page's meta description (not in the fingerprint)
 //
 // Unlike convert, nothing is judged here: buttons and labels stay in, and the
-// writing step decides what to leave out. NODE ONLY (linkedom).
+// writing step decides what to leave out. Accordion panels are opened and
+// loading placeholders dropped the same way as convert, so the model sees the
+// answers and not "Loading…". NODE ONLY (linkedom).
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.formatLinks = void 0;
 exports.extractInput = extractInput;
@@ -39,7 +42,14 @@ async function extractInput(html, opts) {
             // Broken JSON-LD is skipped, as in convert.
         }
     }
+    const metaContent = (sel) => {
+        const v = document.querySelector(sel)?.getAttribute("content");
+        return v ? String(v).replace(/\s+/g, " ").trim() : "";
+    };
+    const description = metaContent('meta[name="description"]') || metaContent('meta[property="og:description"]');
     (0, convert_1.unfoldNextStreaming)(document);
+    (0, convert_1.openDisclosures)(document);
+    (0, convert_1.dropPlaceholders)(document);
     for (const el of Array.from(document.querySelectorAll(DROP)))
         el.parentNode?.removeChild(el);
     for (const el of Array.from(document.querySelectorAll('[style*="display:none"], [style*="display: none"]'))) {
@@ -51,7 +61,15 @@ async function extractInput(html, opts) {
     const mainSource = main ? "main" : article ? "article" : "body";
     const text = readableText(root);
     const links = collectLinks(root, opts.url);
-    return { url: opts.url, text, links, jsonLd, sourceHash: await sourceHashOf(text, jsonLd), mainSource };
+    return {
+        url: opts.url,
+        text,
+        links,
+        jsonLd,
+        sourceHash: await sourceHashOf(text, jsonLd),
+        mainSource,
+        ...(description ? { description } : {}),
+    };
 }
 /** The page fingerprint: SHA-256 hex of the text and the JSON-LD. */
 async function sourceHashOf(text, jsonLd) {

package/dist/refine/finalize.d.ts

@@ -0,0 +1,2 @@
+import type { PageInput } from "./types";
+export declare function finalizeDoc(doc: string, input: PageInput): string;

package/dist/refine/finalize.js

@@ -0,0 +1,32 @@
+"use strict";
+// Refine, between the model and the check: the code's own edits to each reply
+// (changes/agent-ready-quality).
+//
+//   description  the page's meta description goes into the front matter
+//                (replacing the model's); with none, the model's stays and
+//                the check fact-checks it
+//   FAQ          the page's FAQPage questions and answers replace the
+//                rewrite's FAQ section, word for word; a rewrite that dropped
+//                the FAQ of a page showing one gets it back; a page that
+//                shows no FAQ never gets one
+//
+// Done right after the model writes, not after a retry, so no model call is
+// spent on rewording (owner, 2026-11-05). EDGE-SAFE (pure).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.finalizeDoc = finalizeDoc;
+const enrich_1 = require("../core/enrich");
+const faq_1 = require("../core/faq");
+function finalizeDoc(doc, input) {
+    let md = doc;
+    if (input.description && (0, enrich_1.hasFrontMatter)(md))
+        md = (0, enrich_1.setFrontMatterField)(md, "description", input.description);
+    const pairs = (0, faq_1.faqPairs)(input.jsonLd);
+    if (pairs.length) {
+        const replaced = (0, faq_1.replaceFaqSection)(md, pairs);
+        if (replaced !== null)
+            md = replaced;
+        else if ((0, faq_1.pageShowsFaq)(input.text))
+            md = (0, faq_1.addFaqSection)(md, pairs);
+    }
+    return md;
+}

package/dist/refine/refine.js

@@ -1,6 +1,7 @@
 "use strict";
 // Refine one page: fetch -> extract -> (unchanged? stop) -> fill the prompt ->
-// model -> check -> retry once with the issues -> result.
+// model -> finalize (page description, page FAQ) -> check -> retry once with
+// the issues -> result.
 // Nothing is stored here; the runner decides (refine/run.ts).
 //
 // NODE ONLY (extract uses linkedom).
@@ -8,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.refinePage = refinePage;
 const extract_1 = require("./extract");
 const check_1 = require("./check");
+const finalize_1 = require("./finalize");
 const model_1 = require("./model");
 const prompts_1 = require("./prompts");
 async function refinePage(opts) {
@@ -41,7 +43,7 @@ async function refinePage(opts) {
             const reply = await opts.model({ system: filled.system, user });
             result.usage.inputTokens += reply.inputTokens ?? 0;
             result.usage.outputTokens += reply.outputTokens ?? 0;
-            const doc = (0, model_1.cleanReply)(reply.text);
+            const doc = (0, finalize_1.finalizeDoc)((0, model_1.cleanReply)(reply.text), input);
             const check = (0, check_1.checkDoc)(doc, input, opts.prompt.settings, opts);
             result.doc = doc;
             result.issues = check.issues;

package/dist/refine/remote.d.ts

@@ -2,7 +2,7 @@ import { PACKAGE_VERSION } from "../core/version";
 export { PACKAGE_VERSION };
 export declare const DEFAULT_REFINE_ENDPOINT = "https://relvehq.com/api/agent-ready/v1/refine";
 export declare const REFINE_MODELS: readonly ["gpt-4o", "gpt-4o-mini", "gpt-4.1", "gpt-4.1-mini"];
-export declare const USER_AGENT = "agent-ready/0.4.2 (+https://relvehq.com/agent-ready)";
+export declare const USER_AGENT = "agent-ready/0.4.3 (+https://relvehq.com/agent-ready)";
 export type RemoteStatus = "ok" | "unchanged" | "rejected" | "error" | "skipped";
 export interface RemoteItem {
     path: string;

package/dist/refine/types.d.ts

@@ -12,6 +12,8 @@ export interface PageInput {
     /** SHA-256 hex of `text + "\n" + JSON.stringify(jsonLd)`. */
     sourceHash: string;
     mainSource: "main" | "article" | "body";
+    /** The page's meta description (else og:description). Not part of the fingerprint. */
+    description?: string;
 }
 /** Links as the prompt's {{PAGE_LINKS}} block: "url | text" per line. */
 export declare function formatLinks(links: PageLink[]): string;