@pseolint/core 0.6.6 → 0.7.1

package/dist/algorithms/fact-extraction.js

@@ -0,0 +1,223 @@
+import { load } from "cheerio";
+import { maskEntities } from "./entity-mask.js";
+// --- Numeric "citable" facts: the frozen subset aeo/citable-facts counts. ---
+// These six patterns are lifted verbatim from rules/aeo/citable-facts.ts and
+// MUST stay byte-identical to preserve the calibration corpus.
+const CITABLE_FACT_PATTERNS = [
+    { name: "dollar", regex: /\$[\d,]+(\.\d{2})?/g },
+    { name: "percent", regex: /\b\d+(\.\d+)?\s*%/g },
+    {
+        name: "timeframe",
+        regex: /\b\d+(?:-\d+)?\s*(business\s+days?|days?|weeks?|months?|years?|hours?|minutes?)\b/gi,
+    },
+    {
+        name: "date",
+        regex: /\b(january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2}(?:,\s*\d{4})?\b/gi,
+    },
+    { name: "isoDate", regex: /\b\d{4}-\d{2}-\d{2}\b/g },
+    { name: "form", regex: /\bForm\s+[A-Z0-9][A-Z0-9-]*\b/g },
+];
+export function extractCitableFacts(text) {
+    const out = new Set();
+    for (const { regex } of CITABLE_FACT_PATTERNS) {
+        const matches = text.match(regex);
+        if (!matches)
+            continue;
+        for (const m of matches)
+            out.add(m.trim().toLowerCase());
+    }
+    return Array.from(out);
+}
+// --- Measurements: NEW numeric kinds, deliberately separate from citableFacts. ---
+const MEASUREMENT_UNITS = "kg|g|lb|lbs|oz|mi|km|cm|mm|ft|in|MB|GB|TB|KB|ms|fps|mph|kWh";
+const MEASUREMENT_PATTERNS = [
+    { kind: "ratio", regex: /\b\d+(?:\.\d+)?\s*(?:out of|in)\s*\d+\b/gi },
+    { kind: "ratio", regex: /\b\d+\s*:\s*\d+\b/g },
+    { kind: "measurement", regex: new RegExp(`\\b\\d+(?:\\.\\d+)?\\s*(?:${MEASUREMENT_UNITS})\\b`, "g") },
+];
+export function extractMeasurements(maskedText) {
+    const seen = new Set();
+    const out = [];
+    for (const { kind, regex } of MEASUREMENT_PATTERNS) {
+        const matches = maskedText.match(regex);
+        if (!matches)
+            continue;
+        for (const m of matches) {
+            const value = m.replace(/\s+/g, " ").trim().toLowerCase();
+            if (seen.has(value))
+                continue;
+            seen.add(value);
+            out.push({ value, kind });
+        }
+    }
+    return out;
+}
+const MULTI_WORD_PROPER_NOUN = /\b[A-Z][a-z]+(?:\s+(?:of\s+|de\s+|and\s+|the\s+)?[A-Z][a-z]+)+\b/g;
+const ACRONYM = /\b(?:ISO|GDPR|HIPAA|FDA|SEC|FTC|EPA|W3C|IETF|RFC|NIST|OSHA|IRS|EU|UN|WHO|CCPA|PCI)\b/g;
+const CUE_WORD = /\b(?:Inc|LLC|Ltd|Corp|GmbH|Act|Regulation|Directive|Agency|Department|Bureau|Commission|Authority|University|Institute|Association|Standard|Protocol)\b/;
+const JSON_LD_ENTITY_TYPES = new Set([
+    "Organization", "GovernmentOrganization", "Corporation", "NGO",
+    "Person", "Product", "Brand",
+]);
+function jsonLdEntities(nodes) {
+    const out = [];
+    const visit = (node) => {
+        if (Array.isArray(node)) {
+            node.forEach(visit);
+            return;
+        }
+        if (typeof node !== "object" || node === null)
+            return;
+        const obj = node;
+        const type = obj["@type"];
+        const name = obj["name"];
+        if (typeof name === "string" && typeof type === "string" && JSON_LD_ENTITY_TYPES.has(type)) {
+            out.push({ value: name.trim().toLowerCase(), source: "json-ld", type: "organization" });
+        }
+        for (const v of Object.values(obj))
+            visit(v);
+    };
+    nodes.forEach(visit);
+    return out;
+}
+export function extractNamedEntities(maskedText, jsonLd = []) {
+    const seen = new Set();
+    const out = [];
+    const push = (value, source) => {
+        const v = value.replace(/\s+/g, " ").trim().toLowerCase();
+        if (v.length < 2 || seen.has(v))
+            return;
+        seen.add(v);
+        out.push({ value: v, source });
+    };
+    for (const m of jsonLdEntities(jsonLd))
+        push(m.value, "json-ld");
+    for (const m of maskedText.match(ACRONYM) ?? [])
+        push(m, "cue-word");
+    for (const m of maskedText.match(MULTI_WORD_PROPER_NOUN) ?? []) {
+        push(m, CUE_WORD.test(m) ? "cue-word" : "proper-noun");
+    }
+    return out;
+}
+export const DEFAULT_CITATION_ALLOWLIST = [
+    "wikipedia.org", "w3.org", "iso.org", "ietf.org", "rfc-editor.org",
+    "doi.org", "nih.gov", "ncbi.nlm.nih.gov", "who.int", "schema.org",
+    "oecd.org", "worldbank.org", "europa.eu",
+    // Google's own published documentation is the primary authoritative source for
+    // claims about Google's ranking and spam systems (Search Essentials, spam
+    // policies, helpful-content guidance) and for Core Web Vitals (web.dev).
+    // Scoped to the docs subdomain — a bare google.com link (Maps, search results)
+    // is deliberately NOT credited as authoritative.
+    "developers.google.com", "web.dev",
+];
+const MULTI_PART_SUFFIXES = new Set([
+    "co.uk", "ac.uk", "gov.uk", "org.uk", "com.au", "gov.au", "edu.au",
+    "co.jp", "co.nz", "co.za", "com.br",
+]);
+function hostOf(url) {
+    try {
+        return new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return null;
+    }
+}
+export function registrableDomain(host) {
+    const labels = host.replace(/^www\./, "").split(".");
+    if (labels.length <= 2)
+        return labels.join(".");
+    const lastTwo = labels.slice(-2).join(".");
+    if (MULTI_PART_SUFFIXES.has(lastTwo))
+        return labels.slice(-3).join(".");
+    return lastTwo;
+}
+function isAuthoritativeTld(host) {
+    return /\.(?:gov|edu|mil|int)$/.test(host) || /\.(?:gov|edu|ac)\.[a-z]{2}$/.test(host);
+}
+export function classifyCitations(resolvedHrefs, pageUrl, allowlist = DEFAULT_CITATION_ALLOWLIST) {
+    const pageHost = hostOf(pageUrl);
+    const pageDomain = pageHost ? registrableDomain(pageHost) : null;
+    const seen = new Set();
+    const out = [];
+    for (const href of resolvedHrefs) {
+        const host = hostOf(href);
+        if (!host)
+            continue;
+        const domain = registrableDomain(host);
+        if (pageDomain && domain === pageDomain)
+            continue; // internal link
+        if (seen.has(href))
+            continue;
+        seen.add(href);
+        if (isAuthoritativeTld(host)) {
+            out.push({ href, domain, authority: "authoritative", reason: "tld" });
+        }
+        else if (allowlist.some((d) => host === d || host.endsWith(`.${d}`))) {
+            out.push({ href, domain, authority: "authoritative", reason: "allowlist" });
+        }
+        else {
+            out.push({ href, domain, authority: "general" });
+        }
+    }
+    return out;
+}
+export function hasAuthoritativeCitation(resolvedHrefs, pageUrl, allowlist = DEFAULT_CITATION_ALLOWLIST) {
+    return classifyCitations(resolvedHrefs, pageUrl, allowlist).some((c) => c.authority === "authoritative");
+}
+const SENTENCE_SPLIT = /(?<=[.!?])\s+(?=[A-Z0-9"'(])/;
+function resolveHrefs(hrefs, base) {
+    const out = [];
+    for (const h of hrefs) {
+        try {
+            out.push(new URL(h, base).href);
+        }
+        catch { /* skip unparseable */ }
+    }
+    return out;
+}
+/**
+ * Deterministic approximation of "a verifiable claim": a block (<p>/<li>) that
+ * contains a statistic AND an outbound citation. Approximated at block level,
+ * not exact sentence level — documented limitation. Detects co-occurrence, not
+ * semantic truth. Consume at `speculative` confidence.
+ */
+export function extractGroundedClaims(html, pageUrl, allowlist = DEFAULT_CITATION_ALLOWLIST) {
+    const $ = load(html);
+    $("nav, header, footer, aside, script, style, noscript").remove();
+    const claims = [];
+    const scope = $("article").length > 0 ? $("article") : $("main").length > 0 ? $("main") : $("body");
+    scope.find("p, li").each((_i, el) => {
+        const $el = $(el);
+        const rawLinks = $el.find("a[href]").map((_j, a) => String($(a).attr("href") ?? "")).get();
+        const citations = classifyCitations(resolveHrefs(rawLinks, pageUrl), pageUrl, allowlist);
+        if (citations.length === 0)
+            return;
+        const text = $el.text().replace(/\s+/g, " ").trim();
+        for (const sentence of text.split(SENTENCE_SPLIT)) {
+            const facts = [
+                ...extractCitableFacts(sentence),
+                ...extractMeasurements(sentence).map((m) => m.value),
+            ];
+            if (facts.length === 0)
+                continue;
+            claims.push({
+                sentence: sentence.trim().slice(0, 240),
+                facts,
+                citations: citations.map((c) => c.href),
+            });
+            break; // one grounded claim per block is enough; avoids over-counting
+        }
+    });
+    return claims;
+}
+export function extractPageFacts(page, entityPatterns, allowlist = DEFAULT_CITATION_ALLOWLIST) {
+    const masked = maskEntities(page.contentText, entityPatterns);
+    return {
+        citableFacts: extractCitableFacts(masked),
+        measurements: extractMeasurements(masked),
+        namedEntities: extractNamedEntities(masked, page.jsonLd),
+        citations: classifyCitations(page.resolvedHrefs, page.url, allowlist),
+        groundedClaims: extractGroundedClaims(page.html, page.url, allowlist),
+    };
+}
+//# sourceMappingURL=fact-extraction.js.map

package/dist/algorithms/fact-extraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fact-extraction.js","sourceRoot":"","sources":["../../src/algorithms/fact-extraction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAwChD,+EAA+E;AAC/E,6EAA6E;AAC7E,+DAA+D;AAC/D,MAAM,qBAAqB,GAA2C;IACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,qBAAqB,EAAE;IAChD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,oBAAoB,EAAE;IAChD;QACE,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,qFAAqF;KAC7F;IACD;QACE,IAAI,EAAE,MAAM;QACZ,KAAK,EACH,uHAAuH;KAC1H;IACD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,wBAAwB,EAAE;IACpD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,gCAAgC,EAAE;CAC1D,CAAC;AAEF,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,qBAAqB,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,oFAAoF;AACpF,MAAM,iBAAiB,GACrB,6DAA6D,CAAC;AAChE,MAAM,oBAAoB,GAA6C;IACrE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,2CAA2C,EAAE;IACrE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE;IAC9C,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,6BAA6B,iBAAiB,MAAM,EAAE,GAAG,CAAC,EAAE;CACtG,CAAC;AAEF,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,oBAAoB,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC1D,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,SAAS;YAC9B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,sBAAsB,GAAG,mEAAmE,CAAC;AACnG,MAAM,OAAO,GAAG,uFAAuF,CAAC;AACxG,MAAM,QAAQ,GAAG,yJAAyJ,CAAC;AAE3K,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,cAAc,EAAE,wBAAwB,EAAE,aAAa,EAAE,KAAK;IAC9D,QAAQ,EAAE,SAAS,EAAE,OAAO;CAC7B,CAAC,CAAC;AAEH,SAAS,cAAc,CAAC,KAAgB;IACtC,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,IAAa,EAAQ,EAAE;QACpC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACzD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO;QACtD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3F,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1F,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;YAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC;IACF,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAkB,EAAE,SAAoB,EAAE;IAC7E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,MAA6B,EAAQ,EAAE;QAClE,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO;QACxC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,MAAM,CAAC;QAAE,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACrE,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/D,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,MAAM,0BAA0B,GAAsB;IAC3D,eAAe,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,gBAAgB;IAClE,SAAS,EAAE,SAAS,EAAE,kBAAkB,EAAE,SAAS,EAAE,YAAY;IACjE,UAAU,EAAE,eAAe,EAAE,WAAW;IACxC,+EAA+E;IAC/E,0EAA0E;IAC1E,yEAAyE;IACzE,+EAA+E;IAC/E,iDAAiD;IACjD,uBAAuB,EAAE,SAAS;CACnC,CAAC;AAEF,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ;IAClE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;CACpC,CAAC,CAAC;AAEH,SAAS,MAAM,CAAC,GAAW;IACzB,IAAI,CAAC;QAAC,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,IAAI,CAAC;IAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzF,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,aAAgC,EAChC,OAAe,EACf,YAA+B,0BAA0B;IAEzD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,UAAU,IAAI,MAAM,KAAK,UAAU;YAAE,SAAS,CAAC,gBAAgB;QACnE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACxE,CAAC;aAAM,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACvE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,aAAgC,EAChC,OAAe,EACf,YAA+B,0BAA0B;IAEzD,OAAO,iBAAiB,CAAC,aAAa,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,CAAC,CAAC;AAC3G,CAAC;AAED,MAAM,cAAc,GAAG,8BAA8B,CAAC;AAEtD,SAAS,YAAY,CAAC,KAAe,EAAE,IAAY;IACjD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC;YAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,OAAe,EACf,YAA+B,0BAA0B;IAEzD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC,CAAC,qDAAqD,CAAC,CAAC,MAAM,EAAE,CAAC;IAClE,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAC3F,MAAM,SAAS,GAAG,iBAAiB,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACzF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACnC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACpD,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG;gBACZ,GAAG,mBAAmB,CAAC,QAAQ,CAAC;gBAChC,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;aACrD,CAAC;YACF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACjC,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACvC,KAAK;gBACL,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aACxC,CAAC,CAAC;YACH,MAAM,CAAC,+DAA+D;QACxE,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAmF,EACnF,cAAmC,EACnC,YAA+B,0BAA0B;IAEzD,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC9D,OAAO;QACL,YAAY,EAAE,mBAAmB,CAAC,MAAM,CAAC;QACzC,YAAY,EAAE,mBAAmB,CAAC,MAAM,CAAC;QACzC,aAAa,EAAE,oBAAoB,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;QACxD,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC;QACrE,cAAc,EAAE,qBAAqB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC;KACtE,CAAC;AACJ,CAAC"}

package/dist/auditor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"auditor.d.ts","sourceRoot":"","sources":["../src/auditor.ts"],"names":[],"mappings":"AAmEA,OAAO,KAAK,EACV,YAAY,EACZ,YAAY,EAGZ,WAAW,EAUX,UAAU,EAIX,MAAM,YAAY,CAAC;AAQpB,OAAO,EAA8D,KAAK,kBAAkB,EAAiB,MAAM,sBAAsB,CAAC;AA+D1I,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEvE;AA6xBD;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,UAAU,EAAE,EACtB,cAAc,EAAE,kBAAkB,GAAG,SAAS,GAC7C,UAAU,EAAE,CAed;AAoYD,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBjG;AA+pBD,wBAAsB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CA2wC/F"}
+{"version":3,"file":"auditor.d.ts","sourceRoot":"","sources":["../src/auditor.ts"],"names":[],"mappings":"AAoEA,OAAO,KAAK,EACV,YAAY,EACZ,YAAY,EAGZ,WAAW,EAUX,UAAU,EAIX,MAAM,YAAY,CAAC;AAQpB,OAAO,EAA8D,KAAK,kBAAkB,EAAiB,MAAM,sBAAsB,CAAC;AAqE1I,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEvE;AA2yBD;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,UAAU,EAAE,EACtB,cAAc,EAAE,kBAAkB,GAAG,SAAS,GAC7C,UAAU,EAAE,CAed;AAoYD,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBjG;AA+pBD,wBAAsB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CA4yC/F"}

package/dist/auditor.js

@@ -5,6 +5,7 @@ import { parseHtmlPage } from "./parser.js";
 import { pageSkipReason } from "./page-filter.js";
 import { mergeNormalizeUrlOptions, normalizeAuditUrl } from "./url-normalize.js";
 import { eeatSignalsRule } from "./rules/content/eeat-signals.js";
+import { citationCoverageRule } from "./rules/content/citation-coverage.js";
 import { metaUniquenessRule } from "./rules/content/meta-uniqueness.js";
 import { missingAuthorRule } from "./rules/content/missing-author.js";
 import { uniqueValueRule } from "./rules/content/unique-value.js";
@@ -72,6 +73,10 @@ import { CORE_RULESET_VERSION } from "./ruleset-version.js";
 import { planScrapeStrategy, DEFAULT_AGE_FLOOR_DAYS } from "./scrape-strategy.js";
 import { detectTemplates, buildUrlToTemplateMap, shouldActivateTemplateScoring } from "./template-detection.js";
 import { scoreTemplates, siteVerdictFromTemplates } from "./per-template-scoring.js";
+import { deriveEntityPatterns } from "./algorithms/auto-entity-mask.js";
+import { CompositeAuthorityProvider } from "./algorithms/authority/provider.js";
+import { OpenPageRankProvider } from "./algorithms/authority/openpagerank.js";
+import { registrableDomain } from "./algorithms/fact-extraction.js";
 const DEFAULTS = {
     nearDuplicateThreshold: 0.85,
     entitySwapThreshold: 0.95,
@@ -80,10 +85,12 @@ const DEFAULTS = {
     publicationVelocityMaxPerDayCorpusFraction: 0.10,
     boilerplateMaxRatio: 0.7,
     templateDiversityMinUniqueRatio: 0.35,
-    uniqueValueMinWords: 100,
+    uniqueValueDensity: { passBelow: 0.20, errorBelow: 0.12 },
     metaUniquenessMinJaccard: 0.9,
     linkDepthMaxClicks: 3,
     templateCoverageMinPages: 5,
+    citationCoverageMinClaims: 4,
+    citationCoverageMinAuthoritative: 1,
     answerFirstMaxWords: 100,
     citableFactsMin: 3,
     citableFactsTarget: 8,
@@ -209,7 +216,7 @@ const SCORING_PROFILES = {
         // first-principles analysis predicts will false-positive on catalog-
         // shaped sites (Zapier integrations, G2 categories, Wise currency pairs,
         // etc.). A reputable-pSEO calibration corpus + runner has been added
-        // (scripts/calibration-reputable-pseo.ts); these overrides will be
+        // (scripts/calibration-corpus.ts); these overrides will be
         // tightened or loosened based on actual fire-rates measured against
         // sites that demonstrably win in production. See
         // docs/superpowers/specs/2026-05-03-calibration-against-reputable-pseo.md.
@@ -414,6 +421,9 @@ const RULE_IMPACTS = {
     "content/title-uniqueness": { baseImpact: 8, perInstance: 2, maxImpact: 25 }, // 2026-05-03 round 11: title is high-impact but the original 50-cap was disproportionate to other content rules and tipped Typeform into critical on a 6-finding cluster. Keep the rule at native error severity (duplicate titles ARE real bugs); just don't let one rule dominate the integrity bucket.
     "content/heading-structure": { baseImpact: 5, perInstance: 1, maxImpact: 20 },
     "content/image-alt-text": { baseImpact: 3, perInstance: 1, maxImpact: 20 },
+    // Citation coverage is low-confidence (block-level grounded-claim heuristic);
+    // keep its impact modest so it nudges rather than dominates the score.
+    "content/citation-coverage": { baseImpact: 3, perInstance: 1, maxImpact: 15 },
     "content/translation-no-op": { baseImpact: 30, perInstance: 10, maxImpact: 60 },
     // v1 warning-severity heuristic; lower than translation-no-op since it's speculative
     "content/regurgitated-content": { baseImpact: 15, perInstance: 5, maxImpact: 35 },
@@ -642,7 +652,7 @@ sampled = false) {
     }
     // Content rules
     if (isEnabled("content/unique-value") && modeOk("content/unique-value")) {
-        pushAll(findings, tag(uniqueValueRule(pages, resolvedRules.uniqueValueMinWords)));
+        pushAll(findings, tag(uniqueValueRule(pages, resolvedRules.uniqueValueDensity)));
     }
     if (isEnabled("content/meta-uniqueness") && modeOk("content/meta-uniqueness")) {
         pushAll(findings, tag(metaUniquenessRule(pages, entityPatterns, resolvedRules.metaUniquenessMinJaccard)));
@@ -653,6 +663,13 @@ sampled = false) {
     if (isEnabled("content/eeat-signals") && modeOk("content/eeat-signals")) {
         pushAll(findings, tag(eeatSignalsRule(pages)));
     }
+    if (isEnabled("content/citation-coverage") && modeOk("content/citation-coverage")) {
+        pushAll(findings, tag(citationCoverageRule(pages, entityPatterns, {
+            minClaims: resolvedRules.citationCoverageMinClaims,
+            minAuthoritative: resolvedRules.citationCoverageMinAuthoritative,
+            allowlist: resolvedRules.citationAllowlist,
+        })));
+    }
     // 2026-05-03 v0.5.2 blind-spot fixes — title uniqueness + heading
     // structure + image alt-text were tier-1 gaps in the blind-spot audit.
     if (isEnabled("content/title-uniqueness") && modeOk("content/title-uniqueness")) {
@@ -678,7 +695,7 @@ sampled = false) {
     }
     // Link rules — use the global link graph
     if (isEnabled("links/orphan-pages") && modeOk("links/orphan-pages")) {
-        pushAll(findings, tag(orphanPagesRule(pages, inbound, rootUrl)));
+        pushAll(findings, tag(orphanPagesRule(pages, inbound, rootUrl, sampled)));
     }
     if (isEnabled("links/dead-ends") && modeOk("links/dead-ends")) {
         pushAll(findings, tag(deadEndsRule(pages, knownUrls, rootUrl)));
@@ -689,7 +706,7 @@ sampled = false) {
         }
     }
     if (isEnabled("links/cluster-connectivity") && modeOk("links/cluster-connectivity")) {
-        pushAll(findings, tag(clusterConnectivityRule(pages, knownUrls)));
+        pushAll(findings, tag(clusterConnectivityRule(pages, knownUrls, sampled)));
     }
     if (isEnabled("links/host-section-divergence") && modeOk("links/host-section-divergence")) {
         pushAll(findings, tag(hostSectionDivergenceRule(pages, adjacency)));
@@ -1883,10 +1900,13 @@ export async function auditSource(source, options) {
             ?? DEFAULTS.publicationVelocityMaxPerDayCorpusFraction,
         boilerplateMaxRatio: options?.rules?.boilerplateMaxRatio ?? DEFAULTS.boilerplateMaxRatio,
         templateDiversityMinUniqueRatio: options?.rules?.templateDiversityMinUniqueRatio ?? DEFAULTS.templateDiversityMinUniqueRatio,
-        uniqueValueMinWords: options?.rules?.uniqueValueMinWords ?? DEFAULTS.uniqueValueMinWords,
+        uniqueValueDensity: options?.rules?.uniqueValueDensity ?? DEFAULTS.uniqueValueDensity,
         metaUniquenessMinJaccard: options?.rules?.metaUniquenessMinJaccard ?? DEFAULTS.metaUniquenessMinJaccard,
         linkDepthMaxClicks: options?.rules?.linkDepthMaxClicks ?? DEFAULTS.linkDepthMaxClicks,
         templateCoverageMinPages: options?.rules?.templateCoverageMinPages ?? DEFAULTS.templateCoverageMinPages,
+        citationCoverageMinClaims: options?.rules?.citationCoverageMinClaims ?? DEFAULTS.citationCoverageMinClaims,
+        citationCoverageMinAuthoritative: options?.rules?.citationCoverageMinAuthoritative ?? DEFAULTS.citationCoverageMinAuthoritative,
+        citationAllowlist: options?.rules?.citationAllowlist,
         answerFirstMaxWords: options?.rules?.answerFirstMaxWords ?? DEFAULTS.answerFirstMaxWords,
         citableFactsMin: options?.rules?.citableFactsMin ?? DEFAULTS.citableFactsMin,
         citableFactsTarget: options?.rules?.citableFactsTarget ?? DEFAULTS.citableFactsTarget,
@@ -2346,7 +2366,10 @@ export async function auditSource(source, options) {
     const auditMode = options?.mode ?? "full";
     // Site-wide rules (run once, outside group loop)
     if (sitemapUrlSet && sitemapUrlSet.size > 0 && auditMode !== "diff") {
-        const sitemapFindings = sitemapCompletenessRule(parsedPages, sitemapUrlSet);
+        const sitemapFindings = sitemapCompletenessRule(parsedPages, sitemapUrlSet, {
+            sampled: isSampledAudit,
+            normalizeUrlOptions,
+        });
         pushAll(allFindings, sitemapFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
         if (robotsTxtContent) {
             const robotsFindings = robotsComplianceRule(parsedPages, sitemapUrlSet, robotsTxtContent);
@@ -2372,6 +2395,7 @@ export async function auditSource(source, options) {
             pushAll(allFindings, dataIdenticalFindings.map((f) => ({ ...f, ref: f.ref ?? RULE_REFERENCES[f.ruleId] })));
         }
     }
+    const derivedEntityPatterns = options?.autoEntityMask === false ? [] : deriveEntityPatterns(parsedPagesAll);
     for (const [groupName, groupPages] of classified) {
         if (groupPages.length === 0)
             continue;
@@ -2380,7 +2404,7 @@ export async function auditSource(source, options) {
             continue;
         const groupRules = resolveGroupRules(resolvedRules, groupConfig?.overrides);
         const enabledCheck = (ruleId) => !suppressedRuleSet.has(ruleId) && isRuleEnabled(ruleId, groupConfig?.rules);
-        const findings = runRulesOnPages(groupPages, parsedPagesAll, groupRules, enabledCheck, groupName, knownUrls, adjacency, inbound, rootUrl, normalizeUrlOptions, source, DEFAULT_ENTITY_PATTERNS, groupConfig?.overrides, options?.mode ?? "full", 
+        const findings = runRulesOnPages(groupPages, parsedPagesAll, groupRules, enabledCheck, groupName, knownUrls, adjacency, inbound, rootUrl, normalizeUrlOptions, source, [...DEFAULT_ENTITY_PATTERNS, ...derivedEntityPatterns], groupConfig?.overrides, options?.mode ?? "full", 
         // 2026-05-06 calibration fix: pinnedUrls mode fetches a hand-picked subset
         // of the full site — the link graph across those pages is structurally
         // incomplete, just like a random-sampled crawl. Pass `true` so
@@ -2486,12 +2510,31 @@ export async function auditSource(source, options) {
     const { risk, categories, bucketCounts } = scoreFromFindings(enriched.findings, siteClassification, parsedPages.length);
     const auditedPageCount = Object.values(groupPageCounts).reduce((a, b) => a + b, 0);
     const issues = bucketIssues(enriched.findings);
+    // Resolve a domain-authority score to moderate the verdict. Explicit option
+    // wins; otherwise a provider (custom, or default OPR composite). null/absent
+    // → no moderation (fail-safe).
+    let resolvedAuthorityScore = options?.authorityScore;
+    let resolvedAuthorityDomain;
+    if (resolvedAuthorityScore === undefined) {
+        const provider = options?.authorityProvider ??
+            new CompositeAuthorityProvider([new OpenPageRankProvider(options?.openPageRankApiKey ?? "")]);
+        try {
+            const host = new URL(source.startsWith("http") ? source : `https://${source}`).hostname;
+            resolvedAuthorityDomain = registrableDomain(host);
+            const a = await provider.authorityFor(resolvedAuthorityDomain);
+            if (a !== null)
+                resolvedAuthorityScore = a;
+        }
+        catch {
+            /* source is a local dir / unparseable → no authority */
+        }
+    }
     // v0.6.0 — spec §15.1: site verdict comes from siteVerdictFromTemplates when
     // ≥1 template has ≥5% coverage. Falls back to the legacy risk-ladder verdict
     // when no template meets the threshold (single-template sites, `unclear`/
     // `small-marketing` classifications, or the long-tail-only case).
     // The `risk` score is intentionally unchanged — §15.1 governs verdict only.
-    const legacyVerdict = shiftVerdictForAuthority(verdictForRisk(risk), options?.authorityScore);
+    const legacyVerdict = shiftVerdictForAuthority(verdictForRisk(risk), resolvedAuthorityScore);
     const templateVerdict = siteVerdictFromTemplates(siteTemplates);
     const verdict = templateVerdict !== null ? templateVerdict : legacyVerdict;
     const headline = buildHeadline(bucketCounts);
@@ -2530,6 +2573,9 @@ export async function auditSource(source, options) {
         auditedUrls: parsedPages.length > 0
             ? [...parsedPages.map((p) => p.url)].sort()
             : undefined,
+        ...(resolvedAuthorityScore !== undefined
+            ? { authority: { score: resolvedAuthorityScore, domain: resolvedAuthorityDomain ?? "" } }
+            : {}),
     };
     // Partial-report flag: the backpressure watchdog aborted mid-crawl and we
     // salvaged whatever pages had been fetched. Consumers MUST treat coverage as