optbrew 0.1.3 → 0.2.0

package/skills/optbrew/scripts/core/derive.mjs

@@ -1,193 +1,17 @@
-import { createHash } from 'node:crypto';
-import { validateEntity } from './contracts.mjs';
-import { auditCollected } from '../../modules/naver-searchadvisor/scripts/audit.mjs';
-import { parseRobots, robotsVerdict } from '../../modules/naver-searchadvisor/scripts/robots.mjs';
-import { anchorLinks, decodeEntities } from './html.mjs';
-import { agentFor } from './crawlers.mjs';
-import { declaredRobotsIndex } from './collector.mjs';
-import { compareFacts, pageFacts } from './facts.mjs';
+import { blocks } from './checks/index.mjs';
+import { prepare } from './checks/prepare.mjs';
+import { publicOnlyFamily } from './config.mjs';
 
 /**
  * Every collector verdict, derived from collected responses alone. The audit writes these observations and the
  * report replays this same function over hashed raw evidence, so a producer and a verifier cannot disagree.
  * `artifacts` carry the collection record fields plus evidence_ref; `textOf(artifact)` returns the raw body text.
+ * One shared `prepare` step computes what the blocks have in common; each planned family's block then judges it.
+ * A family that only matters for indexed pages (`public_only` in the registry) sees only the pages meant to be indexed.
  */
-/** Link and sitemap target comparison key: no fragment, and a directory index file equals its directory. */
-function canonicalUrl(value) {
-  try {
-    const url = new URL(value); url.hash = '';
-    url.pathname = url.pathname.replace(/\/index\.(?:html?|php)$/i, '/');
-    return url.href;
-  } catch { return null; }
-}
-
-export function deriveObservations({ context, artifacts, collection = {}, textOf }) {
-  const protocol = context.protocol;
-  const planned = new Set(protocol.families.filter(x => x.applicable).map(x => x.family_id));
-  const selected = new Map(protocol.page_types.flatMap(type => type.urls).map(url => [url.normalized, url]));
-  const declared = kind => new Set((protocol.auxiliary_resources ?? []).filter(x => x.kind === kind).map(x => x.normalized));
-  const declaredRobots = declared('robots'), declaredSitemaps = declared('sitemap');
-  const out = [];
-  const add = (family, engine, url, state, reason, evidence_ref) => {
-    const unit_key = `${family}|${engine ? `${engine}|` : ''}${url}`;
-    out.push(validateEntity('CheckObservation', { schema_version: '1.0.0',
-      observation_id: createHash('sha256').update(unit_key).digest('hex'), check_id: family, family_id: family, unit_key,
-      state, q: state === 'pass' ? 1 : state === 'fail' ? 0 : null, partial_basis: null, reason, evaluator_type: 'code',
-      source_ref: 'optbrew:collector', evidence_ref }));
-  };
-  const pageRows = [];
-  for (const artifact of artifacts) {
-    const url = artifact.normalized_url;
-    if (!selected.has(url)) continue;
-    if (planned.has('fetch-response')) add('fetch-response', null, url,
-      artifact.error || artifact.status < 200 || artifact.status >= 300 && artifact.status < 400 ? 'unknown' : artifact.status >= 400 ? 'fail' : 'pass',
-      artifact.error ? 'Request did not finish' : `HTTP ${artifact.status}`, artifact.evidence_ref);
-    if (artifact.capability !== 'raw_html' || artifact.status !== 200) continue;
-    const html = textOf(artifact);
-    const robotsIndex = declaredRobotsIndex(artifacts, declaredRobots, artifact.final_url);
-    const robots = robotsIndex < 0 ? null : artifacts[robotsIndex];
-    const info = auditCollected({ artifact, html, engines: protocol.engines,
-      robotsArtifact: robots?.status === 200 ? { ...robots, body: textOf(robots) } : null });
-    pageRows.push({ url, artifact, html, info, robots, publicIndexing: selected.get(url).public_indexing !== false });
-  }
-  const normalizeTitle = title => title.toLowerCase().replace(/\s+/g, ' ').trim();
-  const titleCounts = new Map();
-  for (const row of pageRows) if (row.info.title) {
-    const key = normalizeTitle(row.info.title);
-    titleCounts.set(key, new Set([...(titleCounts.get(key) ?? []), row.artifact.final_url]));
-  }
-  const facts = pageRows.map(row => ({ url: row.url, row, facts: pageFacts(row.html) }));
-  const comparison = compareFacts(facts);
-  for (const row of pageRows) {
-    const { url, artifact, info } = row, ref = artifact.evidence_ref;
-    if (planned.has('document-title')) {
-      const dup = info.title ? pageRows.filter(x => x !== row && x.info.title && normalizeTitle(x.info.title) === normalizeTitle(info.title) &&
-        x.artifact.final_url !== artifact.final_url) : [];
-      add('document-title', null, url, !info.title || dup.length ? 'fail' : 'unknown',
-        !info.title ? 'Raw HTML has no title' : dup.length ? `Title duplicates ${dup.map(x => x.url).join(', ')}` :
-          'Raw HTML has a title; uniqueness and descriptiveness are not established', ref);
-    }
-    if (planned.has('index-policy') && row.publicIndexing)
-      add('index-policy', null, url, info.index_blocked ? 'fail' : 'pass',
-        info.index_blocked ? 'Raw response blocks indexing for the target engine' : 'Raw response has no applicable noindex directive', ref);
-    if (planned.has('search-crawl-policy') && row.publicIndexing && row.robots?.status === 200) {
-      const parsed = parseRobots(textOf(row.robots));
-      const target = new URL(artifact.final_url);
-      for (const engine of protocol.engines) {
-        const agent = agentFor(engine);
-        if (!agent) continue;
-        const verdict = robotsVerdict(parsed, agent, target.pathname + target.search);
-        add('search-crawl-policy', engine, url, verdict.allowed ? 'pass' : 'fail',
-          verdict.allowed ? `Collected robots.txt allows ${agent} path` : `Collected robots.txt disallows ${agent} path`, row.robots.evidence_ref);
-      }
-    }
-    if (planned.has('naver-markup') && protocol.engines.includes('naver'))
-      add('naver-markup', 'naver', url, info.markup_findings.length ? 'fail' : 'pass',
-        info.markup_findings.length ? info.markup_findings.map(x => x.message).join('; ') : 'Open Graph, description, canonical, viewport and title checks pass', ref);
-    if (planned.has('structured-data-truth') && info.jsonld_parse_errors > 0)
-      add('structured-data-truth', null, url, 'fail', `${info.jsonld_parse_errors} JSON-LD block(s) fail to parse`, ref);
-    if (planned.has('canonical-consistency')) {
-      const hops = artifact.redirect_chain?.length ?? 0;
-      let canonicalState = null, canonicalNote = '';
-      if (info.canonical) {
-        try {
-          const target = new URL(info.canonical, artifact.final_url); target.hash = '';
-          const other = artifacts.find(x => x.normalized_url === target.href || x.final_url === target.href);
-          if (target.href === artifact.final_url || target.href === artifact.normalized_url) canonicalState = 'pass';
-          else if (other && other.status >= 200 && other.status < 300) canonicalState = 'pass';
-          else if (other && Number.isInteger(other.status)) { canonicalState = 'fail'; canonicalNote = `canonical target ${target.href} returns HTTP ${other.status}`; }
-        } catch { canonicalState = 'fail'; canonicalNote = 'canonical is not a valid URL'; }
-      }
-      if (hops >= 2) add('canonical-consistency', null, url, 'fail', `Redirect chain of ${hops} hops${canonicalNote ? `; ${canonicalNote}` : ''}`, ref);
-      else if (canonicalState === 'fail') add('canonical-consistency', null, url, 'fail', canonicalNote, ref);
-      else if (canonicalState === 'pass') add('canonical-consistency', null, url, 'pass', 'Canonical target is this page or a reachable page and redirects stay within one hop', ref);
-    }
-  }
-  /** True when any other collected page's raw bytes name this page's path (whole path segment, not a longer path). */
-  const pathMentioned = ({ url, artifact }, bodies) => {
-    const own = canonicalUrl(url);
-    for (const target of new Set([url, artifact.final_url])) {
-      let path;
-      try { path = decodeURIComponent(new URL(target).pathname); } catch { continue; }
-      if (path === '/') continue;
-      const pattern = new RegExp(path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?![A-Za-z0-9_-])');
-      for (const [page, body] of bodies) if (page !== own && pattern.test(body)) return true;
-    }
-    return false;
-  };
-  if (planned.has('discovery')) {
-    const sitemapUrls = new Set();
-    const brokenSitemaps = [];
-    let sitemapIndex = false;
-    for (const artifact of artifacts) {
-      if (!declaredSitemaps.has(artifact.normalized_url)) continue;
-      if (artifact.status >= 200 && artifact.status < 300) {
-        const body = String(textOf(artifact) ?? '');
-        // Child sitemaps of an index are not fetched, so pages missing from the index prove nothing.
-        if (/<sitemapindex\b/i.test(body)) sitemapIndex = true;
-        for (const match of body.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) sitemapUrls.add(canonicalUrl(decodeEntities(match[1])));
-      } else brokenSitemaps.push(`${artifact.normalized_url} (${artifact.status ?? artifact.error ?? 'no response'})`);
-    }
-    const htmlPages = artifacts.filter(x => x.capability === 'raw_html' && x.status === 200);
-    const linked = new Map();
-    // A page with scripts and no anchors may build its navigation in the browser. That only hides an orphan when the
-    // script can be read: an inline script is searched for the path below, and a same-origin script file that was not
-    // collected cannot be read at all.
-    const collected = new Set(artifacts.flatMap(x => [x.normalized_url, x.final_url].filter(Boolean).map(canonicalUrl)));
-    let uncollectedScript = false;
-    const bodies = new Map();
-    for (const page of htmlPages) {
-      const body = textOf(page);
-      bodies.set(canonicalUrl(page.normalized_url), body);
-      const hrefs = anchorLinks(body, page.final_url);
-      if (!hrefs.length && /<script\b/i.test(body)) {
-        for (const match of body.matchAll(/<script\b[^>]*\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)')/gi)) {
-          try {
-            const source = new URL(decodeEntities(match[1] ?? match[2]), page.final_url);
-            if (source.origin === new URL(page.final_url).origin && !collected.has(canonicalUrl(source.href))) uncollectedScript = true;
-          } catch { /* malformed script URL */ }
-        }
-      }
-      for (const href of hrefs) {
-        const target = canonicalUrl(href);
-        if (target && target !== canonicalUrl(page.final_url) && target !== canonicalUrl(page.normalized_url)) linked.set(target, (linked.get(target) ?? 0) + 1);
-      }
-    }
-    const complete = !collection.limit_reached && !(collection.skipped ?? []).some(x => ['depth_limit', 'url_limit', 'request_limit'].includes(x.reason));
-    for (const row of pageRows) {
-      if (!row.publicIndexing) continue;
-      const { url, artifact } = row;
-      const keys = [canonicalUrl(url), canonicalUrl(artifact.final_url)];
-      const inSitemap = keys.some(key => sitemapUrls.has(key));
-      if (inSitemap || keys.some(key => linked.has(key)))
-        add('discovery', null, url, 'pass', inSitemap ? 'Listed in a declared sitemap' : 'Linked from another collected page', artifact.evidence_ref);
-      // An unlinked page is unknown when a collected page's own bytes (inline scripts included) name its path, or when a
-      // script file that could build the link was not collected, unless a fetched sitemap that lists other pages leaves
-      // it out. Anything else is an orphan.
-      else if (htmlPages.length >= 2 && complete && !sitemapIndex && (sitemapUrls.size || !(uncollectedScript || pathMentioned(row, bodies))))
-        add('discovery', null, url, 'fail', `Not linked from any collected page and not in a sitemap${brokenSitemaps.length ? `; declared sitemap unusable: ${brokenSitemaps.join(', ')}` : ''}`, artifact.evidence_ref);
-    }
-  }
-  if (planned.has('fact-consistency')) {
-    for (const page of facts) {
-      if (comparison.separate.has(page.url)) continue;
-      const found = comparison.deviations.get(page.url) ?? [];
-      const comparable = Object.keys(comparison.references).some(type => page.facts.visible[type].size || page.facts.linked[type].size || (type === 'phone' && page.facts.tel.phone.size));
-      if (found.length) add('fact-consistency', null, page.url, 'fail', `Cross-page fact comparison (no approved fact list): ${found.join('; ')}`, page.row.artifact.evidence_ref);
-      else if (comparable) add('fact-consistency', null, page.url, 'pass', 'Facts on this page match the cross-page reference values (no approved fact list; consistency is not accuracy)', page.row.artifact.evidence_ref);
-    }
-  }
-  if (planned.has('entity-identity') && facts.length) {
-    const reasons = [];
-    const home = facts.find(x => new URL(x.row.artifact.final_url).pathname === '/') ?? facts[0];
-    const orgPages = facts.filter(x => x.facts.orgs.length && !comparison.separate.has(x.url));
-    const nameReference = comparison.references.name;
-    const nameDeviants = facts.filter(x => (comparison.deviations.get(x.url) ?? []).some(text => text.startsWith('name ')));
-    if (nameDeviants.length) reasons.push(`name differs from "${nameReference.label}" on ${nameDeviants.map(x => x.url).join(', ')}`);
-    for (const page of orgPages) if (page.facts.orgs.some(org => !org.sameAs)) reasons.push(`Organization JSON-LD without sameAs on ${page.url}`);
-    if (orgPages.length && !home.facts.orgs.length) reasons.push(`no Organization or LocalBusiness JSON-LD on ${home.url}`);
-    if (reasons.length) add('entity-identity', null, 'site', 'fail', `Entity identity: ${reasons.join('; ')}`, (nameDeviants[0] ?? home).row.artifact.evidence_ref);
-    else if (orgPages.length) add('entity-identity', null, 'site', 'pass', 'Organization JSON-LD with name and sameAs is present on the home page and names agree', home.row.artifact.evidence_ref);
-  }
-  return out;
+export function deriveObservations(input) {
+  const prepared = prepare(input);
+  const publicOnly = { ...prepared, pageRows: prepared.pageRows.filter(row => row.publicIndexing) };
+  return blocks.filter(block => prepared.planned.has(block.family_id))
+    .flatMap(block => block.derive(publicOnlyFamily(block.family_id) ? publicOnly : prepared));
 }

package/skills/optbrew/scripts/core/evidence-store.mjs

@@ -0,0 +1,112 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { readFileSync, realpathSync } from 'node:fs';
+import { readFile, rename, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const INDEX_FILE = 'evidence-index.json';
+// Raw collected bodies keep their own extension; every other kind is a JSON record.
+const RAW_KINDS = new Set(['html', 'bin', 'render-dom', 'owner-export']);
+// Written with the index so an audit that fetched nothing still has them; the others appear on first append.
+const BASE_SECTIONS = ['raw', 'responses', 'failures'];
+const ALL_SECTIONS = [...BASE_SECTIONS, 'renders', 'render_ledger', 'cwv'];
+// One queue per index file for the whole process, so two stores on one directory never interleave a read-modify-write.
+const queues = new Map();
+const sha256Of = bytes => createHash('sha256').update(bytes).digest('hex');
+const within = (root, file) => file === root || file.startsWith(`${root}${path.sep}`);
+
+/**
+ * Read evidence bytes after checking that the file stays inside root and matches its recorded hash.
+ * `ref` is absolute or relative to root. Report replay and Store.read both read through this.
+ */
+export function readVerified(root, ref, sha256) {
+  const file = path.resolve(root, ref);
+  if (!within(path.resolve(root), file) || !within(realpathSync(root), realpathSync(file)))
+    throw new Error(`evidence path outside run directory: ${ref}`);
+  const bytes = readFileSync(file);
+  if (sha256Of(bytes) !== sha256) throw new Error(`evidence hash mismatch: ${ref}`);
+  return bytes;
+}
+
+/**
+ * One directory of run evidence. Objects are content-addressed and written once; the index is a JSON file replaced
+ * atomically on every append, so a crash leaves either the old index or the new one, never a half-written file.
+ * `header` holds the index's leading fields and is used only when the run has no index yet.
+ */
+export function openStore(dir, header = null) {
+  const indexRef = path.join(dir, INDEX_FILE);
+  const empty = () => ({ ...header, ...Object.fromEntries(BASE_SECTIONS.map(name => [name, []])) });
+  const queueKey = path.resolve(indexRef);
+  const serial = job => {
+    const run = (queues.get(queueKey) ?? Promise.resolve()).then(job);
+    const tail = run.catch(() => {}).then(() => { if (queues.get(queueKey) === tail) queues.delete(queueKey); });
+    queues.set(queueKey, tail);
+    return run;
+  };
+  const load = async () => {
+    try { return JSON.parse(await readFile(indexRef, 'utf8')); }
+    catch (error) { if (error.code === 'ENOENT') return null; throw error; }
+  };
+  const save = async index => {
+    const temporary = `${indexRef}.${randomUUID()}.tmp`;
+    try {
+      await writeFile(temporary, `${JSON.stringify(index, null, 2)}\n`);
+      await rename(temporary, indexRef);
+    } catch (error) {
+      await rm(temporary, { force: true }).catch(() => {});
+      throw error;
+    }
+  };
+
+  return {
+    indexRef,
+    /** Write `bytes` once. The file is named by its hash, or by `name` for a fixed-name run document. */
+    async write(kind, bytes, { name } = {}) {
+      if (!/^[a-z][a-z0-9-]*$/.test(kind)) throw new Error(`invalid evidence kind: ${kind}`);
+      if (name !== undefined && (!name || path.basename(name) !== name)) throw new Error(`invalid evidence name: ${name}`);
+      const data = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
+      const sha256 = sha256Of(data);
+      const ref = path.join(dir, name ?? `${sha256}.${RAW_KINDS.has(kind) ? kind : `${kind}.json`}`);
+      try { await writeFile(ref, data, { flag: 'wx' }); }
+      catch (error) {
+        if (error.code !== 'EEXIST') throw error;
+        if (!(await readFile(ref)).equals(data)) throw new Error(`evidence ${path.basename(ref)} already exists with different content`);
+      }
+      return { ref, sha256 };
+    },
+    /** Add one entry to a section of the index (created on first use) and replace the index file. */
+    append(section, entry) {
+      if (!/^[a-z][a-z0-9_]*$/.test(section)) throw new Error(`invalid index section: ${section}`);
+      if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) throw new Error(`index entry must be an object: ${section}`);
+      return serial(async () => {
+        const index = (await load()) ?? empty();
+        if (!Object.hasOwn(index, section)) index[section] = [];
+        else if (!Array.isArray(index[section])) throw new Error(`not an index section: ${section}`);
+        index[section].push(entry);
+        await save(index);
+      });
+    },
+    /** Record one value that is a single object, not a list (such as the Chrome result), replacing an earlier value of the same name. */
+    set(field, value) {
+      if (!/^[a-z][a-z0-9_]*$/.test(field) || ALL_SECTIONS.includes(field) || Object.hasOwn(header ?? {}, field)) throw new Error(`invalid index field: ${field}`);
+      return serial(async () => {
+        const index = (await load()) ?? empty();
+        index[field] = value;
+        await save(index);
+      });
+    },
+    /** The current index with every section present. With a header and no index file yet, the index is created first. */
+    index() {
+      return serial(async () => {
+        let index = await load();
+        if (!index && header) await save(index = empty());
+        return { ...Object.fromEntries(ALL_SECTIONS.map(name => [name, []])), ...index };
+      });
+    },
+    /** Bytes of a ref returned by write or listed in the index; throws if the file is outside the store or its hash differs. */
+    async read(ref, sha256) {
+      // A ref from write() is dir-joined (cwd-relative when dir is); any other relative ref is relative to dir, as in readVerified.
+      const returned = path.resolve(ref);
+      return readVerified(dir, within(path.resolve(dir), returned) ? returned : ref, sha256);
+    },
+  };
+}

package/skills/optbrew/scripts/core/facts.mjs

@@ -1,4 +1,7 @@
 import { decodeEntities, jsonLdBlocks, stripComments, stripInert, visibleText } from './html.mjs';
+import { endpoints } from './config.mjs';
+import { canonicalUrl } from './link-graph.mjs';
+import { messageOf } from './messages.mjs';
 
 /**
  * Cross-page entity fact extraction and comparison, pure functions of the raw HTML of the selected pages.
@@ -7,13 +10,17 @@ import { decodeEntities, jsonLdBlocks, stripComments, stripInert, visibleText }
  * result is unknown, not a failure.
  */
 const FACT_TYPES = ['name', 'phone', 'year', 'address'];
+/** The catalog keys of the two ways a page's fact can conflict; the check that states them lists them as its own. */
+export const FACT_MESSAGE_KEYS = { differs: 'fact-consistency.differs', jsonld: 'fact-consistency.jsonld-differs' };
 const ORG_TYPES = /^(?:Organization|Corporation|LocalBusiness|Store|Restaurant|CafeOrCoffeeShop|OnlineStore|NGO|[A-Za-z]*Business)$/;
 const squash = value => String(value).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
 const BRANCH = /\b(?:branch)\b|지점|지사/i;
 // URL path segments that list branches: /branch/, /branches/, /locations/, /stores/, /지점/.
 const BRANCH_PATH = /^(?:branch|branches|locations|stores|지점)$/i;
+/** A node that states a sub-organization: a branch or department of another entity, by its own link or by a branch word in its name. */
+const isBranchNode = node => Boolean(node.branchOf || node.parentOrganization || BRANCH.test(typeof node.name === 'string' ? node.name : ''));
 const branchPath = url => {
-  try { return new URL(url, 'http://page.invalid').pathname.split('/').some(part => BRANCH_PATH.test(decodeURIComponent(part))); }
+  try { return new URL(url, endpoints().relative_url_base).pathname.split('/').some(part => BRANCH_PATH.test(decodeURIComponent(part))); }
   catch { return false; }
 };
 
@@ -103,8 +110,11 @@ function readPhone(rest) {
   return null;
 }
 
-/** Facts on one page: visible labelled values and JSON-LD / tel: values, keyed by fact type. */
-export function pageFacts(html) {
+/**
+ * Facts on one page: visible labelled values and JSON-LD / tel: values, keyed by fact type. `pageUrls` are the addresses the page
+ * was requested at and served from; `entities` are the JSON-LD nodes that speak for the page.
+ */
+export function pageFacts(html, pageUrls = []) {
   const live = stripComments(html);
   const visible = Object.fromEntries(FACT_TYPES.map(type => [type, new Map()]));
   const linked = Object.fromEntries(FACT_TYPES.map(type => [type, new Map()]));
@@ -125,29 +135,53 @@ export function pageFacts(html) {
     put(visible, 'year', match[1] ?? match[2], match[1] ?? match[2]);
   for (const match of text.matchAll(/\bAddress\b\s*[:：]\s*([^\n]+?)(?=\.\s|\.?\s*$|\n)/gim)) put(visible, 'address', match[1], addressKey(match[1]));
   for (const match of text.matchAll(/주소\s*[:：]\s*([^\n.]+)/g)) put(visible, 'address', match[1], addressKey(match[1]));
-  const orgs = [];
+  const described = [];
+  // The @id values a node of the page names as what it is about (WebPage.about, mainEntity).
+  const aboutIds = new Set();
   for (const block of jsonLdBlocks(live)) {
     if (!block.ok) continue;
     for (const node of nodesOf(block.value)) {
+      for (const key of ['about', 'mainEntity']) for (const ref of [].concat(node[key] ?? []))
+        if (typeof ref === 'string' ? ref : ref?.['@id']) aboutIds.add(typeof ref === 'string' ? ref : ref['@id']);
       if (!typesOf(node).some(type => ORG_TYPES.test(type))) continue;
       const name = typeof node.name === 'string' ? node.name : '';
-      const org = { name, id: typeof node['@id'] === 'string' ? node['@id'] : null,
+      described.push({ node, org: { name, id: typeof node['@id'] === 'string' ? node['@id'] : null,
         sameAs: [].concat(node.sameAs ?? []).some(x => typeof x === 'string' && x.trim()),
-        branch: Boolean(node.branchOf || node.parentOrganization || BRANCH.test(name)) };
-      if (name) put(linked, 'name', name, squash(name));
-      if (typeof node.telephone === 'string') put(linked, 'phone', node.telephone, phoneKey(node.telephone));
-      const address = addressText(node.address);
-      if (address) put(linked, 'address', address, addressKey(address));
-      const year = /\b((?:19|20)\d{2})\b/.exec(String(node.foundingDate ?? ''))?.[1];
-      if (year) put(linked, 'year', year, year);
-      orgs.push(org);
+        branch: isBranchNode(node) } });
     }
   }
+  // A sub-organization (a branch, a department) states facts of its own, not the page's. A sub-organization whose own url is this
+  // page, or that this page names as what it is about, is the page's entity: the page is that branch's page, whatever else its
+  // graph carries (the site's main organization is on every page of many sites). An @id counts only when it is the page address
+  // itself. The site home never belongs to one branch: an address that is the root path, with or without a fragment (`/#annex`), is
+  // the home page's. What the page is about names a branch only when it names exactly one sub-organization (a locations page lists
+  // several).
+  const ownKeys = new Set(pageUrls.map(canonicalUrl).filter(Boolean));
+  const atRoot = value => { try { return new URL(value).pathname === '/'; } catch { return false; } };
+  const claimsRoot = ({ node, org }) => [node.url, org.id].some(value => typeof value === 'string' && atRoot(value));
+  const aboutSubs = described.filter(item => item.org.branch && item.org.id && aboutIds.has(item.org.id));
+  const isOwn = item => item.org.branch && !claimsRoot(item) &&
+    (ownKeys.has(canonicalUrl(item.node.url)) || pageUrls.includes(item.org.id) || (aboutSubs.length === 1 && aboutSubs[0] === item));
+  const own = described.filter(isOwn);
+  // Otherwise, when the page also carries a node that is not a sub-organization, that node is the page's entity and only it
+  // speaks for the page; a page whose every node is a sub-organization is a separate branch page.
+  const speaking = own.length ? own : described.some(item => !item.org.branch) ? described.filter(item => !item.org.branch) : described;
+  const orgs = [];
+  for (const { node, org } of speaking) {
+    if (org.name) put(linked, 'name', org.name, squash(org.name));
+    if (typeof node.telephone === 'string') put(linked, 'phone', node.telephone, phoneKey(node.telephone));
+    const address = addressText(node.address);
+    if (address) put(linked, 'address', address, addressKey(address));
+    const year = /\b((?:19|20)\d{2})\b/.exec(String(node.foundingDate ?? ''))?.[1];
+    if (year) put(linked, 'year', year, year);
+    orgs.push(org);
+  }
   // A branch word in the title or first heading is only a hint (titleBranch). A page is a separate entity when its
   // JSON-LD says so (branch), when its URL is a branch list, or when the hint comes with an address of its own.
   const heading = /<(?:title|h1)\b[^>]*>([\s\S]*?)<\/(?:title|h1)>/gi;
   const headings = [...live.matchAll(heading)].map(match => match[1]).join(' ');
-  return { visible, linked, tel, orgs, branch: orgs.some(org => org.branch), titleBranch: BRANCH.test(headings) };
+  return { visible, linked, tel, orgs, entities: speaking.map(item => item.node), branch: orgs.length > 0 && orgs.every(org => org.branch),
+    titleBranch: BRANCH.test(headings) };
 }
 
 /**
@@ -180,6 +214,8 @@ export function compareFacts(allPages) {
   const pages = allPages.filter(page => !separate(page));
   const references = {};
   const deviations = new Map(allPages.map(page => [page.url, []]));
+  // What `deviations` says, in the form the report's Korean text fills: one entry for each text, in the same order.
+  const deviation_messages = new Map(allPages.map(page => [page.url, []]));
   for (const type of FACT_TYPES) {
     const counts = new Map();
     const carrying = pages.filter(page => keysOf(page, type).size);
@@ -201,15 +237,137 @@ export function compareFacts(allPages) {
         if (![...keysOf(page, type)].some(agrees)) {
           const found = [...evidenceOf(page, type).values()].join(' / ');
           deviations.get(page.url).push(`${type} "${found}" differs from "${label}" on ${count} pages`);
+          deviation_messages.get(page.url).push(messageOf(FACT_MESSAGE_KEYS.differs, { type, found, label, count }));
         }
       }
     }
     for (const page of pages) {
       const v = [...page.facts.visible[type].keys()], l = [...page.facts.linked[type].keys()];
       const same = type === 'address' ? v.some(a => l.some(b => sameAddress(a, b))) : v.some(key => l.includes(key));
-      if (v.length && l.length && !same && !deviations.get(page.url).some(x => x.startsWith(`${type} `)))
-        deviations.get(page.url).push(`${type} in JSON-LD "${[...page.facts.linked[type].values()].join(' / ')}" differs from visible "${[...page.facts.visible[type].values()].join(' / ')}"`);
+      if (v.length && l.length && !same && !deviations.get(page.url).some(x => x.startsWith(`${type} `))) {
+        const linked = [...page.facts.linked[type].values()].join(' / '), visible = [...page.facts.visible[type].values()].join(' / ');
+        deviations.get(page.url).push(`${type} in JSON-LD "${linked}" differs from visible "${visible}"`);
+        deviation_messages.get(page.url).push(messageOf(FACT_MESSAGE_KEYS.jsonld, { type, linked, visible }));
+      }
+    }
+  }
+  return { references, deviations, deviation_messages, separate: new Set(allPages.filter(separate).map(page => page.url)) };
+}
+
+// A phone number with its separators is never longer than this, so a number is read from this many characters after its first digit.
+const PHONE_SPAN = 32;
+/** The phone keys of every number `text` shows, read with the same rules as a labelled number. */
+function numbersIn(text) {
+  const keys = new Set();
+  for (const match of text.matchAll(/(?<!\d)[+(]?\d/g)) {
+    const number = readPhone(text.slice(match.index, match.index + PHONE_SPAN));
+    const key = number && phoneKey(number);
+    if (key) keys.add(key);
+  }
+  return keys;
+}
+/** True when `tokens` stand next to each other, in that order, among `words`. */
+const runIn = (words, tokens) => words.some((_, i) => tokens.every((token, k) => words[i + k] === token));
+/** True when `tokens` (at least two, one of them a word) stand next to each other, in that order, among `words`. */
+const hasRun = (words, tokens) => tokens.length > 1 && tokens.some(token => /^\p{L}/u.test(token)) && runIn(words, tokens);
+const wordsOf = text => String(text).toLowerCase().match(/\p{L}+|\p{N}+/gu) ?? [];
+// Scripts written without spaces between words: a name in them is found as a run of characters, not of words.
+const UNSPACED = /[\p{Script=Hangul}\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
+/** True when the page text shows the name: as whole words (three letters at least), or for an unspaced script as a run of characters. */
+function showsName(name, squashed, words) {
+  if (UNSPACED.test(name)) return squash(name).length > 1 && squashed.includes(squash(name));
+  const tokens = wordsOf(name);
+  return tokens.join('').length > 2 && runIn(words, tokens);
+}
+
+const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
+const YEAR = '((?:19|20)\\d{2})';
+const MONTH_NAME = `(${MONTHS.join('|')})[a-z]*\\.?`;
+/**
+ * The full calendar dates the page shows, by yyyy-mm-dd, each with the text it was read from and where: in the visible text
+ * (`text`: 2026-09-16, 2026.9.16, 2026년 9월 16일, September 16, 2026, 16 Sep 2026) or as the datetime of a `<time>` element that has
+ * visible text (`time`; not inside script, template, noscript or a hidden element). A day without a year is not a date, and a
+ * `time` element is close to circular, because the site writes it and its JSON-LD from one source: the source is recorded.
+ */
+function datesIn(text, html) {
+  const found = new Map();
+  const add = (year, month, day, shown, source) => {
+    const key = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
+    if (!found.has(key)) found.set(key, { text: shown.replace(/\s+/g, ' ').trim(), source });
+  };
+  for (const m of text.matchAll(new RegExp(`\\b${YEAR}\\s*[-./]\\s*(\\d{1,2})\\s*[-./]\\s*(\\d{1,2})\\b`, 'g'))) add(m[1], m[2], m[3], m[0], 'text');
+  for (const m of text.matchAll(new RegExp(`${YEAR}\\s*년\\s*(\\d{1,2})\\s*월\\s*(\\d{1,2})\\s*일`, 'g'))) add(m[1], m[2], m[3], m[0], 'text');
+  for (const m of text.matchAll(new RegExp(`\\b${MONTH_NAME}\\s+(\\d{1,2})(?:st|nd|rd|th)?,?\\s+${YEAR}\\b`, 'gi'))) add(m[3], MONTHS.indexOf(m[1].toLowerCase()) + 1, m[2], m[0], 'text');
+  for (const m of text.matchAll(new RegExp(`\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+${MONTH_NAME},?\\s+${YEAR}\\b`, 'gi'))) add(m[3], MONTHS.indexOf(m[2].toLowerCase()) + 1, m[1], m[0], 'text');
+  const rendered = stripInert(String(html).replace(/<(template|noscript)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, ''));
+  for (const m of rendered.matchAll(/<time\b([^>]*)>([\s\S]*?)<\/time\s*>/gi)) {
+    const shown = decodeEntities(m[2].replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
+    const datetime = /\bdatetime\s*=\s*["'](\d{4})-(\d{2})-(\d{2})/i.exec(m[1]);
+    if (shown && datetime && !/\bhidden\b|display\s*:\s*none/i.test(m[1])) add(datetime[1], datetime[2], datetime[3], shown, 'time');
+  }
+  return found;
+}
+/** True when two headings are the same words: one holds the other and is at least four fifths of its length, in letters and digits only. */
+function sameHeadline(a, b) {
+  const [short, long] = [squash(a), squash(b)].sort((x, y) => x.length - y.length);
+  return short.length > 2 && short.length * 5 >= long.length * 4 && long.includes(short);
+}
+const ARTICLE_TYPES = /^(?:[A-Za-z]*Article|BlogPosting|LiveBlogPosting|Report)$/;
+/** The texts of the page's h1 elements, then of its title: where a headline is expected to appear. */
+const headingTexts = html => ['h1', 'title'].flatMap(tag => [...stripInert(html).matchAll(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, 'gi'))]
+  .map(match => decodeEntities(match[1].replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim()).filter(Boolean));
+
+/**
+ * How the facts a page's JSON-LD states about its own selected type stand against the page's visible text. Only the page's own
+ * nodes are read: top-level and @graph nodes, never an organization nested inside another node (a publisher, a parent). An
+ * Organization or LocalBusiness that speaks for the page (`facts.entities`) states a name, a phone number and a street address;
+ * an Article-family node states a headline and a publication date. Every fact is in exactly one list.
+ *
+ * `matched`: the page shows it (a name as whole words, a phone number, a street run, a headline in the page's h1 or title, a date).
+ * `mismatched`: the page shows a different value in the field that would hold it (a labelled phone or address, or an h1/title that
+ * does not contain the headline). `unevaluated`: the page neither shows nor contradicts it. Other types (Service, FAQPage, Event, ...)
+ * state nothing this compares. The date and the name have no contradicting field, so they only match or stay unevaluated.
+ */
+export function jsonLdTruth(html, facts) {
+  const live = stripComments(html);
+  const text = visibleText(live);
+  const squashed = squash(text);
+  const words = wordsOf(text);
+  const numbers = new Set([...numbersIn(hardText(live)), ...facts.visible.phone.keys(), ...facts.tel.phone.keys()]);
+  const addressWords = addressKey(text).split(' ');
+  const labelled = [...facts.visible.address.keys()];
+  const found = new Map();
+  // `visible` is the value the page shows in the field that contradicts; `shown` is the text a matched date or headline was read from, and where.
+  const note = (list, type, jsonld, visible, shown) => found.has(`${type}|${jsonld}`) ||
+    found.set(`${type}|${jsonld}`, { list, type, jsonld, ...(visible ? { visible } : {}), ...(list === 'matched' && shown ? { shown } : {}) });
+  for (const node of facts.entities) {
+    if (typeof node.name === 'string' && squash(node.name)) note(showsName(node.name, squashed, words) ? 'matched' : 'unevaluated', 'name', node.name);
+    const key = typeof node.telephone === 'string' ? phoneKey(node.telephone) : null;
+    if (key) {
+      if (numbers.has(key)) note('matched', 'phone', node.telephone);
+      else note(facts.visible.phone.size ? 'mismatched' : 'unevaluated', 'phone', node.telephone, [...facts.visible.phone.values()].join(' / '));
+    }
+    const street = typeof node.address === 'string' ? node.address : node.address?.streetAddress;
+    const streetKey = typeof street === 'string' ? addressKey(street) : '';
+    if (streetKey) {
+      if (hasRun(addressWords, streetKey.split(' ')) || labelled.some(other => sameAddress(other, streetKey))) note('matched', 'address', street);
+      else note(labelled.length ? 'mismatched' : 'unevaluated', 'address', street, [...facts.visible.address.values()].join(' / '));
+    }
+  }
+  const headings = headingTexts(live);
+  const hasH1 = /<h1[\s>]/i.test(stripInert(live));
+  const shownDates = datesIn(text, live);
+  for (const block of jsonLdBlocks(live)) if (block.ok) for (const node of nodesOf(block.value)) {
+    if (!typesOf(node).some(type => ARTICLE_TYPES.test(type))) continue;
+    const headline = typeof node.headline === 'string' ? node.headline.trim() : '';
+    if (squash(headline).length > 2) {
+      const same = headings.find(heading => sameHeadline(heading, headline));
+      // A title often differs from the headline on purpose, so only a page that has an h1 can contradict it.
+      note(same ? 'matched' : hasH1 ? 'mismatched' : 'unevaluated', 'headline', headline, headings[0], same && { text: same, source: 'text' });
     }
+    const published = /^(\d{4}-\d{2}-\d{2})/.exec(String(node.datePublished ?? ''))?.[1];
+    if (published) note(shownDates.has(published) ? 'matched' : 'unevaluated', 'date', published, undefined, shownDates.get(published));
   }
-  return { references, deviations, separate: new Set(allPages.filter(separate).map(page => page.url)) };
+  const listed = list => [...found.values()].filter(item => item.list === list).map(({ list: _, ...fact }) => fact);
+  return { matched: listed('matched'), mismatched: listed('mismatched'), unevaluated: listed('unevaluated') };
 }

package/skills/optbrew/scripts/core/findings.mjs

@@ -3,6 +3,22 @@ import { ContractError, validateEntity } from './contracts.mjs';
 const rank = { P0: 0, P1: 1, P2: 2, P3: 3 };
 const evidenceRank = { official: 0, reproduced: 1, research: 2, case: 3, hypothesis: 4 };
 
+/** The text two findings must share, ignoring case and spacing, to be the same issue. */
+export const messageKey = message => message.trim().replace(/\s+/g, ' ').toLowerCase();
+
+/** A finding's text with its own unit's page address replaced, so the same problem on two pages reads the same. */
+export const pageNeutral = (text, unitKey) => {
+  const page = unitKey.split('|').at(-1);
+  // Only a page address is replaced: the unit of a site-level check ends in a word such as "site", which is just text.
+  return /^https?:\/\//.test(page) ? text.split(page).join('{page}') : text;
+};
+
+/**
+ * The distinct problems one failed verdict states. A check that finds several fixed-sentence problems on one page joins them in
+ * its reason and lists them in `measurement.problems`; each is its own fix. Any other verdict states one problem, its reason.
+ */
+export const problemsOf = observation => observation.measurement?.problems ?? [observation.reason];
+
 /** Merge the same check, unit and issue while preserving every module source. */
 export function mergeFindings(findings) {
   if (!Array.isArray(findings)) throw new ContractError('findings must be an array');
@@ -11,7 +27,7 @@ export function mergeFindings(findings) {
     validateEntity('Finding', item);
     if (item.check_id !== item.unit_key.split('|')[0] || !Object.hasOwn(rank, item.priority))
       throw new ContractError('finding check or priority invalid');
-    const key = `${item.check_id}\0${item.unit_key}\0${item.message.trim().replace(/\s+/g, ' ').toLowerCase()}`;
+    const key = `${item.check_id}\0${item.unit_key}\0${messageKey(item.message)}`;
     const source = { source_ref: item.source_ref, evidence_ref: item.evidence_ref };
     if (!byIssue.has(key)) byIssue.set(key, { ...item, priority: item.blocker ? 'P0' : item.priority,
       finding_ids: [item.finding_id], sources: [source] });
@@ -23,6 +39,8 @@ export function mergeFindings(findings) {
       existing.blocker ||= item.blocker;
       if (existing.blocker) existing.priority = 'P0';
       existing.finding_ids = [...new Set([...existing.finding_ids, item.finding_id])];
+      // A finding that names several pages keeps all of them when two sources of the same issue are merged.
+      if (existing.unit_keys || item.unit_keys) existing.unit_keys = [...new Set([...(existing.unit_keys ?? [existing.unit_key]), ...(item.unit_keys ?? [item.unit_key])])];
       existing.remediation_ids = [...new Set([...existing.remediation_ids, ...item.remediation_ids])];
     }
   }