optbrew 0.1.1 → 0.1.3

package/skills/optbrew/scripts/core/derive.mjs

@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto';
+import { validateEntity } from './contracts.mjs';
+import { auditCollected } from '../../modules/naver-searchadvisor/scripts/audit.mjs';
+import { parseRobots, robotsVerdict } from '../../modules/naver-searchadvisor/scripts/robots.mjs';
+import { anchorLinks, decodeEntities } from './html.mjs';
+import { agentFor } from './crawlers.mjs';
+import { declaredRobotsIndex } from './collector.mjs';
+import { compareFacts, pageFacts } from './facts.mjs';
+
+/**
+ * Every collector verdict, derived from collected responses alone. The audit writes these observations and the
+ * report replays this same function over hashed raw evidence, so a producer and a verifier cannot disagree.
+ * `artifacts` carry the collection record fields plus evidence_ref; `textOf(artifact)` returns the raw body text.
+ */
+/** Link and sitemap target comparison key: no fragment, and a directory index file equals its directory. */
+function canonicalUrl(value) {
+  try {
+    const url = new URL(value); url.hash = '';
+    url.pathname = url.pathname.replace(/\/index\.(?:html?|php)$/i, '/');
+    return url.href;
+  } catch { return null; }
+}
+
+export function deriveObservations({ context, artifacts, collection = {}, textOf }) {
+  const protocol = context.protocol;
+  const planned = new Set(protocol.families.filter(x => x.applicable).map(x => x.family_id));
+  const selected = new Map(protocol.page_types.flatMap(type => type.urls).map(url => [url.normalized, url]));
+  const declared = kind => new Set((protocol.auxiliary_resources ?? []).filter(x => x.kind === kind).map(x => x.normalized));
+  const declaredRobots = declared('robots'), declaredSitemaps = declared('sitemap');
+  const out = [];
+  const add = (family, engine, url, state, reason, evidence_ref) => {
+    const unit_key = `${family}|${engine ? `${engine}|` : ''}${url}`;
+    out.push(validateEntity('CheckObservation', { schema_version: '1.0.0',
+      observation_id: createHash('sha256').update(unit_key).digest('hex'), check_id: family, family_id: family, unit_key,
+      state, q: state === 'pass' ? 1 : state === 'fail' ? 0 : null, partial_basis: null, reason, evaluator_type: 'code',
+      source_ref: 'optbrew:collector', evidence_ref }));
+  };
+  const pageRows = [];
+  for (const artifact of artifacts) {
+    const url = artifact.normalized_url;
+    if (!selected.has(url)) continue;
+    if (planned.has('fetch-response')) add('fetch-response', null, url,
+      artifact.error || artifact.status < 200 || artifact.status >= 300 && artifact.status < 400 ? 'unknown' : artifact.status >= 400 ? 'fail' : 'pass',
+      artifact.error ? 'Request did not finish' : `HTTP ${artifact.status}`, artifact.evidence_ref);
+    if (artifact.capability !== 'raw_html' || artifact.status !== 200) continue;
+    const html = textOf(artifact);
+    const robotsIndex = declaredRobotsIndex(artifacts, declaredRobots, artifact.final_url);
+    const robots = robotsIndex < 0 ? null : artifacts[robotsIndex];
+    const info = auditCollected({ artifact, html, engines: protocol.engines,
+      robotsArtifact: robots?.status === 200 ? { ...robots, body: textOf(robots) } : null });
+    pageRows.push({ url, artifact, html, info, robots, publicIndexing: selected.get(url).public_indexing !== false });
+  }
+  const normalizeTitle = title => title.toLowerCase().replace(/\s+/g, ' ').trim();
+  const titleCounts = new Map();
+  for (const row of pageRows) if (row.info.title) {
+    const key = normalizeTitle(row.info.title);
+    titleCounts.set(key, new Set([...(titleCounts.get(key) ?? []), row.artifact.final_url]));
+  }
+  const facts = pageRows.map(row => ({ url: row.url, row, facts: pageFacts(row.html) }));
+  const comparison = compareFacts(facts);
+  for (const row of pageRows) {
+    const { url, artifact, info } = row, ref = artifact.evidence_ref;
+    if (planned.has('document-title')) {
+      const dup = info.title ? pageRows.filter(x => x !== row && x.info.title && normalizeTitle(x.info.title) === normalizeTitle(info.title) &&
+        x.artifact.final_url !== artifact.final_url) : [];
+      add('document-title', null, url, !info.title || dup.length ? 'fail' : 'unknown',
+        !info.title ? 'Raw HTML has no title' : dup.length ? `Title duplicates ${dup.map(x => x.url).join(', ')}` :
+          'Raw HTML has a title; uniqueness and descriptiveness are not established', ref);
+    }
+    if (planned.has('index-policy') && row.publicIndexing)
+      add('index-policy', null, url, info.index_blocked ? 'fail' : 'pass',
+        info.index_blocked ? 'Raw response blocks indexing for the target engine' : 'Raw response has no applicable noindex directive', ref);
+    if (planned.has('search-crawl-policy') && row.publicIndexing && row.robots?.status === 200) {
+      const parsed = parseRobots(textOf(row.robots));
+      const target = new URL(artifact.final_url);
+      for (const engine of protocol.engines) {
+        const agent = agentFor(engine);
+        if (!agent) continue;
+        const verdict = robotsVerdict(parsed, agent, target.pathname + target.search);
+        add('search-crawl-policy', engine, url, verdict.allowed ? 'pass' : 'fail',
+          verdict.allowed ? `Collected robots.txt allows ${agent} path` : `Collected robots.txt disallows ${agent} path`, row.robots.evidence_ref);
+      }
+    }
+    if (planned.has('naver-markup') && protocol.engines.includes('naver'))
+      add('naver-markup', 'naver', url, info.markup_findings.length ? 'fail' : 'pass',
+        info.markup_findings.length ? info.markup_findings.map(x => x.message).join('; ') : 'Open Graph, description, canonical, viewport and title checks pass', ref);
+    if (planned.has('structured-data-truth') && info.jsonld_parse_errors > 0)
+      add('structured-data-truth', null, url, 'fail', `${info.jsonld_parse_errors} JSON-LD block(s) fail to parse`, ref);
+    if (planned.has('canonical-consistency')) {
+      const hops = artifact.redirect_chain?.length ?? 0;
+      let canonicalState = null, canonicalNote = '';
+      if (info.canonical) {
+        try {
+          const target = new URL(info.canonical, artifact.final_url); target.hash = '';
+          const other = artifacts.find(x => x.normalized_url === target.href || x.final_url === target.href);
+          if (target.href === artifact.final_url || target.href === artifact.normalized_url) canonicalState = 'pass';
+          else if (other && other.status >= 200 && other.status < 300) canonicalState = 'pass';
+          else if (other && Number.isInteger(other.status)) { canonicalState = 'fail'; canonicalNote = `canonical target ${target.href} returns HTTP ${other.status}`; }
+        } catch { canonicalState = 'fail'; canonicalNote = 'canonical is not a valid URL'; }
+      }
+      if (hops >= 2) add('canonical-consistency', null, url, 'fail', `Redirect chain of ${hops} hops${canonicalNote ? `; ${canonicalNote}` : ''}`, ref);
+      else if (canonicalState === 'fail') add('canonical-consistency', null, url, 'fail', canonicalNote, ref);
+      else if (canonicalState === 'pass') add('canonical-consistency', null, url, 'pass', 'Canonical target is this page or a reachable page and redirects stay within one hop', ref);
+    }
+  }
+  /** True when any other collected page's raw bytes name this page's path (whole path segment, not a longer path). */
+  const pathMentioned = ({ url, artifact }, bodies) => {
+    const own = canonicalUrl(url);
+    for (const target of new Set([url, artifact.final_url])) {
+      let path;
+      try { path = decodeURIComponent(new URL(target).pathname); } catch { continue; }
+      if (path === '/') continue;
+      const pattern = new RegExp(path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?![A-Za-z0-9_-])');
+      for (const [page, body] of bodies) if (page !== own && pattern.test(body)) return true;
+    }
+    return false;
+  };
+  if (planned.has('discovery')) {
+    const sitemapUrls = new Set();
+    const brokenSitemaps = [];
+    let sitemapIndex = false;
+    for (const artifact of artifacts) {
+      if (!declaredSitemaps.has(artifact.normalized_url)) continue;
+      if (artifact.status >= 200 && artifact.status < 300) {
+        const body = String(textOf(artifact) ?? '');
+        // Child sitemaps of an index are not fetched, so pages missing from the index prove nothing.
+        if (/<sitemapindex\b/i.test(body)) sitemapIndex = true;
+        for (const match of body.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) sitemapUrls.add(canonicalUrl(decodeEntities(match[1])));
+      } else brokenSitemaps.push(`${artifact.normalized_url} (${artifact.status ?? artifact.error ?? 'no response'})`);
+    }
+    const htmlPages = artifacts.filter(x => x.capability === 'raw_html' && x.status === 200);
+    const linked = new Map();
+    // A page with scripts and no anchors may build its navigation in the browser. That only hides an orphan when the
+    // script can be read: an inline script is searched for the path below, and a same-origin script file that was not
+    // collected cannot be read at all.
+    const collected = new Set(artifacts.flatMap(x => [x.normalized_url, x.final_url].filter(Boolean).map(canonicalUrl)));
+    let uncollectedScript = false;
+    const bodies = new Map();
+    for (const page of htmlPages) {
+      const body = textOf(page);
+      bodies.set(canonicalUrl(page.normalized_url), body);
+      const hrefs = anchorLinks(body, page.final_url);
+      if (!hrefs.length && /<script\b/i.test(body)) {
+        for (const match of body.matchAll(/<script\b[^>]*\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)')/gi)) {
+          try {
+            const source = new URL(decodeEntities(match[1] ?? match[2]), page.final_url);
+            if (source.origin === new URL(page.final_url).origin && !collected.has(canonicalUrl(source.href))) uncollectedScript = true;
+          } catch { /* malformed script URL */ }
+        }
+      }
+      for (const href of hrefs) {
+        const target = canonicalUrl(href);
+        if (target && target !== canonicalUrl(page.final_url) && target !== canonicalUrl(page.normalized_url)) linked.set(target, (linked.get(target) ?? 0) + 1);
+      }
+    }
+    const complete = !collection.limit_reached && !(collection.skipped ?? []).some(x => ['depth_limit', 'url_limit', 'request_limit'].includes(x.reason));
+    for (const row of pageRows) {
+      if (!row.publicIndexing) continue;
+      const { url, artifact } = row;
+      const keys = [canonicalUrl(url), canonicalUrl(artifact.final_url)];
+      const inSitemap = keys.some(key => sitemapUrls.has(key));
+      if (inSitemap || keys.some(key => linked.has(key)))
+        add('discovery', null, url, 'pass', inSitemap ? 'Listed in a declared sitemap' : 'Linked from another collected page', artifact.evidence_ref);
+      // An unlinked page is unknown when a collected page's own bytes (inline scripts included) name its path, or when a
+      // script file that could build the link was not collected, unless a fetched sitemap that lists other pages leaves
+      // it out. Anything else is an orphan.
+      else if (htmlPages.length >= 2 && complete && !sitemapIndex && (sitemapUrls.size || !(uncollectedScript || pathMentioned(row, bodies))))
+        add('discovery', null, url, 'fail', `Not linked from any collected page and not in a sitemap${brokenSitemaps.length ? `; declared sitemap unusable: ${brokenSitemaps.join(', ')}` : ''}`, artifact.evidence_ref);
+    }
+  }
+  if (planned.has('fact-consistency')) {
+    for (const page of facts) {
+      if (comparison.separate.has(page.url)) continue;
+      const found = comparison.deviations.get(page.url) ?? [];
+      const comparable = Object.keys(comparison.references).some(type => page.facts.visible[type].size || page.facts.linked[type].size || (type === 'phone' && page.facts.tel.phone.size));
+      if (found.length) add('fact-consistency', null, page.url, 'fail', `Cross-page fact comparison (no approved fact list): ${found.join('; ')}`, page.row.artifact.evidence_ref);
+      else if (comparable) add('fact-consistency', null, page.url, 'pass', 'Facts on this page match the cross-page reference values (no approved fact list; consistency is not accuracy)', page.row.artifact.evidence_ref);
+    }
+  }
+  if (planned.has('entity-identity') && facts.length) {
+    const reasons = [];
+    const home = facts.find(x => new URL(x.row.artifact.final_url).pathname === '/') ?? facts[0];
+    const orgPages = facts.filter(x => x.facts.orgs.length && !comparison.separate.has(x.url));
+    const nameReference = comparison.references.name;
+    const nameDeviants = facts.filter(x => (comparison.deviations.get(x.url) ?? []).some(text => text.startsWith('name ')));
+    if (nameDeviants.length) reasons.push(`name differs from "${nameReference.label}" on ${nameDeviants.map(x => x.url).join(', ')}`);
+    for (const page of orgPages) if (page.facts.orgs.some(org => !org.sameAs)) reasons.push(`Organization JSON-LD without sameAs on ${page.url}`);
+    if (orgPages.length && !home.facts.orgs.length) reasons.push(`no Organization or LocalBusiness JSON-LD on ${home.url}`);
+    if (reasons.length) add('entity-identity', null, 'site', 'fail', `Entity identity: ${reasons.join('; ')}`, (nameDeviants[0] ?? home).row.artifact.evidence_ref);
+    else if (orgPages.length) add('entity-identity', null, 'site', 'pass', 'Organization JSON-LD with name and sameAs is present on the home page and names agree', home.row.artifact.evidence_ref);
+  }
+  return out;
+}

package/skills/optbrew/scripts/core/facts.mjs

@@ -0,0 +1,215 @@
+import { decodeEntities, jsonLdBlocks, stripComments, stripInert, visibleText } from './html.mjs';
+
+/**
+ * Cross-page entity fact extraction and comparison, pure functions of the raw HTML of the selected pages.
+ * Facts come only from Organization or LocalBusiness JSON-LD, tel: links and explicitly labelled contact fields.
+ * Free prose, dates and numbers elsewhere on a page are never read as facts; when a comparison is not clear the
+ * result is unknown, not a failure.
+ */
+const FACT_TYPES = ['name', 'phone', 'year', 'address'];
+const ORG_TYPES = /^(?:Organization|Corporation|LocalBusiness|Store|Restaurant|CafeOrCoffeeShop|OnlineStore|NGO|[A-Za-z]*Business)$/;
+const squash = value => String(value).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
+const BRANCH = /\b(?:branch)\b|지점|지사/i;
+// URL path segments that list branches: /branch/, /branches/, /locations/, /stores/, /지점/.
+const BRANCH_PATH = /^(?:branch|branches|locations|stores|지점)$/i;
+const branchPath = url => {
+  try { return new URL(url, 'http://page.invalid').pathname.split('/').some(part => BRANCH_PATH.test(decodeURIComponent(part))); }
+  catch { return false; }
+};
+
+const ABBREVIATIONS = [[/\bstreet\b/g, 'st'], [/\broad\b/g, 'rd'], [/\bavenue\b/g, 'ave'], [/\bboulevard\b/g, 'blvd'],
+  [/\bdrive\b/g, 'dr'], [/\blane\b/g, 'ln'], [/\bsuite\b/g, 'ste'], [/\bfloor\b/g, 'fl'], [/\bbuilding\b/g, 'bldg']];
+/**
+ * Address key: normalized tokens joined by one space. Lower case, abbreviations unified, trailing country dropped, Korean
+ * 시/도 forms unified. A postal code is dropped only in its own position (in parentheses or brackets, after a label, or as
+ * the last token after the city); a five-digit street number such as "12345 Ocean Avenue" stays part of the address.
+ */
+export function addressKey(value) {
+  let text = String(value).toLowerCase().replace(/[,;]+/g, ' ');
+  text = text.replace(/\b(?:south korea|republic of korea|korea|kr|usa|us|united states|대한민국|한국)\b\s*$/u, '');
+  text = text.replace(/[(\[]\s*\d{3}-?\d{2,3}(?:-\d{4})?\s*[)\]]/g, ' ');
+  text = text.replace(/(?:우편번호|postal code|post code|zip code|zip)\s*[:：]?\s*\d[\d-]*/giu, ' ');
+  text = text.replace(/(?<=\p{L}\s+(?:[a-z]{2}\s+)?)(?:\d{5}(?:-\d{4})?|\d{3}-\d{3}|\d{6})\s*$/u, ' ');
+  for (const [pattern, replacement] of ABBREVIATIONS) text = text.replace(pattern, replacement);
+  text = text.replace(/(특별자치시|특별자치도|특별시|광역시)/g, '').replace(/^\s*(\p{Script=Hangul}+?)(시|도)(?=\s|$)/u, '$1');
+  return (text.match(/\p{L}+|\p{N}+/gu) ?? []).join(' ');
+}
+/**
+ * Two address keys agree when they are equal or one is a run of whole tokens of the other (a street address with or
+ * without city detail). Tokens are compared exactly, so "10 main st" never matches "110 main st".
+ */
+function sameAddress(a, b) {
+  if (a === b) return true;
+  // The same words in another order (an English and a Korean address order) are not a difference.
+  if (a.split(' ').sort().join(' ') === b.split(' ').sort().join(' ')) return true;
+  const [short, long] = a.length <= b.length ? [a, b] : [b, a];
+  const shortTokens = short.split(' '), longTokens = long.split(' ');
+  if (shortTokens.length < 2 || !shortTokens.some(token => /^\p{L}/u.test(token))) return false;
+  for (let i = 0; i + shortTokens.length <= longTokens.length; i++)
+    if (shortTokens.every((token, k) => token === longTokens[i + k])) return true;
+  return false;
+}
+
+function nodesOf(value, out = []) {
+  if (Array.isArray(value)) value.forEach(item => nodesOf(item, out));
+  else if (value && typeof value === 'object') {
+    out.push(value);
+    if (value['@graph']) nodesOf(value['@graph'], out);
+  }
+  return out;
+}
+const typesOf = node => [].concat(node['@type'] ?? []).map(String);
+const addressText = address => typeof address === 'string' ? address : address && typeof address === 'object' ?
+  [address.streetAddress, address.addressLocality, address.addressRegion]
+    .filter(x => typeof x === 'string').join(', ') : '';
+
+function phoneKey(text) {
+  const clean = String(text).replace(/\(0\)/g, '');
+  const digits = clean.replace(/\D/g, '');
+  // Korean nationwide representative numbers (15xx, 16xx, 18xx plus four digits) have eight digits.
+  if (/^1[568]\d{6}$/.test(digits)) return digits;
+  if (digits.length < 9 || digits.length > 15) return null;
+  return /^\s*\+?\s*82/.test(clean) && clean.trim().startsWith('+') && digits.startsWith('82') ? `0${digits.slice(2)}` : digits;
+}
+
+/**
+ * Text for phone extraction: every tag, inline or not, ends the text before it, so a figure in the next element
+ * (a table cell, a span, a list item) is never read as the rest of a number.
+ */
+const hardText = html => decodeEntities(stripInert(html).replace(/<[^>]+>/g, '\n')).replace(/[ \t\f\v\r ]+/g, ' ');
+// An international or parenthesised number may use spaces between its groups; a plain national number only dashes and dots.
+const LOOSE_PHONE = /^[+(]\s?[\d()]{1,6}(?:[-. ]?\(?\d{1,6}\)?){1,5}/;
+const STRICT_PHONE = /^\+?\d{1,4}(?:[-.]\d{1,6}){1,4}/;
+// In a label, a national number may use spaces only (02 555 0142, 010 1234 5678), and a nationwide representative number
+// is 15xx, 16xx or 18xx plus four digits. Neither is read in prose.
+const SPACED_NATIONAL_PHONE = /^0\d{1,2}(?: \d{3,4}){2}(?!\d)/;
+const REPRESENTATIVE_PHONE = /^1[568]\d{2}[- ]\d{4}(?!\d)/;
+/**
+ * The number at the start of `rest`, ending at its own last digit group. A spaced number that is followed by more
+ * digits or a bracketed digit is ambiguous, so only its dashed or dotted part is read, and a number longer than twelve
+ * digits is not read at all.
+ */
+function readPhone(rest) {
+  const usable = text => text && (phoneKey(text)?.length ?? 99) <= 13 ? text.trim() : null;
+  const loose = LOOSE_PHONE.exec(rest)?.[0].trimEnd();
+  if (loose && !/^[ \t]*[(\[]?\d/.test(rest.slice(loose.length)) && usable(loose)) return loose.trim();
+  const strict = usable(STRICT_PHONE.exec(rest)?.[0]);
+  if (strict) return strict;
+  // Digits straight after a spaced number are ambiguous (02 555 0142 42 seats); a bracket after it is not (hours).
+  for (const pattern of [SPACED_NATIONAL_PHONE, REPRESENTATIVE_PHONE]) {
+    const match = pattern.exec(rest)?.[0];
+    if (match && !/^[ \t]*\d/.test(rest.slice(match.length)) && usable(match)) return match.trim();
+  }
+  return null;
+}
+
+/** Facts on one page: visible labelled values and JSON-LD / tel: values, keyed by fact type. */
+export function pageFacts(html) {
+  const live = stripComments(html);
+  const visible = Object.fromEntries(FACT_TYPES.map(type => [type, new Map()]));
+  const linked = Object.fromEntries(FACT_TYPES.map(type => [type, new Map()]));
+  // tel: links are their own evidence class: supporting evidence that never contradicts a page's labelled or JSON-LD phone.
+  const tel = { phone: new Map() };
+  const put = (bucket, type, raw, key) => { if (key) bucket[type].set(key, String(raw).trim()); };
+  const text = visibleText(live);
+  // The text after a label is read in a lookahead, so a label alone (a heading that says "phone") never consumes the next line.
+  for (const match of hardText(live).matchAll(/\b(?:Phone|Tel(?:ephone)?|Mobile)\b(?=\s*[:：]?\s*([^\n]*))|(?:전화(?:번호)?|연락처|대표번호)(?=\s*[:：]?\s*([^\n]*))/gi)) {
+    const number = readPhone(match[1] ?? match[2]);
+    if (number) put(visible, 'phone', number, phoneKey(number));
+  }
+  for (const match of live.matchAll(/href\s*=\s*["']tel:([^"']+)["']/gi)) {
+    const value = decodeURIComponent(match[1]);
+    put(tel, 'phone', value, phoneKey(value));
+  }
+  for (const match of text.matchAll(/\b(?:Founded|Established)\b\s*[:：]\s*((?:19|20)\d{2})\b|(?:설립(?:연도)?|창립(?:연도)?)\s*[:：]?\s*((?:19|20)\d{2})/gi))
+    put(visible, 'year', match[1] ?? match[2], match[1] ?? match[2]);
+  for (const match of text.matchAll(/\bAddress\b\s*[:：]\s*([^\n]+?)(?=\.\s|\.?\s*$|\n)/gim)) put(visible, 'address', match[1], addressKey(match[1]));
+  for (const match of text.matchAll(/주소\s*[:：]\s*([^\n.]+)/g)) put(visible, 'address', match[1], addressKey(match[1]));
+  const orgs = [];
+  for (const block of jsonLdBlocks(live)) {
+    if (!block.ok) continue;
+    for (const node of nodesOf(block.value)) {
+      if (!typesOf(node).some(type => ORG_TYPES.test(type))) continue;
+      const name = typeof node.name === 'string' ? node.name : '';
+      const org = { name, id: typeof node['@id'] === 'string' ? node['@id'] : null,
+        sameAs: [].concat(node.sameAs ?? []).some(x => typeof x === 'string' && x.trim()),
+        branch: Boolean(node.branchOf || node.parentOrganization || BRANCH.test(name)) };
+      if (name) put(linked, 'name', name, squash(name));
+      if (typeof node.telephone === 'string') put(linked, 'phone', node.telephone, phoneKey(node.telephone));
+      const address = addressText(node.address);
+      if (address) put(linked, 'address', address, addressKey(address));
+      const year = /\b((?:19|20)\d{2})\b/.exec(String(node.foundingDate ?? ''))?.[1];
+      if (year) put(linked, 'year', year, year);
+      orgs.push(org);
+    }
+  }
+  // A branch word in the title or first heading is only a hint (titleBranch). A page is a separate entity when its
+  // JSON-LD says so (branch), when its URL is a branch list, or when the hint comes with an address of its own.
+  const heading = /<(?:title|h1)\b[^>]*>([\s\S]*?)<\/(?:title|h1)>/gi;
+  const headings = [...live.matchAll(heading)].map(match => match[1]).join(' ');
+  return { visible, linked, tel, orgs, branch: orgs.some(org => org.branch), titleBranch: BRANCH.test(headings) };
+}
+
+/**
+ * Values compared across pages for one fact type: the labelled and JSON-LD values. A tel: link joins only when the page has
+ * no labelled or JSON-LD phone, so a mobile inquiry link beside the page's landline never counts against it.
+ */
+function evidenceOf(page, type) {
+  const found = new Map([...page.facts.visible[type], ...page.facts.linked[type]]);
+  if (type === 'phone' && !found.size) for (const [key, raw] of page.facts.tel.phone) found.set(key, raw);
+  return found;
+}
+const keysOf = (page, type) => new Set(evidenceOf(page, type).keys());
+
+/**
+ * Compare facts across the selected pages. A reference value needs at least two pages and a unique top count;
+ * otherwise the type is insufficient and stays unknown. Branch pages and entities with a different JSON-LD @id are skipped.
+ */
+export function compareFacts(allPages) {
+  const ids = new Map();
+  for (const page of allPages) for (const id of new Set(page.facts.orgs.map(org => org.id).filter(Boolean))) ids.set(id, (ids.get(id) ?? 0) + 1);
+  const ranked = [...ids.entries()].sort((a, b) => b[1] - a[1]);
+  const mainId = ranked.length && ranked[0][1] >= 2 && (ranked.length === 1 || ranked[0][1] > ranked[1][1]) ? ranked[0][0] : null;
+  const structural = page => page.facts.branch || branchPath(page.url) ||
+    (mainId && page.facts.orgs.some(org => org.id) && !page.facts.orgs.some(org => org.id === mainId));
+  // The main entity's addresses come from the pages with neither structure nor a branch word in the title.
+  const mainAddresses = [...new Set(allPages.filter(page => !structural(page) && !page.facts.titleBranch).flatMap(page => [...keysOf(page, 'address')]))];
+  // A branch word in the title counts only with a labelled address that is not the main entity's.
+  const ownAddress = page => { const own = keysOf(page, 'address'); return own.size > 0 && mainAddresses.length > 0 && ![...own].some(key => mainAddresses.some(main => sameAddress(key, main))); };
+  const separate = page => structural(page) || (page.facts.titleBranch && ownAddress(page));
+  const pages = allPages.filter(page => !separate(page));
+  const references = {};
+  const deviations = new Map(allPages.map(page => [page.url, []]));
+  for (const type of FACT_TYPES) {
+    const counts = new Map();
+    const carrying = pages.filter(page => keysOf(page, type).size);
+    for (const page of carrying) for (const key of keysOf(page, type)) counts.set(key, (counts.get(key) ?? 0) + 1);
+    if (type === 'address') {
+      // Merge keys that agree under address matching so abbreviations and city detail do not split the count.
+      const keys = [...counts.keys()];
+      for (const key of keys) for (const other of keys) if (key !== other && counts.has(key) && counts.has(other) && sameAddress(key, other) && key.length >= other.length) {
+        counts.set(other, counts.get(other) + counts.get(key)); counts.delete(key);
+      }
+    }
+    const top = [...counts.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
+    if (top.length && top[0][1] >= 2 && (top.length === 1 || top[0][1] > top[1][1])) {
+      const [modal, count] = top[0];
+      const agrees = key => type === 'address' ? sameAddress(key, modal) || [...counts.keys()].some(k => sameAddress(k, modal) && sameAddress(k, key)) : key === modal;
+      const label = pages.flatMap(page => [...evidenceOf(page, type).entries()]).find(([key]) => agrees(key))?.[1] ?? modal;
+      references[type] = { key: modal, label, pages: count };
+      for (const page of carrying) {
+        if (![...keysOf(page, type)].some(agrees)) {
+          const found = [...evidenceOf(page, type).values()].join(' / ');
+          deviations.get(page.url).push(`${type} "${found}" differs from "${label}" on ${count} pages`);
+        }
+      }
+    }
+    for (const page of pages) {
+      const v = [...page.facts.visible[type].keys()], l = [...page.facts.linked[type].keys()];
+      const same = type === 'address' ? v.some(a => l.some(b => sameAddress(a, b))) : v.some(key => l.includes(key));
+      if (v.length && l.length && !same && !deviations.get(page.url).some(x => x.startsWith(`${type} `)))
+        deviations.get(page.url).push(`${type} in JSON-LD "${[...page.facts.linked[type].values()].join(' / ')}" differs from visible "${[...page.facts.visible[type].values()].join(' / ')}"`);
+    }
+  }
+  return { references, deviations, separate: new Set(allPages.filter(separate).map(page => page.url)) };
+}

package/skills/optbrew/scripts/core/html.mjs

@@ -0,0 +1,53 @@
+/** Small HTML helpers shared by the collector and the verdict derivation. Pure functions of raw bytes. */
+
+/** Drop comments and raw text elements whose content is not markup. */
+export function stripInert(html) {
+  return String(html ?? '').replace(/<!--[\s\S]*?(?:-->|$)|<(script|style|textarea)\b[^>]*>[\s\S]*?(?:<\/\1\s*>|$)/gi, '');
+}
+
+/** Markup without HTML comments. */
+export const stripComments = html => String(html ?? '').replace(/<!--[\s\S]*?(?:-->|$)/g, '');
+
+const named = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
+/** Decode the character references that appear in attribute values. */
+export const decodeEntities = value => String(value).replace(/&(?:#(\d+)|#x([0-9a-f]+)|(amp|lt|gt|quot|apos|nbsp));/gi,
+  (match, dec, hex, name) => dec ? String.fromCodePoint(Number(dec)) : hex ? String.fromCodePoint(parseInt(hex, 16)) : named[name.toLowerCase()]);
+
+/** Absolute hrefs of anchors, resolved against the response URL. */
+export function anchorLinks(html, base) {
+  const out = [];
+  for (const match of String(html).matchAll(/<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)')/gi)) {
+    try { out.push(new URL(decodeEntities(match[1] ?? match[2]), base).href); } catch { /* malformed link */ }
+  }
+  return out;
+}
+
+/** The raw href of the first rel=canonical link in live markup, or null. */
+export function canonicalLink(html) {
+  for (const match of stripInert(html).matchAll(/<link\b([^>]*)>/gi)) {
+    const attrs = {};
+    for (const a of match[1].matchAll(/([a-zA-Z:_-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/g)) attrs[a[1].toLowerCase()] = a[2] ?? a[3] ?? a[4] ?? '';
+    if (/(^|\s)canonical(\s|$)/i.test(attrs.rel ?? '') && attrs.href) return attrs.href;
+  }
+  return null;
+}
+
+/** JSON-LD blocks with their parse result. */
+export function jsonLdBlocks(html) {
+  // Blocks inside HTML comments are not live markup, and an empty placeholder block is not an error.
+  return [...stripComments(html).matchAll(/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)]
+    .filter(match => match[1].trim()).map(match => {
+      try { return { ok: true, value: JSON.parse(match[1]) }; } catch { return { ok: false, value: null }; }
+    });
+}
+
+const entities = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&#39;': "'", '&nbsp;': ' ' };
+
+/** Visible text with block boundaries kept as line breaks. */
+export function visibleText(html) {
+  return stripInert(html)
+    .replace(/<\/?(?:h[1-6]|p|div|li|ul|ol|br|tr|section|article|header|footer|main|nav|title)\b[^>]*>/gi, '\n')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&(?:amp|lt|gt|quot|#39|nbsp);/g, token => entities[token])
+    .replace(/[ \t\f\v\r]+/g, ' ').replace(/ ?\n ?/g, '\n').replace(/\n{2,}/g, '\n').trim();
+}