optbrew 0.1.2 → 0.2.0

package/skills/optbrew/scripts/core/checks/cwv-field.mjs

@@ -0,0 +1,77 @@
+import { cwvVerdict } from '../contracts.mjs';
+import { thresholds } from '../config.mjs';
+import { HTTP_OK } from '../cwv.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const UNAVAILABLE = 'field_data_unavailable';
+const P75 = 75;
+// PageSpeed Insights gives the CLS percentile as an integer, 100 times the score (0.1 reads 10).
+const PSI_CLS_PERCENTILE_SCALE = 100;
+const SCALE_TOLERANCE = 1e-9;
+
+/**
+ * Where each family's 75th percentile sits in a stored answer. CrUX gives the value itself (a string for CLS); PageSpeed
+ * Insights gives an integer, and for CLS one scaled by PSI_CLS_PERCENTILE_SCALE. Google does not document that scale, so it is
+ * trusted only while the first distribution bucket ends at the good bound times the scale (10 for 0.1); otherwise there is no value.
+ */
+const FIELDS = {
+  'lcp-field': { crux: 'largest_contentful_paint', psi: 'LARGEST_CONTENTFUL_PAINT_MS', scaled: false },
+  'inp-field': { crux: 'interaction_to_next_paint', psi: 'INTERACTION_TO_NEXT_PAINT', scaled: false },
+  'cls-field': { crux: 'cumulative_layout_shift', psi: 'CUMULATIVE_LAYOUT_SHIFT_SCORE', scaled: true },
+};
+
+/** The families that need visitor field data; a run without a Google key leaves them unknown. */
+export const CWV_FAMILIES = Object.keys(FIELDS);
+
+const finite = value => value !== null && value !== '' && Number.isFinite(Number(value)) ? Number(value) : null;
+
+/** The p75 a stored answer holds for a family, or null when it holds none. */
+function p75Of(record, familyId) {
+  const names = FIELDS[familyId];
+  if (record.status !== HTTP_OK || !record.body || typeof record.body !== 'object') return null;
+  if (record.source === 'crux') return finite(record.body.record?.metrics?.[names.crux]?.percentiles?.p75);
+  const metric = record.body.originLoadingExperience?.metrics?.[names.psi];
+  const value = finite(metric?.percentile);
+  if (value === null || !names.scaled) return value;
+  const bound = finite(metric.distributions?.[0]?.max);
+  return Math.abs(bound - thresholds(familyId).good * PSI_CLS_PERCENTILE_SCALE) < SCALE_TOLERANCE ? value / PSI_CLS_PERCENTILE_SCALE : null;
+}
+
+/**
+ * With no CWV answer to point at, an unknown points at a collected response of the origin: collector evidence, as every collector verdict needs.
+ * A page of the origin comes before its robots or sitemap, so the anchor does not depend on the order the collector fetched them in.
+ */
+function anchorOf(artifacts, origin) {
+  const ofOrigin = artifacts.filter(artifact => artifact.normalized_url && new URL(artifact.normalized_url).origin === origin);
+  return (ofOrigin.find(artifact => artifact.capability === 'raw_html') ?? ofOrigin[0])?.evidence_ref;
+}
+
+/**
+ * One family's verdict at the 75th percentile of real users, one per origin, from the stored CWV answers (`prepared.cwv`)
+ * alone. An origin with no usable p75 is `unknown`: absent field data is not a failure. The first stored answer of an
+ * origin that holds a value decides, so replay picks the same one the audit did.
+ */
+const fieldBlock = familyId => ({
+  family_id: familyId,
+  message_keys: [`${familyId}.slow`],
+  derive({ context, artifacts, selected, cwv }) {
+    const { unit, good, poor } = thresholds(familyId);
+    return [...new Set([...selected.keys()].map(url => new URL(url).origin))].map(origin => {
+      const mine = cwv.filter(record => record.origin === origin);
+      const source = mine.find(record => p75Of(record, familyId) !== null);
+      const observation = { family_id: familyId, unit_key: unitKey(familyId, origin) };
+      if (!source) return makeObservation({ ...observation, state: 'unknown', reason: UNAVAILABLE,
+        evidence_ref: mine[0]?.evidence_ref ?? anchorOf(artifacts, origin) ?? context.evidence_ref });
+      const value = p75Of(source, familyId);
+      const measurement = { value, percentile: P75, source_type: 'field', unit };
+      const { state, q, partial_basis } = cwvVerdict(familyId, value, measurement);
+      const reason = `p75 ${value} ${unit} from ${source.source}`;
+      return makeObservation({ ...observation, state, q, partial_basis, evidence_ref: source.evidence_ref, extra: { measurement },
+        ...(state === 'fail' ? { problems: [problem(reason, `${familyId}.slow`, { value, source: source.source, good, poor })] } : { reason }) });
+    });
+  },
+});
+
+export const lcpField = fieldBlock('lcp-field');
+export const inpField = fieldBlock('inp-field');
+export const clsField = fieldBlock('cls-field');

package/skills/optbrew/scripts/core/checks/discovery.mjs

@@ -0,0 +1,43 @@
+import { canonicalUrl } from '../link-graph.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'discovery';
+const ORPHAN = 'discovery.orphan';
+const SITEMAP_UNUSABLE = 'discovery.orphan-sitemap-unusable';
+
+// A declared sitemap that was never fetched may list the page, so its absence from the sitemaps that were read proves nothing.
+const unreadReason = unread => `declared sitemap not read: ${[...new Set(unread.map(x => x.reason))].join(', ')}; not linked from a collected page, and the evidence cannot show that nothing else leads to it`;
+
+const UNPROVEN = 'Not in a read sitemap and not linked from a collected page, but the evidence cannot show that nothing else leads to it';
+
+/** A page listed in a declared sitemap or linked from another collected page passes; a proven orphan fails; the rest is unknown. */
+export const discovery = {
+  family_id: FAMILY,
+  message_keys: [ORPHAN, SITEMAP_UNUSABLE],
+  derive({ pageRows, linkGraph, orphans, sitemaps, crawlComplete, htmlPages }) {
+    const out = [];
+    for (const row of pageRows) {
+      if (!row.publicIndexing) continue;
+      const { url, artifact } = row;
+      const keys = [canonicalUrl(url), canonicalUrl(artifact.final_url)];
+      const inSitemap = keys.some(key => sitemaps.urls.has(key));
+      const observe = (state, reason) => out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state, reason,
+        evidence_ref: artifact.evidence_ref }));
+      if (inSitemap || keys.some(key => linkGraph.inbound(key))) observe('pass', inSitemap ? 'Listed in a declared sitemap' : 'Linked from another collected page');
+      else if (orphans.has(url)) {
+        const unlinked = 'Not linked from any collected page and not in a sitemap';
+        const broken = sitemaps.broken.join(', ');
+        out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state: 'fail', evidence_ref: artifact.evidence_ref,
+          problems: [broken ? problem(`${unlinked}; declared sitemap unusable: ${broken}`, SITEMAP_UNUSABLE, { broken }) : problem(unlinked, ORPHAN)] }));
+      } else {
+        // Reaching a page because the run selected it is not discoverability, and an orphan is only proven on complete evidence
+        // (see findOrphans). A page neither listed nor linked that cannot be proven an orphan is unknown, and says what leaves it open.
+        out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state: 'unknown', evidence_ref: artifact.evidence_ref,
+          reason: sitemaps.unread.length ? unreadReason(sitemaps.unread) : UNPROVEN,
+          extra: { measurement: { crawl_complete: crawlComplete, sitemap_index_unread: sitemaps.index, script_gap: linkGraph.scriptGap, collected_pages: htmlPages,
+            ...(sitemaps.unread.length ? { sitemap_unread: sitemaps.unread } : {}) } } }));
+      }
+    }
+    return out;
+  },
+};

package/skills/optbrew/scripts/core/checks/document-title.mjs

@@ -0,0 +1,62 @@
+import { thresholds } from '../config.mjs';
+import { messageOf } from '../messages.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'document-title';
+const MISSING = 'document-title.missing';
+const DUPLICATE = 'document-title.duplicate';
+const SAME_DOCUMENT = 'document-title.same-document';
+
+/** A title or host name as letters and digits only, so case, spacing and punctuation never tell two spellings apart. */
+const plain = value => String(value).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '');
+const OG_SITE_NAME = [/<meta\b[^>]*\bproperty\s*=\s*["']og:site_name["'][^>]*\bcontent\s*=\s*["']([^"']*)["']/i, /<meta\b[^>]*\bcontent\s*=\s*["']([^"']*)["'][^>]*\bproperty\s*=\s*["']og:site_name["']/i];
+/**
+ * The forms of the site's own name a title can consist of alone: the host and its first label, the `og:site_name` of any audited
+ * page, and the name of the organization any audited page's JSON-LD speaks for.
+ */
+function siteNames(pageRows, facts) {
+  const names = pageRows.flatMap(({ artifact, html }) => {
+    const host = new URL(artifact.final_url).hostname.replace(/^www\./, '');
+    return [host, host.split('.')[0], ...OG_SITE_NAME.map(pattern => pattern.exec(html)?.[1])];
+  });
+  for (const { facts: page } of facts) names.push(...page.linked.name.values());
+  return new Set(names.filter(Boolean).map(plain).filter(Boolean));
+}
+
+/**
+ * A missing or duplicated title fails. A title that is present, shared with no other audited page (and there are at least two audited
+ * pages) and not only the site's name or a generic word passes, on those facts alone. With a single audited page, or a title that is
+ * only a name or a generic word, it stays unknown, because whether it describes the page is not established by code.
+ */
+export const documentTitle = {
+  family_id: FAMILY,
+  message_keys: [MISSING, DUPLICATE],
+  derive({ pageRows, titleCounts, facts }) {
+    const generic = new Set(thresholds(FAMILY).generic_titles.map(plain));
+    const names = siteNames(pageRows, facts);
+    return pageRows.map(row => {
+      const { url, artifact, info } = row;
+      const others = info.title ? titleCounts.get(row.titleKey).filter(other => other !== row && other.artifact.final_url !== artifact.final_url) : [];
+      // Query variants of one page (same document, R98) share their title by being one page; they are not duplicates of each other.
+      const variants = others.filter(other => row.documentKey && other.documentKey === row.documentKey);
+      const duplicates = others.filter(other => !variants.includes(other));
+      const observed = { family_id: FAMILY, unit_key: unitKey(FAMILY, url), evidence_ref: artifact.evidence_ref };
+      if (!info.title) return makeObservation({ ...observed, state: 'fail', problems: [problem('Raw HTML has no title', MISSING)] });
+      // The other pages are page-specific, so they travel in the measurement and the reason stays the same for every page with the problem.
+      if (duplicates.length) return makeObservation({ ...observed, state: 'fail', problems: [problem('Title duplicates the title of other pages', DUPLICATE)],
+        extra: { measurement: { duplicate_pages: duplicates.map(x => x.url) } } });
+      const key = plain(info.title);
+      const measurement = { title: info.title, audited_pages: pageRows.length, site_name_only: names.has(key), generic_word_only: key === '' || generic.has(key) };
+      // With one audited page, "no other page has this title" is true of any title and proves nothing.
+      if (pageRows.length < 2 || measurement.site_name_only || measurement.generic_word_only) return makeObservation({ ...observed, state: 'unknown', extra: { measurement },
+        reason: 'Raw HTML has a title; uniqueness and descriptiveness are not established' });
+      // A title shared only with query variants of the same page says so: it is not "different from every other page".
+      if (variants.length) return makeObservation({ ...observed, state: 'pass',
+        extra: { measurement: { ...measurement, same_document: { document: row.documentKey, pages: variants.map(x => x.url) },
+          ...messageOf(SAME_DOCUMENT, { pages: variants.map(x => x.url).join(', ') }) } },
+        reason: 'The title is present, shared only with query variants of the same page (the same canonical URL), and is not only the site name or a generic word' });
+      return makeObservation({ ...observed, state: 'pass', extra: { measurement },
+        reason: 'The title is present, differs from the title of every other audited page, and is not only the site name or a generic word' });
+    });
+  },
+};

package/skills/optbrew/scripts/core/checks/entity-identity.mjs

@@ -0,0 +1,31 @@
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'entity-identity';
+const NAME_DIFFERS = problem('Entity identity: the business name differs between pages', 'entity-identity.name-differs');
+const NO_SAME_AS = problem('Entity identity: Organization JSON-LD lacks sameAs on some pages', 'entity-identity.no-same-as');
+const NO_HOME_ORGANIZATION = problem('Entity identity: no Organization or LocalBusiness JSON-LD on the home page', 'entity-identity.no-home-organization');
+
+/** One site-level verdict: Organization JSON-LD with sameAs on the home page, and one name across pages. */
+export const entityIdentity = {
+  family_id: FAMILY,
+  message_keys: [NAME_DIFFERS, NO_SAME_AS, NO_HOME_ORGANIZATION].map(item => item.message_key),
+  derive({ facts, comparison }) {
+    if (!facts.length) return [];
+    const problems = [];
+    const home = facts.find(x => new URL(x.row.artifact.final_url).pathname === '/') ?? facts[0];
+    const orgPages = facts.filter(x => x.facts.orgs.length && !comparison.separate.has(x.url));
+    const nameReference = comparison.references.name;
+    const nameDeviants = facts.filter(x => (comparison.deviations.get(x.url) ?? []).some(text => text.startsWith('name ')));
+    // The sentences are fixed; which names and pages they concern travel in the measurement.
+    const withoutSameAs = orgPages.filter(page => page.facts.orgs.some(org => !org.sameAs));
+    if (nameDeviants.length) problems.push(NAME_DIFFERS);
+    if (withoutSameAs.length) problems.push(NO_SAME_AS);
+    if (orgPages.length && !home.facts.orgs.length) problems.push(NO_HOME_ORGANIZATION);
+    const measurement = { name_reference: nameReference?.label ?? null, name_differs_on: nameDeviants.map(x => x.url),
+      without_same_as_on: withoutSameAs.map(x => x.url), home_page: home.url };
+    const observe = (state, stated, evidence_ref) => [makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, 'site'), state, evidence_ref, ...stated, extra: { measurement } })];
+    if (problems.length) return observe('fail', { problems }, (nameDeviants[0] ?? home).row.artifact.evidence_ref);
+    if (orgPages.length) return observe('pass', { reason: 'Organization JSON-LD with name and sameAs is present on the home page and names agree' }, home.row.artifact.evidence_ref);
+    return [];
+  },
+};

package/skills/optbrew/scripts/core/checks/fact-consistency.mjs

@@ -0,0 +1,31 @@
+import { FACT_MESSAGE_KEYS } from '../facts.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'fact-consistency';
+const PREFIX = 'Cross-page fact comparison (no approved fact list): ';
+const NO_FACTS = 'The page has no labelled, marked-up or tel: contact fact (name, phone number, address, founding year) that this check can compare';
+
+/** Contact facts compared across pages. Agreement is consistency, not accuracy, and a branch page is never compared. */
+export const factConsistency = {
+  family_id: FAMILY,
+  message_keys: Object.values(FACT_MESSAGE_KEYS),
+  derive({ facts, comparison }) {
+    const out = [];
+    for (const page of facts) {
+      if (comparison.separate.has(page.url)) continue;
+      const found = comparison.deviations.get(page.url) ?? [];
+      const stated = comparison.deviation_messages.get(page.url) ?? [];
+      const comparable = Object.keys(comparison.references).some(type => page.facts.visible[type].size || page.facts.linked[type].size || (type === 'phone' && page.facts.tel.phone.size));
+      // A page that states no contact fact has nothing to compare. A page that states one but has no reference to meet stays unknown:
+      // a missing reference is a missing fact list, not an inapplicable page.
+      const stating = Object.keys(page.facts.visible).some(type => page.facts.visible[type].size || page.facts.linked[type].size) || page.facts.tel.phone.size > 0;
+      const observe = (state, judged) => out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, page.url), state, evidence_ref: page.row.artifact.evidence_ref, ...judged }));
+      // Each conflict is its own problem: the values are the problem itself (R63), so a page with two conflicts has two fixes.
+      if (found.length) observe('fail', { problems: found.map((text, index) => problem(`${PREFIX}${text}`, stated[index].message_key, stated[index].message_values)) });
+      else if (comparable) observe('pass', { reason: 'Facts on this page match the cross-page reference values (no approved fact list; consistency is not accuracy)' });
+      else if (!stating) observe('not_applicable', { reason: NO_FACTS, extra: { measurement: { facts_found: 0 },
+        applicability_decision: { condition: 'no_evaluable_facts', reason: NO_FACTS, evidence_ref: page.row.artifact.evidence_ref } } });
+    }
+    return out;
+  },
+};

package/skills/optbrew/scripts/core/checks/fetch-response.mjs

@@ -0,0 +1,18 @@
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'fetch-response';
+const HTTP_ERROR = 'fetch-response.http-error';
+
+/** Every selected page, collected or not: a finished 2xx passes, a 4xx or 5xx fails, anything else is unknown. */
+export const fetchResponse = {
+  family_id: FAMILY,
+  message_keys: [HTTP_ERROR],
+  derive({ artifacts, selected }) {
+    return artifacts.filter(artifact => selected.has(artifact.normalized_url)).map(artifact => {
+      const state = artifact.error || artifact.status < 200 || artifact.status >= 300 && artifact.status < 400 ? 'unknown' : artifact.status >= 400 ? 'fail' : 'pass';
+      return makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, artifact.normalized_url), state, evidence_ref: artifact.evidence_ref,
+        ...(state === 'fail' ? { problems: [problem(`HTTP ${artifact.status}`, HTTP_ERROR, { status: artifact.status })] }
+          : { reason: artifact.error ? 'Request did not finish' : `HTTP ${artifact.status}` }) });
+    });
+  },
+};

package/skills/optbrew/scripts/core/checks/index-policy.mjs

@@ -0,0 +1,16 @@
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'index-policy';
+const NOINDEX = 'index-policy.noindex';
+
+/** Whether the raw response blocks indexing, for pages meant to be indexed. */
+export const indexPolicy = {
+  family_id: FAMILY,
+  message_keys: [NOINDEX],
+  derive({ pageRows }) {
+    return pageRows.filter(row => row.publicIndexing).map(({ url, artifact, info }) => makeObservation({
+      family_id: FAMILY, unit_key: unitKey(FAMILY, url), state: info.index_blocked ? 'fail' : 'pass', evidence_ref: artifact.evidence_ref,
+      ...(info.index_blocked ? { problems: [problem('Raw response blocks indexing for the target engine', NOINDEX)] }
+        : { reason: 'Raw response has no applicable noindex directive' }) }));
+  },
+};

package/skills/optbrew/scripts/core/checks/index.mjs

@@ -0,0 +1,18 @@
+import { fetchResponse } from './fetch-response.mjs';
+import { searchCrawlPolicy } from './search-crawl-policy.mjs';
+import { indexPolicy } from './index-policy.mjs';
+import { renderContent } from './render-content.mjs';
+import { discovery } from './discovery.mjs';
+import { documentTitle } from './document-title.mjs';
+import { canonicalConsistency } from './canonical-consistency.mjs';
+import { internalNavigation } from './internal-navigation.mjs';
+import { structuredDataTruth } from './structured-data-truth.mjs';
+import { localeLinks } from './locale-links.mjs';
+import { naverMarkup } from './naver-markup.mjs';
+import { entityIdentity } from './entity-identity.mjs';
+import { factConsistency } from './fact-consistency.mjs';
+import { lcpField, inpField, clsField } from './cwv-field.mjs';
+
+/** The one registration list: every check block, in the order of check-registry.json. A block is `{ family_id, derive(prepared) }`. */
+export const blocks = [fetchResponse, searchCrawlPolicy, indexPolicy, renderContent, discovery, documentTitle, canonicalConsistency,
+  internalNavigation, structuredDataTruth, localeLinks, naverMarkup, lcpField, inpField, clsField, entityIdentity, factConsistency];

package/skills/optbrew/scripts/core/checks/internal-navigation.mjs

@@ -0,0 +1,67 @@
+import { thresholds } from '../config.mjs';
+import { anchorTexts } from '../html.mjs';
+import { canonicalUrl, sameOriginLinks } from '../link-graph.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'internal-navigation';
+const TOO_DEEP = problem('The home page reaches this page in more clicks than the limit', 'internal-navigation.too-deep');
+const WEAK_ANCHORS = problem('Linked pages have too many empty or generic anchors', 'internal-navigation.weak-anchors');
+
+/** Same-origin links of a page to other pages, each with its anchor label. */
+function internalAnchors({ artifact, html }) {
+  const anchors = anchorTexts(html, artifact.final_url);
+  const internal = new Set(sameOriginLinks(anchors.map(({ href }) => href), artifact.final_url));
+  return anchors.filter(({ href }) => {
+    const target = new URL(href); target.hash = '';
+    return internal.has(target.href) && canonicalUrl(href) !== canonicalUrl(artifact.final_url);
+  });
+}
+
+/**
+ * A page that is too many clicks from the home page, or whose linked pages mostly have no descriptive anchor, fails. A linked page
+ * counts as weak only when every anchor to it on the page is empty or generic: an icon or a "more" link beside a descriptive link
+ * to the same target is normal navigation. A page nothing links to has no click depth: it is left to `discovery`, so it is judged
+ * on its anchors only. A site whose pages carry no internal link is not link based, unless a script that was not collected could be
+ * building the links.
+ */
+export const internalNavigation = {
+  family_id: FAMILY,
+  message_keys: [TOO_DEEP, WEAK_ANCHORS].map(item => item.message_key),
+  derive({ pageRows, linkGraph }) {
+    const { max_click_depth, empty_anchor_ratio, generic_anchor_terms } = thresholds(FAMILY);
+    const generic = new Set(generic_anchor_terms.map(term => term.toLowerCase()));
+    const rows = pageRows.map(row => ({ row, anchors: internalAnchors(row) }));
+    const linkBased = rows.some(({ anchors }) => anchors.length > 0);
+    const out = [];
+    for (const { row: { url, artifact }, anchors } of rows) {
+      const observe = (state, reason, extra = {}) => out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state, reason,
+        evidence_ref: artifact.evidence_ref, extra }));
+      if (!linkBased) {
+        if (linkGraph.scriptGap) observe('unknown', 'No links in the raw HTML; a script that was not collected may build the navigation');
+        else {
+          const reason = 'No collected page links to another page of the site';
+          observe('not_applicable', reason, { applicability_decision: { condition: 'not_link_based', reason, evidence_ref: artifact.evidence_ref } });
+        }
+        continue;
+      }
+      const descriptive = new Map();
+      for (const { href, label } of anchors) {
+        const target = canonicalUrl(href);
+        descriptive.set(target, descriptive.get(target) || (Boolean(label) && !generic.has(label.toLowerCase())));
+      }
+      const weak = [...descriptive.values()].filter(ok => !ok).length;
+      const depth = linkGraph.depthOf(url);
+      // The sentences are fixed, so the same problem reads the same on every page; the numbers are page-specific and travel in the measurement.
+      const measurement = { click_depth: depth, max_click_depth, weak_anchor_pages: weak, linked_pages: descriptive.size, allowed_weak_ratio: empty_anchor_ratio };
+      const problems = [];
+      if (depth !== null && depth > max_click_depth) problems.push(TOO_DEEP);
+      if (descriptive.size && weak / descriptive.size > empty_anchor_ratio) problems.push(WEAK_ANCHORS);
+      if (problems.length) out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state: 'fail', evidence_ref: artifact.evidence_ref, problems,
+        extra: { measurement: { ...measurement, problems: problems.map(item => item.text) } } }));
+      // A page with no click depth and no internal link has nothing to judge: it gets no observation on purpose (discovery covers it).
+      else if (depth !== null || descriptive.size)
+        observe('pass', depth === null ? 'Click depth not measured (left to discovery); anchors are within the limit' : 'Reached within the click limit; anchors are within the limit', { measurement });
+    }
+    return out;
+  },
+};

package/skills/optbrew/scripts/core/checks/locale-links.mjs

@@ -0,0 +1,55 @@
+import { canonicalUrl } from '../link-graph.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'locale-links';
+const X_DEFAULT = 'x-default';
+const NO_X_DEFAULT = problem('No x-default alternate is declared', 'locale-links.no-x-default');
+const NO_RETURN = problem('An alternate page does not link back to this page', 'locale-links.no-return');
+const NOT_LISTED_BACK = problem('A page that lists this page as an alternate is not listed back', 'locale-links.not-listed-back');
+
+const points = (href, page) => page.urls.includes(canonicalUrl(href));
+
+/**
+ * A page that declares hreflang alternates needs an x-default and a return link from every alternate it names; a page that
+ * another page lists as its alternate must list that page in turn. An alternate that was not collected cannot be checked,
+ * so the page is unknown rather than passed. A page with no hreflang, or only one that names itself, that no page lists belongs to a single-locale site.
+ */
+export const localeLinks = {
+  family_id: FAMILY,
+  message_keys: [NO_X_DEFAULT, NO_RETURN, NOT_LISTED_BACK].map(item => item.message_key),
+  derive({ pageRows, alternates }) {
+    const pages = new Set(alternates.values());
+    const out = [];
+    for (const { url, artifact } of pageRows) {
+      const self = alternates.get(canonicalUrl(artifact.normalized_url));
+      const mine = self.alternates;
+      const listedBy = [...pages].filter(other => other !== self && other.alternates.some(alternate => points(alternate.href, self)));
+      const observe = (state, reason, extra = {}) => out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state, reason,
+        evidence_ref: artifact.evidence_ref, extra }));
+      if (mine.every(alternate => points(alternate.href, self)) && !listedBy.length) {
+        const reason = 'No hreflang link on this page and no collected page lists it as an alternate';
+        observe('not_applicable', reason, { applicability_decision: { condition: 'single_locale', reason, evidence_ref: artifact.evidence_ref } });
+        continue;
+      }
+      // Fixed sentences; which pages are meant travels in the measurement.
+      const problems = new Map();
+      const add = item => problems.set(item.text, item);
+      const noReturn = [], listedWithoutReturn = [];
+      if (mine.length && !mine.some(alternate => alternate.hreflang === X_DEFAULT)) add(NO_X_DEFAULT);
+      let unchecked = 0;
+      for (const alternate of mine) {
+        if (points(alternate.href, self)) continue;
+        const target = alternates.get(canonicalUrl(alternate.href));
+        if (!target) unchecked += 1;
+        else if (!target.alternates.some(back => points(back.href, self))) { add(NO_RETURN); noReturn.push(target.urls[0]); }
+      }
+      for (const other of listedBy) if (!mine.some(alternate => points(alternate.href, other))) { add(NOT_LISTED_BACK); listedWithoutReturn.push(other.urls[0]); }
+      const measurement = { alternates_without_return_link: noReturn, pages_listing_this_page_without_return: listedWithoutReturn, unchecked_alternates: unchecked };
+      if (problems.size) out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), state: 'fail', evidence_ref: artifact.evidence_ref, problems: [...problems.values()],
+        extra: { measurement: { ...measurement, problems: [...problems.keys()] } } }));
+      else if (unchecked) observe('unknown', 'Some alternate pages were not collected, so their return links are unchecked', { measurement });
+      else observe('pass', 'An x-default alternate is declared and every alternate links back', { measurement });
+    }
+    return out;
+  },
+};

package/skills/optbrew/scripts/core/checks/naver-markup.mjs

@@ -0,0 +1,21 @@
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'naver-markup';
+const OG_PREFIX = 'no-og-';
+/** The Korean message each markup finding code of the Naver module has; the five Open Graph codes share one. */
+const CODES = ['no-og', 'no-description', 'no-canonical', 'canonical-relative', 'title-keyword-repeat', 'no-viewport', 'resource-blocked'];
+const keyOf = code => `naver-markup.${code.startsWith(OG_PREFIX) ? 'no-og' : code}`;
+
+/** Open Graph, description, canonical, viewport and title markup that Naver reads, judged only when Naver is a target engine. */
+export const naverMarkup = {
+  family_id: FAMILY,
+  message_keys: CODES.map(code => keyOf(code)),
+  derive({ context, pageRows }) {
+    if (!context.protocol.engines.includes('naver')) return [];
+    // Each distinct problem is its own finding, as in the other checks that state several: the joined reason is only the verdict's summary.
+    return pageRows.map(({ url, artifact, info }) => makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url, 'naver'),
+      state: info.markup_findings.length ? 'fail' : 'pass', evidence_ref: artifact.evidence_ref,
+      ...(info.markup_findings.length ? { problems: info.markup_findings.map(x => problem(x.message, keyOf(x.code), x.values)) }
+        : { reason: 'Open Graph, description, canonical, viewport and title checks pass' }) }));
+  },
+};

package/skills/optbrew/scripts/core/checks/observation.mjs

@@ -0,0 +1,37 @@
+import { createHash } from 'node:crypto';
+import { validateEntity } from '../contracts.mjs';
+import { versions } from '../config.mjs';
+import { messageOf } from '../messages.mjs';
+
+const FIXED_Q = { pass: 1, fail: 0 };
+
+/** Identity of one judged unit: the family, the engine when the family is judged per engine, and the url. */
+export const unitKey = (family_id, url, engine = null) => `${family_id}|${engine ? `${engine}|` : ''}${url}`;
+
+/**
+ * One problem a failed check states: its English sentence (the grouping key, never translated), the catalog key of its Korean
+ * sentence, and the page-specific values that sentence fills.
+ */
+export const problem = (text, message_key, message_values = {}) => ({ text, ...messageOf(message_key, message_values) });
+
+/** What a failed verdict records for the problems it states: the joined reason, the list when there are several, and each problem's Korean key. */
+function statedProblems(problems, extra) {
+  const texts = problems.map(item => item.text);
+  const measurement = texts.length > 1 && !extra.measurement?.problems ? { ...extra.measurement, problems: texts } : extra.measurement;
+  return { reason: texts.join('; '), extra: { ...extra, ...(measurement ? { measurement } : {}),
+    problem_messages: problems.map(({ message_key, message_values }) => messageOf(message_key, message_values)) } };
+}
+
+/**
+ * One code-judged CheckObservation. `observation_id` is the sha256 of `unit_key`, as in 0.1.4. `q` follows the state
+ * (pass 1, fail 0, anything else null) unless a block passes its own; `check_id` defaults to the family. A failed verdict
+ * passes the `problems` it states (see `problem`) instead of a reason: the reason is their texts joined, and
+ * `problem_messages` lines up with them for the report's Korean text.
+ */
+export function makeObservation({ family_id, check_id = family_id, unit_key, state, q = FIXED_Q[state] ?? null, reason, evidence_ref,
+  partial_basis = null, problems = null, extra = {} }) {
+  const stated = problems ? statedProblems(problems, extra) : { reason, extra };
+  return validateEntity('CheckObservation', { schema_version: versions().contract_schema,
+    observation_id: createHash('sha256').update(unit_key).digest('hex'), check_id, family_id, unit_key,
+    state, q, partial_basis, reason: stated.reason, evaluator_type: 'code', source_ref: 'optbrew:collector', evidence_ref, ...stated.extra });
+}

package/skills/optbrew/scripts/core/checks/prepare.mjs

@@ -0,0 +1,103 @@
+import { auditCollected } from '../../../modules/naver-searchadvisor/scripts/audit.mjs';
+import { alternateLinks, decodeEntities } from '../html.mjs';
+import { declaredRobotsIndex } from '../collector.mjs';
+import { compareFacts, pageFacts } from '../facts.mjs';
+import { canonicalUrl, findOrphans, linkGraph } from '../link-graph.mjs';
+import { sameSite } from '../page-select.mjs';
+import { selectRenderPages } from '../render.mjs';
+
+const SKIP_REASONS = ['depth_limit', 'url_limit', 'request_limit'];
+const NOT_COLLECTED = 'not_collected';
+const normalizeTitle = title => title.toLowerCase().replace(/\s+/g, ' ').trim();
+const BASE_HREF = /<base\b[^>]*\bhref\s*=\s*["']([^"']*)["']/i;
+const pathOf = url => url.pathname.length > 1 ? url.pathname.replace(/\/+$/, '') || '/' : url.pathname;
+/** A page's address as its site and path, and its query when asked: the spelling documents are compared by. */
+const documentAddress = (url, withQuery) => `${url.origin}${pathOf(url)}${withQuery ? url.search : ''}`;
+/**
+ * The document a page stands for (R95, R98). A page whose canonical link names the same site (`www.` and http/https folded) and the same
+ * path is one document with every page that names the same canonical, query included: `/bookings?service=x` -> `/bookings` groups with
+ * `/bookings`, while `/products?cat=shoes` and `/products?cat=bags` that each name themselves stay two documents.
+ * Any other canonical (the home page, another path, another site), an unusable one, or none leaves the page its own document.
+ * The canonical is resolved against `<base href>` when the head has one, else against where the page was served.
+ */
+function documentKeyOf(artifact, info, html) {
+  const own = new URL(artifact.final_url);
+  if (info.canonical) {
+    try {
+      const live = String(html ?? '').replace(/<!--[\s\S]*?(?:-->|$)/g, '').replace(/<(script|style|template|noscript)\b[\s\S]*?(?:<\/\1\s*>|$)/gi, '');
+      const baseHref = BASE_HREF.exec(/<head\b[\s\S]*?<\/head\s*>/i.exec(live)?.[0] ?? live)?.[1];
+      const target = new URL(info.canonical, baseHref ? new URL(baseHref, own).href : own.href);
+      if (sameSite(target, own) && pathOf(target) === pathOf(own)) return `${documentAddress(own, false)}${target.search}`;
+    } catch { /* an unusable canonical declares nothing */ }
+  }
+  return documentAddress(own, true);
+}
+/** The pages the run renders: the ones its protocol fixed (none when rendering is off), else, before render plans existed, the default sample. */
+const renderPlan = protocol => protocol.render ? (protocol.render.enabled ? protocol.render.pages : []) : selectRenderPages(protocol.page_types);
+
+/**
+ * The declared sitemaps among the collected artifacts: every listed url, whether one is an index, the unusable ones, and the ones the
+ * run never fetched (`unread`, each with the reason the collector skipped it, else `not_collected`).
+ */
+function readSitemaps(artifacts, declared, textOf, skipped) {
+  const sitemaps = { urls: new Set(), index: false, broken: [], unread: [] };
+  const fetched = new Set(artifacts.map(artifact => artifact.normalized_url));
+  for (const url of declared) if (!fetched.has(url)) sitemaps.unread.push({ url, reason: skipped.find(x => x.url === url)?.reason ?? NOT_COLLECTED });
+  for (const artifact of artifacts) {
+    if (!declared.has(artifact.normalized_url)) continue;
+    if (artifact.status >= 200 && artifact.status < 300) {
+      const body = String(textOf(artifact) ?? '');
+      // Child sitemaps of an index are not fetched, so pages missing from the index prove nothing.
+      if (/<sitemapindex\b/i.test(body)) sitemaps.index = true;
+      for (const match of body.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) sitemaps.urls.add(canonicalUrl(decodeEntities(match[1])));
+    } else sitemaps.broken.push(`${artifact.normalized_url} (${artifact.status ?? artifact.error ?? 'no response'})`);
+  }
+  return sitemaps;
+}
+
+/**
+ * The values several check blocks share, computed once from the collected evidence.
+ * `pageRows` are the selected pages with a 200 raw HTML body, each with its module audit (`info`), its declared robots
+ * artifact and its normalized title key. `titleCounts` maps a title key to the rows that carry it. `facts` and `comparison`
+ * are the cross-page entity facts. `linkGraph` and `orphans` come from every collected HTML page, `sitemaps` from the
+ * declared sitemaps; `crawlComplete` is false when the crawl was cut short and `htmlPages` counts the collected HTML pages. `alternates` maps a page url (as requested and as finally served) to `{ urls, alternates }`, the hreflang
+ * links of every collected HTML page. `renderSample` holds the canonical urls of the pages the run renders: the protocol's fixed render pages, or for a run from before the render plan the first URL of each page type. `renders` are the stored render records, `renderTextOf(record)` the final DOM of one, and
+ * `chrome` is `{ found, reason }` for the run's Chrome, or null when the run recorded none. `cwv` holds the stored
+ * field-data answers, each `{ source, origin, status, body | error, evidence_ref }`, empty when the run collected none.
+ */
+export function prepare({ context, artifacts, collection = {}, textOf, renders = [], renderTextOf = () => '', chrome = null, cwv = [] }) {
+  const { protocol } = context;
+  const planned = new Set(protocol.families.filter(x => x.applicable).map(x => x.family_id));
+  const selected = new Map(protocol.page_types.flatMap(type => type.urls).map(url => [url.normalized, url]));
+  const declared = kind => new Set((protocol.auxiliary_resources ?? []).filter(x => x.kind === kind).map(x => x.normalized));
+  const declaredRobots = declared('robots');
+  const pageRows = [];
+  for (const artifact of artifacts) {
+    const url = artifact.normalized_url;
+    if (!selected.has(url) || artifact.capability !== 'raw_html' || artifact.status !== 200) continue;
+    const html = textOf(artifact);
+    const robotsIndex = declaredRobotsIndex(artifacts, declaredRobots, artifact.final_url);
+    const robots = robotsIndex < 0 ? null : artifacts[robotsIndex];
+    const info = auditCollected({ artifact, html, engines: protocol.engines,
+      robotsArtifact: robots?.status === 200 ? { ...robots, body: textOf(robots) } : null });
+    pageRows.push({ url, artifact, html, info, robots, publicIndexing: selected.get(url).public_indexing !== false,
+      titleKey: info.title ? normalizeTitle(info.title) : null, documentKey: documentKeyOf(artifact, info, html) });
+  }
+  const titleCounts = new Map();
+  // A title shared with a page that is not meant to be indexed is no duplicate: only indexed pages compete for a title.
+  for (const row of pageRows) if (row.titleKey !== null && row.publicIndexing) titleCounts.set(row.titleKey, [...titleCounts.get(row.titleKey) ?? [], row]);
+  const facts = pageRows.map(row => ({ url: row.url, row, facts: pageFacts(row.html, [row.url, row.artifact.final_url]) }));
+  const htmlRows = artifacts.filter(x => x.capability === 'raw_html' && x.status === 200)
+    .map(artifact => ({ url: artifact.normalized_url, artifact, html: textOf(artifact) }));
+  const graph = linkGraph(htmlRows, artifacts);
+  const alternates = new Map();
+  for (const { artifact, html } of htmlRows) {
+    const entry = { urls: [...new Set([artifact.normalized_url, artifact.final_url].map(canonicalUrl))], alternates: alternateLinks(html, artifact.final_url) };
+    for (const key of entry.urls) alternates.set(key, entry);
+  }
+  const sitemaps = readSitemaps(artifacts, declared('sitemap'), textOf, collection.skipped ?? []);
+  const complete = !collection.limit_reached && !(collection.skipped ?? []).some(x => SKIP_REASONS.includes(x.reason));
+  return { context, artifacts, textOf, planned, selected, pageRows, titleCounts, facts, comparison: compareFacts(facts), sitemaps,
+    linkGraph: graph, orphans: findOrphans({ pageRows, rows: htmlRows, graph, sitemaps, complete }), crawlComplete: complete, htmlPages: htmlRows.length, alternates,
+    renderSample: new Set(renderPlan(protocol).map(page => canonicalUrl(page.normalized))), renders, renderTextOf, chrome, cwv };
+}