optbrew 0.1.3 → 0.2.0

package/skills/optbrew/scripts/core/checks/render-content.mjs

@@ -0,0 +1,104 @@
+import { thresholds } from '../config.mjs';
+import { anchorLinks, visibleText } from '../html.mjs';
+import { canonicalUrl, sameOriginLinks } from '../link-graph.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'render-content';
+const NO_BODY_TEXT = 'render-content.no-body-text';
+const CHROME_MISSING = 'chrome_missing';
+const RENDER_BLOCKED = 'render_blocked';
+const RENDER_FAILED = 'render_failed';
+/** How the reason of a page that passes only after rendering begins; the report keys its JavaScript note on it. */
+export const CONTENT_AFTER_RENDER = 'Content shows after rendering';
+
+const squash = value => String(value ?? '').replace(/\s+/g, ' ').trim();
+/** Visible characters of the body, whitespace not counted. */
+const bodyLength = html => visibleText(/<body\b[^>]*>([\s\S]*)/i.exec(String(html ?? ''))?.[1] ?? html).replace(/\s/g, '').length;
+
+const assetOf = (value, base) => { try { const url = new URL(value, base); return url.origin + url.pathname; } catch { return String(value); } };
+/**
+ * What a page's template loads: the external scripts and the head assets (stylesheets, preloads), each without query or fragment.
+ * Two pages made by one template load the same set; null when the page loads nothing external, which is no signal at all.
+ */
+function templateAssets(html, base) {
+  const live = String(html ?? '').replace(/<!--[\s\S]*?(?:-->|$)/g, '');
+  const scripts = [...live.matchAll(/<script\b[^>]*\bsrc\s*=\s*["']([^"']+)["']/gi)].map(match => assetOf(match[1], base));
+  const head = [...live.matchAll(/<link\b[^>]*>/gi)].map(match => match[0]).filter(tag => /\brel\s*=\s*["'][^"']*\b(?:stylesheet|preload|modulepreload)\b/i.test(tag))
+    .map(tag => /\bhref\s*=\s*["']([^"']+)["']/i.exec(tag)?.[1]).filter(Boolean).map(href => assetOf(href, base));
+  const sorted = list => [...new Set(list)].sort();
+  return scripts.length || head.length ? { scripts: sorted(scripts), head: sorted(head) } : null;
+}
+const sameTemplate = (a, b) => a !== null && b !== null && JSON.stringify(a) === JSON.stringify(b);
+
+/**
+ * The raw HTML against the stored render of the same page. A page whose raw body, title and link count already match the render
+ * does not depend on rendering. One that depends on it passes when the render shows content and fails when neither the raw HTML
+ * nor the render has any. A challenge page (a challenge status, or a marker in the title or in a short body) and a render that did
+ * not finish are unknown, and so is a page of the render sample with no render when the run did not record a Chrome.
+ *
+ * A page outside the render sample inherits the verdict of the rendered page of its own page type when every rendered page of the
+ * type gives the same pass or not-applicable, and that page loads the same external scripts and head assets as this one (a page type
+ * is only the first path segment, so it can hold leftovers and list pages beside detail pages). The page must have body text in its
+ * own raw HTML, and the reason names the rendered page; the evidence is that page's render record. Otherwise the page gets no
+ * verdict here.
+ */
+export const renderContent = {
+  family_id: FAMILY,
+  message_keys: [NO_BODY_TEXT],
+  derive({ context, pageRows, renders, renderTextOf, chrome, renderSample }) {
+    const { min_body_ratio, challenge_status, challenge_markers, challenge_max_body_chars } = thresholds(FAMILY);
+    const judgeOwn = ({ url, artifact, html, info }) => {
+      const own = [canonicalUrl(url), canonicalUrl(artifact.final_url)];
+      const stored = renders.find(render => [render.url, render.final_url].some(value => value && own.includes(canonicalUrl(value))));
+      if (!stored) return chrome?.found !== true && renderSample.has(canonicalUrl(url)) ? { state: 'unknown', reason: CHROME_MISSING } : null;
+      if (challenge_status.includes(stored.status)) return { state: 'unknown', reason: RENDER_BLOCKED, stored };
+      if (!stored.dom_ref || stored.blocked_reason) return { state: 'unknown', reason: RENDER_FAILED, stored };
+      const dom = renderTextOf(stored);
+      const rawLength = bodyLength(html), renderedLength = bodyLength(dom);
+      // A challenge page is short; a long page that happens to quote a marker is judged on its content.
+      const markerText = `${stored.title ?? ''} ${renderedLength <= challenge_max_body_chars ? visibleText(dom) : ''}`.toLowerCase();
+      if (challenge_markers.some(marker => markerText.includes(marker))) return { state: 'unknown', reason: RENDER_BLOCKED, stored };
+      // The counts belong to the page, so they travel in the measurement and the reason stays the same for every page.
+      const measurement = { raw_body_characters: rawLength, rendered_body_characters: renderedLength };
+      const ratio = renderedLength === 0 ? (rawLength > 0 ? Infinity : 0) : rawLength / renderedLength;
+      if (ratio >= min_body_ratio && squash(info.title) === squash(stored.title)
+          && sameOriginLinks(anchorLinks(html, artifact.final_url), artifact.final_url).length === stored.links.length) {
+        const reason = 'Raw HTML holds the rendered body text, with the same title and link count';
+        return { state: 'not_applicable', reason, stored, extra: { measurement, applicability_decision: { condition: 'not_render_dependent', reason, evidence_ref: artifact.evidence_ref } } };
+      }
+      if (renderedLength > 0) return { state: 'pass', reason: CONTENT_AFTER_RENDER, stored, extra: { measurement } };
+      if (rawLength > 0) return { state: 'pass', reason: 'The raw HTML holds body text and the render is empty', stored, extra: { measurement } };
+      return { state: 'fail', stored, problems: [problem('Neither the raw HTML nor the rendered page has any body text', NO_BODY_TEXT)] };
+    };
+
+    const judged = new Map(pageRows.map(row => [row.url, judgeOwn(row)]));
+    // Every page of the audit with the verdict its own render gave, by canonical url, and the pages by canonical url.
+    const rendered = new Map(pageRows.filter(row => judged.get(row.url)?.stored).map(row => [canonicalUrl(row.url), { row, ...judged.get(row.url) }]));
+    const templates = new Map(pageRows.map(row => [row.url, templateAssets(row.html, row.artifact.final_url)]));
+    const typeOf = new Map(context.protocol.page_types.flatMap(type => type.urls.map(item => [canonicalUrl(item.normalized), type])));
+    const inherit = ({ url, artifact, html }) => {
+      const type = typeOf.get(canonicalUrl(url));
+      // Only when every rendered page of the type gives one usable verdict, and from the rendered page that loads the same scripts and
+      // stylesheets as this page (the first path segment groups leftovers and list and detail pages as well as templates).
+      const peers = (type?.urls ?? []).map(item => rendered.get(canonicalUrl(item.normalized))).filter(Boolean);
+      const usable = peers.length > 0 && ['pass', 'not_applicable'].includes(peers[0].state) && peers.every(peer => peer.state === peers[0].state);
+      const from = usable ? peers.find(peer => sameTemplate(templates.get(peer.row.url), templates.get(url))) : null;
+      const rawLength = bodyLength(html);
+      if (!from || !from.stored.evidence_ref || renderSample.has(canonicalUrl(url)) || rawLength === 0) return null;
+      const evidence_ref = from.stored.evidence_ref;
+      const measurement = { raw_body_characters: rawLength, template: { scripts: templates.get(url).scripts.length, head_assets: templates.get(url).head.length },
+        inherited_from: { url: from.row.url, state: from.state, page_type: type.id, render_ref: evidence_ref } };
+      const basis = `inferred from ${from.row.url}, the rendered page of the same type that loads the same scripts and stylesheets; this page's raw HTML holds body text too`;
+      if (from.state === 'pass') return { state: 'pass', evidence_ref, reason: `${from.reason} (${basis})`, extra: { measurement } };
+      const reason = `Inferred from ${from.row.url}, the rendered page of the same type that loads the same scripts and stylesheets, which does not depend on rendering; this page's raw HTML holds body text too`;
+      return { state: 'not_applicable', evidence_ref, reason, extra: { measurement, applicability_decision: { condition: 'not_render_dependent', reason, evidence_ref } } };
+    };
+
+    return pageRows.flatMap(row => {
+      const { url, artifact } = row;
+      const own = judged.get(url);
+      const verdict = own ? (({ stored, ...rest }) => rest)(own) : inherit(row);
+      return verdict ? [makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, url), evidence_ref: artifact.evidence_ref, ...verdict })] : [];
+    });
+  },
+};

package/skills/optbrew/scripts/core/checks/search-crawl-policy.mjs

@@ -0,0 +1,29 @@
+import { parseRobots, robotsVerdict } from '../../../modules/naver-searchadvisor/scripts/robots.mjs';
+import { agentFor } from '../crawlers.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'search-crawl-policy';
+const DISALLOWED = 'search-crawl-policy.disallowed';
+
+/** The collected robots.txt against each target engine's crawler, for pages meant to be indexed. */
+export const searchCrawlPolicy = {
+  family_id: FAMILY,
+  message_keys: [DISALLOWED],
+  derive({ context, pageRows, textOf }) {
+    const out = [];
+    for (const row of pageRows) {
+      if (!row.publicIndexing || row.robots?.status !== 200) continue;
+      const parsed = parseRobots(textOf(row.robots));
+      const target = new URL(row.artifact.final_url);
+      for (const engine of context.protocol.engines) {
+        const agent = agentFor(engine);
+        if (!agent) continue;
+        const verdict = robotsVerdict(parsed, agent, target.pathname + target.search);
+        out.push(makeObservation({ family_id: FAMILY, unit_key: unitKey(FAMILY, row.url, engine), state: verdict.allowed ? 'pass' : 'fail',
+          evidence_ref: row.robots.evidence_ref, ...(verdict.allowed ? { reason: `Collected robots.txt allows ${agent} path` }
+            : { problems: [problem(`Collected robots.txt disallows ${agent} path`, DISALLOWED, { agent })] }) }));
+      }
+    }
+    return out;
+  },
+};

package/skills/optbrew/scripts/core/checks/structured-data-truth.mjs

@@ -0,0 +1,46 @@
+import { jsonLdBlocks, stripComments } from '../html.mjs';
+import { jsonLdTruth } from '../facts.mjs';
+import { makeObservation, problem, unitKey } from './observation.mjs';
+
+const FAMILY = 'structured-data-truth';
+const PARSE_ERROR = 'structured-data-truth.parse-error';
+const FACT_DIFFERS = 'structured-data-truth.fact-differs';
+// Microdata and RDFa mark a page up without a JSON-LD block; this check does not read them.
+const OTHER_MARKUP = /<[a-z][^>]*\s(?:itemscope|vocab\s*=|typeof\s*=)/i;
+const NO_MARKUP = 'The page has no JSON-LD, microdata or RDFa markup';
+
+/**
+ * JSON-LD that does not parse fails. A page with no markup at all is not applicable. A page whose own JSON-LD (an organization's name,
+ * phone number and street address; an article's headline and publication date) states facts passes when the page shows every one and
+ * contradicts none, and fails when the page shows a different value in the field that holds it. A fact the page neither shows nor
+ * contradicts leaves the page unknown, and so does JSON-LD of a type that states nothing this compares. A publisher or author name
+ * alone never passes.
+ */
+export const structuredDataTruth = {
+  family_id: FAMILY,
+  message_keys: [PARSE_ERROR, FACT_DIFFERS],
+  derive({ pageRows, facts }) {
+    const factsOf = new Map(facts.map(item => [item.url, item.facts]));
+    return pageRows.map(({ url, artifact, html, info }) => {
+      const observed = { family_id: FAMILY, unit_key: unitKey(FAMILY, url), evidence_ref: artifact.evidence_ref };
+      if (info.jsonld_parse_errors > 0) return makeObservation({ ...observed, state: 'fail', problems: [problem('JSON-LD block(s) fail to parse', PARSE_ERROR)],
+        extra: { measurement: { parse_errors: info.jsonld_parse_errors } } });
+      const blocks = jsonLdBlocks(html).length;
+      if (!blocks) {
+        if (OTHER_MARKUP.test(stripComments(html))) return makeObservation({ ...observed, state: 'unknown', reason: 'The page uses microdata or RDFa, which this check does not read' });
+        return makeObservation({ ...observed, state: 'not_applicable', reason: NO_MARKUP, extra: { measurement: { jsonld_blocks: 0 },
+          applicability_decision: { condition: 'markup_not_used_or_selected', reason: NO_MARKUP, evidence_ref: artifact.evidence_ref } } });
+      }
+      const { matched, mismatched, unevaluated } = jsonLdTruth(html, factsOf.get(url));
+      const measurement = { jsonld_blocks: blocks, matched: matched.map(({ type, jsonld, shown }) => ({ type, jsonld, ...(shown ? { shown } : {}) })), mismatched,
+        unevaluated: unevaluated.map(({ type, jsonld }) => ({ type, jsonld })) };
+      if (mismatched.length) return makeObservation({ ...observed, state: 'fail', extra: { measurement },
+        problems: mismatched.map(({ type, jsonld, visible }) => problem(`JSON-LD ${type} "${jsonld}" differs from visible "${visible}"`, FACT_DIFFERS, { type, linked: jsonld, visible })) });
+      if (matched.length && !unevaluated.length) return makeObservation({ ...observed, state: 'pass', extra: { measurement },
+        reason: 'Every name, phone number, address, headline and date the JSON-LD states is shown on the page, and none is contradicted' });
+      return makeObservation({ ...observed, state: 'unknown', extra: { measurement }, reason: unevaluated.length
+        ? 'The JSON-LD states a fact that the page text neither shows nor contradicts, so not every fact is confirmed'
+        : 'The JSON-LD states no organization or article fact that the page text could confirm' });
+    });
+  },
+};

package/skills/optbrew/scripts/core/chrome-temp.mjs

@@ -0,0 +1,68 @@
+import { execFile } from 'node:child_process';
+import { readdir, rm, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { limits } from './config.mjs';
+
+/** Chrome's temp folders: the environment it runs in, and the sweep of profile folders a killed launcher left (T22 M-7). */
+
+const LIST_LINES_WINDOWS = '$ErrorActionPreference = "Stop"; Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -and $_.CommandLine.Contains($env:OPTBREW_MARKER) } | ForEach-Object { $_.CommandLine }';
+
+/** Command lines (whole lines) that name `marker`; null when the listing failed or timed out, so a caller never mistakes "could not look" for "nothing there". */
+function commandLinesNaming(marker, timeoutMs) {
+  const [command, args] = process.platform === 'win32' ?
+    ['powershell', ['-NoProfile', '-NonInteractive', '-Command', LIST_LINES_WINDOWS]] : ['ps', ['-ww', '-eo', 'args=']];
+  return new Promise(resolve => {
+    execFile(command, args, { env: { ...process.env, OPTBREW_MARKER: marker }, maxBuffer: 1 << 24, windowsHide: true, timeout: timeoutMs }, (error, stdout) => {
+      resolve(error ? null : stdout.split(/\r?\n/).filter(line => line.includes(marker)));
+    });
+  });
+}
+
+export const PROFILE_PREFIX = 'optbrew-chrome-';
+const TEMP_VARIABLES = /^(?:temp|tmp|tmpdir)$/i;
+/**
+ * A profile folder younger than this may belong to a run still going (one whose command line this process cannot read, e.g. an elevated
+ * Chrome), so the sweep leaves it. An audit takes minutes; a folder left for half a day is from a launcher that was killed.
+ */
+const YOUNG_PROFILE_MS = 12 * 60 * 60_000;
+
+/**
+ * The environment Chrome runs in: `tempDir`, a folder inside this run's profile folder, in place of the user's temporary folder.
+ * What Chrome writes there (its `scoped_dir*` folders and component updates) is removed with the profile, even after a forced kill
+ * skipped Chrome's own cleanup. Existing TEMP, TMP and TMPDIR entries are dropped in any letter case, so the child sees one value each.
+ *
+ * Off Windows, Chrome binds its single-instance socket at `<temp folder>/.org.chromium.Chromium.XXXXXX/SingletonSocket`, and a Unix
+ * socket path is limited to 107 bytes on Linux and 104 on macOS; Chrome aborts at start when it is longer (Chromium
+ * `process_singleton_posix.cc`: "Socket path too long"). A profile under a long folder (a macOS `/var/folders/...` temp folder, a
+ * folder with Hangul, which takes three bytes a character) would put `tempDir` past that, so then Chrome keeps the user's own temp
+ * folder, as it did before this redirect. `platform` is injectable for tests.
+ */
+const MAX_REDIRECTED_TEMP_BYTES = 48;
+export function chromeEnv(env, tempDir, platform = process.platform) {
+  if (platform !== 'win32' && Buffer.byteLength(tempDir) > MAX_REDIRECTED_TEMP_BYTES) return { ...env };
+  const kept = Object.fromEntries(Object.entries(env).filter(([name]) => !TEMP_VARIABLES.test(name)));
+  return { ...kept, TEMP: tempDir, TMP: tempDir, TMPDIR: tempDir };
+}
+
+/**
+ * Remove the profile folders of earlier launches whose launcher was killed before it could clean up. A folder goes only when it is not
+ * young and no running process names it; when the process list cannot be read nothing is removed. `listLines(marker)` returns the
+ * command lines that name the marker, or null (injectable for tests). Returns the names removed.
+ */
+export async function sweepStaleProfiles(tmpRoot, { now = Date.now(), listLines = marker => commandLinesNaming(marker, limits().chrome_close_grace_ms) } = {}) {
+  let names;
+  try { names = (await readdir(tmpRoot)).filter(name => name.startsWith(PROFILE_PREFIX)); } catch { return []; }
+  const old = [];
+  for (const name of names) {
+    try { if (now - (await stat(path.join(tmpRoot, name))).mtimeMs > YOUNG_PROFILE_MS) old.push(name); } catch { /* gone already */ }
+  }
+  if (!old.length) return [];
+  const lines = await listLines(PROFILE_PREFIX);
+  if (lines === null) return [];
+  const removed = [];
+  for (const name of old) {
+    if (lines.some(line => line.includes(name))) continue;
+    try { await rm(path.join(tmpRoot, name), { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); removed.push(name); } catch { /* in use or locked: the next start tries again */ }
+  }
+  return removed;
+}