optbrew 0.1.2 → 0.2.0

package/skills/optbrew/scripts/core/collector.mjs

@@ -1,14 +1,16 @@
 import { lookup } from 'node:dns/promises';
 import { createHash } from 'node:crypto';
-import http from 'node:http';
-import https from 'node:https';
 import { isIP } from 'node:net';
 import { performance } from 'node:perf_hooks';
-import { assertInScope, assertUrlInScope } from './scope.mjs';
+import { assertInScope, assertUrlInScope, urlTarget } from './scope.mjs';
 import { anchorLinks, canonicalLink } from './html.mjs';
+import { limits, requestLimit } from './config.mjs';
+import { directTransport } from './transport.mjs';
 
 const HEADER_NAMES = ['content-type', 'content-length', 'content-range', 'content-encoding', 'location', 'x-robots-tag', 'last-modified', 'etag'];
 const REDIRECTS = new Set([301, 302, 303, 307, 308]);
+// An API call follows no redirect: zero hops.
+const FOLLOW_NONE = 0;
 const text = value => Buffer.isBuffer(value) ? value : Buffer.from(value ?? '');
 
 function auxiliaryDocument(url) {
@@ -24,14 +26,14 @@ function undeclaredAuxiliary(url, allowed) {
   throw error;
 }
 
-function safeHeaders(headers) {
+function safeHeaders(headers, extra = []) {
   const source = headers instanceof Headers ? Object.fromEntries(headers) : headers ?? {};
-  return Object.fromEntries(Object.entries(source).filter(([name]) => HEADER_NAMES.includes(name.toLowerCase()))
+  return Object.fromEntries(Object.entries(source).filter(([name]) => HEADER_NAMES.includes(name.toLowerCase()) || extra.includes(name.toLowerCase()))
     .map(([name, value]) => [name.toLowerCase(), String(value)]));
 }
 
 /** Response header lines in wire order; repeated lines (for example two X-Robots-Tag lines) stay separate. */
-export function headerLines(response) {
+function headerLines(response) {
   const lines = [];
   if (Array.isArray(response.rawHeaders)) {
     for (let i = 0; i + 1 < response.rawHeaders.length; i += 2) {
@@ -47,38 +49,8 @@ export function headerLines(response) {
   return lines;
 }
 
-/** Node's lookup callback pins the already-vetted address while TLS still verifies the URL hostname. */
-async function nodeTransport({ url, address, timeoutMs, maxBytes, headers = {}, signal }) {
-  const client = url.protocol === 'https:' ? https : http;
-  const hostname = url.hostname.replace(/^\[|\]$/g, '');
-  return new Promise((resolve, reject) => {
-    const req = client.request(url, {
-      method: 'GET', agent: false, signal, headers: { 'User-Agent': 'optbrew/0.1 (bounded public audit)', Accept: '*/*', 'Accept-Encoding': 'identity', ...headers },
-      lookup: (_hostname, options, callback) => options.all ?
-        callback(null, [{ address: address.address, family: address.family }]) : callback(null, address.address, address.family),
-      ...(url.protocol === 'https:' ? { rejectUnauthorized: true,
-        ...(!isIP(hostname) ? { servername: hostname } : {}) } : {}),
-    }, res => {
-      const chunks = []; let size = 0;
-      if (Number(res.headers['content-length']) > maxBytes) {
-        req.destroy(new Error('response byte limit exceeded')); return;
-      }
-      res.on('data', chunk => {
-        size += chunk.length;
-        if (size > maxBytes) { req.destroy(new Error('response byte limit exceeded')); return; }
-        chunks.push(chunk);
-      });
-      res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, rawHeaders: res.rawHeaders, body: Buffer.concat(chunks) }));
-      res.on('error', reject);
-    });
-    req.setTimeout(timeoutMs, () => req.destroy(new Error('request timeout')));
-    req.on('error', reject);
-    req.end();
-  });
-}
-
 async function addressesFor(url, resolve) {
-  const host = url.hostname.replace(/^\[|\]$/g, '');
+  const { host } = urlTarget(url);
   if (isIP(host)) return [{ address: host, family: isIP(host) }];
   const values = await resolve(host);
   return (Array.isArray(values) ? values : [values]).map(x => {
@@ -88,13 +60,20 @@ async function addressesFor(url, resolve) {
 }
 
 /** Fetch one URL with fresh DNS validation at every redirect and a pinned connection. */
-export async function fetchScoped(input, { context, transport = nodeTransport, resolve = host => lookup(host, { all: true, verbatim: true }), timeoutMs = 20000, maxBytes = 5242880, maxRedirects = 10, headers = {}, allowedAuxiliary = null, reserveRequest = null }) {
-  if (![timeoutMs, maxBytes, maxRedirects].every(Number.isInteger) || timeoutMs < 1 || timeoutMs > 20000 || maxBytes < 1 || maxBytes > 5242880 || maxRedirects < 0 || maxRedirects > 10)
+export async function fetchScoped(input, { context, transport = directTransport, resolve = host => lookup(host, { all: true, verbatim: true }), timeoutMs = limits().timeout_ms, maxBytes = limits().max_bytes, maxRedirects = limits().max_redirects, headers = {}, allowedAuxiliary = null, reserveRequest = null, addressVetting = 'direct', extraHeaders = [], method = 'GET', body: requestBody = null, ceiling = { timeoutMs: limits().timeout_ms, maxBytes: limits().max_bytes }, fieldData = false }) {
+  // A caller may ask for a lower ceiling, never a higher one than the config fixes: the crawl's limits, or the field-data limits
+  // for the one adapter that says it is a field-data call (`transportFetch`).
+  const fixed = fieldData ? { timeoutMs: limits().cwv_timeout_ms, maxBytes: limits().cwv_max_bytes } : { timeoutMs: limits().timeout_ms, maxBytes: limits().max_bytes };
+  if (![timeoutMs, maxBytes, maxRedirects].every(Number.isInteger) || timeoutMs < 1 || timeoutMs > Math.min(ceiling.timeoutMs, fixed.timeoutMs) ||
+      maxBytes < 1 || maxBytes > Math.min(ceiling.maxBytes, fixed.maxBytes) || maxRedirects < 0 || maxRedirects > limits().max_redirects)
     throw new Error('fetch limits exceed context ceiling');
+  // Proxy vetting skips DNS-name checks, which is only safe when the proxy transport (not a direct connection) carries the request.
+  if (addressVetting === 'proxy' && transport.addressVetting !== 'proxy') throw new Error('proxy address vetting requires the proxy transport');
   let url = assertUrlInScope(input, context.scope);
   if (allowedAuxiliary) undeclaredAuxiliary(url, allowedAuxiliary);
   const requested_url = String(input);
   const redirect_chain = [];
+  let made = 0; // requests actually sent; an answered hop is in redirect_chain, an unanswered attempt is made - chain length
   const controller = new AbortController();
   const deadline = performance.now() + timeoutMs;
   let rejectTimeout;
@@ -104,7 +83,7 @@ export async function fetchScoped(input, { context, transport = nodeTransport, r
   try {
     for (let hop = 0; hop <= maxRedirects; hop++) {
       const addresses = await withinDeadline(addressesFor(url, resolve));
-      try { assertInScope(url.href, context.scope, addresses); }
+      try { assertInScope(url.href, context.scope, addresses, { addressVetting }); }
       catch (error) { error.code = 'scope_rejected'; error.redirect_chain = redirect_chain; throw error; }
       // Pick only a vetted address. The transport must use this address, not resolve again.
       const remainingMs = Math.ceil(deadline - performance.now());
@@ -116,18 +95,23 @@ export async function fetchScoped(input, { context, transport = nodeTransport, r
         error.redirect_chain = redirect_chain;
         throw error;
       }
-      const response = await withinDeadline(transport({ url, address: addresses[0], timeoutMs: remainingMs, maxBytes, headers, signal: controller.signal }));
+      made++;
+      const response = await withinDeadline(transport({ url, address: addresses[0], timeoutMs: remainingMs, maxBytes, headers, signal: controller.signal, method, body: requestBody }));
       if (!Number.isInteger(response?.status) || response.status < 100 || response.status > 599) throw new Error('invalid transport status');
       const body = text(response.body);
       if (body.length > maxBytes) throw new Error('response byte limit exceeded');
-      const savedHeaders = safeHeaders(response.headers);
+      const savedHeaders = safeHeaders(response.headers, extraHeaders);
       if (REDIRECTS.has(response.status) && savedHeaders.location) {
-        if (hop === maxRedirects) throw new Error('redirect limit exceeded');
+        if (hop === maxRedirects) {
+          redirect_chain.push({ url: url.href, status: response.status, location: '[limit]' });
+          throw new Error('redirect limit exceeded');
+        }
         let next;
         try { next = assertUrlInScope(new URL(savedHeaders.location, url).href, context.scope); }
         catch {
           const error = new Error('redirect outside scope');
           error.code = 'redirect_outside_scope';
+          try { error.redirect_to = new URL(savedHeaders.location, url).href; } catch { /* an unparsable location names no site */ }
           error.redirect_chain = [...redirect_chain, { url: url.href, status: response.status, location: '[blocked]' }];
           throw error;
         }
@@ -149,10 +133,29 @@ export async function fetchScoped(input, { context, transport = nodeTransport, r
       };
     }
     throw new Error('redirect limit exceeded');
-  } catch (error) { error.redirect_chain ??= [...redirect_chain]; throw error; }
+  } catch (error) {
+    error.redirect_chain ??= [...redirect_chain];
+    error.unanswered = Math.max(0, made - error.redirect_chain.length);
+    throw error;
+  }
   finally { clearTimeout(timer); }
 }
 
+/**
+ * A fetch-like function over the transport `selectTransport` chose, for API calls the audit makes itself (field CWV): the
+ * host is resolved and vetted exactly as `fetchScoped` does it (direct: pinned vetted address; proxy: the proxy resolves,
+ * literals are vetted), a redirect is never followed, and the limits are the caller's, not the page collection's.
+ * It answers `{ status, text() }` and throws what `fetchScoped` throws.
+ */
+export function transportFetch(selected, { timeoutMs, maxBytes }) {
+  return async (url, { method = 'GET', headers = {}, body = null } = {}) => {
+    const result = await fetchScoped(url, { context: { scope: { allowed: [{ origin: new URL(url).origin }] } }, transport: selected.transport,
+      resolve: selected.resolve, addressVetting: selected.address_vetting, method, headers, body, maxRedirects: FOLLOW_NONE,
+      timeoutMs, maxBytes, ceiling: { timeoutMs, maxBytes }, fieldData: true });
+    return { status: result.status, text: async () => result.body.toString('utf8') };
+  };
+}
+
 /** Index of the declared robots.txt artifact on the same origin as the page, as the audit and the report both select it. */
 export function declaredRobotsIndex(artifacts, declaredRobots, pageFinalUrl) {
   return artifacts.findIndex(x => {
@@ -170,11 +173,11 @@ export function links(html, base) {
 }
 
 /** Bounded crawl of fixed protocol URLs and their in-scope links. */
-export async function collect(context, { transport, resolve } = {}) {
+export async function collect(context, { transport, resolve, addressVetting } = {}) {
   const scope = context?.scope;
-  const limit = Math.min(scope?.max_urls ?? 50, 50);
-  const depthLimit = Math.min(scope?.max_depth ?? 3, 3);
-  const concurrency = Math.min(scope?.concurrency ?? 2, 2);
+  const limit = requestLimit(scope);
+  const depthLimit = Math.min(scope?.max_depth ?? limits().max_depth, limits().max_depth);
+  const concurrency = Math.min(scope?.concurrency ?? limits().concurrency, limits().concurrency);
   if (![limit, depthLimit, concurrency].every(Number.isInteger) || limit < 1 || depthLimit < 0 || concurrency < 1)
     throw new Error('invalid collection limits');
   const initial = context.protocol.page_types.flatMap(type => type.urls.map(ref => ref.normalized));
@@ -200,39 +203,42 @@ export async function collect(context, { transport, resolve } = {}) {
     if (depth > depthLimit) { skip(normalized, 'depth_limit'); return; }
     queue.push({ url: normalized, depth, auxiliaryResource });
   };
-  initial.forEach(url => enqueue(url, 0));
+  if (auxiliary.some(resource => initial.includes(resource.normalized))) throw new Error('auxiliary resource duplicates selected URL');
+  // Declared resources (robots, sitemap) are evidence for every page, so they go first: a selected page's redirect hop
+  // takes a request the page budget did not count, and that overrun must cost a page, never the site-wide evidence.
   for (const resource of auxiliary) {
-    if (seen.has(resource.normalized)) throw new Error('auxiliary resource duplicates selected URL');
     const normalized = assertUrlInScope(resource.normalized, scope).href;
     if (normalized !== resource.normalized || assertUrlInScope(resource.original, scope).href !== normalized)
       throw new Error('auxiliary resource URL mismatch');
     enqueue(normalized, 0, true);
   }
+  initial.forEach(url => enqueue(url, 0));
   const artifacts = []; let cursor = 0; let requestCount = 0;
   async function worker() {
     while (cursor < queue.length) {
       const item = queue[cursor++];
       if (requestCount >= limit) { skip(item.url, 'url_limit'); continue; }
       let used = 0;
-      // The budget decides whether a fetch may start. A redirect chain that has started is allowed to finish (at most
-      // maxRedirects further requests), so a chain is never cut in the middle by the limit.
+      // The budget is a hard ceiling: every request, redirect hops included, takes one slot immediately before it is sent.
+      // A chain the budget stops is a request_limit unknown whose answered hops are recorded.
       const reserveRequest = () => {
-        if (!used && requestCount >= limit) return false;
+        if (requestCount >= limit) return false;
         requestCount++; used++;
         return true;
       };
       let artifact;
-      // A chain is at most min(10, limit) redirects long, so the requests a started chain adds beyond the limit scale with
-      // the limit: at most concurrency x min(10, limit) in all.
-      try { artifact = await fetchScoped(item.url, { context, transport, resolve, allowedAuxiliary: allowedDocuments, reserveRequest, maxRedirects: Math.min(10, limit) }); }
+      try { artifact = await fetchScoped(item.url, { context, transport, resolve, addressVetting, allowedAuxiliary: allowedDocuments, reserveRequest }); }
       catch (error) {
         const reason = ['scope_rejected', 'redirect_outside_scope', 'auxiliary_undeclared', 'request_limit'].includes(error.code) ? error.code : 'fetch_failed';
         if (reason === 'request_limit' && !used) { skip(item.url, 'url_limit'); continue; }
         if (reason !== 'fetch_failed') skipped.push({ url: item.url, reason });
-        // A failed fetch has no hashed response record, so the requests it made cannot be proven later: it counts as one.
-        ledger.push({ url: item.url, requests: 1, outcome: 'error', skipped_reason: reason !== 'fetch_failed' ? reason : null, error: reason });
+        // The requests a failed fetch made are its answered hops (kept in the hashed failure record) plus at most one
+        // attempt that got no response, so the ledger can prove them.
+        const chain = error.redirect_chain ?? [];
+        const unanswered = error.unanswered ?? 0;
+        ledger.push({ url: item.url, requests: chain.length + unanswered, outcome: 'error', skipped_reason: reason !== 'fetch_failed' ? reason : null, error: reason });
         artifact = { requested_url: item.url, normalized_url: item.url, final_url: null, status: null,
-          headers: {}, header_lines: [], redirect_chain: error.redirect_chain ?? [], raw_html_ref: null, content_hash: null, capability: 'none',
+          headers: {}, header_lines: [], redirect_chain: chain, unanswered, raw_html_ref: null, content_hash: null, capability: 'none',
           source_ref: 'optbrew:collector', evidence_ref: null, error: reason, body: Buffer.alloc(0) };
       }
       if (!artifact.error) ledger.push({ url: item.url, requests: used, outcome: 'response', skipped_reason: null, error: null });

package/skills/optbrew/scripts/core/compare.mjs

@@ -2,6 +2,7 @@ import { validateEntity } from './contracts.mjs';
 import { freezeProtocol } from './protocol.mjs';
 import { compileUnits } from './weights.mjs';
 import { score } from './score.mjs';
+import { olderVersion } from './migration.mjs';
 
 const close = (a, b) => Math.abs(a - b) <= 1e-9;
 const rawScore = result => result.M === 0 ? null : 100 * result.N / result.M;
@@ -9,11 +10,27 @@ const rawCoverage = result => result.A === 0 ? null : 100 * result.M / result.A;
 const difference = (before, after) => before === null || after === null ? null : after - before;
 const relative = (before, after) => before === null || after === null || close(before, 0) ? null : (after - before) / before;
 
+/**
+ * An input made by an older optbrew was a valid input then, so it is reported as not comparable in plain words instead of
+ * failing the schema. It is found by its version labels, never by the shape the older protocol lacked.
+ */
+const olderReason = (input, label) => {
+  const older = olderVersion(input);
+  return older && `the ${label} was made with ${older.name} ${older.found}, which is older than the current one; run a new audit to compare`;
+};
+
 /** Compare two ScoreInputs only when their frozen conditions and applicable units match. */
 export function compare(before, after) {
+  const oldBaseline = olderReason(before, 'baseline');
+  if (after === null || after === undefined) {
+    if (!oldBaseline) { validateEntity('ScoreInput', before); freezeProtocol(before.run_context); }
+    return { status: 'no_after', score_deltas: [], reasons: ['after_missing'] };
+  }
+  if (oldBaseline) return { status: 'not_comparable', score_deltas: [], reasons: [oldBaseline] };
+  const oldFollowup = olderReason(after, 'follow-up');
+  if (oldFollowup) return { status: 'not_comparable', score_deltas: [], reasons: [oldFollowup] };
   validateEntity('ScoreInput', before);
   const baseline = freezeProtocol(before.run_context);
-  if (after === null || after === undefined) return { status: 'no_after', score_deltas: [], reasons: ['after_missing'] };
   validateEntity('ScoreInput', after);
   const followup = freezeProtocol(after.run_context);
   const reasons = [];

package/skills/optbrew/scripts/core/config.mjs

@@ -0,0 +1,31 @@
+import { readFileSync } from 'node:fs';
+
+const cache = new Map();
+
+/** The one place that reads a config block: first call parses it, every call returns the same frozen object. */
+const block = name => () => {
+  if (!cache.has(name)) {
+    const file = new URL(`../../references/${name}.json`, import.meta.url);
+    cache.set(name, Object.freeze(JSON.parse(readFileSync(file, 'utf8'))));
+  }
+  return cache.get(name);
+};
+
+export const limits = block('limits');
+export const versions = block('versions');
+export const endpoints = block('endpoints');
+export const chromePaths = block('chrome-paths');
+export const harnessSignals = block('harness-signals');
+export const messagesKo = block('messages.ko');
+
+/** The request ceiling of one run: the start question's `max_urls` (default 50), never above the cap (200). */
+export const requestLimit = scope => Math.min(scope?.max_urls ?? limits().max_urls_default, limits().max_urls_cap);
+
+export const checkRegistry = block('check-registry');
+/**
+ * Whether a family matters only for pages meant to be indexed (`public_only` in `check-registry.json`, the one list): a page with
+ * `public_indexing:false` has no unit of it, no verdict and no fix. Protocol, unit compilation and the derivation all read this.
+ */
+export const publicOnlyFamily = familyId => checkRegistry().families.find(family => family.family_id === familyId)?.public_only === true;
+/** The judgment thresholds one family's check block reads from `check-registry.json`. */
+export const thresholds = familyId => checkRegistry().families.find(family => family.family_id === familyId).thresholds;

package/skills/optbrew/scripts/core/contracts.mjs

@@ -1,10 +1,13 @@
 import { readFileSync } from 'node:fs';
+import { limits, versions } from './config.mjs';
 
 const schema = JSON.parse(readFileSync(new URL('../../references/schemas/contract.schema.json', import.meta.url)));
 const registry = JSON.parse(readFileSync(new URL('../../references/check-registry.json', import.meta.url)));
 const families = new Map(registry.families.map(family => [family.family_id, family]));
 const tolerance = 1e-9;
-const cwv = { 'lcp-field': [2500, 4000, 'ms'], 'inp-field': [200, 500, 'ms'], 'cls-field': [0.1, 0.25, 'unitless'] };
+// [good, poor, unit] of each field CWV family, read from the registry so the thresholds are part of the evaluation source.
+const cwv = Object.fromEntries(registry.families.filter(family => family.thresholds?.good !== undefined)
+  .map(({ family_id, thresholds }) => [family_id, [thresholds.good, thresholds.poor, thresholds.unit]]));
 
 export class ContractError extends Error {
   constructor(message) { super(message); this.name = 'ContractError'; }
@@ -26,13 +29,19 @@ function jsonSafe(value, path = '$', seen = new Set()) {
   }
 }
 
+/** Whether a value satisfies a rule; the `if` and `not` keywords test this instead of failing. */
+function matches(rule, value) {
+  try { checkRule(rule, value, '$'); return true; } catch (error) { if (error instanceof ContractError) return false; throw error; }
+}
+
 function checkRule(rule, value, path) {
   if (rule.$ref) return checkRule(schema.$defs[rule.$ref.split('/').at(-1)], value, path);
   if ('const' in rule && value !== rule.const) fail(path, `expected ${JSON.stringify(rule.const)}`);
+  if (rule.not && matches(rule.not, value)) fail(path, 'value not allowed here');
   if (rule.enum && !rule.enum.includes(value)) fail(path, 'unrecognized value');
   if (rule.type) {
     const types = Array.isArray(rule.type) ? rule.type : [rule.type];
-    const valid = types.some(type => type === 'null' ? value === null : type === 'array' ? Array.isArray(value) : type === 'object' ? plainObject(value) : typeof value === type);
+    const valid = types.some(type => type === 'null' ? value === null : type === 'integer' ? Number.isInteger(value) : type === 'array' ? Array.isArray(value) : type === 'object' ? plainObject(value) : typeof value === type);
     if (!valid) fail(path, `expected ${types.join(' or ')}`);
   }
   if (typeof value === 'number') {
@@ -44,7 +53,10 @@ function checkRule(rule, value, path) {
     if (rule.additionalProperties === false) for (const key of Object.keys(value))
       if (!Object.hasOwn(rule.properties ?? {}, key)) fail(`${path}.${key}`, 'unexpected field');
     for (const [key, child] of Object.entries(rule.properties ?? {})) if (Object.hasOwn(value, key)) checkRule(child, value[key], `${path}.${key}`);
+    if (rule.if && rule.then && matches(rule.if, value)) checkRule(rule.then, value, path);
   }
+  if (typeof value === 'string' && rule.minLength !== undefined && value.length < rule.minLength) fail(path, 'too short');
+  if (Array.isArray(value) && rule.minItems !== undefined && value.length < rule.minItems) fail(path, 'too few items');
   if (Array.isArray(value) && rule.items) value.forEach((item, i) => checkRule(rule.items, item, `${path}[${i}]`));
 }
 
@@ -131,12 +143,29 @@ function checkMetric(value) {
       /(?:^|:)citation_response_rate$/.test(value.metric_id)) fail(path, 'positive-only sample cannot estimate all-question citation rate');
 }
 
+/** The request limit a run may ask for: a whole number from 1 up to the configured cap. */
+function checkMaxUrls(value, path) {
+  if (!Number.isInteger(value) || value < 1 || value > limits().max_urls_cap) fail(path, `must be a whole number from 1 to ${limits().max_urls_cap}`);
+}
+
 /** Validate a persisted entity and return its original value. */
 export function validateEntity(typeName, value) {
   const rule = schema.$defs[typeName];
   if (!rule) throw new ContractError(`unknown entity type: ${typeName}`);
   jsonSafe(value);
+  // Answers are written by an agent from the installed skill's text: a different contract version means the text and the scripts disagree.
+  // Checked before the schema so the user reads one plain sentence, not a schema path.
+  if (typeName === 'IntakeAnswers' && value?.schema_version !== versions().contract_schema)
+    fail('IntakeAnswers.schema_version', `the answers file ${value?.schema_version === undefined ? 'has no schema_version' : `says ${JSON.stringify(value.schema_version)}`} but this optbrew needs ${JSON.stringify(versions().contract_schema)}; copy the contract_schema value from references/versions.json`);
   checkRule(rule, value, typeName);
+  if (typeName === 'RunContext' && value.scope?.max_urls !== undefined) checkMaxUrls(value.scope.max_urls, 'RunContext.scope.max_urls');
+  if (typeName === 'IntakeAnswers') {
+    checkMaxUrls(value.max_urls, 'IntakeAnswers.max_urls');
+    // The start question is "sample or raise the limit": the two answers and the number must agree.
+    const raised = value.sample_mode === 'raise';
+    if (raised ? value.max_urls <= limits().max_urls_default : value.max_urls !== limits().max_urls_default)
+      fail('IntakeAnswers.max_urls', raised ? 'a raised limit must be above the default' : `a sample uses the default limit (${limits().max_urls_default}); answer "raise" to go higher`);
+  }
   if (typeName === 'CheckObservation') checkObservation(value);
   if (typeName === 'ScoreInput') value.observations.forEach(checkObservation);
   if (typeName === 'MetricObservation') checkMetric(value);

package/skills/optbrew/scripts/core/cwv.mjs

@@ -0,0 +1,100 @@
+import { endpoints, limits } from './config.mjs';
+
+export const HTTP_OK = 200;
+const HTTP_NOT_FOUND = 404;
+const NOT_FOUND = /chrome ux report data not found/i;
+// Google API keys are letters, digits, ".", "_" and "-" (AI Studio's AQ. keys have a dot); anything else is not a key optbrew can send, and a short string could not be removed safely.
+const KEY_SHAPE = /^[A-Za-z0-9._-]{20,}$/;
+// CrUX is asked for phones, as PageSpeed Insights is (strategy=mobile in endpoints.json).
+const FORM_FACTOR = 'PHONE';
+// CLS counts shifts in session windows: a gap of 1 s ends a window and a window ends after 5 s (the web.dev definition).
+const SESSION_GAP_MS = 1000;
+const SESSION_CAP_MS = 5000;
+
+/** `text` with the key removed, in the form it was sent and in its URL-encoded form. */
+const without = (text, key) => key ? [key, encodeURIComponent(key)].reduce((rest, secret) => rest.split(secret).join(''), String(text)) : String(text);
+
+/** One request. A network failure is a result too (`status: null`), so a dead network leaves evidence instead of an exception. */
+async function ask(fetchFn, url, init, key) {
+  try {
+    const response = await fetchFn(url, init);
+    return { status: response.status, text: without(await response.text(), key) };
+  } catch (error) {
+    return { status: null, error: without(error?.message ?? error, key) };
+  }
+}
+
+/** Store what the source answered, cleaned of the key, and list it in the index's `cwv` section. `key` says what became of the user's key. */
+async function keep(store, source, origin, keyState, result) {
+  let body = result.text;
+  try { body = JSON.parse(result.text); } catch { /* a body that is not JSON is kept as text */ }
+  const record = { source, origin, key: keyState, status: result.status, ...(result.error === undefined ? { body } : { error: result.error }) };
+  const { ref, sha256 } = await store.write('cwv', `${JSON.stringify(record, null, 2)}\n`);
+  await store.append('cwv', { source, evidence_ref: ref, sha256 });
+}
+
+/**
+ * Field Core Web Vitals of one origin, stored as evidence and never judged here. With the user's own Google key the
+ * Chrome UX Report is asked first and PageSpeed Insights only when CrUX has no data for the origin; without a key (or with
+ * one that is not shaped like a Google key, which the record says) PageSpeed Insights is tried `psi_attempts` times, keyless.
+ * `fetchFn(url, { method, headers, body })` answers `{ status, text() }`: pass `transportFetch` of the run's transport, so a
+ * proxy-only network works and the host is vetted. The key travels in the `x-goog-api-key` header only: no endpoint and no URL
+ * holds it, and every stored body or error has it removed.
+ */
+export async function collectCwv({ origin, key = null, fetchFn, store }) {
+  if (typeof fetchFn !== 'function') throw new TypeError('collectCwv needs fetchFn');
+  const usable = typeof key === 'string' && KEY_SHAPE.test(key);
+  const keyState = key ? (usable ? 'used' : 'key_ignored_malformed') : 'absent';
+  const auth = usable ? { 'x-goog-api-key': key } : {};
+  const secret = usable ? key : null;
+  if (usable) {
+    const crux = await ask(fetchFn, endpoints().crux, { method: 'POST', headers: { 'content-type': 'application/json', ...auth },
+      body: JSON.stringify({ origin, formFactor: FORM_FACTOR }) }, secret);
+    await keep(store, 'crux', origin, keyState, crux);
+    if (!(crux.status === HTTP_NOT_FOUND && NOT_FOUND.test(crux.text))) return;
+  }
+  for (let attempt = 0; attempt < limits().psi_attempts; attempt++) {
+    const url = new URL(endpoints().psi);
+    url.searchParams.set('url', `${origin}/`);
+    const psi = await ask(fetchFn, url.href, { method: 'GET', headers: auth }, secret);
+    await keep(store, 'psi', origin, keyState, psi);
+    if (psi.status === HTTP_OK) return;
+  }
+}
+
+/** Evaluated in the page: the buffered LCP and layout-shift entries, as plain data. */
+const READ_LAB = `new Promise(resolve => {
+  const lcp = [], shifts = [];
+  const addLcp = entry => lcp.push(entry.startTime);
+  const addShift = entry => shifts.push({ startTime: entry.startTime, value: entry.value, hadRecentInput: entry.hadRecentInput });
+  const watch = (type, add) => {
+    const observer = new PerformanceObserver(list => list.getEntries().forEach(add));
+    observer.observe({ type, buffered: true });
+    return () => observer.takeRecords().forEach(add);
+  };
+  const drain = [watch('largest-contentful-paint', addLcp), watch('layout-shift', addShift)];
+  // Buffered entries arrive in a task of their own; one more task later they have all been delivered.
+  setTimeout(() => { drain.forEach(run => run()); resolve(JSON.stringify({ lcp, shifts })); }, 0);
+})`;
+
+/** The largest sum of shift values over session windows; shifts right after user input do not count. */
+function layoutShift(shifts) {
+  let best = 0, sum = 0, windowStart = null, last = null;
+  for (const { startTime, value } of shifts.filter(shift => !shift.hadRecentInput).sort((a, b) => a.startTime - b.startTime)) {
+    if (windowStart === null || startTime - last > SESSION_GAP_MS || startTime - windowStart > SESSION_CAP_MS) { sum = 0; windowStart = startTime; }
+    sum += value;
+    last = startTime;
+    best = Math.max(best, sum);
+  }
+  return best;
+}
+
+/**
+ * Lab Core Web Vitals of a page a `scopedPage` has already loaded, read with PerformanceObserver. `lcp` is the last
+ * largest-contentful-paint entry in ms (null when the page painted none), `cls` the layout shift score. The caller keeps
+ * the result in `RunState.lab_cwv`: it is shown as reference only and never becomes an observation.
+ */
+export async function labCwv({ page }) {
+  const { lcp, shifts } = JSON.parse(await page.evaluate(READ_LAB));
+  return { lcp: lcp.length ? lcp.at(-1) : null, cls: layoutShift(shifts) };
+}