optbrew 0.1.1 → 0.1.2

package/skills/optbrew/scripts/core/report.mjs

@@ -10,7 +10,9 @@ import { measurePanel } from './metrics.mjs';
 import { normalizeImport } from '../adapters/imports.mjs';
 import { mergeFindings, selectRemediations } from './findings.mjs';
 import { auditCollected, indexingDirectives } from '../../modules/naver-searchadvisor/scripts/audit.mjs';
-import { declaredRobotsIndex } from './collector.mjs';
+import { deriveObservations } from './derive.mjs';
+import { links } from './collector.mjs';
+import { assertUrlInScope } from './scope.mjs';
 import { parseRobots, robotsVerdict } from '../../modules/naver-searchadvisor/scripts/robots.mjs';
 
 const catalog = JSON.parse(readFileSync(new URL('../../references/remediations.json', import.meta.url))).items;
@@ -34,6 +36,28 @@ function completeWindow(metric) {
   return completeDate <= dayBefore(localCollectionDate) && metric.window.end <= completeDate;
 }
 
+/** The optbrew operating rule (design spec, scoring.md): at least 80% of the applicable weight must be measured. */
+export const COVERAGE_THRESHOLD = 80;
+
+/** One headline per profile. Below the coverage threshold the single number is withheld and only the measured subset is shown. */
+export function scoreHeadline(result) {
+  const exact = result.A > 0 ? 100 * result.M / result.A : null;
+  const coverageOk = result.score !== null && exact !== null && exact >= COVERAGE_THRESHOLD - 1e-9;
+  // The design spec also asks for verdicts on the core access and indexing checks before a score counts as sufficient.
+  const coreOk = (result.unknown_core ?? []).length === 0;
+  const status = !coverageOk ? 'insufficient_coverage' : !coreOk ? 'insufficient_core' : 'sufficient';
+  return { profile: result.profile, status, headline_score: status === 'sufficient' ? result.score : null,
+    measured_subset_score: result.score, coverage: result.coverage, coverage_exact: exact, coverage_threshold: COVERAGE_THRESHOLD,
+    unresolved_core: (result.unknown_core ?? []).length, missing_bounds: result.missing_bounds, readiness_status: result.readiness_status };
+}
+const floorTenth = value => value === null ? 'unknown' : Math.floor(value * 10 + 1e-9) / 10;
+const headlineCell = result => {
+  const headline = scoreHeadline(result);
+  return headline.status === 'sufficient' ? `${headline.headline_score} (coverage ${headline.coverage}%)` :
+    headline.status === 'insufficient_core' ? `insufficient core checks (${headline.unresolved_core} unresolved; measured subset ${headline.measured_subset_score ?? 'unknown'}; coverage ${headline.coverage ?? 'unknown'}%; bounds ${headline.missing_bounds?.join('–') ?? 'unknown'})` :
+    `insufficient coverage (measured subset ${headline.measured_subset_score ?? 'unknown'}; coverage ${headline.coverage ?? 'unknown'}%; bounds ${headline.missing_bounds?.join('–') ?? 'unknown'})`;
+};
+
 function metricRows(metrics) {
   const groups = new Map();
   for (const metric of metrics) {
@@ -327,6 +351,74 @@ function statusFor(finding, changes, afterInput, target, requireRef) {
   return linked.every(change => change.environment === 'local') && localResolved ? 'verified_local' : 'modified_unverified';
 }
 
+/**
+ * The request ledger accounts for every queued or rejected URL. Collection counts and the skipped list must equal what
+ * the ledger and the hashed response records show, so a fetched page cannot be moved to skipped and a request limit
+ * cannot be claimed without the requests that exhaust it.
+ */
+function checkLedger(input, audit, collection, artifacts, requireRef) {
+  const ledger = JSON.parse(requireRef(audit.ledger_ref).text);
+  const entries = ledger.entries;
+  const fail = message => { throw new ContractError(`request ledger: ${message}`); };
+  if (!Array.isArray(entries)) fail('entries missing');
+  const limit = Math.min(input.run_context.scope?.max_urls ?? 50, 50);
+  if (ledger.limit !== limit) fail('request limit differs from the run scope');
+  const total = entries.reduce((sum, entry) => sum + entry.requests, 0);
+  // The limit decides whether a fetch may start; a redirect chain that has started finishes. A chain is at most
+  // min(10, limit) redirects long and at most `concurrency` chains are in flight, so the overshoot is bounded by both.
+  const concurrency = Math.min(input.run_context.scope?.concurrency ?? 2, 2);
+  if (total > limit + concurrency * Math.min(10, limit)) fail('more requests than the limit');
+  const responded = artifacts.filter(x => Number.isInteger(x.status));
+  const failed = artifacts.filter(x => !Number.isInteger(x.status));
+  const byUrl = outcome => new Map(entries.filter(x => x.outcome === outcome).map(x => [x.url, x]));
+  const responses = byUrl('response'), errors = byUrl('error');
+  if (responses.size !== responded.length || audit.responses.length !== responded.length || errors.size !== failed.length)
+    fail('collected artifacts differ from the ledger or the response records');
+  for (const artifact of responded) {
+    const entry = responses.get(artifact.normalized_url);
+    if (!entry || entry.requests !== artifact.redirect_chain.length + 1) fail(`response for ${artifact.normalized_url} is not recorded`);
+  }
+  // A skipped entry never reached the network. Requests booked on it would only pad the total toward a claimed limit.
+  if (entries.some(x => x.outcome === 'skipped' && x.requests !== 0)) fail('a skipped entry carries requests');
+  for (const artifact of failed) {
+    const entry = errors.get(artifact.normalized_url);
+    if (!entry) fail(`failed fetch of ${artifact.normalized_url} is not recorded`);
+    // A failed fetch has no hashed response record, so nothing proves how many requests it made: it is always one.
+    // Neither the redirect chain nor any count inside collection.json can raise it, so a forger cannot pad the total.
+    if (entry.requests !== 1) fail(`failed fetch of ${artifact.normalized_url} is not booked as exactly one request`);
+  }
+  const key = x => `${x.url}|${x.reason ?? x.skipped_reason}`;
+  const ledgerSkips = entries.filter(x => x.skipped_reason).map(key).sort();
+  if (JSON.stringify((collection.skipped ?? []).map(key).sort()) !== JSON.stringify(ledgerSkips)) fail('skipped list differs from the ledger');
+  const limited = entries.some(x => ['url_limit', 'request_limit'].includes(x.skipped_reason));
+  if (collection.limit_reached !== limited) fail('limit_reached differs from the ledger');
+  if (limited && total < limit) fail('a request limit is claimed but fewer requests than the limit were made');
+  if (collection.request_count !== total || collection.evaluated_count !== entries.filter(x => x.requests > 0).length) fail('request or evaluated count differs from the ledger');
+  const selected = new Set(input.run_context.protocol.page_types.flatMap(type => type.urls.map(x => x.normalized)));
+  if (entries.some(x => selected.has(x.url) && x.skipped_reason && x.skipped_reason !== 'url_limit' && x.outcome === 'skipped')) fail('a selected page is skipped for a reason other than the request limit');
+  // The protocol caps selected pages plus auxiliary resources at the limit, and the collector queues them before any
+  // discovered page. A selected page can only hit the request limit after redirects used the budget, so no page that was
+  // merely discovered may have been fetched in the same run.
+  const auxiliary = new Set((input.run_context.protocol.auxiliary_resources ?? []).map(x => x.normalized));
+  const planned = new Set([...selected, ...auxiliary]);
+  // Provenance: a failed fetch or a request-limit skip belongs to a URL the run was given (selected or auxiliary) or one
+  // that a hashed page of this collection links to (anchors and canonical, as the collector follows them). A URL nothing
+  // links to cannot be a failed fetch or a skipped one, so fabricated entries cannot pad the total or claim a limit.
+  const known = new Set(planned);
+  for (const artifact of responded) {
+    if (artifact.status !== 200 || artifact.capability !== 'raw_html' || auxiliary.has(artifact.normalized_url)) continue;
+    for (const link of links(requireRef(artifact.evidence_ref).text, artifact.final_url)) {
+      try { known.add(assertUrlInScope(link, input.run_context.scope).href); } catch { /* out of scope: the collector skipped it */ }
+    }
+  }
+  for (const entry of entries) {
+    if ((entry.outcome === 'error' || (entry.outcome === 'skipped' && entry.skipped_reason === 'url_limit')) && !known.has(entry.url))
+      fail(`${entry.url} has no provenance: it is neither a selected or auxiliary URL nor linked from a collected page`);
+  }
+  if (entries.some(x => selected.has(x.url) && x.outcome === 'skipped' && x.skipped_reason === 'url_limit') &&
+      entries.some(x => x.requests > 0 && !planned.has(x.url))) fail('a selected page is skipped for the request limit although a discovered page was fetched');
+}
+
 function checkAuditPhase(input, phase, snapshot, requireRef) {
   if (!input) return;
   const observations = input.observations.filter(x => x.source_ref === 'optbrew:collector');
@@ -341,6 +433,10 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
   const savedObservations = JSON.parse(requireRef(audit.observations_ref).text);
   if (!Array.isArray(collection.artifacts) || !Array.isArray(audit.raw) || !Array.isArray(savedObservations))
     throw new ContractError('audit snapshot files incomplete');
+  if (!Array.isArray(audit.modules) || !audit.ledger_ref || !Array.isArray(audit.responses))
+    throw new ContractError('audit evidence was produced by an older optbrew release (it has no request ledger or module list); re-run the audit with optbrew 0.1.2 and build a new packet');
+  for (const name of audit.modules) if (!Array.isArray(moduleOutputs[name])) throw new ContractError(`planned module output missing: ${name}`);
+  const naverPlanned = audit.modules.includes('naver-searchadvisor');
   const rawRefs = new Set();
   for (const raw of audit.raw) {
     if (raw.content_hash === null) {
@@ -355,6 +451,7 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
   }
   // Final URL, redirect chain and header lines come from the hashed response records, not from collection.json alone.
   const artifacts = collection.artifacts.map(x => boundArtifact(x, audit, requireRef));
+  checkLedger(input, audit, collection, artifacts, requireRef);
   for (const obs of observations) {
     if (!savedObservations.some(x => same(x, obs)) || !rawRefs.has(obs.evidence_ref))
       throw new ContractError('collector observation is not bound to audit index and raw evidence');
@@ -374,21 +471,20 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
     if (['index-policy', 'document-title'].includes(obs.check_id) &&
         (['pass', 'fail'].includes(obs.state) || obs.check_id === 'document-title' && obs.state === 'unknown')) {
       const raw = artifact && requireRef(obs.evidence_ref);
-      if (!module || !artifact || artifact.status !== 200 || artifact.capability !== 'raw_html' ||
+      if ((naverPlanned && !module) || !artifact || artifact.status !== 200 || artifact.capability !== 'raw_html' ||
           artifact.raw_html_ref !== obs.evidence_ref || artifact.content_hash !== raw.sha256 || raw.kind !== 'raw_html')
         throw new ContractError('title/index verdict lacks bound raw response');
       const replay = auditCollected({ artifact, html: raw.text, engines });
-      if (!same([module.title, module.robots_meta, module.x_robots_tag],
+      if (naverPlanned && (!same([module.title, module.robots_meta, module.x_robots_tag],
         [replay.title, replay.robots_meta, replay.x_robots_tag]) ||
         (module.requested_url !== undefined && module.requested_url !== artifact.normalized_url) ||
-        (module.redirect_chain !== undefined && !same(module.redirect_chain, artifact.redirect_chain ?? [])))
+        (module.redirect_chain !== undefined && !same(module.redirect_chain, artifact.redirect_chain ?? []))))
         throw new ContractError('title/index module differs from raw response and headers');
-      // A present title proves existence only; uniqueness and descriptiveness need separate evidence.
-      const expected = obs.check_id === 'document-title' ? replay.title ? 'unknown' : 'fail' : replay.index_blocked ? 'fail' : 'pass';
-      if (obs.state !== expected)
+      // Title states (missing, duplicated, unknown) are replayed by the derivation below.
+      if (obs.check_id === 'index-policy' && obs.state !== (replay.index_blocked ? 'fail' : 'pass'))
         throw new ContractError('title/index verdict differs from raw response and headers');
     }
-    if (obs.check_id === 'search-crawl-policy' && ['pass', 'fail'].includes(obs.state)) {
+    if (obs.check_id === 'search-crawl-policy' && ['pass', 'fail'].includes(obs.state) && obs.unit_key.split('|')[1] === 'naver') {
       const [, engine] = obs.unit_key.split('|');
       const robotsArtifact = artifacts.find(x => x.evidence_ref === obs.evidence_ref && x.status === 200);
       const declared = robotsArtifact && (input.run_context.protocol.auxiliary_resources ?? []).some(resource =>
@@ -409,40 +505,26 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
         throw new ContractError('crawl verdict differs from bound robots evidence and module output');
     }
   }
-  // Every observation the collector plans for this phase must be in the packet and in the saved observation file;
-  // deleting a failed unit from both would otherwise silently remove its blocker.
-  if (!Array.isArray(moduleOutputs['naver-searchadvisor'])) return;
-  const planned = new Set(input.run_context.protocol.families.filter(x => x.applicable).map(x => x.family_id));
-  const pages = input.run_context.protocol.page_types.flatMap(type => type.urls);
-  const declaredRobots = new Set((input.run_context.protocol.auxiliary_resources ?? [])
-    .filter(resource => resource.kind === 'robots').map(resource => resource.normalized));
-  const expect = (family, engine, url) => {
-    const unitKey = `${family}|${engine ? `${engine}|` : ''}${url}`;
-    if (!observations.some(x => x.unit_key === unitKey) || !savedObservations.some(x => x.unit_key === unitKey))
-      throw new ContractError(`planned collector observation missing: ${unitKey}`);
-  };
-  for (const { normalized: url, public_indexing: publicIndexing } of pages) {
-    const artifact = artifacts.find(x => x.normalized_url === url);
-    if (!artifact) {
-      // A selected page is either collected or recorded as skipped; vanishing from both hides its observations.
-      if (!(collection.skipped ?? []).some(x => x.url === url))
-        throw new ContractError(`selected page missing from the collection and its skipped list: ${url}`);
-      continue;
-    }
-    if (planned.has('fetch-response')) expect('fetch-response', null, url);
-    if (artifact.status !== 200 || artifact.capability !== 'raw_html') continue;
-    if (planned.has('document-title')) expect('document-title', null, url);
-    if (publicIndexing === false) continue;
-    if (planned.has('index-policy')) expect('index-policy', null, url);
-    if (planned.has('search-crawl-policy') && input.run_context.protocol.engines.includes('naver')) {
-      const robots = artifacts[declaredRobotsIndex(artifacts, declaredRobots, artifact.final_url)];
-      if (robots?.status === 200) {
-        const target = new URL(artifact.final_url);
-        if (!robotsVerdict(parseRobots(requireRef(robots.evidence_ref).text), 'yeti', target.pathname + target.search).allowed)
-          expect('search-crawl-policy', 'naver', url);
-      }
-    }
+  // Every selected page is either collected or recorded as skipped; vanishing from both hides its observations.
+  for (const { normalized: url } of input.run_context.protocol.page_types.flatMap(type => type.urls))
+    if (!artifacts.some(x => x.normalized_url === url) && !(collection.skipped ?? []).some(x => x.url === url))
+      throw new ContractError(`selected page missing from the collection and its skipped list: ${url}`);
+  // Re-derive every collector verdict from the hashed raw evidence. Each derived observation must be present in the
+  // packet and in the saved observation file, and no collector observation may lack a derivation.
+  const derived = deriveObservations({ context: input.run_context, artifacts, collection,
+    textOf: artifact => artifact.content_hash ? requireRef(artifact.evidence_ref).text : null });
+  const shape = x => [x.state, x.q, x.partial_basis, x.evidence_ref, x.reason, x.evaluator_type, x.source_ref];
+  for (const expected of derived) {
+    if (!observations.some(x => x.unit_key === expected.unit_key) || !savedObservations.some(x => x.unit_key === expected.unit_key))
+      throw new ContractError(`planned collector observation missing: ${expected.unit_key}`);
+    const have = observations.find(x => x.unit_key === expected.unit_key);
+    if (!same(shape(have), shape(expected)))
+      throw new ContractError(expected.family_id === 'search-crawl-policy' ? 'crawl verdict differs from bound robots evidence and module output' :
+        ['index-policy', 'document-title'].includes(expected.family_id) ? 'title/index verdict differs from raw response and headers' :
+          `collector verdict differs from raw evidence: ${expected.unit_key}`);
   }
+  for (const have of observations) if (!derived.some(x => x.unit_key === have.unit_key))
+    throw new ContractError(`collector observation is not derivable from raw evidence: ${have.unit_key}`);
 }
 
 /** Recompute scores and metrics from contained run evidence; render four design tables. */
@@ -599,11 +681,18 @@ export function buildReport(packet) {
   const scoreRows = baselineScores.map(before => {
     const after = byProfile.get(before.profile);
     const delta = comparison.score_deltas.find(x => x.profile === before.profile);
-    return [before.profile, before.score, after?.score ?? 'unknown', delta?.percentage_point_delta ?? 'unknown',
+    const cautious = scoreHeadline(before).status !== 'sufficient' || (after && scoreHeadline(after).status !== 'sufficient');
+    return [before.profile, headlineCell(before), after ? headlineCell(after) : 'unknown',
+      `${delta?.percentage_point_delta ?? 'unknown'}${cautious && delta ? ' (between measured subsets)' : ''}`,
       `${before.coverage ?? 'unknown'} → ${after?.coverage ?? 'unknown'}`,
       `${before.missing_bounds?.join('–') ?? 'unknown'} → ${after?.missing_bounds?.join('–') ?? 'unknown'}`,
       `${before.readiness_status} → ${after?.readiness_status ?? 'unknown'}`];
   });
+  const headlines = baselineScores.map(scoreHeadline);
+  result.headline = headlines;
+  const headlineText = headlines.map(h => h.status === 'sufficient' ? `${h.profile}: ${h.headline_score} (coverage ${h.coverage}%)` :
+    h.status === 'insufficient_core' ? `${h.profile}: insufficient core checks (${h.unresolved_core} access or indexing check(s) have no verdict, although ${h.coverage}% is measured); measured subset ${h.measured_subset_score ?? 'unknown'}, bounds ${h.missing_bounds?.join('–') ?? 'unknown'}` :
+    `${h.profile}: insufficient coverage (${floorTenth(h.coverage_exact)}% of the applicable weight is measured; at least ${h.coverage_threshold}% is needed); measured subset ${h.measured_subset_score ?? 'unknown'}, bounds ${h.missing_bounds?.join('–') ?? 'unknown'}`);
   const remediationRows = findings.map(f => [f.priority, `${f.message} ${f.unit_key}`, f.sources.map(x => `${x.source_ref} ${x.evidence_ref}`).join('; '),
     f.remediation ? `${f.remediation.mechanism} (${f.remediation.source_url}, checked ${f.remediation.checked_date})` : 'unknown',
     packet.changes.filter(x => x.finding_ids.some(id => f.finding_ids.includes(id))).map(x => x.path).join('; ') || f.remediation?.location_hint || 'owner implementation location required',
@@ -614,6 +703,7 @@ export function buildReport(packet) {
   const markdown = `# optbrew report\n\n${packet.data_class === 'synthetic' ? '**Synthetic example — no real site or provider outcome.**\n\n' : ''}` +
     `Target: ${html(packet.run_context.target)}  \nEnvironment: ${html(packet.run_context.environment)}  \nGoals: ${html(packet.run_context.goals.join(', '))}  \n` +
     `Comparison: ${html(comparison.status)} (${html(comparison.reasons.join(', ') || 'none')})  \n` +
+    `Headline:  \n${headlineText.map(line => `- ${html(line)}  \n`).join('')}` +
     `Baseline blockers: ${html(baselineScores[0]?.blockers.length ?? 0)} (${html(baselineScores[0]?.blockers.map(x => `${x.family_id} ${x.unit_key}`).join('; ') || 'none')}) → ` +
     `After blockers: ${afterScores ? html(afterScores[0]?.blockers.length ?? 0) : 'unknown'} (${afterScores ? html(afterScores[0]?.blockers.map(x => `${x.family_id} ${x.unit_key}`).join('; ') || 'none') : 'after not observed'}); ` +
     `after core unknown: ${afterScores ? html(afterScores[0]?.unknown_core.length ?? 0) : 'unknown'}; actual outcomes: ${packet.metrics.length ? 'see measured and missing states below' : 'not_connected / unknown'}  \n` +

package/skills/optbrew/source-lock.json

@@ -395,7 +395,7 @@
       "package_path": "skills/optbrew/modules/naver-searchadvisor/scripts/audit.mjs",
       "original_git_blob_sha": "de8c4a3430633cb2445e339d7a943d944cbf45a9",
       "original_sha256": "3fd83141a00a9d45d4fbffa9d8ff515c7a49e7b1ea4637844e92375ea7797ff9",
-      "distributed_sha256": "616ea1e0e9295bd3f4c53374d36ce14dcb5923650c94c63c3d4032590fb93a8a",
+      "distributed_sha256": "fa70051e07b429ad9477b8d778232cc4b3ca0bae8d2a1352074dfd4cb6f5f884",
       "transformations": [
         "Audit output routes policy evidence to official URL index",
         "reviewed distribution patch naver-robots-rfc9309"

package/source-lock.json

@@ -395,7 +395,7 @@
       "package_path": "skills/optbrew/modules/naver-searchadvisor/scripts/audit.mjs",
       "original_git_blob_sha": "de8c4a3430633cb2445e339d7a943d944cbf45a9",
       "original_sha256": "3fd83141a00a9d45d4fbffa9d8ff515c7a49e7b1ea4637844e92375ea7797ff9",
-      "distributed_sha256": "616ea1e0e9295bd3f4c53374d36ce14dcb5923650c94c63c3d4032590fb93a8a",
+      "distributed_sha256": "fa70051e07b429ad9477b8d778232cc4b3ca0bae8d2a1352074dfd4cb6f5f884",
       "transformations": [
         "Audit output routes policy evidence to official URL index",
         "reviewed distribution patch naver-robots-rfc9309"