@nebulacomponents/citable 1.6.0 → 1.8.0

package/src/commands/governance.js

@@ -0,0 +1,144 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createRun } from '../evidence/run.js';
+import { loadRegistries, checkReferentialIntegrity } from '../registries/index.js';
+import { readJson, sha256, parseRefDate } from '../shared/io.js';
+
+function recordHash(value) {
+  return sha256(JSON.stringify(value));
+}
+
+function assigned(exception, role) {
+  return exception.reviewer_assignments.filter((item) => item.role === role).map((item) => item.reviewer_id);
+}
+
+function reviewerCanAct(reviewer, role, finding) {
+  if (!reviewer || reviewer.status !== 'active' || !reviewer.roles.includes(role)) return false;
+  const scopes = reviewer.authorized_scopes || [];
+  return scopes.includes('*') || scopes.includes(finding.detector_id) || scopes.includes(finding.subject.identifier);
+}
+
+function exceptionProblems(exception, policy, reviewers, findings, evidence, refDate) {
+  const problems = [];
+  if (!policy || policy.status !== 'active') problems.push('review policy is missing or inactive');
+  if (exception.status !== 'approved') problems.push(`exception status is ${exception.status}`);
+  if (exception.residual_risk !== 'documented') problems.push('residual risk is not documented');
+  if (new Date(exception.expires_at) < refDate) problems.push('exception is expired');
+  if (exception.renewal_count > exception.renewal_limit) problems.push('exception renewal limit exceeded');
+  if (policy && exception.renewal_limit > policy.max_renewals) problems.push('exception renewal limit exceeds policy');
+  if (policy && new Date(exception.expires_at) - new Date(exception.created_at) > policy.max_exception_days * 86400000) problems.push('exception duration exceeds policy');
+  if (policy && recordHash(policy) !== exception.policy_hash) problems.push('policy changed since approval');
+  if (exception.superseded_by_exception_id) problems.push('exception is superseded');
+  const owner = reviewers.get(exception.owner_reviewer_id);
+  if (!owner || owner.status !== 'active') problems.push('exception owner is missing or inactive');
+
+  for (const findingId of exception.finding_ids) {
+    const finding = findings.get(findingId);
+    if (!finding) problems.push(`source finding ${findingId} is unavailable`);
+    else if (recordHash(finding) !== exception.finding_hashes[findingId]) problems.push(`source finding ${findingId} changed since approval`);
+  }
+  for (const evidenceId of exception.evidence_ids) {
+    const item = evidence.get(evidenceId);
+    if (!item) problems.push(`evidence ${evidenceId} is unavailable`);
+    else if (recordHash(item) !== exception.evidence_hashes[evidenceId]) problems.push(`evidence ${evidenceId} changed since approval`);
+  }
+
+  if (policy) {
+    for (const role of policy.required_roles) {
+      const actors = assigned(exception, role);
+      if (actors.length === 0) problems.push(`required role ${role} is unassigned`);
+      for (const actor of actors) {
+        const reviewer = reviewers.get(actor);
+        const scoped = exception.finding_ids.every((id) => findings.has(id) && reviewerCanAct(reviewer, role, findings.get(id)));
+        if (!scoped) problems.push(`${actor} is not active and authorized for role ${role} across the exception scope`);
+      }
+    }
+    const authors = new Set(assigned(exception, 'content_author'));
+    const implementers = new Set(assigned(exception, 'remediation_implementer'));
+    const verifiers = new Set([...assigned(exception, 'technical_reviewer'), ...assigned(exception, 'subject_matter_reviewer'), ...assigned(exception, 'independent_reviewer')]);
+    if (policy.separation_rules.includes('author_cannot_verify_own_claim') && [...authors].some((id) => verifiers.has(id))) problems.push('content author also verifies the scoped finding');
+    if (policy.separation_rules.includes('implementer_cannot_solely_verify') && verifiers.size === 1 && [...implementers].some((id) => verifiers.has(id))) problems.push('remediation implementer is the sole verifier');
+    if (policy.separation_rules.includes('commercial_owner_requires_independent_benefit_review') && exception.reviewer_independence !== 'established') problems.push('independent review is not established');
+    if (policy.separation_rules.includes('risk_acceptor_must_be_authorized') && assigned(exception, 'risk_acceptor').length === 0) problems.push('authorized risk acceptor is missing');
+  }
+  for (const assignment of exception.reviewer_assignments) {
+    const reviewer = reviewers.get(assignment.reviewer_id);
+    if (!reviewer) continue;
+    if (reviewer.status !== 'active' || !reviewer.roles.includes(assignment.role)) problems.push(`${reviewer.reviewer_id} is not active in assigned role ${assignment.role}`);
+    if ((reviewer.conflicts || []).some((scope) => exception.finding_ids.some((id) => findings.get(id)?.subject.identifier === scope))) problems.push(`${reviewer.reviewer_id} has a declared conflict with the exception scope`);
+  }
+  return [...new Set(problems)];
+}
+
+export function validateGovernance(root, { refDate } = {}) {
+  const { registries, problems: schemaProblems } = loadRegistries(root);
+  const problems = [...schemaProblems, ...checkReferentialIntegrity(registries)];
+  const reference = parseRefDate(refDate);
+  const reviewers = new Map(registries.reviewers.entries.map((item) => [item.reviewer_id, item]));
+  const policies = new Map(registries.review_policies.entries.map((item) => [item.policy_id, item]));
+  const evidence = new Map(registries.evidence.entries.map((item) => [item.evidence_id, item]));
+  const activeByFinding = new Map();
+  for (const exception of registries.exceptions.entries) {
+    const file = path.join(root, '.citable', 'runs', exception.source_run_id, 'findings.json');
+    const findings = new Map(fs.existsSync(file) ? readJson(file).map((item) => [item.finding_id, item]) : []);
+    const exceptionIssues = exceptionProblems(exception, policies.get(exception.policy_id), reviewers, findings, evidence, reference);
+    problems.push(...exceptionIssues.map((problem) => `exceptions/${exception.exception_id}: ${problem}`));
+    if (exceptionIssues.length === 0) {
+      for (const findingId of exception.finding_ids) {
+        const key = `${exception.source_run_id}:${findingId}`;
+        activeByFinding.set(key, [...(activeByFinding.get(key) || []), exception.exception_id]);
+      }
+    }
+  }
+  for (const [scope, exceptionIds] of activeByFinding) {
+    if (exceptionIds.length > 1) problems.push(`ambiguous active exceptions for ${scope}: ${exceptionIds.join(', ')}`);
+  }
+  return { ok: problems.length === 0, problems, counts: { reviewers: reviewers.size, policies: policies.size, exceptions: registries.exceptions.entries.length } };
+}
+
+export function evaluateDispositions(root, { runId, refDate } = {}) {
+  if (!runId) throw new Error('source run id is required');
+  const findingsFile = path.join(root, '.citable', 'runs', runId, 'findings.json');
+  if (!fs.existsSync(findingsFile)) throw new Error(`source findings not found for run ${runId}`);
+  const sourceBytes = fs.readFileSync(findingsFile);
+  const sourceFindings = JSON.parse(sourceBytes);
+  const findings = new Map(sourceFindings.map((item) => [item.finding_id, item]));
+  const { registries, problems } = loadRegistries(root);
+  const integrity = checkReferentialIntegrity(registries);
+  if (problems.length || integrity.length) throw new Error(`registry validation failed: ${[...problems, ...integrity].join('; ')}`);
+  const reviewers = new Map(registries.reviewers.entries.map((item) => [item.reviewer_id, item]));
+  const policies = new Map(registries.review_policies.entries.map((item) => [item.policy_id, item]));
+  const evidence = new Map(registries.evidence.entries.map((item) => [item.evidence_id, item]));
+  const reference = parseRefDate(refDate);
+  const applicable = registries.exceptions.entries.filter((item) => item.source_run_id === runId);
+  const evaluations = applicable.map((exception) => {
+    const exceptionProblemsList = exceptionProblems(exception, policies.get(exception.policy_id), reviewers, findings, evidence, reference);
+    return { exception, validity: exceptionProblemsList.length ? 'invalid' : 'active', problems: exceptionProblemsList };
+  });
+  const dispositions = sourceFindings.map((finding) => {
+    const matches = evaluations.filter((item) => item.exception.finding_ids.includes(finding.finding_id));
+    const active = matches.filter((item) => item.validity === 'active');
+    return {
+      finding_id: finding.finding_id,
+      detector_id: finding.detector_id,
+      subject: finding.subject,
+      technical_state: 'failed',
+      enforcement_disposition: active.length === 1 ? 'accepted_exception' : active.length > 1 ? 'blocked_ambiguous_exception' : 'enforce',
+      exception_validity: active.length === 1 ? 'active' : matches.length ? (active.length > 1 ? 'ambiguous' : 'invalid') : 'not_applicable',
+      exception_id: active.length === 1 ? active[0].exception.exception_id : null,
+      residual_risk: active.length === 1 ? active[0].exception.residual_risk : 'not_documented',
+      problems: matches.flatMap((item) => item.problems),
+    };
+  });
+  const run = createRun(root, { command: 'governance evaluate', argv: [runId], target: { kind: 'registries', location: findingsFile } });
+  run.addInput('source-findings', sourceBytes);
+  run.addInput('exceptions', registries.exceptions);
+  run.addInput('review-policies', registries.review_policies);
+  run.writeArtifact('dispositions.json', dispositions);
+  run.writeArtifact('exception-evaluations.json', evaluations.map(({ exception, ...evaluation }) => ({ exception_id: exception.exception_id, ...evaluation })));
+  run.manifest.warnings.push('Accepted exceptions alter enforcement disposition only; source findings remain failed and immutable.');
+  run.finalize(dispositions.some((item) => item.enforcement_disposition === 'blocked_ambiguous_exception') ? 'incomplete' : 'completed_with_warnings');
+  return { runId: run.runId, dir: run.dir, source_run_id: runId, dispositions, exception_evaluations: evaluations.length };
+}
+
+export { recordHash };

package/src/commands/observe.js

@@ -2,10 +2,15 @@ import { buildContext } from './context.js';
 import { envelope, observationRun, readInput } from '../observations/common.js';
 import { sha256 } from '../shared/io.js';
 import { fetchUrl } from '../crawler/fetch.js';
-import net from 'node:net';
+import fs from 'node:fs';
+import path from 'node:path';
+import { parse as parseCsv } from 'csv-parse/sync';
+import { crawlerIdentity } from '../observations/crawlerIdentity.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
 
 const originOf = (value) => { try { return new URL(value).origin; } catch { return null; } };
 const words = (text) => String(text || '').trim().split(/\s+/).filter(Boolean);
+const strictDate = (value) => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && new Date(`${value}T00:00:00Z`).toISOString().slice(0, 10) === value;
 
 function canonicalReview(raw, targetOrigin) {
   const citations = raw.citations || [];
@@ -125,32 +130,85 @@ async function observeCitations(root, options) {
 }
 
 function observeLogs(root, options) {
-  const input = readInput(options.input);
-  const rows = Array.isArray(input.value) ? input.value : input.value.requests || [];
-  const ranges = input.value.provider_ranges || {};
-  const verify = (row) => {
-    const cidrs = ranges[row.user_agent] || [];
-    if (!row.source_ip || !cidrs.length) return { verified: false, method: null };
-    const family = net.isIP(row.source_ip);
-    if (!family) return { verified: false, method: 'invalid source IP' };
-    const block = new net.BlockList();
-    for (const cidr of cidrs) {
-      const [network, prefix] = cidr.split('/');
-      const rangeFamily = net.isIP(network);
-      if (rangeFamily === family && Number.isInteger(Number(prefix))) block.addSubnet(network, Number(prefix), family === 4 ? 'ipv4' : 'ipv6');
-    }
-    return { verified: block.check(row.source_ip, family === 4 ? 'ipv4' : 'ipv6'), method: `matched imported ${row.user_agent} CIDR set` };
-  };
+  if (!options.input || !fs.existsSync(options.input)) throw new Error('logs requires --input <json|csv>');
+  const raw = fs.readFileSync(options.input, 'utf8'), ext = path.extname(options.input).toLowerCase();
+  const value = ext === '.csv' ? { requests: parseCsv(raw, { columns: true, skip_empty_lines: true, trim: true }) } : JSON.parse(raw);
+  const input = { raw, value, file: path.resolve(options.input) };
+  const rows = (Array.isArray(value) ? value : value.requests || []).map((row) => ({
+    ...row,
+    timestamp: row.timestamp || row.datetime || row.date,
+    url: row.url || row.request_url || row['cs-uri-stem'],
+    user_agent: row.user_agent || row.userAgent || row['cs(User-Agent)'],
+    source_ip: row.source_ip || row.clientIP || row['c-ip'],
+    status: Number(row.status || row.status_code || row['sc-status']),
+    bytes: row.bytes == null ? null : Number(row.bytes),
+    latency_ms: row.latency_ms == null ? null : Number(row.latency_ms),
+    edge_observed: row.edge_observed === true || row.edge_observed === 'true',
+    origin_observed: row.origin_observed === true || row.origin_observed === 'true',
+    origin_status: row.origin_status == null || row.origin_status === '' ? null : Number(row.origin_status),
+    claimed_verified: row.claimed_verified === true || row.claimed_verified === 'true',
+  }));
+  const sensitive = (Array.isArray(value) ? value : value.requests || []).flatMap((row) => Object.keys(row).filter((key) => /(^|[-_])(authorization|cookie|set-cookie|access[-_]?token|refresh[-_]?token|api[-_]?key|password|secret)($|[-_])/i.test(key)));
+  if (sensitive.length) throw new Error(`log import contains sensitive fields that must be removed before collection: ${[...new Set(sensitive)].join(', ')}`);
+  const ranges = value.provider_ranges || {}, rangeSources = value.range_sources || {}, metadata = value.metadata || {};
   const observations = rows.map((row) => {
-    const identity = verify(row);
+    if (!row.timestamp || Number.isNaN(Date.parse(row.timestamp)) || !row.url || !row.user_agent || !Number.isInteger(row.status)) throw new Error('each log row requires valid timestamp, url, user_agent, and integer status');
+    const identity = crawlerIdentity(row, { ranges, rangeSources, collector: metadata.collector || 'owner_import' });
+    const fullyVerified = identity.verification_status === 'fully_verified';
     return envelope('crawler_log', {
     timestamp: row.timestamp, url: row.url, user_agent: row.user_agent, source_ip: row.source_ip,
     status: row.status, bytes: row.bytes ?? null, latency_ms: row.latency_ms ?? null,
     cache_status: row.cache_status ?? null, region: row.region ?? null,
-    identity_verified: identity.verified, verification_method: identity.method,
-  }, { method: 'owner_import', source: input.file, raw: JSON.stringify(row), confidence: identity.verified ? 'high' : 'low', limitations: identity.verified ? ['CIDR ranges were imported with the log evidence; their provider currency must be established by source metadata.'] : ['User-agent strings alone do not verify crawler identity.'] });
+    crawler_identity: identity,
+  }, { method: 'owner_import', source: input.file, raw: JSON.stringify(row), confidence: fullyVerified ? 'confirmed' : identity.verification_status === 'contradictory' ? 'high' : 'low',
+    authority: { collection_authority: 'production_log', authenticity_status: fullyVerified ? 'provider_range_verified' : 'checksum_protected_only', representativeness: metadata.representativeness || 'unknown' },
+    limitations: fullyVerified ? ['Verification applies only to this imported production event and captured verification chain.'] : ['The crawler identity chain is incomplete or contradictory; user agent and CIDR matching alone are insufficient.'] });
+  });
+  return observationRun(root, 'observe logs', input.file, observations, { rawInputs: { server_logs: input.raw }, incomplete: observations.some((o) => o.data.crawler_identity.verification_status !== 'fully_verified') ? ['Some crawler identities are not fully verified.'] : [] });
+}
+
+function observeBing(root, options) {
+  if (!options.input || !fs.existsSync(options.input)) throw new Error('bing requires --input <json|csv> --dataset <search_performance|ai_performance>');
+  if (!['search_performance', 'ai_performance'].includes(options.dataset)) throw new Error('bing dataset must be search_performance or ai_performance');
+  const raw = fs.readFileSync(options.input, 'utf8'), ext = path.extname(options.input).toLowerCase();
+  const value = ext === '.csv' ? parseCsv(raw, { columns: true, skip_empty_lines: true, trim: true }) : JSON.parse(raw);
+  const document = Array.isArray(value) ? { rows: value } : value;
+  const rows = document.rows || document.items || [];
+  if (!rows.length) throw new Error('Bing owner export contains no rows');
+  const numeric = (row, ...names) => {
+    const rawValue = names.map((name) => row[name]).find((item) => item !== '' && item != null);
+    if (rawValue == null) return null;
+    const result = Number(String(rawValue).replace(/%$/, ''));
+    if (!Number.isFinite(result)) throw new Error(`Bing metric ${names[0]} must be numeric`);
+    return String(rawValue).endsWith('%') ? result / 100 : result;
+  };
+  const observations = rows.map((row) => {
+    const date = row.date || row.Date;
+    if (!strictDate(date)) throw new Error('each Bing row requires a valid date in YYYY-MM-DD format');
+    const metrics = options.dataset === 'ai_performance' ? {
+      total_citations: numeric(row, 'total_citations', 'Total Citations', 'citations'),
+      average_cited_pages: numeric(row, 'average_cited_pages', 'Average Cited Pages'),
+    } : {
+      clicks: numeric(row, 'clicks', 'Clicks'), impressions: numeric(row, 'impressions', 'Impressions'),
+      ctr: numeric(row, 'ctr', 'CTR', 'Average CTR'), position: numeric(row, 'position', 'Average Position'),
+      crawl_requests: numeric(row, 'crawl_requests', 'Crawl Requests'), crawl_errors: numeric(row, 'crawl_errors', 'Crawl Errors'),
+      indexed_pages: numeric(row, 'indexed_pages', 'Indexed Pages'),
+    };
+    const boundaries = options.dataset === 'ai_performance'
+      ? ['Citation counts do not indicate ranking, authority, placement, page importance, or material support.', 'Grounding queries are sampled and the dashboard aggregates supported Microsoft AI surfaces.']
+      : ['Bing search performance can combine traffic sources; source dimensions must be preserved.', 'Observed performance does not establish that an intervention caused a change.'];
+    if (!Object.values(metrics).some((value) => value != null)) throw new Error(`Bing ${options.dataset} row contains no recognized metrics`);
+    const data = {
+      dataset: options.dataset, date, url: row.url || row.URL || row.page || null, metrics,
+      dimensions: { query: row.query || row.Query || null, grounding_query: row.grounding_query || row['Grounding Query'] || null, source: row.source || row.Source || null, country: row.country || row.Country || null, device: row.device || row.Device || null },
+      interpretation_boundary: boundaries,
+    };
+    const check = validateAgainst('bing-webmaster.schema.json', data);
+    if (!check.valid) throw new Error(`Bing observation violates contract: ${check.errors.join('; ')}`);
+    return envelope('bing_webmaster', data, { method: 'owner_import', source: path.resolve(options.input), raw: JSON.stringify(row),
+      authority: { representativeness: document.metadata?.representativeness || 'unknown' }, limitations: boundaries });
   });
-  return observationRun(root, 'observe logs', input.file, observations, { rawInputs: { server_logs: input.raw }, incomplete: observations.some((o) => !o.data.identity_verified) ? ['Some crawler identities are not IP-verified.'] : [] });
+  return observationRun(root, 'observe bing', path.resolve(options.input), observations, { rawInputs: { bing_owner_export: raw }, warnings: ['Bing AI Performance has no supported API contract captured by this release; this evidence is an owner export.'] });
 }
 
 async function observePassages(root, options) {
@@ -226,10 +284,11 @@ export async function observe(root, mode, options = {}) {
     case 'index': return observeIndex(root, options);
     case 'citations': return observeCitations(root, options);
     case 'logs': return observeLogs(root, options);
+    case 'bing': return observeBing(root, options);
     case 'passages': return observePassages(root, options);
     case 'consensus': return observeConsensus(root, options);
     case 'performance': return observePerformance(root, options);
     case 'corroboration': return observeCorroboration(root, options);
-    default: throw new Error('observe mode must be render, index, citations, logs, passages, consensus, performance, or corroboration');
+    default: throw new Error('observe mode must be render, index, citations, logs, bing, passages, consensus, performance, or corroboration');
   }
 }

package/src/commands/selfUpgrade.js

@@ -21,7 +21,7 @@ const FETCH_TIMEOUT_MS = 10_000;
 
 /**
  * Fetch the latest published version from the npm registry.
- * Uses Node's built-in fetch (Node 22+); no extra dependencies.
+ * Uses Node's built-in fetch (Node 24+); no extra dependencies.
  *
  * @returns {Promise<string>} latest semver string
  */

package/src/observations/authority.js

@@ -0,0 +1,13 @@
+const DEFAULTS = {
+  live_api: ['provider_published', 'direct_api', 'transport_authenticated', 'unknown'],
+  browser: ['synthetic', 'browser_capture', 'checksum_protected_only', 'single_observation'],
+  owner_import: ['owner_controlled', 'owner_export', 'checksum_protected_only', 'unknown'],
+  synthetic_fetch: ['synthetic', 'synthetic_probe', 'checksum_protected_only', 'single_observation'],
+  static_analysis: ['inferred', 'static_analysis', 'checksum_protected_only', 'single_observation'],
+  human_review: ['owner_controlled', 'manual_entry', 'owner_attested', 'single_observation'],
+};
+
+export function evidenceAuthority(method, overrides = {}) {
+  const [source_authority, collection_authority, authenticity_status, representativeness] = DEFAULTS[method] || DEFAULTS.static_analysis;
+  return { source_authority, collection_authority, authenticity_status, representativeness, ...overrides };
+}

package/src/observations/common.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { createRun } from '../evidence/run.js';
 import { nowIso, readJson, sha256 } from '../shared/io.js';
 import { validateAgainst } from '../shared/schemaValidator.js';
+import { evidenceAuthority } from './authority.js';
 
 export function readInput(input) {
   if (!input) throw new Error('--input <json> is required for this collector');
@@ -10,12 +11,13 @@ export function readInput(input) {
   return { raw: fs.readFileSync(input, 'utf8'), value: readJson(input), file: path.resolve(input) };
 }
 
-export function envelope(kind, data, { method, source, state = 'observed', confidence = 'confirmed', limitations = [], raw } = {}) {
+export function envelope(kind, data, { method, source, state = 'observed', confidence = 'confirmed', limitations = [], authority, raw } = {}) {
   const evidence = raw ?? JSON.stringify(data);
   const item = {
     observation_id: `OBS-${kind.toUpperCase()}-${sha256(evidence).slice(0, 16)}`,
     kind, state, collected_at: nowIso(), collection_method: method,
     confidence, source, evidence_hash: sha256(evidence), data, limitations,
+    authority: evidenceAuthority(method, authority),
   };
   const check = validateAgainst('observation.schema.json', item);
   if (!check.valid) throw new Error(`observation violates contract: ${check.errors.join('; ')}`);

package/src/observations/crawlerIdentity.js

@@ -0,0 +1,60 @@
+import net from 'node:net';
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+function cidrMembership(sourceIp, cidrs) {
+  if (!sourceIp) return 'not_tested';
+  const family = net.isIP(sourceIp);
+  if (!family) return 'invalid';
+  const block = new net.BlockList();
+  for (const cidr of cidrs || []) {
+    const [network, prefixText] = String(cidr).split('/');
+    const prefix = Number(prefixText), rangeFamily = net.isIP(network);
+    if (rangeFamily === family && Number.isInteger(prefix)) block.addSubnet(network, prefix, family === 4 ? 'ipv4' : 'ipv6');
+  }
+  return block.check(sourceIp, family === 4 ? 'ipv4' : 'ipv6') ? 'matched' : 'not_matched';
+}
+
+export function crawlerIdentity(row, { ranges = {}, rangeSources = {}, collector = 'owner_import' } = {}) {
+  const ua = row.user_agent, cidrs = ranges[ua] || [], source = rangeSources[ua] || {};
+  const sourceSnapshot = source.source_snapshot;
+  const expectedChecksum = sourceSnapshot == null ? null : sha256(typeof sourceSnapshot === 'string' ? sourceSnapshot : JSON.stringify(sourceSnapshot));
+  const rangeSource = {
+    url: source.url || null,
+    retrieved_at: source.retrieved_at || null,
+    authority: source.authority || 'unknown',
+    checksum: source.checksum || null,
+    checksum_scope: sourceSnapshot == null ? 'not_captured' : 'source_snapshot',
+    checksum_valid: Boolean(source.checksum && expectedChecksum && source.checksum === expectedChecksum),
+  };
+  const membership = cidrMembership(row.source_ip, cidrs);
+  const dns = { status: row.dns_status || 'not_tested', method: row.dns_method || null };
+  const edge = { observed: row.edge_observed === true, decision: row.edge_decision || null };
+  const origin = { observed: row.origin_observed === true, status: row.origin_status ?? null };
+  const stages = ['declared'];
+  if (row.synthetic === true) stages.push('synthetically_observed');
+  if (membership === 'matched') stages.push('range_matched');
+  if (dns.status === 'verified') stages.push('dns_verified');
+  if (edge.observed) stages.push('edge_observed');
+  if (origin.observed) stages.push('origin_observed');
+  const contradictory = membership === 'not_matched' || dns.status === 'failed' || (row.claimed_verified === true && (!rangeSource.checksum_valid || membership !== 'matched'));
+  const complete = identityProvider(row, source) !== 'unknown' && row.synthetic !== true && rangeSource.url && rangeSource.retrieved_at && rangeSource.authority === 'provider_published' && rangeSource.checksum_scope === 'source_snapshot' && rangeSource.checksum_valid && membership === 'matched' && dns.status === 'verified' && edge.observed && origin.observed;
+  if (contradictory) stages.push('contradictory');
+  else if (complete) stages.push('fully_verified');
+  else stages.push('insufficient_evidence');
+  const precedence = ['contradictory', 'fully_verified', 'origin_observed', 'edge_observed', 'dns_verified', 'range_matched', 'synthetically_observed', 'declared'];
+  const verificationStatus = precedence.find((state) => stages.includes(state));
+  const identity = {
+    provider: identityProvider(row, source), declared_user_agent: ua,
+    observed_source_ip: row.source_ip || null, range_source: rangeSource,
+    cidr_membership: membership, dns, edge, origin, region: row.region || null,
+    collector, stages, verification_status: verificationStatus,
+  };
+  const check = validateAgainst('crawler-identity.schema.json', identity);
+  if (!check.valid) throw new Error(`crawler identity violates contract: ${check.errors.join('; ')}`);
+  return identity;
+}
+
+function identityProvider(row, source) {
+  return row.provider || source.provider || 'unknown';
+}

package/src/registries/index.js

@@ -17,6 +17,9 @@ export const REGISTRY_SPECS = [
   { file: 'objectives.yaml', kind: 'objectives', schema: 'objective.schema.json', idField: 'objective_id' },
   { file: 'interventions.yaml', kind: 'interventions', schema: 'intervention.schema.json', idField: 'intervention_id' },
   { file: 'connections.yaml', kind: 'connections', schema: 'connection.schema.json', idField: 'connection_id' },
+  { file: 'reviewers.yaml', kind: 'reviewers', schema: 'reviewer.schema.json', idField: 'reviewer_id' },
+  { file: 'review-policies.yaml', kind: 'review_policies', schema: 'review-policy.schema.json', idField: 'policy_id' },
+  { file: 'exceptions.yaml', kind: 'exceptions', schema: 'exception.schema.json', idField: 'exception_id' },
 ];
 
 export function contextDir(root) {
@@ -104,6 +107,20 @@ export function checkReferentialIntegrity(registries) {
   for (const intervention of registries.interventions?.entries || []) {
     for (const ref of intervention.objective_ids || []) if (!objectiveIds.has(ref)) problems.push(`interventions/${intervention.intervention_id}: references unknown objective id "${ref}"`);
   }
+  const reviewerIds = ids.reviewers || new Set();
+  const policyIds = ids.review_policies || new Set();
+  const evidenceIds = ids.evidence || new Set();
+  const interventionIds = ids.interventions || new Set();
+  const exceptionIds = ids.exceptions || new Set();
+  for (const exception of registries.exceptions?.entries || []) {
+    const prefix = `exceptions/${exception.exception_id}`;
+    if (!policyIds.has(exception.policy_id)) problems.push(`${prefix}: references unknown review policy id "${exception.policy_id}"`);
+    const reviewerRefs = [exception.owner_reviewer_id, ...(exception.reviewer_assignments || []).map((item) => item.reviewer_id), ...(exception.audit_history || []).map((item) => item.actor_reviewer_id)];
+    for (const ref of reviewerRefs) if (ref && !reviewerIds.has(ref)) problems.push(`${prefix}: references unknown reviewer id "${ref}"`);
+    for (const ref of exception.evidence_ids || []) if (!evidenceIds.has(ref)) problems.push(`${prefix}: references unknown evidence id "${ref}"`);
+    if (exception.related_intervention_id && !interventionIds.has(exception.related_intervention_id)) problems.push(`${prefix}: references unknown intervention id "${exception.related_intervention_id}"`);
+    for (const ref of [exception.supersedes_exception_id, exception.superseded_by_exception_id]) if (ref && !exceptionIds.has(ref)) problems.push(`${prefix}: references unknown exception id "${ref}"`);
+  }
   return problems;
 }