@nebulacomponents/citable 1.6.0 → 1.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nebulacomponents/citable",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "description": "Production-grade SEO, AEO, and GEO agent skill with auditable detection, claim governance, evidence validation, and multi-agent installation.",
   "keywords": [
     "seo",

package/schemas/bing-webmaster.schema.json

@@ -0,0 +1,25 @@
+{
+  "$id": "citable://schemas/bing-webmaster.schema.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Normalized Bing Webmaster observation",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["dataset", "date", "url", "metrics", "dimensions", "interpretation_boundary"],
+  "properties": {
+    "dataset": { "type": "string", "enum": ["search_performance", "ai_performance"] },
+    "date": { "type": "string", "format": "date" },
+    "url": { "type": ["string", "null"], "format": "uri", "pattern": "^https?://" },
+    "metrics": { "type": "object", "additionalProperties": { "type": ["number", "null"] } },
+    "dimensions": {
+      "type": "object", "additionalProperties": false,
+      "properties": {
+        "query": { "type": ["string", "null"] },
+        "grounding_query": { "type": ["string", "null"] },
+        "source": { "type": ["string", "null"] },
+        "country": { "type": ["string", "null"] },
+        "device": { "type": ["string", "null"] }
+      }
+    },
+    "interpretation_boundary": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }
+  }
+}

package/schemas/crawler-identity.schema.json

@@ -0,0 +1,42 @@
+{
+  "$id": "citable://schemas/crawler-identity.schema.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Crawler identity evidence contract",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["provider", "declared_user_agent", "observed_source_ip", "range_source", "cidr_membership", "dns", "edge", "origin", "region", "collector", "stages", "verification_status"],
+  "properties": {
+    "provider": { "type": "string", "minLength": 1 },
+    "declared_user_agent": { "type": "string", "minLength": 1 },
+    "observed_source_ip": { "type": ["string", "null"] },
+    "range_source": {
+      "type": "object", "additionalProperties": false,
+      "required": ["url", "retrieved_at", "authority", "checksum", "checksum_scope", "checksum_valid"],
+      "properties": {
+        "url": { "type": ["string", "null"], "format": "uri" },
+        "retrieved_at": { "type": ["string", "null"], "format": "date-time" },
+        "authority": { "type": "string", "enum": ["provider_published", "owner_controlled", "third_party", "unknown"] },
+        "checksum": { "type": ["string", "null"], "pattern": "^[a-f0-9]{64}$" },
+        "checksum_scope": { "type": "string", "enum": ["source_snapshot", "not_captured"] },
+        "checksum_valid": { "type": "boolean" }
+      }
+    },
+    "cidr_membership": { "type": "string", "enum": ["matched", "not_matched", "not_tested", "invalid"] },
+    "dns": {
+      "type": "object", "additionalProperties": false, "required": ["status", "method"],
+      "properties": { "status": { "type": "string", "enum": ["verified", "failed", "not_tested"] }, "method": { "type": ["string", "null"] } }
+    },
+    "edge": {
+      "type": "object", "additionalProperties": false, "required": ["observed", "decision"],
+      "properties": { "observed": { "type": "boolean" }, "decision": { "type": ["string", "null"] } }
+    },
+    "origin": {
+      "type": "object", "additionalProperties": false, "required": ["observed", "status"],
+      "properties": { "observed": { "type": "boolean" }, "status": { "type": ["integer", "null"], "minimum": 100, "maximum": 599 } }
+    },
+    "region": { "type": ["string", "null"] },
+    "collector": { "type": "string", "minLength": 1 },
+    "stages": { "type": "array", "uniqueItems": true, "items": { "type": "string", "enum": ["declared", "synthetically_observed", "range_matched", "dns_verified", "edge_observed", "origin_observed", "fully_verified", "contradictory", "insufficient_evidence"] } },
+    "verification_status": { "type": "string", "enum": ["declared", "synthetically_observed", "range_matched", "dns_verified", "edge_observed", "origin_observed", "fully_verified", "contradictory", "insufficient_evidence"] }
+  }
+}

package/schemas/evidence-authority.schema.json

@@ -0,0 +1,15 @@
+{
+  "$id": "citable://schemas/evidence-authority.schema.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Multidimensional evidence authority",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["source_authority", "collection_authority", "authenticity_status", "representativeness"],
+  "properties": {
+    "source_authority": { "type": "string", "enum": ["provider_published", "standards_body", "owner_controlled", "independently_controlled", "third_party_commercial", "synthetic", "inferred", "unknown"] },
+    "collection_authority": { "type": "string", "enum": ["direct_api", "production_log", "browser_capture", "owner_export", "third_party_export", "manual_entry", "model_generated_inference", "static_analysis", "synthetic_probe"] },
+    "authenticity_status": { "type": "string", "enum": ["cryptographically_authenticated", "provider_range_verified", "owner_attested", "transport_authenticated", "checksum_protected_only", "unverified"] },
+    "representativeness": { "type": "string", "enum": ["production_population", "complete_export", "statistically_sampled", "controlled_experiment", "convenience_sample", "single_observation", "unknown"] },
+    "limitations": { "type": "array", "items": { "type": "string", "minLength": 1 } }
+  }
+}

package/schemas/observation.schema.json

@@ -7,7 +7,7 @@
   "required": ["observation_id", "kind", "state", "collected_at", "collection_method", "confidence", "source", "evidence_hash", "data"],
   "properties": {
     "observation_id": { "type": "string", "minLength": 1 },
-    "kind": { "type": "string", "enum": ["render", "index", "citation", "citation_review", "crawler_log", "passage", "canonical_freshness", "performance", "corroboration", "metric"] },
+    "kind": { "type": "string", "enum": ["render", "index", "citation", "citation_review", "crawler_log", "bing_webmaster", "passage", "canonical_freshness", "performance", "corroboration", "metric"] },
     "state": { "type": "string", "enum": ["observed", "not_observed", "not_evidenced", "incomplete", "review_required", "failed"] },
     "collected_at": { "type": "string", "format": "date-time" },
     "collection_method": { "type": "string", "enum": ["live_api", "browser", "owner_import", "synthetic_fetch", "static_analysis", "human_review"] },
@@ -15,6 +15,7 @@
     "source": { "type": "string", "minLength": 1 },
     "evidence_hash": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
     "data": { "type": "object" },
-    "limitations": { "type": "array", "items": { "type": "string" } }
+    "limitations": { "type": "array", "items": { "type": "string" } },
+    "authority": { "$ref": "citable://schemas/evidence-authority.schema.json" }
   }
 }

package/skill/SKILL.md

@@ -7,12 +7,16 @@ description: >
   generative engine optimization, structured data governance, claim substantiation,
   crawler policy, entity consistency, content discoverability, or wants a site audited
   for how search and AI systems will retrieve, understand, cite, or recommend it.
-version: 1.6.0
+version: 1.7.0
 ---
 
-# Citable — search & generative discoverability governance
+# Citable — evidence and change control for search and AI citation readiness
 
 Citable is an operational quality and governance layer, not a content generator.
+It records what a property makes technically available, what its content can
+support, and what external systems have actually been observed doing. It does
+not score visibility; it establishes what is eligible, supportable, observed,
+changed, and still unknown.
 It treats discoverability as an engineering system: registries as the source of
 truth, deterministic detectors for observable conditions, rubrics for semantic
 judgment, evidence packages for every run, and fail-closed behaviour wherever a
@@ -70,6 +74,11 @@ Within retrieval, distinguish `allowed_by_policy`, `synthetic_fetch_succeeded`,
 citation testing, distinguish mention, citation, material support, canonical
 source selection, and recommendation. These states are not interchangeable.
 
+Evidence preservation, authority, authenticity, collection method, and
+representativeness are also independent. Follow
+`references/evidence-authority.md`; a checksum or CIDR match alone never proves
+source authenticity, complete coverage, or verified crawler identity.
+
 ## Tooling
 
 The `citable` CLI in this repository performs the deterministic work. Always
@@ -86,6 +95,7 @@ citable validate [registries|claims|evidence|schema|links]
 citable compare-snapshots [runA runB]
 citable action-plan [run-id]          # ordered actions, blockers, semantic gates, verification
 citable observe <mode> [options]      # render/index/citation/log/passage/consensus/performance evidence
+citable observe bing --dataset <search_performance|ai_performance> --input <csv|json>
 citable apply --input <spec> [--write] # reviewed, hash-locked remediation; dry run by default
 citable monitor [runA runB]           # observation regression alerts
 citable metrics import --provider <name> --input <csv|json>

package/skill/commands/measurement.md

@@ -70,6 +70,21 @@ agent × URL × status; flag 403/429/5xx concentrations against declared-allow
 crawlers; note that user-agent strings are spoofable and IP validation is the
 confirmation step (crawler registry ip_validation_method). Compare observed
 access against `crawlers.yaml` decisions (CRAWL-001 logic applied to logs).
+Normalize JSON or CSV edge/origin events into the crawler identity contract.
+Do not label CIDR membership alone verified: preserve range-source provenance,
+DNS, edge, origin, region, collector, stages, and contradiction state.
+Remove credentials, cookies, secrets, and unnecessary personal data before
+import. Citable rejects common sensitive field names but cannot establish that
+arbitrary URL or query values are lawful or appropriately minimized.
+
+# Bing owner observations
+
+Use `citable observe bing --dataset search_performance|ai_performance --input
+<csv|json>`. This release normalizes owner exports because no supported Bing AI
+Performance API contract has been captured. Citation counts do not indicate
+ranking, authority, placement, page importance, or material support. Grounding
+queries may be sampled. Preserve traffic-source dimensions and never infer
+causation from a temporal change.
 
 # monitor-contradictions
 Compare approved entity + claim records against: owned pages (automated, via

package/skill/references/evidence-authority.md

@@ -0,0 +1,43 @@
+# Evidence authority and crawler identity
+
+An evidence checksum establishes preservation only. It does not establish that
+the source was authentic, the collection was complete, or the observation was
+representative. Report four axes independently on every new observation:
+
+1. `source_authority`: provider-published, standards-body, owner-controlled,
+   independently controlled, third-party commercial, synthetic, inferred, or
+   unknown.
+2. `collection_authority`: direct API, production log, browser capture, owner
+   export, third-party export, manual entry, model-generated inference, static
+   analysis, or synthetic probe.
+3. `authenticity_status`: cryptographically authenticated, provider-range
+   verified, owner-attested, transport-authenticated, checksum-protected only,
+   or unverified.
+4. `representativeness`: production population, complete export,
+   statistically sampled, controlled experiment, convenience sample, single
+   observation, or unknown.
+
+Never convert these axes into one authority score.
+
+## Crawler identity
+
+Contract support and verified observations are different outcomes. The
+`crawler-identity` schema can exist without any production event satisfying it.
+A user-agent match is `declared`. A synthetic request is
+`synthetically_observed`. An observed IP in an imported range is only
+`range_matched`.
+
+`fully_verified` requires all of:
+
+- a provider and declared user agent;
+- observed source IP matching the captured CIDR set;
+- provider-published range source URL, retrieval time, captured source snapshot,
+  and valid snapshot checksum;
+- documented forward/reverse DNS verification where applicable;
+- an explicit edge observation and decision;
+- a correlated origin observation and status;
+- region and collector identity.
+
+A failed range or DNS check, or a claimed verification without the required
+chain, is `contradictory`. Missing links are `insufficient_evidence`. Verification
+applies only to the captured event; it does not establish all crawler traffic.

package/src/cli/index.js

@@ -39,7 +39,7 @@ Commands
   validate [mode]           registries (default) | claims | evidence | schema | links
   compare-snapshots [a b]   Regression diff between two audit runs
   action-plan [run]         Turn audit findings into ordered remediation work
-  observe <mode>            Collect render, index, citation, log, passage,
+  observe <mode>            Collect render, index, citation, log, Bing, passage,
                             consensus, performance, or corroboration evidence
   apply                     Apply a reviewed, hash-locked remediation spec
   monitor [runA runB]       Compare observation runs and emit regression alerts
@@ -61,6 +61,7 @@ Options
   --ref-date <YYYY-MM-DD>   Reference date for expiry/staleness checks (default: today)
   --input <file>            Import file or remediation specification
   --provider <name>         Provider label for imported observations
+  --dataset <name>          Provider export dataset (for example ai_performance)
   --connection-id <id>      Connection registry identifier
   --property-id <id>        Provider property or site identifier
   --credential-env <name>   Environment variable containing the access token
@@ -88,6 +89,7 @@ function parseArgs(argv) {
     else if (a === '--ref-date') args.refDate = argv[++i];
     else if (a === '--input') args.input = argv[++i];
     else if (a === '--provider') args.provider = argv[++i];
+    else if (a === '--dataset') args.dataset = argv[++i];
     else if (a === '--connection-id') args.connectionId = argv[++i];
     else if (a === '--property-id') args.propertyId = argv[++i];
     else if (a === '--credential-env') args.credentialEnv = argv[++i];

package/src/commands/observe.js

@@ -2,10 +2,15 @@ import { buildContext } from './context.js';
 import { envelope, observationRun, readInput } from '../observations/common.js';
 import { sha256 } from '../shared/io.js';
 import { fetchUrl } from '../crawler/fetch.js';
-import net from 'node:net';
+import fs from 'node:fs';
+import path from 'node:path';
+import { parse as parseCsv } from 'csv-parse/sync';
+import { crawlerIdentity } from '../observations/crawlerIdentity.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
 
 const originOf = (value) => { try { return new URL(value).origin; } catch { return null; } };
 const words = (text) => String(text || '').trim().split(/\s+/).filter(Boolean);
+const strictDate = (value) => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && new Date(`${value}T00:00:00Z`).toISOString().slice(0, 10) === value;
 
 function canonicalReview(raw, targetOrigin) {
   const citations = raw.citations || [];
@@ -125,32 +130,85 @@ async function observeCitations(root, options) {
 }
 
 function observeLogs(root, options) {
-  const input = readInput(options.input);
-  const rows = Array.isArray(input.value) ? input.value : input.value.requests || [];
-  const ranges = input.value.provider_ranges || {};
-  const verify = (row) => {
-    const cidrs = ranges[row.user_agent] || [];
-    if (!row.source_ip || !cidrs.length) return { verified: false, method: null };
-    const family = net.isIP(row.source_ip);
-    if (!family) return { verified: false, method: 'invalid source IP' };
-    const block = new net.BlockList();
-    for (const cidr of cidrs) {
-      const [network, prefix] = cidr.split('/');
-      const rangeFamily = net.isIP(network);
-      if (rangeFamily === family && Number.isInteger(Number(prefix))) block.addSubnet(network, Number(prefix), family === 4 ? 'ipv4' : 'ipv6');
-    }
-    return { verified: block.check(row.source_ip, family === 4 ? 'ipv4' : 'ipv6'), method: `matched imported ${row.user_agent} CIDR set` };
-  };
+  if (!options.input || !fs.existsSync(options.input)) throw new Error('logs requires --input <json|csv>');
+  const raw = fs.readFileSync(options.input, 'utf8'), ext = path.extname(options.input).toLowerCase();
+  const value = ext === '.csv' ? { requests: parseCsv(raw, { columns: true, skip_empty_lines: true, trim: true }) } : JSON.parse(raw);
+  const input = { raw, value, file: path.resolve(options.input) };
+  const rows = (Array.isArray(value) ? value : value.requests || []).map((row) => ({
+    ...row,
+    timestamp: row.timestamp || row.datetime || row.date,
+    url: row.url || row.request_url || row['cs-uri-stem'],
+    user_agent: row.user_agent || row.userAgent || row['cs(User-Agent)'],
+    source_ip: row.source_ip || row.clientIP || row['c-ip'],
+    status: Number(row.status || row.status_code || row['sc-status']),
+    bytes: row.bytes == null ? null : Number(row.bytes),
+    latency_ms: row.latency_ms == null ? null : Number(row.latency_ms),
+    edge_observed: row.edge_observed === true || row.edge_observed === 'true',
+    origin_observed: row.origin_observed === true || row.origin_observed === 'true',
+    origin_status: row.origin_status == null || row.origin_status === '' ? null : Number(row.origin_status),
+    claimed_verified: row.claimed_verified === true || row.claimed_verified === 'true',
+  }));
+  const sensitive = (Array.isArray(value) ? value : value.requests || []).flatMap((row) => Object.keys(row).filter((key) => /(^|[-_])(authorization|cookie|set-cookie|access[-_]?token|refresh[-_]?token|api[-_]?key|password|secret)($|[-_])/i.test(key)));
+  if (sensitive.length) throw new Error(`log import contains sensitive fields that must be removed before collection: ${[...new Set(sensitive)].join(', ')}`);
+  const ranges = value.provider_ranges || {}, rangeSources = value.range_sources || {}, metadata = value.metadata || {};
   const observations = rows.map((row) => {
-    const identity = verify(row);
+    if (!row.timestamp || Number.isNaN(Date.parse(row.timestamp)) || !row.url || !row.user_agent || !Number.isInteger(row.status)) throw new Error('each log row requires valid timestamp, url, user_agent, and integer status');
+    const identity = crawlerIdentity(row, { ranges, rangeSources, collector: metadata.collector || 'owner_import' });
+    const fullyVerified = identity.verification_status === 'fully_verified';
     return envelope('crawler_log', {
     timestamp: row.timestamp, url: row.url, user_agent: row.user_agent, source_ip: row.source_ip,
     status: row.status, bytes: row.bytes ?? null, latency_ms: row.latency_ms ?? null,
     cache_status: row.cache_status ?? null, region: row.region ?? null,
-    identity_verified: identity.verified, verification_method: identity.method,
-  }, { method: 'owner_import', source: input.file, raw: JSON.stringify(row), confidence: identity.verified ? 'high' : 'low', limitations: identity.verified ? ['CIDR ranges were imported with the log evidence; their provider currency must be established by source metadata.'] : ['User-agent strings alone do not verify crawler identity.'] });
+    crawler_identity: identity,
+  }, { method: 'owner_import', source: input.file, raw: JSON.stringify(row), confidence: fullyVerified ? 'confirmed' : identity.verification_status === 'contradictory' ? 'high' : 'low',
+    authority: { collection_authority: 'production_log', authenticity_status: fullyVerified ? 'provider_range_verified' : 'checksum_protected_only', representativeness: metadata.representativeness || 'unknown' },
+    limitations: fullyVerified ? ['Verification applies only to this imported production event and captured verification chain.'] : ['The crawler identity chain is incomplete or contradictory; user agent and CIDR matching alone are insufficient.'] });
+  });
+  return observationRun(root, 'observe logs', input.file, observations, { rawInputs: { server_logs: input.raw }, incomplete: observations.some((o) => o.data.crawler_identity.verification_status !== 'fully_verified') ? ['Some crawler identities are not fully verified.'] : [] });
+}
+
+function observeBing(root, options) {
+  if (!options.input || !fs.existsSync(options.input)) throw new Error('bing requires --input <json|csv> --dataset <search_performance|ai_performance>');
+  if (!['search_performance', 'ai_performance'].includes(options.dataset)) throw new Error('bing dataset must be search_performance or ai_performance');
+  const raw = fs.readFileSync(options.input, 'utf8'), ext = path.extname(options.input).toLowerCase();
+  const value = ext === '.csv' ? parseCsv(raw, { columns: true, skip_empty_lines: true, trim: true }) : JSON.parse(raw);
+  const document = Array.isArray(value) ? { rows: value } : value;
+  const rows = document.rows || document.items || [];
+  if (!rows.length) throw new Error('Bing owner export contains no rows');
+  const numeric = (row, ...names) => {
+    const rawValue = names.map((name) => row[name]).find((item) => item !== '' && item != null);
+    if (rawValue == null) return null;
+    const result = Number(String(rawValue).replace(/%$/, ''));
+    if (!Number.isFinite(result)) throw new Error(`Bing metric ${names[0]} must be numeric`);
+    return String(rawValue).endsWith('%') ? result / 100 : result;
+  };
+  const observations = rows.map((row) => {
+    const date = row.date || row.Date;
+    if (!strictDate(date)) throw new Error('each Bing row requires a valid date in YYYY-MM-DD format');
+    const metrics = options.dataset === 'ai_performance' ? {
+      total_citations: numeric(row, 'total_citations', 'Total Citations', 'citations'),
+      average_cited_pages: numeric(row, 'average_cited_pages', 'Average Cited Pages'),
+    } : {
+      clicks: numeric(row, 'clicks', 'Clicks'), impressions: numeric(row, 'impressions', 'Impressions'),
+      ctr: numeric(row, 'ctr', 'CTR', 'Average CTR'), position: numeric(row, 'position', 'Average Position'),
+      crawl_requests: numeric(row, 'crawl_requests', 'Crawl Requests'), crawl_errors: numeric(row, 'crawl_errors', 'Crawl Errors'),
+      indexed_pages: numeric(row, 'indexed_pages', 'Indexed Pages'),
+    };
+    const boundaries = options.dataset === 'ai_performance'
+      ? ['Citation counts do not indicate ranking, authority, placement, page importance, or material support.', 'Grounding queries are sampled and the dashboard aggregates supported Microsoft AI surfaces.']
+      : ['Bing search performance can combine traffic sources; source dimensions must be preserved.', 'Observed performance does not establish that an intervention caused a change.'];
+    if (!Object.values(metrics).some((value) => value != null)) throw new Error(`Bing ${options.dataset} row contains no recognized metrics`);
+    const data = {
+      dataset: options.dataset, date, url: row.url || row.URL || row.page || null, metrics,
+      dimensions: { query: row.query || row.Query || null, grounding_query: row.grounding_query || row['Grounding Query'] || null, source: row.source || row.Source || null, country: row.country || row.Country || null, device: row.device || row.Device || null },
+      interpretation_boundary: boundaries,
+    };
+    const check = validateAgainst('bing-webmaster.schema.json', data);
+    if (!check.valid) throw new Error(`Bing observation violates contract: ${check.errors.join('; ')}`);
+    return envelope('bing_webmaster', data, { method: 'owner_import', source: path.resolve(options.input), raw: JSON.stringify(row),
+      authority: { representativeness: document.metadata?.representativeness || 'unknown' }, limitations: boundaries });
   });
-  return observationRun(root, 'observe logs', input.file, observations, { rawInputs: { server_logs: input.raw }, incomplete: observations.some((o) => !o.data.identity_verified) ? ['Some crawler identities are not IP-verified.'] : [] });
+  return observationRun(root, 'observe bing', path.resolve(options.input), observations, { rawInputs: { bing_owner_export: raw }, warnings: ['Bing AI Performance has no supported API contract captured by this release; this evidence is an owner export.'] });
 }
 
 async function observePassages(root, options) {
@@ -226,10 +284,11 @@ export async function observe(root, mode, options = {}) {
     case 'index': return observeIndex(root, options);
     case 'citations': return observeCitations(root, options);
     case 'logs': return observeLogs(root, options);
+    case 'bing': return observeBing(root, options);
     case 'passages': return observePassages(root, options);
     case 'consensus': return observeConsensus(root, options);
     case 'performance': return observePerformance(root, options);
     case 'corroboration': return observeCorroboration(root, options);
-    default: throw new Error('observe mode must be render, index, citations, logs, passages, consensus, performance, or corroboration');
+    default: throw new Error('observe mode must be render, index, citations, logs, bing, passages, consensus, performance, or corroboration');
   }
 }

package/src/observations/authority.js

@@ -0,0 +1,13 @@
+const DEFAULTS = {
+  live_api: ['provider_published', 'direct_api', 'transport_authenticated', 'unknown'],
+  browser: ['synthetic', 'browser_capture', 'checksum_protected_only', 'single_observation'],
+  owner_import: ['owner_controlled', 'owner_export', 'checksum_protected_only', 'unknown'],
+  synthetic_fetch: ['synthetic', 'synthetic_probe', 'checksum_protected_only', 'single_observation'],
+  static_analysis: ['inferred', 'static_analysis', 'checksum_protected_only', 'single_observation'],
+  human_review: ['owner_controlled', 'manual_entry', 'owner_attested', 'single_observation'],
+};
+
+export function evidenceAuthority(method, overrides = {}) {
+  const [source_authority, collection_authority, authenticity_status, representativeness] = DEFAULTS[method] || DEFAULTS.static_analysis;
+  return { source_authority, collection_authority, authenticity_status, representativeness, ...overrides };
+}

package/src/observations/common.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { createRun } from '../evidence/run.js';
 import { nowIso, readJson, sha256 } from '../shared/io.js';
 import { validateAgainst } from '../shared/schemaValidator.js';
+import { evidenceAuthority } from './authority.js';
 
 export function readInput(input) {
   if (!input) throw new Error('--input <json> is required for this collector');
@@ -10,12 +11,13 @@ export function readInput(input) {
   return { raw: fs.readFileSync(input, 'utf8'), value: readJson(input), file: path.resolve(input) };
 }
 
-export function envelope(kind, data, { method, source, state = 'observed', confidence = 'confirmed', limitations = [], raw } = {}) {
+export function envelope(kind, data, { method, source, state = 'observed', confidence = 'confirmed', limitations = [], authority, raw } = {}) {
   const evidence = raw ?? JSON.stringify(data);
   const item = {
     observation_id: `OBS-${kind.toUpperCase()}-${sha256(evidence).slice(0, 16)}`,
     kind, state, collected_at: nowIso(), collection_method: method,
     confidence, source, evidence_hash: sha256(evidence), data, limitations,
+    authority: evidenceAuthority(method, authority),
   };
   const check = validateAgainst('observation.schema.json', item);
   if (!check.valid) throw new Error(`observation violates contract: ${check.errors.join('; ')}`);

package/src/observations/crawlerIdentity.js

@@ -0,0 +1,60 @@
+import net from 'node:net';
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+function cidrMembership(sourceIp, cidrs) {
+  if (!sourceIp) return 'not_tested';
+  const family = net.isIP(sourceIp);
+  if (!family) return 'invalid';
+  const block = new net.BlockList();
+  for (const cidr of cidrs || []) {
+    const [network, prefixText] = String(cidr).split('/');
+    const prefix = Number(prefixText), rangeFamily = net.isIP(network);
+    if (rangeFamily === family && Number.isInteger(prefix)) block.addSubnet(network, prefix, family === 4 ? 'ipv4' : 'ipv6');
+  }
+  return block.check(sourceIp, family === 4 ? 'ipv4' : 'ipv6') ? 'matched' : 'not_matched';
+}
+
+export function crawlerIdentity(row, { ranges = {}, rangeSources = {}, collector = 'owner_import' } = {}) {
+  const ua = row.user_agent, cidrs = ranges[ua] || [], source = rangeSources[ua] || {};
+  const sourceSnapshot = source.source_snapshot;
+  const expectedChecksum = sourceSnapshot == null ? null : sha256(typeof sourceSnapshot === 'string' ? sourceSnapshot : JSON.stringify(sourceSnapshot));
+  const rangeSource = {
+    url: source.url || null,
+    retrieved_at: source.retrieved_at || null,
+    authority: source.authority || 'unknown',
+    checksum: source.checksum || null,
+    checksum_scope: sourceSnapshot == null ? 'not_captured' : 'source_snapshot',
+    checksum_valid: Boolean(source.checksum && expectedChecksum && source.checksum === expectedChecksum),
+  };
+  const membership = cidrMembership(row.source_ip, cidrs);
+  const dns = { status: row.dns_status || 'not_tested', method: row.dns_method || null };
+  const edge = { observed: row.edge_observed === true, decision: row.edge_decision || null };
+  const origin = { observed: row.origin_observed === true, status: row.origin_status ?? null };
+  const stages = ['declared'];
+  if (row.synthetic === true) stages.push('synthetically_observed');
+  if (membership === 'matched') stages.push('range_matched');
+  if (dns.status === 'verified') stages.push('dns_verified');
+  if (edge.observed) stages.push('edge_observed');
+  if (origin.observed) stages.push('origin_observed');
+  const contradictory = membership === 'not_matched' || dns.status === 'failed' || (row.claimed_verified === true && (!rangeSource.checksum_valid || membership !== 'matched'));
+  const complete = identityProvider(row, source) !== 'unknown' && row.synthetic !== true && rangeSource.url && rangeSource.retrieved_at && rangeSource.authority === 'provider_published' && rangeSource.checksum_scope === 'source_snapshot' && rangeSource.checksum_valid && membership === 'matched' && dns.status === 'verified' && edge.observed && origin.observed;
+  if (contradictory) stages.push('contradictory');
+  else if (complete) stages.push('fully_verified');
+  else stages.push('insufficient_evidence');
+  const precedence = ['contradictory', 'fully_verified', 'origin_observed', 'edge_observed', 'dns_verified', 'range_matched', 'synthetically_observed', 'declared'];
+  const verificationStatus = precedence.find((state) => stages.includes(state));
+  const identity = {
+    provider: identityProvider(row, source), declared_user_agent: ua,
+    observed_source_ip: row.source_ip || null, range_source: rangeSource,
+    cidr_membership: membership, dns, edge, origin, region: row.region || null,
+    collector, stages, verification_status: verificationStatus,
+  };
+  const check = validateAgainst('crawler-identity.schema.json', identity);
+  if (!check.valid) throw new Error(`crawler identity violates contract: ${check.errors.join('; ')}`);
+  return identity;
+}
+
+function identityProvider(row, source) {
+  return row.provider || source.provider || 'unknown';
+}