@nebulacomponents/citable 1.20.0 → 1.21.0

package/src/evidence/external.js

@@ -0,0 +1,251 @@
+import { canonicalEvidenceJson, createEvidenceHashes } from './hashes.js';
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+const OBSERVATION_SCHEMA = 'external-observation.schema.json';
+const DATASET_SCHEMA = 'research-dataset.schema.json';
+const DERIVATION_SCHEMA = 'derivation.schema.json';
+const CLAIM_SCHEMA = 'claim-evidence.schema.json';
+const COMPARISON_SCHEMA = 'external-comparison.schema.json';
+const VERIFICATION_SCHEMA = 'claim-verification.schema.json';
+
+function without(value, key) {
+  const copy = { ...value };
+  delete copy[key];
+  return copy;
+}
+
+function integrity(value, key) {
+  return sha256(canonicalEvidenceJson(without(value, key)));
+}
+
+function validate(schema, value, label) {
+  const result = validateAgainst(schema, value);
+  if (!result.valid) throw new Error(`${label} violates contract: ${result.errors.join('; ')}`);
+  return value;
+}
+
+export function createExternalObservation(input) {
+  const item = { ...input };
+  item.evidence_hash = integrity(item, 'evidence_hash');
+  return validate(OBSERVATION_SCHEMA, item, 'external observation');
+}
+
+export function verifyExternalObservation(item) {
+  const failures = [];
+  const check = validateAgainst(OBSERVATION_SCHEMA, item);
+  if (!check.valid) failures.push(...check.errors);
+  if (item?.evidence_hash !== integrity(item || {}, 'evidence_hash')) failures.push('evidence hash is inconsistent');
+  return { valid: failures.length === 0, failures };
+}
+
+export function createResearchDataset(input, { previous = null } = {}) {
+  const item = { ...input };
+  item.dataset_hash = integrity(item, 'dataset_hash');
+  validate(DATASET_SCHEMA, item, 'research dataset');
+  if (previous && previous.dataset_id === item.dataset_id && previous.dataset_version === item.dataset_version
+    && previous.dataset_hash !== item.dataset_hash) {
+    throw new Error('research dataset version is immutable; create a new dataset_version');
+  }
+  return item;
+}
+
+export function verifyResearchDataset(item, { previous = null } = {}) {
+  const failures = [];
+  const check = validateAgainst(DATASET_SCHEMA, item);
+  if (!check.valid) failures.push(...check.errors);
+  if (item?.dataset_hash !== integrity(item || {}, 'dataset_hash')) failures.push('dataset hash is inconsistent');
+  if (previous && previous.dataset_id === item?.dataset_id && previous.dataset_version === item?.dataset_version
+    && previous.dataset_hash !== item?.dataset_hash) failures.push('research dataset version is immutable');
+  if (failures.length) throw new Error(`research dataset verification failed: ${failures.join('; ')}`);
+  return { valid: true, failures: [] };
+}
+
+export function createDerivation(input) {
+  const item = { ...input };
+  item.derivation_hash = integrity(item, 'derivation_hash');
+  return validate(DERIVATION_SCHEMA, item, 'derivation');
+}
+
+function verifyDerivation(item) {
+  const check = validateAgainst(DERIVATION_SCHEMA, item);
+  if (!check.valid || item.derivation_hash !== integrity(item, 'derivation_hash')) return false;
+  return true;
+}
+
+export function createEvidenceClaim(input) {
+  const item = { ...input };
+  item.claim_hash = integrity(item, 'claim_hash');
+  return validate(CLAIM_SCHEMA, item, 'claim evidence');
+}
+
+function verifyClaimIdentity(item) {
+  const check = validateAgainst(CLAIM_SCHEMA, item);
+  return check.valid && item.claim_hash === integrity(item, 'claim_hash');
+}
+
+function valueAt(object, path) {
+  return String(path).split('.').reduce((value, key) => value == null ? undefined : value[key], object);
+}
+
+function reason(code, message) {
+  return { code, message };
+}
+
+function evaluatePredicate(assertion, observations) {
+  if (!observations.length) return { status: 'NOT_OBSERVED', reasons: [reason('observation_missing', 'No referenced observation was available.')] };
+  const results = observations.map((item) => valueAt(item, assertion.path) === assertion.value);
+  if (results.every(Boolean)) return { status: 'SUPPORTED', reasons: [] };
+  if (results.some(Boolean)) return { status: 'PARTIALLY_SUPPORTED', reasons: [reason('predicate_partial', 'The predicate matched only part of the referenced population.')] };
+  return { status: 'CONTRADICTED', reasons: [reason('predicate_contradicted', 'In-scope evidence establishes an incompatible predicate value.')] };
+}
+
+function evaluateAssertion(assertion, observations, derivations) {
+  if (!assertion || typeof assertion !== 'object') return { status: 'INDETERMINATE', reasons: [reason('assertion_missing', 'The claim has no structured assertion.')] };
+  if (assertion.type === 'predicate' || assertion.type === 'all') return evaluatePredicate(assertion, observations);
+  if (assertion.type === 'count') {
+    const derivation = derivations[0];
+    if (!derivation || !verifyDerivation(derivation)) return { status: 'INDETERMINATE', reasons: [reason('derivation_unavailable', 'The referenced derivation is missing or unverifiable.')] };
+    if (derivation.result?.count === assertion.expected) return { status: 'SUPPORTED', reasons: [] };
+    return { status: 'CONTRADICTED', reasons: [reason('count_contradicted', 'The preserved derivation result differs from the claimed value.')] };
+  }
+  if (assertion.type === 'compound') {
+    const results = (assertion.assertions || []).map((part) => evaluateAssertion(part, observations, derivations));
+    if (!results.length) return { status: 'INDETERMINATE', reasons: [reason('compound_empty', 'The compound claim has no components.')] };
+    const reasons = results.flatMap((item) => item.reasons);
+    if (results.every((item) => item.status === 'SUPPORTED')) return { status: 'SUPPORTED', reasons };
+    if (results.some((item) => item.status === 'CONTRADICTED')) return { status: 'PARTIALLY_SUPPORTED', reasons: [...reasons, reason('compound_component_unsupported', 'At least one compound component is contradicted.')] };
+    if (results.some((item) => item.status === 'SUPPORTED')) return { status: 'PARTIALLY_SUPPORTED', reasons: [...reasons, reason('compound_component_unsupported', 'At least one compound component is unsupported.')] };
+    return { status: results.some((item) => item.status === 'NOT_OBSERVED') ? 'NOT_OBSERVED' : 'UNSUPPORTED', reasons };
+  }
+  return { status: 'INDETERMINATE', reasons: [reason('assertion_unsupported', `Assertion type ${assertion.type || 'unknown'} is not supported.`)] };
+}
+
+function scopeSupport(claim, datasets, observations) {
+  const scope = claim.declared_scope;
+  if (scope.generalization === 'universal') return { status: 'UNSUPPORTED', reasons: [reason('generalization_outside_population', 'The evidence population does not support a universal claim.')] };
+  if (scope.type === 'observation' && scope.observation_reference
+    && !(claim.observation_references || []).includes(scope.observation_reference)) {
+    return { status: 'INDETERMINATE', reasons: [reason('scope_reference_mismatch', 'The declared observation scope is not among the claim evidence references.')] };
+  }
+  if (scope.type === 'dataset' && scope.dataset_reference
+    && !(claim.dataset_references || []).includes(scope.dataset_reference)) {
+    return { status: 'INDETERMINATE', reasons: [reason('scope_reference_mismatch', 'The declared dataset scope is not among the claim evidence references.')] };
+  }
+  if (scope.type === 'dataset') {
+    const dataset = datasets[0];
+    if (!dataset || !verifyDatasetSafe(dataset)) return { status: 'INDETERMINATE', reasons: [reason('dataset_unavailable', 'The declared dataset is missing or unverifiable.')] };
+    if (dataset.coverage_status !== 'complete_for_declared_population') return { status: 'PARTIALLY_SUPPORTED', reasons: [reason('dataset_coverage_incomplete', `Dataset coverage is ${dataset.coverage_status}, not complete for the declared population.`)] };
+    const referenced = new Set(claim.observation_references || []);
+    const missingMembers = (dataset.observation_references || []).filter((id) => !referenced.has(id));
+    if (missingMembers.length) return { status: 'PARTIALLY_SUPPORTED', reasons: [reason('dataset_members_missing', `The claim does not reference every observation in the declared dataset: ${missingMembers.join(', ')}`)] };
+  }
+  if (!observations.length) return { status: 'NOT_OBSERVED', reasons: [reason('observation_missing', 'The declared scope has no observed members.')] };
+  return { status: 'SUPPORTED', reasons: [] };
+}
+
+function verifyDatasetSafe(dataset) {
+  try { return verifyResearchDataset(dataset).valid; } catch { return false; }
+}
+
+function datasetReference(dataset) {
+  return `${dataset.dataset_id}@${dataset.dataset_version}`;
+}
+
+function resolveDatasetReference(reference, datasets) {
+  const exact = datasets.filter((item) => datasetReference(item) === reference);
+  if (exact.length === 1) return exact[0];
+  // Legacy id-only references remain valid only while the id has one version.
+  // Once versions diverge, silently selecting one would rewrite historical
+  // claim meaning.
+  if (!String(reference).includes('@')) {
+    const byId = datasets.filter((item) => item.dataset_id === reference);
+    return byId.length === 1 ? byId[0] : null;
+  }
+  return null;
+}
+
+function combine(proposition, scope) {
+  if (proposition.status === 'NOT_OBSERVED') return 'NOT_OBSERVED';
+  if (proposition.status === 'INDETERMINATE' || scope.status === 'INDETERMINATE') return 'INDETERMINATE';
+  if (proposition.status === 'CONTRADICTED') return 'CONTRADICTED';
+  if (proposition.status === 'PARTIALLY_SUPPORTED' || scope.status === 'PARTIALLY_SUPPORTED' || scope.status === 'UNSUPPORTED') return 'PARTIALLY_SUPPORTED';
+  if (proposition.status === 'UNSUPPORTED') return 'UNSUPPORTED';
+  return 'SUPPORTED';
+}
+
+export function verifyClaim(claim, { observations = [], datasets = [], derivations = [] } = {}) {
+  if (!verifyClaimIdentity(claim)) {
+    return {
+      claim_id: claim?.claim_id || 'unknown', claim_version: claim?.claim_version || 1,
+      overall: 'INDETERMINATE', proposition_support: 'INDETERMINATE', scope_support: 'INDETERMINATE',
+      reasons: [reason('claim_unverifiable', 'Claim identity, schema, or integrity hash could not be verified.')],
+      lineage: { claim_reference: claim?.claim_id || 'unknown', observation_references: claim?.observation_references || [], dataset_references: claim?.dataset_references || [], derivation_references: claim?.derivation_references || [] }, limitations: ['Claim evidence could not be verified.'],
+    };
+  }
+  const byId = new Map(observations.map((item) => [item.observation_id, item]));
+  const referenced = (claim.observation_references || []).map((id) => byId.get(id)).filter(Boolean);
+  const missing = (claim.observation_references || []).filter((id) => !byId.has(id));
+  const integrityFailure = referenced.some((item) => !verifyExternalObservation(item).valid);
+  let proposition;
+  if (integrityFailure) proposition = { status: 'INDETERMINATE', reasons: [reason('evidence_unverifiable', 'Referenced observation integrity verification failed.')] };
+  else if (missing.length) proposition = { status: 'NOT_OBSERVED', reasons: [reason('observation_missing', `Missing referenced observations: ${missing.join(', ')}`)] };
+  else proposition = evaluateAssertion(claim.assertion, referenced, (claim.derivation_references || []).map((id) => derivations.find((item) => item.derivation_id === id)).filter(Boolean));
+  const selectedDatasets = (claim.dataset_references || []).map((reference) => resolveDatasetReference(reference, datasets)).filter(Boolean);
+  const missingDatasets = (claim.dataset_references || []).filter((reference) => !resolveDatasetReference(reference, datasets));
+  const scope = scopeSupport(claim, selectedDatasets, referenced);
+  if (missingDatasets.length && claim.declared_scope.type === 'dataset') {
+    scope.status = 'INDETERMINATE';
+    scope.reasons.push(reason('dataset_reference_ambiguous_or_missing', `Dataset references could not be resolved exactly: ${missingDatasets.join(', ')}`));
+  }
+  const result = {
+    claim_id: claim.claim_id, claim_version: claim.claim_version, overall: combine(proposition, scope),
+    proposition_support: proposition.status, scope_support: scope.status,
+    reasons: [...proposition.reasons, ...scope.reasons],
+    lineage: { claim_reference: claim.claim_id, observation_references: claim.observation_references || [], dataset_references: claim.dataset_references || [], derivation_references: claim.derivation_references || [] },
+    limitations: claim.limitations || [],
+  };
+  return validate(VERIFICATION_SCHEMA, result, 'claim verification');
+}
+
+const COMPARISON_DIMENSIONS = ['source', 'subject', 'collector', 'parser', 'configuration', 'population', 'normalization', 'dataset_version', 'derivation_version'];
+
+function dimensionValue(item, dimension) {
+  if (dimension === 'source') return item.source?.provider;
+  if (dimension === 'subject') return item.subject?.id;
+  if (dimension === 'collector') return `${item.collector?.id || ''}@${item.collector?.version || ''}` || null;
+  if (dimension === 'parser') return item.parser ? `${item.parser.id}@${item.parser.version}` : null;
+  if (dimension === 'configuration') return item.configuration ? `${item.configuration.id}@${item.configuration.version}` : null;
+  if (dimension === 'population') return item.population?.declared?.id || item.declared_population?.id;
+  if (dimension === 'normalization') return item.normalization ? `${item.normalization.method}@${item.normalization.version}` : null;
+  if (dimension === 'dataset_version') return item.dataset_id ? `${item.dataset_id}@${item.dataset_version}` : null;
+  if (dimension === 'derivation_version') return item.derivation_id ? `${item.derivation_id}@${item.derivation_version}` : null;
+  return null;
+}
+
+function compareItems(left, right, leftReference, rightReference, dimensionsToCompare = COMPARISON_DIMENSIONS) {
+  const dimensions = {};
+  for (const key of dimensionsToCompare) {
+    const a = dimensionValue(left, key); const b = dimensionValue(right, key);
+    dimensions[key] = a == null || b == null ? 'missing' : a === b ? 'match' : 'mismatch';
+  }
+  const mismatches = Object.values(dimensions).filter((x) => x === 'mismatch').length;
+  const missing = Object.values(dimensions).filter((x) => x === 'missing').length;
+  const status = mismatches ? 'NOT_COMPARABLE' : missing ? 'PARTIALLY_COMPARABLE' : 'COMPARABLE';
+  const result = { comparison_id: `CMP-${sha256(canonicalEvidenceJson({ leftReference, rightReference, dimensions })).slice(0, 16)}`, left_reference: leftReference, right_reference: rightReference, status, dimensions, limitations: status === 'COMPARABLE' ? [] : ['No delta should be generalized beyond the aligned dimensions.'] };
+  return validate(COMPARISON_SCHEMA, result, 'external comparison');
+}
+
+export function compareExternalObservations(left, right) {
+  if (!verifyExternalObservation(left).valid || !verifyExternalObservation(right).valid) {
+    return { comparison_id: `CMP-${sha256(`${left?.observation_id || ''}:${right?.observation_id || ''}`).slice(0, 16)}`, left_reference: left?.observation_id || 'unknown', right_reference: right?.observation_id || 'unknown', status: 'INDETERMINATE', dimensions: {}, limitations: ['Observation integrity could not be verified.'] };
+  }
+  return compareItems(left, right, left.observation_id, right.observation_id,
+    ['source', 'subject', 'collector', 'parser', 'configuration', 'population']);
+}
+
+export function compareResearchDatasets(left, right) {
+  if (!verifyDatasetSafe(left) || !verifyDatasetSafe(right)) return { comparison_id: 'CMP-invalid', left_reference: left?.dataset_id || 'unknown', right_reference: right?.dataset_id || 'unknown', status: 'INDETERMINATE', dimensions: {}, limitations: ['Dataset integrity could not be verified.'] };
+  return compareItems(left, right, `${left.dataset_id}@${left.dataset_version}`, `${right.dataset_id}@${right.dataset_version}`,
+    ['dataset_version', 'normalization', 'population']);
+}

package/src/representation/prompts.js

@@ -0,0 +1,117 @@
+import { canonicalEvidenceJson } from '../evidence/hashes.js';
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+const PROMPT_SCHEMA = 'prompt-set.schema.json';
+
+function without(value, key) {
+  const copy = { ...value };
+  delete copy[key];
+  return copy;
+}
+
+function promptId({ prompt_text, context = null, locale = null }) {
+  return `PROMPT-${sha256(canonicalEvidenceJson({ prompt_text, context, locale })).slice(0, 24)}`;
+}
+
+export function createPrompt(input) {
+  if (!input || typeof input.prompt_text !== 'string' || !input.prompt_text.trim()) throw new TypeError('prompt_text is required');
+  const prompt = {
+    prompt_id: input.prompt_id || promptId(input),
+    prompt_text: input.prompt_text,
+    ...(input.context == null ? {} : { context: input.context }),
+    ...(input.locale == null ? {} : { locale: input.locale }),
+  };
+  return prompt;
+}
+
+function normalizeIncluded(prompts = []) {
+  if (!Array.isArray(prompts)) throw new TypeError('included_prompts must be an array');
+  const normalized = prompts.map(createPrompt);
+  const ids = new Set();
+  for (const item of normalized) {
+    if (ids.has(item.prompt_id)) throw new Error(`duplicate prompt_id: ${item.prompt_id}`);
+    ids.add(item.prompt_id);
+  }
+  return normalized.sort((left, right) => left.prompt_id.localeCompare(right.prompt_id));
+}
+
+function normalizeExcluded(prompts = []) {
+  if (!Array.isArray(prompts)) throw new TypeError('excluded_prompts must be an array');
+  return prompts.map((item) => ({
+    prompt_id: item.prompt_id || promptId(item),
+    ...(item.prompt_text == null ? {} : { prompt_text: item.prompt_text }),
+    exclusion_reason: item.exclusion_reason,
+  })).sort((left, right) => left.prompt_id.localeCompare(right.prompt_id));
+}
+
+function hashPromptSet(item) {
+  const value = without(item, 'prompt_set_hash');
+  return sha256(canonicalEvidenceJson({
+    ...value,
+    included_prompts: [...(value.included_prompts || [])].sort((left, right) => left.prompt_id.localeCompare(right.prompt_id)),
+    excluded_prompts: [...(value.excluded_prompts || [])].sort((left, right) => left.prompt_id.localeCompare(right.prompt_id)),
+  }));
+}
+
+export function createPromptSet(input, { previous = null } = {}) {
+  if (!input || typeof input.prompt_set_id !== 'string' || !input.prompt_set_id) throw new TypeError('prompt_set_id is required');
+  if (!Number.isInteger(input.prompt_set_version) || input.prompt_set_version < 1) throw new TypeError('prompt_set_version must be a positive integer');
+  const included = normalizeIncluded(input.included_prompts);
+  const excluded = normalizeExcluded(input.excluded_prompts || []);
+  const item = {
+    prompt_set_id: input.prompt_set_id,
+    prompt_set_version: input.prompt_set_version,
+    purpose: input.purpose,
+    generation_method: input.generation_method,
+    generation_method_version: input.generation_method_version ?? null,
+    generation_input: input.generation_input ?? null,
+    generated_at: input.generated_at,
+    population_size: included.length,
+    sampling_method: input.sampling_method,
+    included_prompts: included,
+    excluded_prompts: excluded,
+    intended_population: input.intended_population,
+    limitations: [...new Set(input.limitations || [])],
+  };
+  item.prompt_set_hash = hashPromptSet(item);
+  const check = validateAgainst(PROMPT_SCHEMA, item);
+  if (!check.valid) throw new TypeError(`prompt set violates contract: ${check.errors.join('; ')}`);
+  if (previous && previous.prompt_set_id === item.prompt_set_id && previous.prompt_set_version === item.prompt_set_version && previous.prompt_set_hash !== item.prompt_set_hash) {
+    throw new Error('prompt set version is immutable; create a new prompt_set_version');
+  }
+  return item;
+}
+
+export function verifyPromptSet(item, { previous = null } = {}) {
+  const failures = [];
+  const check = validateAgainst(PROMPT_SCHEMA, item);
+  if (!check.valid) failures.push(...check.errors);
+  if (item?.prompt_set_hash !== hashPromptSet(item || {})) failures.push('prompt set hash is inconsistent');
+  if (previous && previous.prompt_set_id === item?.prompt_set_id && previous.prompt_set_version === item?.prompt_set_version && previous.prompt_set_hash !== item?.prompt_set_hash) failures.push('prompt set version is immutable');
+  return { valid: failures.length === 0, failures };
+}
+
+export function promptReference(promptSet, promptIdValue) {
+  if (!verifyPromptSet(promptSet).valid) throw new Error('cannot reference an unverifiable prompt set');
+  const prompt = promptSet.included_prompts.find((item) => item.prompt_id === promptIdValue);
+  if (!prompt) throw new Error(`prompt ${promptIdValue} is not included in prompt set ${promptSet.prompt_set_id}@${promptSet.prompt_set_version}`);
+  return `${promptSet.prompt_set_id}@${promptSet.prompt_set_version}:${prompt.prompt_id}`;
+}
+
+export function comparePromptSets(left, right) {
+  const validLeft = verifyPromptSet(left).valid;
+  const validRight = verifyPromptSet(right).valid;
+  if (!validLeft || !validRight) return { status: 'INDETERMINATE', dimensions: {}, limitations: ['Prompt-set integrity could not be verified.'] };
+  const dimensions = {
+    prompt_set_id: left.prompt_set_id === right.prompt_set_id ? 'match' : 'mismatch',
+    prompt_set_version: left.prompt_set_version === right.prompt_set_version ? 'match' : 'mismatch',
+    population: left.population_size === right.population_size ? 'match' : 'mismatch',
+    sampling_method: left.sampling_method === right.sampling_method ? 'match' : 'mismatch',
+  };
+  const status = dimensions.prompt_set_id === 'match' && dimensions.prompt_set_version === 'match' && dimensions.population === 'match' && dimensions.sampling_method === 'match'
+    ? 'COMPARABLE' : 'NOT_COMPARABLE';
+  return { status, left_reference: `${left.prompt_set_id}@${left.prompt_set_version}`, right_reference: `${right.prompt_set_id}@${right.prompt_set_version}`, dimensions, limitations: status === 'COMPARABLE' ? [] : ['Prompt population, sampling, or version changed; no denominator is reused silently.'] };
+}
+
+export { promptId };

package/src/representation/representation.js

@@ -0,0 +1,186 @@
+import { createExternalObservation, verifyExternalObservation } from '../evidence/external.js';
+import { canonicalEvidenceJson } from '../evidence/hashes.js';
+import { sha256, nowIso } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+import { verifyPromptSet, promptReference } from './prompts.js';
+
+const REPRESENTATION_SCHEMA = 'representation-data.schema.json';
+const COLLECTOR = { id: 'citable-representation-observations', version: '1.0.0' };
+const PARSER = { id: 'provider-neutral-representation-parser', version: '1.0.0' };
+
+function responseHash(rawResponse, response) {
+  return sha256(typeof rawResponse === 'string' ? rawResponse : canonicalEvidenceJson(response ?? null));
+}
+
+function modelIdentity(model) {
+  if (model && typeof model === 'string') return { value: model, status: 'known' };
+  if (model && typeof model === 'object' && ['known', 'UNKNOWN', 'UNAVAILABLE', 'NOT_DISCLOSED'].includes(model.status)) return { value: model.value ?? null, status: model.status };
+  return { value: null, status: 'UNKNOWN' };
+}
+
+function hostOf(value) {
+  try { return new URL(value).hostname.toLowerCase(); } catch { return null; }
+}
+
+function normalizeUrls(value) {
+  if (!Array.isArray(value)) throw new TypeError('provider citations must be an array');
+  const urls = value.map((item) => typeof item === 'string' ? item : item?.url);
+  if (urls.some((item) => typeof item !== 'string' || !hostOf(item))) throw new TypeError('provider citation URL is malformed');
+  return [...new Set(urls)];
+}
+
+const SENSITIVE_KEY = /(^|[-_])(authorization|cookie|set-cookie|access[-_]?token|refresh[-_]?token|api[-_]?key|password|secret)($|[-_])/i;
+
+function safeMetadata(value, depth = 0) {
+  if (depth > 4 || value == null) return value == null ? null : '[bounded]';
+  if (Array.isArray(value)) return value.slice(0, 50).map((item) => safeMetadata(item, depth + 1));
+  if (typeof value !== 'object') return typeof value === 'string' ? value.slice(0, 500) : value;
+  return Object.fromEntries(Object.entries(value).slice(0, 50).filter(([key]) => !SENSITIVE_KEY.test(key)).map(([key, item]) => [key, safeMetadata(item, depth + 1)]));
+}
+
+function executionStatusFor(status) {
+  if (status === 'succeeded') return { observation_status: 'observed', retrieval_status: 'retrieved' };
+  if (status === 'malformed' || status === 'parser_failed') return { observation_status: 'indeterminate', retrieval_status: 'indeterminate' };
+  return { observation_status: 'failed', retrieval_status: 'failed' };
+}
+
+function parserErrorObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt, rawResponse, status, errorMessage }) {
+  const hash = responseHash(rawResponse, null);
+  const execution = executionConfiguration || { id: 'default', version: '1.0.0' };
+  const state = executionStatusFor(status);
+  return createRepresentationObservation({
+    provider, product, surface, prompt, promptSet, executionConfiguration: execution, model, evidenceOrigin, executedAt,
+    response: { status, response_hash: hash, parser: PARSER, raw_response_retained: false, provider_metadata: {} },
+    representation: { brand_mentioned: null, domain_mentioned: null, domain_cited: null, cited_urls: [], mentioned_entities: [], cited_domains: [] },
+    observationStatus: state.observation_status, retrievalStatus: state.retrieval_status,
+    limitations: [`Representation response could not be normalized: ${String(errorMessage || status).slice(0, 240)}`, 'Raw provider response is not retained; response_hash is retained for identity.'],
+  });
+}
+
+export function createRepresentationObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt = nowIso(), response, representation, observationStatus = 'observed', retrievalStatus = 'retrieved', limitations = [] }) {
+  if (!provider || !product || !surface) throw new TypeError('provider, product, and surface are required');
+  if (!prompt || !prompt.prompt_id || !prompt.prompt_text) throw new TypeError('prompt identity and text are required');
+  if (!promptSet || !verifyPromptSet(promptSet).valid) throw new TypeError('prompt set is missing or unverifiable');
+  if (!promptSet.included_prompts.some((item) => item.prompt_id === prompt.prompt_id)) throw new Error(`prompt ${prompt.prompt_id} is not included in the prompt set`);
+  if (!['LIVE_PROVIDER', 'RECORDED_FIXTURE', 'SYNTHETIC_TEST'].includes(evidenceOrigin)) throw new TypeError('evidenceOrigin is invalid');
+  const config = executionConfiguration || { id: 'default', version: '1.0.0' };
+  const normalizedModel = modelIdentity(model);
+  const data = {
+    provider,
+    product,
+    surface,
+    model: normalizedModel,
+    prompt: { prompt_id: prompt.prompt_id, prompt_version: promptSet.prompt_set_version, prompt_text: prompt.prompt_text },
+    prompt_set: { prompt_set_id: promptSet.prompt_set_id, prompt_set_version: promptSet.prompt_set_version },
+    execution: { executed_at: executedAt, configuration: config },
+    response,
+    representation,
+    evidence_origin: evidenceOrigin,
+    limitations: [...new Set(limitations)],
+  };
+  const check = validateAgainst(REPRESENTATION_SCHEMA, data);
+  if (!check.valid) throw new TypeError(`representation data violates contract: ${check.errors.join('; ')}`);
+  const state = executionStatusFor(response.status);
+  const observation = createExternalObservation({
+    observation_id: `REP-${sha256(canonicalEvidenceJson({ provider, product, surface, prompt, promptSet: promptSet.prompt_set_hash, executedAt, response })).slice(0, 24)}`,
+    observation_type: 'EXTERNAL_OBSERVATION',
+    subject: { type: 'prompt_execution', id: promptReference(promptSet, prompt.prompt_id) },
+    source: { provider, method: 'representation_adapter', authority: 'external_provider' },
+    collector: COLLECTOR,
+    parser: PARSER,
+    configuration: config,
+    observed_at: executedAt,
+    retrieved_at: executedAt,
+    executed_at: executedAt,
+    population: { declared: { id: `${promptSet.prompt_set_id}@${promptSet.prompt_set_version}`, size: promptSet.population_size }, observed: 1, retrieved: response.status === 'succeeded' ? 1 : 0, evaluated: response.status === 'succeeded' ? 1 : 0, unavailable: response.status === 'unavailable' ? 1 : 0, failed: response.status === 'succeeded' ? 0 : 1 },
+    observation_status: observationStatus || state.observation_status,
+    retrieval_status: retrievalStatus || state.retrieval_status,
+    data,
+    lineage: { source_evidence: response.response_hash ? [response.response_hash] : [] },
+    limitations: data.limitations,
+  });
+  return observation;
+}
+
+export function normalizeRepresentationResponse({ provider, product, surface, prompt, promptSet, executionConfiguration, model, response, rawResponse = null, evidenceOrigin = 'RECORDED_FIXTURE', executedAt = nowIso(), brand = null, firstPartyDomains = [], retainRawResponse = false }) {
+  const base = { provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt };
+  const rawHash = responseHash(rawResponse, response);
+  try {
+    if (!response || typeof response !== 'object' || Array.isArray(response)) throw new TypeError('provider response must be an object');
+    const answerText = response.answer_text ?? response.answer ?? response.text;
+    if (typeof answerText !== 'string') throw new TypeError('provider response requires answer_text');
+    const citations = normalizeUrls(response.citations || []);
+    const mentionedEntities = Array.isArray(response.mentions) ? response.mentions.filter((item) => typeof item === 'string') : [];
+    const firstParty = new Set(firstPartyDomains.map((item) => String(item).toLowerCase()));
+    const citedDomains = [...new Set(citations.map(hostOf).filter(Boolean))];
+    const domainMentioned = firstParty.size ? [...firstParty].some((domain) => answerText.toLowerCase().includes(domain)) : null;
+    const domainCited = firstParty.size ? citedDomains.some((domain) => firstParty.has(domain)) : null;
+    const brandMentioned = typeof brand === 'string' ? answerText.toLowerCase().includes(brand.toLowerCase()) : null;
+    return createRepresentationObservation({
+      ...base,
+      response: { status: 'succeeded', response_hash: rawHash, parser: PARSER, raw_response_retained: retainRawResponse, provider_metadata: { ...safeMetadata(response.provider_metadata || {}), ...(retainRawResponse ? { raw_response: rawResponse ?? response } : {}) } },
+      representation: { brand_mentioned: brandMentioned, domain_mentioned: domainMentioned, domain_cited: domainCited, cited_urls: citations, mentioned_entities: mentionedEntities, cited_domains: citedDomains },
+      limitations: retainRawResponse ? [] : ['Raw provider response is not retained; response_hash is retained for identity.'],
+    });
+  } catch (error) {
+    return parserErrorObservation({ ...base, rawResponse, status: 'parser_failed', errorMessage: error.message });
+  }
+}
+
+export async function collectRepresentationObservations({ promptSet, adapter, provider, product = 'unknown', surface = 'unknown', model = null, executionConfiguration = { id: 'default', version: '1.0.0' }, evidenceOrigin = 'LIVE_PROVIDER', brand = null, firstPartyDomains = [] } = {}) {
+  if (!promptSet || !verifyPromptSet(promptSet).valid) throw new TypeError('prompt set is missing or unverifiable');
+  if (typeof adapter !== 'function') throw new TypeError('representation adapter is required');
+  const observations = [];
+  for (const prompt of promptSet.included_prompts) {
+    const executedAt = nowIso();
+    try {
+      const result = await adapter(prompt, { provider, product, surface, model, promptSet, executionConfiguration });
+      if (!result || typeof result !== 'object') throw new Error('adapter returned an invalid result');
+      const status = result.status || 'succeeded';
+      if (status !== 'succeeded') {
+        observations.push(parserErrorObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt, rawResponse: result.raw ?? result.body ?? null, status, errorMessage: result.error || status }));
+      } else {
+        observations.push(normalizeRepresentationResponse({ provider, product, surface, prompt, promptSet, executionConfiguration, model, response: result.body ?? result.response ?? result, rawResponse: result.raw ?? null, evidenceOrigin, executedAt, brand, firstPartyDomains }));
+      }
+    } catch (error) {
+      observations.push(parserErrorObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt, rawResponse: null, status: error.connectorState === 'quota_limited' ? 'rate_limited' : error.name === 'AbortError' ? 'timeout' : 'error', errorMessage: error.message }));
+    }
+  }
+  return { observations, coverage: { declared: promptSet.population_size, executed: observations.filter((item) => item.data.response.status === 'succeeded').length, failed: observations.filter((item) => item.data.response.status !== 'succeeded').length, coverage_status: observations.every((item) => item.data.response.status === 'succeeded') ? 'complete_for_declared_prompt_set' : 'incomplete' }, prompt_set_reference: `${promptSet.prompt_set_id}@${promptSet.prompt_set_version}` };
+}
+
+export function createHttpRepresentationAdapter({ endpoint, fetchImpl = globalThis.fetch, accessToken = null } = {}) {
+  if (typeof endpoint !== 'string' || !/^https:\/\//.test(endpoint)) throw new TypeError('representation adapter endpoint must use HTTPS');
+  return async (prompt) => {
+    const response = await fetchImpl(endpoint, { method: 'POST', redirect: 'error', headers: { accept: 'application/json', 'content-type': 'application/json', ...(accessToken ? { authorization: `Bearer ${accessToken}` } : {}) }, body: JSON.stringify({ prompt_id: prompt.prompt_id, prompt_text: prompt.prompt_text }) });
+    const raw = await response.text();
+    if (!response.ok) {
+      const status = response.status === 429 ? 'rate_limited' : response.status === 408 ? 'timeout' : response.status >= 500 ? 'unavailable' : 'refused';
+      return { status, raw, error: `provider returned ${response.status}` };
+    }
+    let body;
+    try { body = JSON.parse(raw); } catch { return { status: 'malformed', raw, error: 'provider response was not valid JSON' }; }
+    return { status: 'succeeded', body, raw };
+  };
+}
+
+export function verifyRepresentationObservation(item) {
+  return verifyExternalObservation(item);
+}
+
+export function compareRepresentationObservations(left, right) {
+  if (!verifyExternalObservation(left).valid || !verifyExternalObservation(right).valid) return { status: 'INDETERMINATE', dimensions: {}, limitations: ['Representation observation integrity could not be verified.'] };
+  const dimensions = {
+    provider: left.data.provider === right.data.provider ? 'match' : 'mismatch',
+    product: left.data.product === right.data.product ? 'match' : 'mismatch',
+    surface: left.data.surface === right.data.surface ? 'match' : 'mismatch',
+    model: left.data.model.status !== 'known' || right.data.model.status !== 'known' ? 'missing' : left.data.model.value === right.data.model.value ? 'match' : 'mismatch',
+    prompt: left.data.prompt.prompt_id === right.data.prompt.prompt_id && left.data.prompt.prompt_text === right.data.prompt.prompt_text ? 'match' : 'mismatch',
+    prompt_set: left.data.prompt_set.prompt_set_id === right.data.prompt_set.prompt_set_id && left.data.prompt_set.prompt_set_version === right.data.prompt_set.prompt_set_version ? 'match' : 'mismatch',
+    configuration: left.data.execution.configuration.id === right.data.execution.configuration.id && left.data.execution.configuration.version === right.data.execution.configuration.version ? 'match' : 'mismatch',
+    parser: left.data.response.parser.id === right.data.response.parser.id && left.data.response.parser.version === right.data.response.parser.version ? 'match' : 'mismatch',
+  };
+  const hardMismatch = ['provider', 'product', 'surface', 'prompt', 'prompt_set', 'configuration', 'parser'].some((key) => dimensions[key] === 'mismatch');
+  const status = hardMismatch ? 'NOT_COMPARABLE' : dimensions.model === 'mismatch' ? 'NOT_COMPARABLE' : dimensions.model === 'missing' ? 'PARTIALLY_COMPARABLE' : 'COMPARABLE';
+  return { comparison_id: `REP-CMP-${sha256(canonicalEvidenceJson({ left: left.observation_id, right: right.observation_id, dimensions })).slice(0, 16)}`, left_reference: left.observation_id, right_reference: right.observation_id, status, dimensions, limitations: status === 'COMPARABLE' ? [] : ['Provider, model, prompt, execution, and parser identity remain explicit; no visibility delta is inferred from non-comparable observations.'] };
+}