@nebulacomponents/citable 1.20.0 → 1.22.0

package/src/representation/prompts.js

@@ -0,0 +1,117 @@
+import { canonicalEvidenceJson } from '../evidence/hashes.js';
+import { sha256 } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+const PROMPT_SCHEMA = 'prompt-set.schema.json';
+
+function without(value, key) {
+  const copy = { ...value };
+  delete copy[key];
+  return copy;
+}
+
+function promptId({ prompt_text, context = null, locale = null }) {
+  return `PROMPT-${sha256(canonicalEvidenceJson({ prompt_text, context, locale })).slice(0, 24)}`;
+}
+
+export function createPrompt(input) {
+  if (!input || typeof input.prompt_text !== 'string' || !input.prompt_text.trim()) throw new TypeError('prompt_text is required');
+  const prompt = {
+    prompt_id: input.prompt_id || promptId(input),
+    prompt_text: input.prompt_text,
+    ...(input.context == null ? {} : { context: input.context }),
+    ...(input.locale == null ? {} : { locale: input.locale }),
+  };
+  return prompt;
+}
+
+function normalizeIncluded(prompts = []) {
+  if (!Array.isArray(prompts)) throw new TypeError('included_prompts must be an array');
+  const normalized = prompts.map(createPrompt);
+  const ids = new Set();
+  for (const item of normalized) {
+    if (ids.has(item.prompt_id)) throw new Error(`duplicate prompt_id: ${item.prompt_id}`);
+    ids.add(item.prompt_id);
+  }
+  return normalized.sort((left, right) => left.prompt_id.localeCompare(right.prompt_id));
+}
+
+function normalizeExcluded(prompts = []) {
+  if (!Array.isArray(prompts)) throw new TypeError('excluded_prompts must be an array');
+  return prompts.map((item) => ({
+    prompt_id: item.prompt_id || promptId(item),
+    ...(item.prompt_text == null ? {} : { prompt_text: item.prompt_text }),
+    exclusion_reason: item.exclusion_reason,
+  })).sort((left, right) => left.prompt_id.localeCompare(right.prompt_id));
+}
+
+function hashPromptSet(item) {
+  const value = without(item, 'prompt_set_hash');
+  return sha256(canonicalEvidenceJson({
+    ...value,
+    included_prompts: [...(value.included_prompts || [])].sort((left, right) => left.prompt_id.localeCompare(right.prompt_id)),
+    excluded_prompts: [...(value.excluded_prompts || [])].sort((left, right) => left.prompt_id.localeCompare(right.prompt_id)),
+  }));
+}
+
+export function createPromptSet(input, { previous = null } = {}) {
+  if (!input || typeof input.prompt_set_id !== 'string' || !input.prompt_set_id) throw new TypeError('prompt_set_id is required');
+  if (!Number.isInteger(input.prompt_set_version) || input.prompt_set_version < 1) throw new TypeError('prompt_set_version must be a positive integer');
+  const included = normalizeIncluded(input.included_prompts);
+  const excluded = normalizeExcluded(input.excluded_prompts || []);
+  const item = {
+    prompt_set_id: input.prompt_set_id,
+    prompt_set_version: input.prompt_set_version,
+    purpose: input.purpose,
+    generation_method: input.generation_method,
+    generation_method_version: input.generation_method_version ?? null,
+    generation_input: input.generation_input ?? null,
+    generated_at: input.generated_at,
+    population_size: included.length,
+    sampling_method: input.sampling_method,
+    included_prompts: included,
+    excluded_prompts: excluded,
+    intended_population: input.intended_population,
+    limitations: [...new Set(input.limitations || [])],
+  };
+  item.prompt_set_hash = hashPromptSet(item);
+  const check = validateAgainst(PROMPT_SCHEMA, item);
+  if (!check.valid) throw new TypeError(`prompt set violates contract: ${check.errors.join('; ')}`);
+  if (previous && previous.prompt_set_id === item.prompt_set_id && previous.prompt_set_version === item.prompt_set_version && previous.prompt_set_hash !== item.prompt_set_hash) {
+    throw new Error('prompt set version is immutable; create a new prompt_set_version');
+  }
+  return item;
+}
+
+export function verifyPromptSet(item, { previous = null } = {}) {
+  const failures = [];
+  const check = validateAgainst(PROMPT_SCHEMA, item);
+  if (!check.valid) failures.push(...check.errors);
+  if (item?.prompt_set_hash !== hashPromptSet(item || {})) failures.push('prompt set hash is inconsistent');
+  if (previous && previous.prompt_set_id === item?.prompt_set_id && previous.prompt_set_version === item?.prompt_set_version && previous.prompt_set_hash !== item?.prompt_set_hash) failures.push('prompt set version is immutable');
+  return { valid: failures.length === 0, failures };
+}
+
+export function promptReference(promptSet, promptIdValue) {
+  if (!verifyPromptSet(promptSet).valid) throw new Error('cannot reference an unverifiable prompt set');
+  const prompt = promptSet.included_prompts.find((item) => item.prompt_id === promptIdValue);
+  if (!prompt) throw new Error(`prompt ${promptIdValue} is not included in prompt set ${promptSet.prompt_set_id}@${promptSet.prompt_set_version}`);
+  return `${promptSet.prompt_set_id}@${promptSet.prompt_set_version}:${prompt.prompt_id}`;
+}
+
+export function comparePromptSets(left, right) {
+  const validLeft = verifyPromptSet(left).valid;
+  const validRight = verifyPromptSet(right).valid;
+  if (!validLeft || !validRight) return { status: 'INDETERMINATE', dimensions: {}, limitations: ['Prompt-set integrity could not be verified.'] };
+  const dimensions = {
+    prompt_set_id: left.prompt_set_id === right.prompt_set_id ? 'match' : 'mismatch',
+    prompt_set_version: left.prompt_set_version === right.prompt_set_version ? 'match' : 'mismatch',
+    population: left.population_size === right.population_size ? 'match' : 'mismatch',
+    sampling_method: left.sampling_method === right.sampling_method ? 'match' : 'mismatch',
+  };
+  const status = dimensions.prompt_set_id === 'match' && dimensions.prompt_set_version === 'match' && dimensions.population === 'match' && dimensions.sampling_method === 'match'
+    ? 'COMPARABLE' : 'NOT_COMPARABLE';
+  return { status, left_reference: `${left.prompt_set_id}@${left.prompt_set_version}`, right_reference: `${right.prompt_set_id}@${right.prompt_set_version}`, dimensions, limitations: status === 'COMPARABLE' ? [] : ['Prompt population, sampling, or version changed; no denominator is reused silently.'] };
+}
+
+export { promptId };

package/src/representation/provenance.js

@@ -0,0 +1,214 @@
+import { createExternalObservation, createDerivation, verifyExternalObservation, verifyClaim } from '../evidence/external.js';
+import { canonicalEvidenceJson } from '../evidence/hashes.js';
+import { sha256, nowIso } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+
+const SPAN_SCHEMA = 'response-span.schema.json';
+const CITATION_SCHEMA = 'citation-provenance.schema.json';
+const SOURCE_SCHEMA = 'source-retrieval.schema.json';
+const COLLECTOR = { id: 'citable-citation-provenance', version: '1.0.0' };
+const PARSER = { id: 'provider-neutral-citation-parser', version: '1.0.0' };
+
+function validate(schema, value, label) {
+  const result = validateAgainst(schema, value);
+  if (!result.valid) throw new TypeError(`${label} violates contract: ${result.errors.join('; ')}`);
+  return value;
+}
+
+function validUrl(value) {
+  try { return new URL(value).toString(); } catch { return null; }
+}
+
+function executionReference(execution) {
+  if (!execution?.observation_id || !verifyExternalObservation(execution).valid) {
+    throw new TypeError('citation execution must be a verified external observation');
+  }
+  if (execution.data?.response?.status !== 'succeeded') throw new Error('citation requires a substantive provider response');
+  return execution.observation_id;
+}
+
+/** Create a stable, exact text span without normalizing provider prose. */
+export function createResponseSpan({ execution, start, end, text, spanId = null } = {}) {
+  const executionId = execution?.observation_id || execution;
+  if (typeof executionId !== 'string' || !executionId) throw new TypeError('execution reference is required');
+  if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start) throw new TypeError('span offsets are invalid');
+  if (typeof text !== 'string') throw new TypeError('span text is required');
+  if (end - start !== text.length) throw new TypeError('span offsets must bound the exact response text');
+  const item = {
+    span_id: spanId || `SPAN-${sha256(canonicalEvidenceJson({ execution_reference: executionId, start, end, text })).slice(0, 24)}`,
+    execution_reference: executionId,
+    start,
+    end,
+    text,
+    text_hash: sha256(text),
+  };
+  return validate(SPAN_SCHEMA, item, 'response span');
+}
+
+function sourceShape(citation, source = {}) {
+  const displayed = citation.displayed_url ?? citation.url ?? null;
+  const resolved = validUrl(source.resolved_url ?? citation.resolved_url);
+  const status = source.identity_status || (resolved ? 'IDENTIFIED' : displayed ? 'AMBIGUOUS' : 'NOT_IDENTIFIED');
+  return {
+    identity_status: status,
+    resolved_url: resolved,
+    redirect_chain: (source.redirect_chain || []).map(validUrl).filter(Boolean),
+    canonical_url: validUrl(source.canonical_url) || null,
+  };
+}
+
+function citationData({ execution, span, citation, proposition, source, retrieval, support }) {
+  const data = {
+    execution_reference: execution.observation_id,
+    response_reference: execution.data.response.response_hash,
+    provider: execution.data.provider,
+    span,
+    citation: {
+      citation_id: citation.citation_id || `CIT-${sha256(canonicalEvidenceJson({ execution: execution.observation_id, span: span.span_id, ordinal: citation.ordinal || 1, marker: citation.raw_marker || null })).slice(0, 24)}`,
+      ordinal: citation.ordinal || 1,
+      raw_marker: citation.raw_marker ?? null,
+      placement: citation.placement || 'unknown',
+      displayed_title: citation.displayed_title ?? citation.title ?? null,
+      displayed_domain: citation.displayed_domain ?? citation.domain ?? null,
+      displayed_url: citation.displayed_url ?? citation.url ?? null,
+    },
+    source: sourceShape(citation, source),
+    retrieval: { status: retrieval?.status || 'NOT_ATTEMPTED', observation_reference: retrieval?.observation_reference || null },
+    proposition: { proposition_id: proposition?.proposition_id || `PROP-${sha256(proposition?.text || span.text).slice(0, 24)}`, text: proposition?.text || span.text },
+    support: {
+      relevance_status: support?.relevance_status || 'NOT_ASSESSED',
+      support_status: support?.support_status || 'NOT_ASSESSED',
+      evidence_references: [...new Set(support?.evidence_references || [])],
+    },
+  };
+  return validate(CITATION_SCHEMA, data, 'citation provenance');
+}
+
+/** Create an immutable citation linked to one exact response span. */
+export function createCitationObservation({ execution, span, citation, proposition, source = {}, retrieval = null, support = null, evidenceOrigin = 'RECORDED_FIXTURE', observedAt = nowIso(), limitations = [] } = {}) {
+  executionReference(execution);
+  const responseSpan = span?.span_id ? validate(SPAN_SCHEMA, span, 'response span') : createResponseSpan({ execution, ...(span || {}) });
+  if (responseSpan.execution_reference !== execution.observation_id) throw new Error('span does not belong to execution');
+  const data = citationData({ execution, span: responseSpan, citation: citation || {}, proposition, source, retrieval, support });
+  if (!['LIVE_PROVIDER', 'RECORDED_FIXTURE', 'SYNTHETIC_TEST'].includes(evidenceOrigin)) throw new TypeError('evidenceOrigin is invalid');
+  return createExternalObservation({
+    observation_id: `CIT-${sha256(canonicalEvidenceJson({ execution: execution.observation_id, data })).slice(0, 24)}`,
+    observation_type: 'EXTERNAL_OBSERVATION',
+    subject: { type: 'citation', id: data.citation.citation_id },
+    source: { provider: execution.data.provider, method: 'citation_provenance', authority: 'external_provider' },
+    collector: COLLECTOR,
+    parser: PARSER,
+    configuration: execution.data.execution.configuration,
+    observed_at: observedAt,
+    retrieved_at: observedAt,
+    executed_at: execution.data.execution.executed_at,
+    population: { declared: execution.population?.declared || null, observed: 1, retrieved: 1, evaluated: 1, unavailable: 0, failed: 0 },
+    observation_status: 'observed',
+    retrieval_status: 'retrieved',
+    data: { ...data, evidence_origin: evidenceOrigin },
+    lineage: { source_evidence: [execution.observation_id], predecessor_references: [execution.observation_id], package_reference: null },
+    limitations: [...new Set(limitations)],
+  });
+}
+
+/** Build a set while refusing duplicate citation ordinals or identifiers. */
+export function createCitationObservations(input = {}) {
+  const citations = input.citations || [];
+  const ids = new Set(), ordinals = new Set();
+  return citations.map((citation) => {
+    const item = createCitationObservation({ ...input, citation });
+    const id = item.data.citation.citation_id;
+    const ordinal = item.data.citation.ordinal;
+    if (ids.has(id)) throw new Error(`duplicate citation_id: ${id}`);
+    if (ordinals.has(ordinal)) throw new Error(`duplicate citation ordinal: ${ordinal}`);
+    ids.add(id); ordinals.add(ordinal);
+    return item;
+  });
+}
+
+export function verifyCitationObservation(item) {
+  return verifyExternalObservation(item);
+}
+
+/** Preserve source retrieval as a separate observation; failed retrieval is never support failure. */
+export function createSourceRetrievalObservation({ citation, url, status = 'SUCCEEDED', body = null, finalUrl = null, redirectChain = [], httpStatus = null, attemptedAt = nowIso(), relevanceStatus = 'NOT_ASSESSED', supportStatus = 'NOT_ASSESSED', evidenceReferences = [], rawRetained = false, evidenceOrigin = 'RECORDED_FIXTURE', limitations = [] } = {}) {
+  if (!citation?.observation_id || !verifyCitationObservation(citation).valid) throw new TypeError('citation must be a verified citation observation');
+  const sourceUrl = validUrl(url || citation.data.source.resolved_url);
+  if (!sourceUrl) throw new TypeError('retrieval URL must be a valid URL');
+  const contentHash = body == null ? null : sha256(body);
+  const data = validate(SOURCE_SCHEMA, {
+    citation_reference: citation.observation_id,
+    url: sourceUrl,
+    retrieval: { status, attempted_at: attemptedAt, final_url: validUrl(finalUrl) || (status === 'SUCCEEDED' ? sourceUrl : null), redirect_chain: redirectChain.map(validUrl).filter(Boolean), http_status: status === 'SUCCEEDED' ? (httpStatus || 200) : httpStatus },
+    content: { observed: status === 'SUCCEEDED' && body != null, content_hash: contentHash, raw_retained: Boolean(rawRetained) },
+    relevance: { status: relevanceStatus, support_status: supportStatus, evidence_references: [...new Set(evidenceReferences)] },
+  }, 'source retrieval');
+  return createExternalObservation({
+    observation_id: `SRC-${sha256(canonicalEvidenceJson({ citation: citation.observation_id, data })).slice(0, 24)}`,
+    observation_type: 'EXTERNAL_OBSERVATION',
+    subject: { type: 'citation_source', id: sourceUrl },
+    source: { provider: citation.data.provider, method: 'citation_source_retrieval', authority: 'third_party' },
+    collector: COLLECTOR,
+    parser: PARSER,
+    configuration: null,
+    observed_at: attemptedAt,
+    retrieved_at: attemptedAt,
+    executed_at: null,
+    population: { declared: null, observed: status === 'SUCCEEDED' ? 1 : 0, retrieved: status === 'SUCCEEDED' ? 1 : 0, evaluated: status === 'SUCCEEDED' ? 1 : 0, unavailable: 0, failed: status === 'FAILED' ? 1 : 0 },
+    observation_status: status === 'SUCCEEDED' ? 'observed' : status === 'FAILED' ? 'failed' : 'indeterminate',
+    retrieval_status: status === 'SUCCEEDED' ? 'retrieved' : status === 'FAILED' ? 'failed' : 'indeterminate',
+    data: { ...data, evidence_origin: evidenceOrigin },
+    lineage: { source_evidence: [citation.observation_id], predecessor_references: [citation.observation_id], package_reference: null },
+    limitations: [...new Set([...(status === 'FAILED' ? ['Source retrieval failed; proposition support is indeterminate.'] : []), ...limitations])],
+  });
+}
+
+export function verifySourceRetrievalObservation(item) {
+  return verifyExternalObservation(item);
+}
+
+export function assessPropositionSupport({ retrieval, relevanceStatus, supportStatus, evidenceReferences = [] } = {}) {
+  if (!retrieval || !verifySourceRetrievalObservation(retrieval).valid) return { status: 'INDETERMINATE', reasons: ['source retrieval evidence is missing or unverifiable'], evidence_references: [] };
+  if (retrieval.data.retrieval.status !== 'SUCCEEDED') return { status: 'INDETERMINATE', reasons: ['source retrieval did not succeed'], evidence_references: [retrieval.observation_id] };
+  if (relevanceStatus === 'IRRELEVANT') return { status: 'UNSUPPORTED', reasons: ['retrieved source was assessed as irrelevant to the proposition'], evidence_references: [retrieval.observation_id, ...evidenceReferences] };
+  const supportedStates = new Set(['SUPPORTED', 'PARTIALLY_SUPPORTED', 'UNSUPPORTED', 'CONTRADICTED', 'INDETERMINATE', 'NOT_OBSERVED']);
+  if (relevanceStatus !== 'RELEVANT' || !supportStatus || supportStatus === 'NOT_ASSESSED' || !supportedStates.has(supportStatus)) return { status: 'INDETERMINATE', reasons: ['source relevance or proposition support was not established'], evidence_references: [retrieval.observation_id, ...evidenceReferences] };
+  return { status: supportStatus, reasons: [], evidence_references: [retrieval.observation_id, ...evidenceReferences] };
+}
+
+/** Derive prevalence over successful/evaluable executions only, retaining exclusions. */
+export function deriveRepresentationPrevalence(observations, { predicatePath = 'data.representation.brand_mentioned', expected = true, derivationId = 'REPRESENTATION-PREVALENCE', derivationVersion = 1 } = {}) {
+  if (!Array.isArray(observations) || !observations.length) throw new TypeError('observations are required');
+  const valid = observations.filter((item) => verifyExternalObservation(item).valid);
+  const evaluable = valid.filter((item) => item.data?.response?.status === 'succeeded');
+  const excluded = observations.filter((item) => !verifyExternalObservation(item).valid || item.data?.response?.status !== 'succeeded');
+  const valueAt = (item) => predicatePath.split('.').reduce((v, key) => v == null ? undefined : v[key], item);
+  const numerator = evaluable.filter((item) => valueAt(item) === expected).length;
+  return createDerivation({
+    derivation_id: derivationId,
+    derivation_version: derivationVersion,
+    operation: 'representation_prevalence',
+    input_observation_references: observations.map((item) => item.observation_id),
+    predicate: { path: predicatePath, operator: 'equals', value: expected },
+    result: {
+      numerator,
+      denominator: evaluable.length,
+      rate: evaluable.length ? numerator / evaluable.length : null,
+      evaluable_observation_references: evaluable.map((item) => item.observation_id),
+      excluded_observation_references: excluded.map((item) => item.observation_id),
+      exclusions: excluded.map((item) => ({ observation_reference: item.observation_id, reason: verifyExternalObservation(item).valid ? (item.data?.response?.status || 'not_evaluable') : 'integrity_unverifiable' })),
+      population_scope: 'successful_representation_executions_in_supplied_observations',
+    },
+    implementation: { id: 'citable-representation-prevalence', version: '1.0.0' },
+  });
+}
+
+export function citationsForExecution(citations, executionId) { return (citations || []).filter((item) => item.data?.execution_reference === executionId); }
+export function citationsForProposition(citations, propositionId) { return (citations || []).filter((item) => item.data?.proposition?.proposition_id === propositionId); }
+export function sourceForCitation(retrievals, citationId) { return (retrievals || []).find((item) => item.data?.citation_reference === citationId) || null; }
+export function evidenceForProposition(retrievals, propositionId, citations = []) {
+  const citationIds = new Set(citationsForProposition(citations, propositionId).map((item) => item.observation_id));
+  return (retrievals || []).filter((item) => citationIds.has(item.data?.citation_reference));
+}
+
+export { verifyClaim };

package/src/representation/representation.js

@@ -0,0 +1,186 @@
+import { createExternalObservation, verifyExternalObservation } from '../evidence/external.js';
+import { canonicalEvidenceJson } from '../evidence/hashes.js';
+import { sha256, nowIso } from '../shared/io.js';
+import { validateAgainst } from '../shared/schemaValidator.js';
+import { verifyPromptSet, promptReference } from './prompts.js';
+
+const REPRESENTATION_SCHEMA = 'representation-data.schema.json';
+const COLLECTOR = { id: 'citable-representation-observations', version: '1.0.0' };
+const PARSER = { id: 'provider-neutral-representation-parser', version: '1.0.0' };
+
+function responseHash(rawResponse, response) {
+  return sha256(typeof rawResponse === 'string' ? rawResponse : canonicalEvidenceJson(response ?? null));
+}
+
+function modelIdentity(model) {
+  if (model && typeof model === 'string') return { value: model, status: 'known' };
+  if (model && typeof model === 'object' && ['known', 'UNKNOWN', 'UNAVAILABLE', 'NOT_DISCLOSED'].includes(model.status)) return { value: model.value ?? null, status: model.status };
+  return { value: null, status: 'UNKNOWN' };
+}
+
+function hostOf(value) {
+  try { return new URL(value).hostname.toLowerCase(); } catch { return null; }
+}
+
+function normalizeUrls(value) {
+  if (!Array.isArray(value)) throw new TypeError('provider citations must be an array');
+  const urls = value.map((item) => typeof item === 'string' ? item : item?.url);
+  if (urls.some((item) => typeof item !== 'string' || !hostOf(item))) throw new TypeError('provider citation URL is malformed');
+  return [...new Set(urls)];
+}
+
+const SENSITIVE_KEY = /(^|[-_])(authorization|cookie|set-cookie|access[-_]?token|refresh[-_]?token|api[-_]?key|password|secret)($|[-_])/i;
+
+function safeMetadata(value, depth = 0) {
+  if (depth > 4 || value == null) return value == null ? null : '[bounded]';
+  if (Array.isArray(value)) return value.slice(0, 50).map((item) => safeMetadata(item, depth + 1));
+  if (typeof value !== 'object') return typeof value === 'string' ? value.slice(0, 500) : value;
+  return Object.fromEntries(Object.entries(value).slice(0, 50).filter(([key]) => !SENSITIVE_KEY.test(key)).map(([key, item]) => [key, safeMetadata(item, depth + 1)]));
+}
+
+function executionStatusFor(status) {
+  if (status === 'succeeded') return { observation_status: 'observed', retrieval_status: 'retrieved' };
+  if (status === 'malformed' || status === 'parser_failed') return { observation_status: 'indeterminate', retrieval_status: 'indeterminate' };
+  return { observation_status: 'failed', retrieval_status: 'failed' };
+}
+
+function parserErrorObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt, rawResponse, status, errorMessage }) {
+  const hash = responseHash(rawResponse, null);
+  const execution = executionConfiguration || { id: 'default', version: '1.0.0' };
+  const state = executionStatusFor(status);
+  return createRepresentationObservation({
+    provider, product, surface, prompt, promptSet, executionConfiguration: execution, model, evidenceOrigin, executedAt,
+    response: { status, response_hash: hash, parser: PARSER, raw_response_retained: false, provider_metadata: {} },
+    representation: { brand_mentioned: null, domain_mentioned: null, domain_cited: null, cited_urls: [], mentioned_entities: [], cited_domains: [] },
+    observationStatus: state.observation_status, retrievalStatus: state.retrieval_status,
+    limitations: [`Representation response could not be normalized: ${String(errorMessage || status).slice(0, 240)}`, 'Raw provider response is not retained; response_hash is retained for identity.'],
+  });
+}
+
+export function createRepresentationObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt = nowIso(), response, representation, observationStatus = 'observed', retrievalStatus = 'retrieved', limitations = [] }) {
+  if (!provider || !product || !surface) throw new TypeError('provider, product, and surface are required');
+  if (!prompt || !prompt.prompt_id || !prompt.prompt_text) throw new TypeError('prompt identity and text are required');
+  if (!promptSet || !verifyPromptSet(promptSet).valid) throw new TypeError('prompt set is missing or unverifiable');
+  if (!promptSet.included_prompts.some((item) => item.prompt_id === prompt.prompt_id)) throw new Error(`prompt ${prompt.prompt_id} is not included in the prompt set`);
+  if (!['LIVE_PROVIDER', 'RECORDED_FIXTURE', 'SYNTHETIC_TEST'].includes(evidenceOrigin)) throw new TypeError('evidenceOrigin is invalid');
+  const config = executionConfiguration || { id: 'default', version: '1.0.0' };
+  const normalizedModel = modelIdentity(model);
+  const data = {
+    provider,
+    product,
+    surface,
+    model: normalizedModel,
+    prompt: { prompt_id: prompt.prompt_id, prompt_version: promptSet.prompt_set_version, prompt_text: prompt.prompt_text },
+    prompt_set: { prompt_set_id: promptSet.prompt_set_id, prompt_set_version: promptSet.prompt_set_version },
+    execution: { executed_at: executedAt, configuration: config },
+    response,
+    representation,
+    evidence_origin: evidenceOrigin,
+    limitations: [...new Set(limitations)],
+  };
+  const check = validateAgainst(REPRESENTATION_SCHEMA, data);
+  if (!check.valid) throw new TypeError(`representation data violates contract: ${check.errors.join('; ')}`);
+  const state = executionStatusFor(response.status);
+  const observation = createExternalObservation({
+    observation_id: `REP-${sha256(canonicalEvidenceJson({ provider, product, surface, prompt, promptSet: promptSet.prompt_set_hash, executedAt, response })).slice(0, 24)}`,
+    observation_type: 'EXTERNAL_OBSERVATION',
+    subject: { type: 'prompt_execution', id: promptReference(promptSet, prompt.prompt_id) },
+    source: { provider, method: 'representation_adapter', authority: 'external_provider' },
+    collector: COLLECTOR,
+    parser: PARSER,
+    configuration: config,
+    observed_at: executedAt,
+    retrieved_at: executedAt,
+    executed_at: executedAt,
+    population: { declared: { id: `${promptSet.prompt_set_id}@${promptSet.prompt_set_version}`, size: promptSet.population_size }, observed: 1, retrieved: response.status === 'succeeded' ? 1 : 0, evaluated: response.status === 'succeeded' ? 1 : 0, unavailable: response.status === 'unavailable' ? 1 : 0, failed: response.status === 'succeeded' ? 0 : 1 },
+    observation_status: observationStatus || state.observation_status,
+    retrieval_status: retrievalStatus || state.retrieval_status,
+    data,
+    lineage: { source_evidence: response.response_hash ? [response.response_hash] : [] },
+    limitations: data.limitations,
+  });
+  return observation;
+}
+
+export function normalizeRepresentationResponse({ provider, product, surface, prompt, promptSet, executionConfiguration, model, response, rawResponse = null, evidenceOrigin = 'RECORDED_FIXTURE', executedAt = nowIso(), brand = null, firstPartyDomains = [], retainRawResponse = false }) {
+  const base = { provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt };
+  const rawHash = responseHash(rawResponse, response);
+  try {
+    if (!response || typeof response !== 'object' || Array.isArray(response)) throw new TypeError('provider response must be an object');
+    const answerText = response.answer_text ?? response.answer ?? response.text;
+    if (typeof answerText !== 'string') throw new TypeError('provider response requires answer_text');
+    const citations = normalizeUrls(response.citations || []);
+    const mentionedEntities = Array.isArray(response.mentions) ? response.mentions.filter((item) => typeof item === 'string') : [];
+    const firstParty = new Set(firstPartyDomains.map((item) => String(item).toLowerCase()));
+    const citedDomains = [...new Set(citations.map(hostOf).filter(Boolean))];
+    const domainMentioned = firstParty.size ? [...firstParty].some((domain) => answerText.toLowerCase().includes(domain)) : null;
+    const domainCited = firstParty.size ? citedDomains.some((domain) => firstParty.has(domain)) : null;
+    const brandMentioned = typeof brand === 'string' ? answerText.toLowerCase().includes(brand.toLowerCase()) : null;
+    return createRepresentationObservation({
+      ...base,
+      response: { status: 'succeeded', response_hash: rawHash, parser: PARSER, raw_response_retained: retainRawResponse, provider_metadata: { ...safeMetadata(response.provider_metadata || {}), ...(retainRawResponse ? { raw_response: rawResponse ?? response } : {}) } },
+      representation: { brand_mentioned: brandMentioned, domain_mentioned: domainMentioned, domain_cited: domainCited, cited_urls: citations, mentioned_entities: mentionedEntities, cited_domains: citedDomains },
+      limitations: retainRawResponse ? [] : ['Raw provider response is not retained; response_hash is retained for identity.'],
+    });
+  } catch (error) {
+    return parserErrorObservation({ ...base, rawResponse, status: 'parser_failed', errorMessage: error.message });
+  }
+}
+
+export async function collectRepresentationObservations({ promptSet, adapter, provider, product = 'unknown', surface = 'unknown', model = null, executionConfiguration = { id: 'default', version: '1.0.0' }, evidenceOrigin = 'LIVE_PROVIDER', brand = null, firstPartyDomains = [] } = {}) {
+  if (!promptSet || !verifyPromptSet(promptSet).valid) throw new TypeError('prompt set is missing or unverifiable');
+  if (typeof adapter !== 'function') throw new TypeError('representation adapter is required');
+  const observations = [];
+  for (const prompt of promptSet.included_prompts) {
+    const executedAt = nowIso();
+    try {
+      const result = await adapter(prompt, { provider, product, surface, model, promptSet, executionConfiguration });
+      if (!result || typeof result !== 'object') throw new Error('adapter returned an invalid result');
+      const status = result.status || 'succeeded';
+      if (status !== 'succeeded') {
+        observations.push(parserErrorObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt, rawResponse: result.raw ?? result.body ?? null, status, errorMessage: result.error || status }));
+      } else {
+        observations.push(normalizeRepresentationResponse({ provider, product, surface, prompt, promptSet, executionConfiguration, model, response: result.body ?? result.response ?? result, rawResponse: result.raw ?? null, evidenceOrigin, executedAt, brand, firstPartyDomains }));
+      }
+    } catch (error) {
+      observations.push(parserErrorObservation({ provider, product, surface, prompt, promptSet, executionConfiguration, model, evidenceOrigin, executedAt, rawResponse: null, status: error.connectorState === 'quota_limited' ? 'rate_limited' : error.name === 'AbortError' ? 'timeout' : 'error', errorMessage: error.message }));
+    }
+  }
+  return { observations, coverage: { declared: promptSet.population_size, executed: observations.filter((item) => item.data.response.status === 'succeeded').length, failed: observations.filter((item) => item.data.response.status !== 'succeeded').length, coverage_status: observations.every((item) => item.data.response.status === 'succeeded') ? 'complete_for_declared_prompt_set' : 'incomplete' }, prompt_set_reference: `${promptSet.prompt_set_id}@${promptSet.prompt_set_version}` };
+}
+
+export function createHttpRepresentationAdapter({ endpoint, fetchImpl = globalThis.fetch, accessToken = null } = {}) {
+  if (typeof endpoint !== 'string' || !/^https:\/\//.test(endpoint)) throw new TypeError('representation adapter endpoint must use HTTPS');
+  return async (prompt) => {
+    const response = await fetchImpl(endpoint, { method: 'POST', redirect: 'error', headers: { accept: 'application/json', 'content-type': 'application/json', ...(accessToken ? { authorization: `Bearer ${accessToken}` } : {}) }, body: JSON.stringify({ prompt_id: prompt.prompt_id, prompt_text: prompt.prompt_text }) });
+    const raw = await response.text();
+    if (!response.ok) {
+      const status = response.status === 429 ? 'rate_limited' : response.status === 408 ? 'timeout' : response.status >= 500 ? 'unavailable' : 'refused';
+      return { status, raw, error: `provider returned ${response.status}` };
+    }
+    let body;
+    try { body = JSON.parse(raw); } catch { return { status: 'malformed', raw, error: 'provider response was not valid JSON' }; }
+    return { status: 'succeeded', body, raw };
+  };
+}
+
+export function verifyRepresentationObservation(item) {
+  return verifyExternalObservation(item);
+}
+
+export function compareRepresentationObservations(left, right) {
+  if (!verifyExternalObservation(left).valid || !verifyExternalObservation(right).valid) return { status: 'INDETERMINATE', dimensions: {}, limitations: ['Representation observation integrity could not be verified.'] };
+  const dimensions = {
+    provider: left.data.provider === right.data.provider ? 'match' : 'mismatch',
+    product: left.data.product === right.data.product ? 'match' : 'mismatch',
+    surface: left.data.surface === right.data.surface ? 'match' : 'mismatch',
+    model: left.data.model.status !== 'known' || right.data.model.status !== 'known' ? 'missing' : left.data.model.value === right.data.model.value ? 'match' : 'mismatch',
+    prompt: left.data.prompt.prompt_id === right.data.prompt.prompt_id && left.data.prompt.prompt_text === right.data.prompt.prompt_text ? 'match' : 'mismatch',
+    prompt_set: left.data.prompt_set.prompt_set_id === right.data.prompt_set.prompt_set_id && left.data.prompt_set.prompt_set_version === right.data.prompt_set.prompt_set_version ? 'match' : 'mismatch',
+    configuration: left.data.execution.configuration.id === right.data.execution.configuration.id && left.data.execution.configuration.version === right.data.execution.configuration.version ? 'match' : 'mismatch',
+    parser: left.data.response.parser.id === right.data.response.parser.id && left.data.response.parser.version === right.data.response.parser.version ? 'match' : 'mismatch',
+  };
+  const hardMismatch = ['provider', 'product', 'surface', 'prompt', 'prompt_set', 'configuration', 'parser'].some((key) => dimensions[key] === 'mismatch');
+  const status = hardMismatch ? 'NOT_COMPARABLE' : dimensions.model === 'mismatch' ? 'NOT_COMPARABLE' : dimensions.model === 'missing' ? 'PARTIALLY_COMPARABLE' : 'COMPARABLE';
+  return { comparison_id: `REP-CMP-${sha256(canonicalEvidenceJson({ left: left.observation_id, right: right.observation_id, dimensions })).slice(0, 16)}`, left_reference: left.observation_id, right_reference: right.observation_id, status, dimensions, limitations: status === 'COMPARABLE' ? [] : ['Provider, model, prompt, execution, and parser identity remain explicit; no visibility delta is inferred from non-comparable observations.'] };
+}