@mnstry/atelier 0.2.0-alpha.12 → 0.2.0-alpha.13

package/src/ingestion/optional/vanilla/native-normalization.mjs

@@ -0,0 +1,187 @@
+import { createHash } from 'node:crypto'
+import { MAX_CAPTURE_BYTES } from './index.mjs'
+
+const sha = bytes => createHash('sha256').update(bytes).digest('hex')
+const candidateId = value => typeof value === 'string' && /^[a-z][a-z0-9-]{0,63}$/.test(value)
+const own = (value, key) => value !== null && typeof value === 'object' && Object.hasOwn(value, key)
+const same = (a, b) => a?.id === b?.id && a?.digest === b?.digest
+const modalities = new Set(['asserted', 'conditional', 'proposed', 'possible', 'uncertain', 'unknown'])
+const nativeAuxiliary = {
+  graphify: new Set(['id', 'source', 'target', 'relation', 'description', 'source_file', 'source_location', 'source_url', 'confidence', 'confidence_score', 'weight']),
+  'lightrag-json': new Set(['id', 'source', 'target', 'description', 'keywords', 'source_id', 'file_path', 'timestamp', 'weight']),
+}
+
+export class NativeHandoffError extends Error {
+  constructor(code, message, normalization = null) {
+    super(message); this.name = 'NativeHandoffError'; this.code = code
+    if (normalization) this.normalization = normalization
+  }
+}
+const check = (condition, code, message) => { if (!condition) throw new NativeHandoffError(code, message) }
+function call(mapper, context) {
+  const value = mapper(structuredClone(context))
+  check(!value || typeof value.then !== 'function', 'NATIVE_MAPPER_ASYNC', 'Mapping callbacks must be cheap synchronous owner mappings')
+  return value
+}
+function atPath(record, path) {
+  check(Array.isArray(path) && path.length > 0 && path.every(key => typeof key === 'string' || Number.isSafeInteger(key)),
+    'NATIVE_QUALIFIER_MISSING', 'Explicit raw-field qualifier paths are required')
+  let value = record
+  for (const key of path) {
+    check(own(value, key), 'NATIVE_QUALIFIER_MISSING', 'The qualifier is absent from the received native object')
+    value = value[key]
+  }
+  return structuredClone(value)
+}
+function approved(mapping, input, vocabulary) {
+  return mapping?.approved === true && same(mapping.ownerRef, input.domainRef) && vocabulary.has(mapping.id)
+}
+function fieldsAccounted(record, paths, auxiliary, prefix = []) {
+  if (paths.some(path => path.length === prefix.length && path.every((key, index) => key === prefix[index]))) return true
+  if (prefix.length === 1 && auxiliary.has(prefix[0])) return true
+  if (!record || typeof record !== 'object') return false
+  return Object.keys(record).every(key => {
+    const next = [...prefix, key]
+    if (next.length === 1 && auxiliary.has(key)) return true
+    return paths.some(path => path.length >= next.length && next.every((item, index) => path[index] === item)) &&
+      fieldsAccounted(record[key], paths, auxiliary, next)
+  })
+}
+
+/** Pure conversion of received raw Graphify nodes/edges or LightRAG model JSON.
+ * It does not run either tool, find evidence, approve mappings or resolve identity.
+ */
+export function normalizeNativeExtraction({ bytes, input, profile, mappers }) {
+  check(bytes instanceof Uint8Array && bytes.length <= MAX_CAPTURE_BYTES, 'NATIVE_RAW_LIMIT', 'Bounded original native bytes are required')
+  check(['graphify', 'lightrag-json'].includes(profile), 'NATIVE_PROFILE_UNSUPPORTED', 'Select an inspected native JSON profile')
+  check(input?.schema === 'atelier.semantic-input/v0' && input.coverage === 'selected-spans-only' && input.readScope === 'all-plan',
+    'NATIVE_INPUT_BINDING', 'The existing pinned semantic input is required')
+  for (const name of ['entityType', 'identity', 'predicate', 'evidence', 'participants', 'qualifiers'])
+    check(typeof mappers?.[name] === 'function', 'NATIVE_MAPPER_MISSING', `Existing owner ${name} mapping callback is required`)
+  let raw
+  try { raw = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) }
+  catch { throw new NativeHandoffError('NATIVE_RAW_INVALID', 'Native output must be exact UTF-8 JSON; original bytes stay with intake') }
+  const entityKey = profile === 'graphify' ? 'nodes' : 'entities', assertionKey = profile === 'graphify' ? 'edges' : 'relationships'
+  check(raw && Array.isArray(raw[entityKey]) && Array.isArray(raw[assertionKey]), 'NATIVE_PROFILE_UNSUPPORTED', 'The selected native collection shape is required')
+  check(raw.hyperedges === undefined || Array.isArray(raw.hyperedges), 'NATIVE_PROFILE_UNSUPPORTED', 'Native hyperedges must retain their actual collection shape')
+  check(raw[entityKey].length <= 64 && raw[assertionKey].length <= 64 && (raw.hyperedges?.length ?? 0) <= 64,
+    'NATIVE_HANDOFF_LIMIT', 'Existing candidate collections require an owner-selected bounded native slice')
+  const candidates = { schema: 'atelier.semantic-candidate/v0', inputDigest: input.digest, entities: [], assertions: [], unknowns: [] }
+  const unresolved = [], mappings = [], ids = new Set(), entities = new Map()
+  const types = new Set(input.domain.data.vocabulary.types.map(x => x.id)), predicates = new Set(input.domain.data.vocabulary.relations.map(x => x.id))
+  const context = (record, kind, index, collection) => ({ native: record, nativeRef: { profile, collection, index }, kind, input })
+  function support(ctx) {
+    const selected = call(mappers.evidence, ctx)
+    check(Array.isArray(selected) && selected.length > 0 && selected.length <= 16, 'NATIVE_SUPPORT_MISSING', 'Exact existing located input support is required')
+    const seen = new Set()
+    return selected.map(value => {
+      const matches = input.evidence.filter(span => {
+        const a = span.reference, b = value.reference
+        return b && a.sourceId === b.sourceId && a.sourceDigest === b.sourceDigest && a.attemptId === b.attemptId &&
+          a.locator.kind === b.locator?.kind && a.locator.value === b.locator.value
+      })
+      check(matches.length === 1 && !seen.has(matches[0].id), 'NATIVE_SUPPORT_BINDING', 'Support must resolve to one exact current selected input reference')
+      const span = matches[0]; seen.add(span.id)
+      check(typeof value.quote === 'string' && value.quote.length > 0 && span.receipt.text.includes(value.quote),
+        'NATIVE_SUPPORT_QUOTE', 'Owner-supplied quote must occur in its exact input span')
+      check(!/[\r\n]/u.test(value.quote), 'NATIVE_LOCATOR_UNSUPPORTED', 'Current runner does not represent multiline assertion quotes')
+      return { id: span.id, quote: value.quote }
+    })
+  }
+  function loss(ctx, code, reason, located = []) {
+    let unknownId = null
+    if (located.length && candidates.unknowns.length < 64) {
+      unknownId = `native-unresolved-${candidates.unknowns.length + 1}`
+      while (ids.has(unknownId)) unknownId += '-x'
+      ids.add(unknownId)
+      candidates.unknowns.push({ id: unknownId, relatedCandidateId: null,
+        reason: `${code}: ${reason}; retained ${profile} ${ctx.nativeRef.collection}[${ctx.nativeRef.index}] in raw sha256:${sha(bytes)}`,
+        evidence: located.map(x => x.id) })
+    }
+    unresolved.push({ code, reason, nativeRef: ctx.nativeRef, native: structuredClone(ctx.native),
+      representedUnknownId: unknownId, support: located })
+  }
+  function identity(ctx, located) {
+    const choice = call(mappers.identity, ctx)
+    check(choice && same(choice.ownerRef, input.domainRef) && candidateId(choice.id) && !ids.has(choice.id),
+      'NATIVE_IDENTITY_UNMAPPED', 'Explicit unique owner-mapped source candidate identity is required; labels do not merge IDs')
+    const selected = ctx.kind === 'entity' ? choice.identity : null
+    check(ctx.kind !== 'entity' || selected && ['source-local', 'existing-candidate', 'unknown'].includes(selected.status) && Array.isArray(selected.candidateIds),
+      'NATIVE_IDENTITY_UNMAPPED', 'Existing candidate identity semantics must be supplied explicitly')
+    // The existing semantic validator checks matching supplied canonical candidates.
+    ids.add(choice.id)
+    return { id: choice.id, identity: structuredClone(selected), evidence: located.map(x => x.id) }
+  }
+  for (const [index, native] of raw[entityKey].entries()) {
+    const ctx = context(native, 'entity', index, entityKey); let located = []
+    if (!native || typeof native !== 'object' || Array.isArray(native)) { loss(ctx, 'NATIVE_OBJECT_UNSUPPORTED', 'A received native entity object is required'); continue }
+    try {
+      located = support(ctx)
+      const type = call(mappers.entityType, ctx)
+      check(approved(type, input, types), 'NATIVE_TYPE_UNMAPPED', 'Explicit owner-approved existing vocabulary entity type is required')
+      const mapped = identity(ctx, located), label = profile === 'graphify' ? native.label : native.name
+      const nativeId = profile === 'graphify' ? native.id : native.name
+      check(typeof label === 'string' && label.length > 0, 'NATIVE_LABEL_MISSING', 'Native display label is required')
+      check(typeof nativeId === 'string' && nativeId.length > 0, 'NATIVE_IDENTITY_UNMAPPED', 'Actual native identifier is required before participant mapping')
+      candidates.entities.push({ ...mapped, label, type: type.id })
+      entities.set(mapped.id, { native, nativeRef: ctx.nativeRef, nativeId })
+      mappings.push({ nativeRef: ctx.nativeRef, candidateId: mapped.id, entityType: type })
+    } catch (error) {
+      if (!(error instanceof NativeHandoffError)) throw error
+      loss(ctx, error.code, error.message, located)
+    }
+  }
+  for (const [index, native] of raw[assertionKey].entries()) {
+    const ctx = context(native, 'assertion', index, assertionKey); let located = []
+    if (!native || typeof native !== 'object' || Array.isArray(native)) { loss(ctx, 'NATIVE_OBJECT_UNSUPPORTED', 'A received native assertion object is required'); continue }
+    try {
+      located = support(ctx)
+      const predicate = call(mappers.predicate, ctx)
+      check(approved(predicate, input, predicates), 'NATIVE_PREDICATE_UNMAPPED', 'Explicit owner-approved existing vocabulary predicate is required')
+      const choice = call(mappers.participants, { ...ctx, entityMappings: [...entities].map(([id, entity]) => ({ id, ...entity })) })
+      check(same(choice?.ownerRef, input.domainRef) && entities.has(choice.subjectId) &&
+        (choice.objectId === null || entities.has(choice.objectId)), 'NATIVE_PARTICIPANT_UNMAPPED', 'Explicit mapped source participants are required')
+      check(native.source === entities.get(choice.subjectId).nativeId &&
+        (choice.objectId === null ? native.target === null : native.target === entities.get(choice.objectId).nativeId),
+        'NATIVE_PARTICIPANT_BINDING', 'Mapped participants differ from actual native source/target fields')
+      const proof = call(mappers.qualifiers, ctx), qualifier = {}
+      check(proof?.paths && Object.keys(proof.paths).length === 5 && ['direction', 'negated', 'modality', 'scope', 'time'].every(field => own(proof.paths, field)),
+        'NATIVE_QUALIFIER_MISSING', 'Provide exactly the five existing qualifier paths; extra qualifiers require a separate supported mapping')
+      for (const field of ['direction', 'negated', 'modality', 'scope', 'time']) qualifier[field] = atPath(native, proof?.paths?.[field])
+      check(qualifier.direction === (choice.objectId === null ? 'subject-only' : 'subject-to-object') && typeof qualifier.negated === 'boolean' &&
+        modalities.has(qualifier.modality), 'NATIVE_QUALIFIER_UNSUPPORTED', 'Explicit native direction, polarity and supported modality are required')
+      check(typeof qualifier.scope === 'string' && qualifier.scope === input.domain.data.scope, 'NATIVE_SCOPE_UNSUPPORTED', 'Native assertion scope must equal the existing domain scope')
+      const time = qualifier.time
+      check(time && ['from', 'until', 'expression', 'unknowns'].every(field => own(time, field)) && Object.keys(time).length === 4,
+        'NATIVE_TIME_UNSUPPORTED', 'Native time must explicitly retain the existing four-field qualification')
+      check(fieldsAccounted(native, Object.values(proof.paths), nativeAuxiliary[profile]),
+        'NATIVE_QUALIFIER_UNMAPPED', 'Additional native fields require an explicit reviewed mapping; retained unresolved rather than dropped')
+      const mapped = identity(ctx, located)
+      candidates.assertions.push({ id: mapped.id, subjectId: choice.subjectId, predicate: predicate.id, objectId: choice.objectId,
+        ...qualifier, evidence: located })
+      mappings.push({ nativeRef: ctx.nativeRef, candidateId: mapped.id, predicate, qualifierPaths: proof.paths })
+    } catch (error) {
+      if (!(error instanceof NativeHandoffError)) throw error
+      loss(ctx, error.code, error.message, located)
+    }
+  }
+  for (const [index, native] of (raw.hyperedges ?? []).entries()) {
+    const ctx = context(native, 'hyperedge', index, 'hyperedges'); let located = []
+    try { located = support(ctx) } catch (error) { if (!(error instanceof NativeHandoffError)) throw error }
+    loss(ctx, 'NATIVE_HYPEREDGE_UNSUPPORTED', 'Current candidate profile has unary/binary assertions only', located)
+  }
+  return { candidates, mappings, unresolved,
+    refused: unresolved.some(item => item.representedUnknownId === null),
+    rawBinding: { bytes: bytes.length, sha256: sha(bytes) },
+    coverage: input.coverage, readScope: input.readScope,
+    semanticAcceptance: 'pending', authority: 'none', canonicalMutation: false }
+}
+
+/** Pass directly as bridge.normalize; immutable provider/tool bytes stay intact. */
+export function createNativeNormalizer({ profile, mappers }) {
+  return ({ bytes, input }) => {
+    const handoff = normalizeNativeExtraction({ bytes, input, profile, mappers })
+    if (handoff.refused) throw new NativeHandoffError('NATIVE_HANDOFF_UNRESOLVED', 'Native objects lack representable existing-profile support; reopen retained raw capture', handoff)
+    return handoff.candidates
+  }
+}

package/src/ingestion/processors.mjs

@@ -0,0 +1,218 @@
+// Pure structural projections. No source reference or embedded instruction is executed.
+const VERSION = '1.0.0';
+const MAX_BYTES = 16 * 1024 * 1024;
+const MAX_ITEMS = 100000;
+const DISCOVERY = 'Processor selection uses the filename extension only; content type is not verified.';
+const REFERENCES = 'Embedded media, external references and instructions are not processed or followed.';
+const UTF8 = 'Input must be valid UTF-8; one leading UTF-8 BOM prefix is removed.';
+
+function refuse(code, message, limitKind) {
+  throw Object.assign(new Error(message), { code, ...(limitKind ? { limitKind } : {}) });
+}
+function limit(message, kind) { refuse('INGESTION_LIMIT', message, kind); }
+
+export function describeIngestionProcessor(ref) {
+  if (typeof ref !== 'string' || !ref || ref.length > 4096 || ref.includes('\0')) {
+    refuse('INGESTION_ARGUMENT', 'A bounded source reference is required');
+  }
+  const filename = ref.split(/[\\/]/).at(-1);
+  const extension = filename.includes('.') ? filename.slice(filename.lastIndexOf('.') + 1).toLowerCase() : '';
+  const definitions = {
+    txt: ['utf8-lines', 'text', 'text', 'Line evidence preserves text and blank lines, excludes line terminators and does not create a final empty line after a terminator.'],
+    text: ['utf8-lines', 'text', 'text', 'Line evidence preserves text and blank lines, excludes line terminators and does not create a final empty line after a terminator.'],
+    md: ['utf8-lines', 'markdown', 'text', 'Markdown is projected as source lines, not rendered or interpreted; line terminators are excluded and a final terminator creates no extra empty line.'],
+    markdown: ['utf8-lines', 'markdown', 'text', 'Markdown is projected as source lines, not rendered or interpreted; line terminators are excluded and a final terminator creates no extra empty line.'],
+    csv: ['csv-cells', 'csv', 'tabular', 'CSV evidence contains decoded cell strings including headers and empty cells; ragged rows are retained, and types, headers and formulas are not interpreted.'],
+    json: ['json-leaves', 'json', 'structured', 'JSON evidence contains primitive leaves only; empty containers have no evidence. Strings are decoded; number tokens, booleans and null retain their source spelling. Duplicate decoded object keys are refused.'],
+  };
+  if (Object.hasOwn(definitions, extension)) {
+    const [id, format, modality, projection] = definitions[extension];
+    return { id, version: VERSION, format, modality, supported: true,
+      limitations: [DISCOVERY, UTF8, REFERENCES, projection, 'Complete coverage applies only to the named structural projection, not semantic acceptance.'] };
+  }
+  const unsupported = {
+    pdf: 'document', doc: 'document', docx: 'document', rtf: 'document', odt: 'document',
+    html: 'document', htm: 'document', xml: 'document', xlsx: 'tabular', xls: 'tabular',
+    png: 'image', jpg: 'image', jpeg: 'image', gif: 'image', webp: 'image', svg: 'image', tiff: 'image',
+    mp3: 'audio', wav: 'audio', m4a: 'audio', opus: 'audio', ogg: 'audio', flac: 'audio',
+    mp4: 'video', mov: 'video', webm: 'video', zip: 'archive', gz: 'archive', tar: 'archive',
+  };
+  const known = Object.hasOwn(unsupported, extension);
+  return { id: 'unsupported-source', version: VERSION, format: known ? extension : 'unknown',
+    modality: known ? unsupported[extension] : 'unknown', supported: false,
+    limitations: [DISCOVERY, REFERENCES, 'No built-in structural processor supports this source type; no content was extracted.'] };
+}
+
+function argumentsFor(bytes, limits) {
+  if (!(bytes instanceof Uint8Array)) refuse('INGESTION_ARGUMENT', 'Input bytes must be a Uint8Array or Buffer');
+  if (bytes.byteLength > MAX_BYTES) limit('Input exceeds the 16 MiB processor ceiling', 'input-bytes');
+  if (!limits || typeof limits !== 'object' || Array.isArray(limits) ||
+      Object.keys(limits).some(key => !['maxOutputBytes', 'maxEvidenceItems'].includes(key))) {
+    refuse('INGESTION_ARGUMENT', 'Explicit output and evidence limits are required');
+  }
+  for (const [key, ceiling] of [['maxOutputBytes', MAX_BYTES], ['maxEvidenceItems', MAX_ITEMS]]) {
+    if (!Number.isSafeInteger(limits[key]) || limits[key] < 0 || limits[key] > ceiling) {
+      refuse('INGESTION_ARGUMENT', 'Processor limits must be nonnegative integers within the built-in ceilings');
+    }
+  }
+}
+
+function lines(text, emit) {
+  const endings = /\r\n|\n|\r/g;
+  let start = 0, line = 1, match;
+  while ((match = endings.exec(text))) {
+    emit('line', String(line++), text.slice(start, match.index));
+    start = match.index + match[0].length;
+  }
+  if (start < text.length) emit('line', String(line), text.slice(start));
+}
+
+function csvCells(text, emit) {
+  let cursor = 0, row = 1, column = 1;
+  const invalid = () => refuse('INGESTION_INVALID_CSV', 'CSV has invalid quoting');
+  while (cursor < text.length) {
+    let value;
+    if (text[cursor] === '"') {
+      cursor++;
+      let start = cursor, closed = false;
+      const parts = [];
+      while (cursor < text.length) {
+        if (text[cursor] !== '"') { cursor++; continue; }
+        parts.push(text.slice(start, cursor));
+        if (text[cursor + 1] === '"') {
+          parts.push('"'); cursor += 2; start = cursor; continue;
+        }
+        cursor++; closed = true; break;
+      }
+      if (!closed) invalid();
+      if (cursor < text.length && ![',', '\r', '\n'].includes(text[cursor])) invalid();
+      value = parts.join('');
+    } else {
+      const start = cursor;
+      while (cursor < text.length && ![',', '\r', '\n'].includes(text[cursor])) {
+        if (text[cursor] === '"') invalid();
+        cursor++;
+      }
+      value = text.slice(start, cursor);
+    }
+    emit('csv-cell', `row:${row},column:${column}`, value);
+    if (cursor === text.length) break;
+    if (text[cursor] === ',') {
+      cursor++; column++;
+      if (cursor === text.length) emit('csv-cell', `row:${row},column:${column}`, '');
+    } else {
+      const crlf = text[cursor] === '\r' && text[cursor + 1] === '\n';
+      cursor += crlf ? 2 : 1; row++; column = 1;
+    }
+  }
+}
+
+function jsonLeaves(text, emit) {
+  let cursor = 0;
+  const invalid = () => refuse('INGESTION_INVALID_JSON', 'JSON is malformed or contains duplicate object keys');
+  const whitespace = () => { while (cursor < text.length && /[ \t\r\n]/.test(text[cursor])) cursor++; };
+  const pointerKey = key => key.replaceAll('~', '~0').replaceAll('/', '~1');
+  function string() {
+    if (text[cursor] !== '"') invalid();
+    const start = cursor++;
+    while (cursor < text.length) {
+      const current = text[cursor++];
+      if (current === '"') {
+        try { return JSON.parse(text.slice(start, cursor)); } catch { invalid(); }
+      }
+      if (current.charCodeAt(0) < 0x20) invalid();
+      if (current !== '\\') continue;
+      if (cursor === text.length) invalid();
+      const escape = text[cursor++];
+      if (escape === 'u') {
+        if (!/^[0-9a-fA-F]{4}$/.test(text.slice(cursor, cursor + 4))) invalid();
+        cursor += 4;
+      } else if (!'"\\/bfnrt'.includes(escape)) invalid();
+    }
+    invalid();
+  }
+  function value(pointer, depth) {
+    if (depth > 128) limit('JSON exceeds the nesting ceiling of 128', 'json-depth');
+    whitespace();
+    const next = text[cursor];
+    if (next === '"') { emit('json-pointer', pointer, string()); return; }
+    if (next === '{') {
+      cursor++; whitespace();
+      if (text[cursor] === '}') { cursor++; return; }
+      const keys = new Set();
+      while (cursor < text.length) {
+        whitespace();
+        const key = string();
+        if (keys.has(key)) invalid();
+        keys.add(key); whitespace();
+        if (text[cursor++] !== ':') invalid();
+        value(`${pointer}/${pointerKey(key)}`, depth + 1); whitespace();
+        const separator = text[cursor++];
+        if (separator === '}') return;
+        if (separator !== ',') invalid();
+      }
+      invalid();
+    }
+    if (next === '[') {
+      cursor++; whitespace();
+      if (text[cursor] === ']') { cursor++; return; }
+      let index = 0;
+      while (cursor < text.length) {
+        value(`${pointer}/${index++}`, depth + 1); whitespace();
+        const separator = text[cursor++];
+        if (separator === ']') return;
+        if (separator !== ',') invalid();
+      }
+      invalid();
+    }
+    for (const literal of ['true', 'false', 'null']) {
+      if (text.startsWith(literal, cursor)) {
+        cursor += literal.length; emit('json-pointer', pointer, literal); return;
+      }
+    }
+    const number = /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/y;
+    number.lastIndex = cursor;
+    const match = number.exec(text);
+    if (!match) invalid();
+    cursor = number.lastIndex;
+    emit('json-pointer', pointer, match[0]);
+  }
+  value('', 0); whitespace();
+  if (cursor !== text.length) invalid();
+}
+
+export function extractIngestionEvidence({ ref, bytes, limits } = {}) {
+  const descriptor = describeIngestionProcessor(ref);
+  argumentsFor(bytes, limits);
+  const evidence = [];
+  let evidenceBytes = 2;
+  const emit = (kind, value, text) => {
+    if (evidence.length >= limits.maxEvidenceItems) limit('Extraction exceeds the evidence item limit', 'evidence-items');
+    const item = { locator: { kind, value }, text };
+    evidenceBytes += Buffer.byteLength(JSON.stringify(item), 'utf8') + (evidence.length ? 1 : 0);
+    if (evidenceBytes > limits.maxOutputBytes) limit('Extraction exceeds the output byte limit', 'output-bytes');
+    evidence.push(item);
+  };
+  let unit = 'source';
+  if (descriptor.supported) {
+    let text;
+    try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); }
+    catch { refuse('INGESTION_INVALID_UTF8', 'Source is not valid UTF-8'); }
+    if (descriptor.id === 'utf8-lines') { unit = 'line'; lines(text, emit); }
+    else if (descriptor.id === 'csv-cells') { unit = 'csv-cell'; csvCells(text, emit); }
+    else { unit = 'json-leaf'; jsonLeaves(text, emit); }
+  }
+  const result = {
+    schema: 'mnstry.atelier-ingestion-extraction@v1',
+    processor: { id: descriptor.id, version: descriptor.version },
+    format: descriptor.format, modality: descriptor.modality,
+    coverage: { unit, total: descriptor.supported ? evidence.length : 1,
+      processed: evidence.length, omitted: descriptor.supported ? 0 : 1,
+      complete: descriptor.supported, limitations: [...descriptor.limitations] },
+    evidence,
+  };
+  if (Buffer.byteLength(JSON.stringify(result), 'utf8') > limits.maxOutputBytes) {
+    limit('Extraction including its coverage envelope exceeds the output byte limit', 'output-bytes');
+  }
+  return result;
+}

package/src/ingestion/semantic.mjs

@@ -0,0 +1,203 @@
+import { ingestionDigest, ingestionJson } from './contracts.mjs'
+import { assertHarness, harnessRef } from '../harnesses/contracts.mjs'
+
+// Internal first-consumer formats. These are not registered public contracts.
+export const SEMANTIC_CANDIDATE_VERSION = 'atelier.semantic-candidate/v0'
+const INPUT_VERSION = 'atelier.semantic-input/v0'
+const PROPOSAL_VERSION = 'atelier.semantic-proposals/v0'
+const modalities = new Set(['asserted', 'conditional', 'proposed', 'possible', 'uncertain', 'unknown'])
+const identifier = value => typeof value === 'string' && /^[a-z][a-z0-9-]{0,63}$/.test(value)
+const string = (value, max = 8192) => {
+  if (typeof value === 'string') check(value.length <= max, 'SEMANTIC_LIMIT', 'Semantic text length exceeded')
+  return typeof value === 'string' && value.length > 0
+}
+const array = (value, max) => {
+  if (Array.isArray(value)) check(value.length <= max, 'SEMANTIC_LIMIT', 'Semantic collection limit exceeded')
+  return Array.isArray(value)
+}
+const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value)
+
+export class SemanticProposalError extends Error {
+  constructor(code, message) { super(message); this.name = 'SemanticProposalError'; this.code = code; this.refusalCount = 1 }
+}
+function check(condition, code, message) { if (!condition) throw new SemanticProposalError(code, message) }
+function plain(value) {
+  try { return ingestionJson(value) }
+  catch (error) {
+    const limit = ['ingestion JSON depth exceeded', 'ingestion member ceiling exceeded', 'ingestion document byte ceiling exceeded'].includes(error?.message)
+    throw new SemanticProposalError(limit ? 'SEMANTIC_LIMIT' : 'SEMANTIC_INVALID', limit ? 'Semantic JSON limit exceeded' : 'Bounded plain JSON is required')
+  }
+}
+function closed(value, fields, code = 'SEMANTIC_INVALID') {
+  check(value && !Array.isArray(value) && typeof value === 'object' && Object.keys(value).length === fields.length && fields.every(field => Object.hasOwn(value, field)), code, 'The declared semantic shape is required')
+}
+function profile(value) {
+  const domain = plain(value)
+  try { assertHarness(domain, 'knowledge') }
+  catch { throw new SemanticProposalError('SEMANTIC_PROFILE', 'A valid existing knowledge domain is required') }
+  check(['domain', 'domain-revision'].includes(domain.kind), 'SEMANTIC_PROFILE', 'A domain definition is required')
+  for (const entries of [domain.data.vocabulary.types, domain.data.vocabulary.relations]) {
+    check(new Set(entries.map(entry => entry.id)).size === entries.length, 'SEMANTIC_PROFILE', 'Vocabulary identifiers must be unambiguous')
+  }
+  return domain
+}
+function evidenceReference(value) {
+  closed(value, ['sourceId', 'sourceDigest', 'attemptId', 'locator'])
+  check(identifier(value.sourceId) && digest(value.sourceDigest) && string(value.attemptId, 256), 'SEMANTIC_EVIDENCE', 'Exact source and attempt bindings are required')
+  closed(value.locator, ['kind', 'value'])
+  check(['line', 'csv-cell', 'json-pointer'].includes(value.locator.kind) && typeof value.locator.value === 'string', 'SEMANTIC_EVIDENCE', 'A supported exact locator is required')
+  check(value.locator.value.length <= 16384, 'SEMANTIC_LIMIT', 'Semantic locator length exceeded')
+}
+function readEvidence(store, plan, reference) {
+  check(typeof store?.getEvidence === 'function', 'SEMANTIC_EVIDENCE', 'The existing exact evidence reader is required')
+  let receipt
+  try { receipt = plain(store.getEvidence(plain({ ...plan, ...reference }))) }
+  catch (error) {
+    if (error instanceof SemanticProposalError) throw error
+    throw new SemanticProposalError(error?.code === 'INGESTION_STALE' ? 'SEMANTIC_STALE' : 'SEMANTIC_EVIDENCE', 'Current verified evidence could not be read')
+  }
+  check(receipt && typeof receipt === 'object' && !Array.isArray(receipt), 'SEMANTIC_EVIDENCE', 'An evidence receipt object is required')
+  check(receipt.readScope === 'all-plan', 'SEMANTIC_READ_SCOPE', 'This profile requires the existing all-plan read scope')
+  check(receipt.schema === 'mnstry.atelier-ingestion-evidence@v1', 'SEMANTIC_EVIDENCE', 'The existing evidence receipt schema is required')
+  check(receipt.freshness === 'current', 'SEMANTIC_EVIDENCE', 'Current source evidence is required')
+  check(receipt.integrity === 'verified', 'SEMANTIC_EVIDENCE', 'Verified source evidence integrity is required')
+  check(receipt.semanticAcceptance === 'pending', 'SEMANTIC_EVIDENCE', 'Evidence semantic acceptance must remain pending')
+  check(receipt.synthesized === false, 'SEMANTIC_EVIDENCE', 'Unsynthesized source evidence is required')
+  check(string(receipt.ref, 16384) && typeof receipt.text === 'string', 'SEMANTIC_EVIDENCE', 'Located source evidence text is required')
+  for (const key of ['planId', 'planDigest', 'sourceId', 'sourceDigest', 'attemptId']) check(receipt[key] === { ...plan, ...reference }[key], 'SEMANTIC_BINDING', 'Evidence differs from its requested binding')
+  closed(receipt.locator, ['kind', 'value'], 'SEMANTIC_EVIDENCE')
+  check(ingestionDigest(receipt.locator) === ingestionDigest(reference.locator), 'SEMANTIC_BINDING', 'Evidence locator differs from its requested binding')
+  return receipt
+}
+function payload(input) { const { digest: ignored, ...value } = input; return value }
+
+// The trusted host supplies a permitted all-plan reader and current domain.
+// It invokes its chosen extractor separately, using the returned pinned input.
+export function prepareSemanticInput({ store, plan, domain, references, identityCandidates = [] }) {
+  const selected = plain({ plan, references, identityCandidates }), definition = profile(domain)
+  closed(selected.plan, ['planId', 'planDigest'])
+  check(string(selected.plan.planId, 256) && /^sha256:[a-f0-9]{64}$/.test(selected.plan.planDigest), 'SEMANTIC_BINDING', 'A bound ingestion plan is required')
+  check(array(selected.references, 64), 'SEMANTIC_INVALID', 'Evidence references must be an array')
+  check(selected.references.length > 0, 'SEMANTIC_EVIDENCE', 'At least one evidence span is required')
+  check(array(selected.identityCandidates, 64), 'SEMANTIC_IDENTITY', 'Identity candidates must be an array')
+  const identities = new Set(), types = new Set(definition.data.vocabulary.types.map(t => t.id))
+  for (const candidate of selected.identityCandidates) {
+    closed(candidate, ['id', 'label', 'type'], 'SEMANTIC_IDENTITY')
+    check(identifier(candidate.id), 'SEMANTIC_IDENTITY', 'A valid supplied identity candidate identifier is required')
+    check(!identities.has(candidate.id), 'SEMANTIC_IDENTITY', 'Supplied identity candidate identifiers must be unique')
+    check(string(candidate.label), 'SEMANTIC_IDENTITY', 'A supplied identity candidate label is required')
+    check(types.has(candidate.type), 'SEMANTIC_IDENTITY', 'Supplied identity candidate types must fit the domain'); identities.add(candidate.id)
+  }
+  const seen = new Set(), evidence = []
+  let sourceId
+  for (const reference of selected.references) {
+    evidenceReference(reference)
+    sourceId ??= reference.sourceId
+    check(sourceId === reference.sourceId, 'SEMANTIC_SCOPE', 'The first semantic profile processes one source per input')
+    const key = ingestionDigest(reference)
+    check(!seen.has(key), 'SEMANTIC_EVIDENCE', 'Duplicate evidence references are refused'); seen.add(key)
+    evidence.push({ id: `span-${evidence.length + 1}`, reference, receipt: readEvidence(store, selected.plan, reference) })
+  }
+  const input = plain({ schema: INPUT_VERSION, plan: selected.plan, domain: definition, domainRef: harnessRef(definition), identityCandidates: selected.identityCandidates, evidence,
+    coverage: 'selected-spans-only', readScope: 'all-plan', authority: 'none' })
+  return { ...input, digest: ingestionDigest(input) }
+}
+function currentInput(store, value) {
+  const input = plain(value)
+  closed(input, ['schema', 'plan', 'domain', 'domainRef', 'identityCandidates', 'evidence', 'coverage', 'readScope', 'authority', 'digest'])
+  check(input.schema === INPUT_VERSION && input.authority === 'none' && input.coverage === 'selected-spans-only' && input.readScope === 'all-plan', 'SEMANTIC_BINDING', 'The pinned semantic input is required')
+  check(input.digest === ingestionDigest(payload(input)), 'SEMANTIC_BINDING', 'Semantic input digest differs')
+  check(array(input.evidence, 64) && input.evidence.length > 0, 'SEMANTIC_BINDING', 'A nonempty evidence array is required')
+  for (const span of input.evidence) closed(span, ['id', 'reference', 'receipt'])
+  const rebuilt = prepareSemanticInput({ store, plan: input.plan, domain: input.domain, references: input.evidence.map(e => e.reference), identityCandidates: input.identityCandidates })
+  check(rebuilt.digest === input.digest, 'SEMANTIC_BINDING', 'Current evidence or profile binding differs')
+  return rebuilt
+}
+function date(value) {
+  if (value === null) return true
+  if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false
+  const parsed = new Date(`${value}T00:00:00Z`)
+  return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value
+}
+
+export function prepareSemanticProposals({ store, input: original, candidates: supplied }) {
+  const candidates = plain(supplied), input = currentInput(store, original)
+  closed(candidates, ['schema', 'inputDigest', 'entities', 'assertions', 'unknowns'])
+  check(candidates.schema === SEMANTIC_CANDIDATE_VERSION && candidates.inputDigest === input.digest, 'SEMANTIC_BINDING', 'Candidates must bind the exact extractor input')
+  for (const key of ['entities', 'assertions', 'unknowns']) check(array(candidates[key], 64), 'SEMANTIC_INVALID', 'Semantic candidate collections must be arrays')
+  const ids = new Set(), entities = new Map(), spans = new Map(input.evidence.map(e => [e.id, e])), identityCandidates = new Map(input.identityCandidates.map(c => [c.id, c]))
+  const types = new Set(input.domain.data.vocabulary.types.map(t => t.id)), predicates = new Set(input.domain.data.vocabulary.relations.map(r => r.id))
+  function identity(id) { check(identifier(id) && !ids.has(id), 'SEMANTIC_IDENTITY', 'Unique source-local candidate identities are required'); ids.add(id) }
+  function supports(references, quoteRequired = false) {
+    check(array(references, 16) && references.length > 0, 'SEMANTIC_EVIDENCE', 'At least one bounded supporting span is required')
+    const seen = new Set()
+    return references.map(value => {
+      if (quoteRequired) closed(value, ['id', 'quote'], 'SEMANTIC_EVIDENCE')
+      const id = quoteRequired ? value.id : value, span = spans.get(id)
+      check(span && !seen.has(id), 'SEMANTIC_EVIDENCE', 'Supporting evidence must resolve uniquely'); seen.add(id)
+      const quote = quoteRequired ? value.quote : span.receipt.text
+      check(string(quote, 65536) && span.receipt.text.includes(quote), 'SEMANTIC_EVIDENCE', 'The quote must occur in the declared span')
+      return { id, ...span.reference, ref: span.receipt.ref, quote }
+    })
+  }
+  const preparedEntities = candidates.entities.map(entity => {
+    closed(entity, ['id', 'label', 'type', 'identity', 'evidence']); identity(entity.id)
+    check(types.has(entity.type) && string(entity.label), 'SEMANTIC_TYPE', 'Entity type and label must fit the supplied domain')
+    closed(entity.identity, ['status', 'candidateIds'], 'SEMANTIC_IDENTITY')
+    check(['source-local', 'existing-candidate', 'unknown'].includes(entity.identity.status) && array(entity.identity.candidateIds, 16), 'SEMANTIC_IDENTITY', 'Explicit candidate identity status is required')
+    const candidateIds = entity.identity.candidateIds
+    check(new Set(candidateIds).size === candidateIds.length, 'SEMANTIC_IDENTITY', 'Identity candidate mappings must be unique')
+    check(candidateIds.every(id => identityCandidates.has(id)), 'SEMANTIC_IDENTITY', 'Identity mappings must use supplied candidates')
+    check(candidateIds.every(id => !identityCandidates.has(id) || identityCandidates.get(id).type === entity.type), 'SEMANTIC_IDENTITY', 'Identity mappings must use matching candidate types')
+    check(entity.identity.status === 'existing-candidate' ? candidateIds.length > 0 : candidateIds.length === 0, 'SEMANTIC_IDENTITY', 'Identity status and candidates differ')
+    const prepared = { ...entity, evidence: supports(entity.evidence), identityAcceptance: 'pending' }
+    entities.set(entity.id, prepared); return prepared
+  })
+  const preparedAssertions = candidates.assertions.map(assertion => {
+    closed(assertion, ['id', 'subjectId', 'predicate', 'objectId', 'direction', 'negated', 'modality', 'scope', 'time', 'evidence']); identity(assertion.id)
+    check(entities.has(assertion.subjectId), 'SEMANTIC_IDENTITY', 'Assertion subject must resolve to a supplied entity')
+    check(assertion.objectId === null || entities.has(assertion.objectId), 'SEMANTIC_IDENTITY', 'Assertion object must resolve to a supplied entity')
+    check(predicates.has(assertion.predicate), 'SEMANTIC_PREDICATE', 'Unmodeled predicates must remain explicit findings')
+    check(assertion.direction === (assertion.objectId === null ? 'subject-only' : 'subject-to-object'), 'SEMANTIC_DIRECTION', 'Unary or directed binary roles must match the supplied endpoints')
+    check(typeof assertion.negated === 'boolean', 'SEMANTIC_NEGATION', 'Explicit Boolean negation is required')
+    check(modalities.has(assertion.modality), 'SEMANTIC_MODALITY', 'Explicit supported modality is required')
+    check(assertion.scope === input.domain.data.scope, 'SEMANTIC_SCOPE', 'Assertion scope differs from its supplied domain')
+    const evidence = supports(assertion.evidence, true), time = assertion.time
+    closed(time, ['from', 'until', 'expression', 'unknowns'], 'SEMANTIC_TIME')
+    check(date(time.from) && date(time.until) && (!time.from || !time.until || time.from <= time.until), 'SEMANTIC_TIME', 'Valid ordered calendar dates are required')
+    check((time.expression === null || string(time.expression)) && array(time.unknowns, 16) && time.unknowns.every(value => string(value)), 'SEMANTIC_TIME', 'Explicit bounded time qualifications are required')
+    check(time.expression === null || evidence.some(item => item.quote.includes(time.expression)), 'SEMANTIC_TIME', 'A temporal expression must be located in its cited passage')
+    check(time.expression === null || time.from !== null || time.until !== null || time.unknowns.length > 0, 'SEMANTIC_TIME', 'Unresolved temporal expressions must preserve their unknowns')
+    return { ...assertion, evidence, proposalAcceptance: 'pending' }
+  })
+  const unknowns = candidates.unknowns.map(unknown => {
+    closed(unknown, ['id', 'relatedCandidateId', 'reason', 'evidence'], 'SEMANTIC_UNKNOWN'); identity(unknown.id)
+    check(string(unknown.reason) && (unknown.relatedCandidateId === null || [...entities.keys(), ...preparedAssertions.map(a => a.id)].includes(unknown.relatedCandidateId)), 'SEMANTIC_UNKNOWN', 'An explicit finding reason and valid optional subject are required')
+    return { ...unknown, evidence: supports(unknown.evidence) }
+  })
+  return plain({ schema: PROPOSAL_VERSION, inputDigest: input.digest, domainRef: input.domainRef,
+    entities: preparedEntities, assertions: preparedAssertions, unknowns, rawCandidates: candidates,
+    counts: { entities: preparedEntities.length, assertions: preparedAssertions.length, abstentions: unknowns.length, refusals: 0 },
+    semanticAcceptance: 'pending', authority: 'none', canonicalMutation: false, coverage: input.coverage, readScope: input.readScope })
+}
+
+// A fresh, derived proposal view. Acceptance is still owned by knowledge review.
+export function readSemanticProposals({ store, input, proposals: supplied, query, limit = 64 }) {
+  check(string(query, 512) && query.trim().length > 0, 'SEMANTIC_INVALID', 'A bounded nonempty query is required')
+  check(Number.isSafeInteger(limit), 'SEMANTIC_INVALID', 'The result limit must be a safe integer')
+  check(limit > 0 && limit <= 64, 'SEMANTIC_LIMIT', 'The result limit must be between one and 64')
+  const proposals = plain(supplied)
+  closed(proposals, ['schema', 'inputDigest', 'domainRef', 'entities', 'assertions', 'unknowns', 'rawCandidates', 'counts', 'semanticAcceptance', 'authority', 'canonicalMutation', 'coverage', 'readScope'])
+  const current = prepareSemanticProposals({ store, input, candidates: proposals.rawCandidates })
+  check(ingestionDigest(current) === ingestionDigest(proposals), 'SEMANTIC_BINDING', 'The proposal view differs from its validated candidates')
+  const byId = new Map(current.entities.map(entity => [entity.id, entity])), terms = query.toLowerCase().trim().split(/\s+/u)
+  const matched = current.assertions.filter(assertion => {
+    const subject = byId.get(assertion.subjectId), object = byId.get(assertion.objectId)
+    const description = [subject.id, subject.label, object?.id ?? '', object?.label ?? '', assertion.predicate,
+      ...assertion.evidence.map(item => item.quote), assertion.time.expression ?? ''].join('\n').toLowerCase()
+    return terms.every(term => description.includes(term))
+  })
+  return plain({ schema: 'atelier.semantic-proposal-view/v0', inputDigest: current.inputDigest, domainRef: current.domainRef, entities: current.entities,
+    assertions: matched.slice(0, limit), unknowns: current.unknowns, totalMatched: matched.length, omitted: Math.max(0, matched.length - limit),
+    counts: current.counts, coverage: current.coverage, readScope: current.readScope, semanticAcceptance: 'pending', synthesized: false, canonicalMutation: false, authority: 'none' })
+}