@ngockhoale/ukit 3.4.23 → 3.4.25

package/src/core/observability/analytics/usageReadiness.js

@@ -0,0 +1,342 @@
+/**
+ * usageReadiness.js (TASK-C98-005, SPEC §5 FR-06 / §9 / §10) — pure, deterministic
+ * usage-readiness report.
+ *
+ *   buildUsageReadiness(records, options): UsageReport
+ *   renderUsageReadiness(report): string
+ *
+ * Pure like analytics/summary.js: no IO, no clock, no randomness — the same
+ * records in any order always produce a byte-identical report, and the input
+ * is never mutated.
+ *
+ * Honesty rules (SPEC §1/§5; docs/OBSERVABILITY.md):
+ *   - Totals are PER SOURCE and PER FAMILY only. Sources are never summed
+ *     together, and model vs subagent usage is never combined (a subagent
+ *     run may include model usage — summing would double-count).
+ *   - A total is null when no numeric value was recorded for that source:
+ *     unknown stays unknown, never 0. A provider-reported 0 stays 0.
+ *   - Record counts count records, not unique runs (ingest may duplicate).
+ *   - cache.hit/cache.miss are UKit's own lookup cache, never a provider
+ *     prompt cache.
+ *   - ESTIMATED / LOCAL_TOKENIZER / UNKNOWN evidence never upgrades the
+ *     verdict to aggregate_only/categorized.
+ *
+ * The five category key names are duplicated locally (SPEC FR-02) so wave-1
+ * tasks stay file-disjoint; TASK-C98-006 guards the drift by feeding a
+ * validator-accepted fixture through this report.
+ */
+
+export const USAGE_REPORT_VERSION = 'df-usage-1';
+
+// Local frozen copy of SPEC FR-02's USAGE_CATEGORY_KEYS (wave-1 parallelism).
+const USAGE_CATEGORY_KEYS = Object.freeze([
+  'fresh_input_tokens',
+  'cache_read_tokens',
+  'cache_write_tokens',
+  'output_tokens',
+  'reasoning_tokens',
+]);
+
+const SOURCE_KEYS = Object.freeze(['PROVIDER', 'LOCAL_TOKENIZER', 'ESTIMATED', 'UNKNOWN']);
+
+const INTERNAL_CACHE_SCOPE = 'ukit_internal_lookup_not_provider_prompt_cache';
+
+const MAX_DEGRADED_DEFAULT = 20;
+const MAX_STRING_LENGTH = 200;
+
+// Fixed canonical gap order (SPEC FR-06); membership is data-driven below.
+const GAP_NO_MODEL_RECORDS = 'no_model_records';
+const GAP_NO_PROVIDER_AGGREGATE = 'no_provider_aggregate';
+const GAP_NO_PROVIDER_CATEGORIES = 'no_provider_categories';
+const GAP_CATEGORIES_PARTIAL = 'categories_partial';
+const GAP_SUBAGENT_USAGE_ABSENT = 'subagent_usage_absent';
+const GAP_PROVIDER_IDENTITY = 'provider_identity_not_recorded';
+const GAP_PROMPT_CACHE = 'prompt_cache_not_provider_reported';
+const GAP_STORE_COVERAGE = 'store_coverage_degraded';
+const GAP_SCAN_TRUNCATED = 'scan_truncated';
+
+const NOTES = Object.freeze([
+  'Record counts are record counts, not unique runs (ingest may duplicate records).',
+  'cache.hit/cache.miss are UKit internal cache lookups, not a provider prompt cache.',
+  'Estimated values are never reported as provider values.',
+]);
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isNumeric(value) {
+  return typeof value === 'number' && Number.isFinite(value);
+}
+
+function truncateString(value) {
+  return typeof value === 'string' && value.length > MAX_STRING_LENGTH
+    ? value.slice(0, MAX_STRING_LENGTH)
+    : value;
+}
+
+function emptyBySource() {
+  const bySource = {};
+  for (const source of SOURCE_KEYS) {
+    bySource[source] = { records: 0, total: null };
+  }
+  return bySource;
+}
+
+/** Bucket one {value, source} entry. UNKNOWN totals stay null by shape. */
+function addUsageEntry(bySource, flags, entry) {
+  const source = isPlainObject(entry) && typeof entry.source === 'string' ? entry.source : null;
+  // Own-property guard: an inherited name like 'toString' must never resolve
+  // to a prototype member (R-A finding — purity/pollution).
+  const bucket = source === null || !Object.hasOwn(bySource, source)
+    ? null
+    : bySource[source];
+  if (!bucket) return;
+  bucket.records += 1;
+  const numeric = isNumeric(entry.value);
+  if (!numeric) return;
+  if (source === 'PROVIDER') flags.providerAggregate = true;
+  else if (source === 'ESTIMATED' || source === 'LOCAL_TOKENIZER') flags.numericNonProvider = true;
+  if (source !== 'UNKNOWN') {
+    bucket.total = (bucket.total === null ? 0 : bucket.total) + entry.value;
+  }
+}
+
+/** Capped, sorted, string-truncated copy of the reader coverage; input untouched. */
+function capCoverage(coverage, maxDegraded) {
+  if (!isPlainObject(coverage)) return null;
+  const capped = { ...coverage };
+  const degraded = Array.isArray(coverage.degraded)
+    ? coverage.degraded.filter(isPlainObject).map((entry) => ({
+        ...entry,
+        reason: truncateString(typeof entry.reason === 'string' ? entry.reason : ''),
+        file: truncateString(typeof entry.file === 'string' ? entry.file : ''),
+      }))
+    : [];
+  degraded.sort((a, b) => a.reason.localeCompare(b.reason) || a.file.localeCompare(b.file));
+  capped.degraded = degraded.slice(0, maxDegraded);
+  if (degraded.length > capped.degraded.length) {
+    capped.degraded_truncated = degraded.length - capped.degraded.length;
+  } else {
+    delete capped.degraded_truncated;
+  }
+  return capped;
+}
+
+function coverageIsDegraded(coverage) {
+  if (!isPlainObject(coverage)) return false;
+  if (Array.isArray(coverage.degraded) && coverage.degraded.length > 0) return true;
+  for (const key of ['corrupt_lines', 'quarantined_segments', 'expired_segments', 'unquarantined_segments']) {
+    if (isNumeric(coverage[key]) && coverage[key] > 0) return true;
+  }
+  return false;
+}
+
+export function buildUsageReadiness(records, options = {}) {
+  const opts = isPlainObject(options) ? options : {};
+  const maxDegraded =
+    Number.isInteger(opts.maxDegraded) && opts.maxDegraded > 0 ? opts.maxDegraded : MAX_DEGRADED_DEFAULT;
+  const scanLimit = Number.isInteger(opts.scanLimit) && opts.scanLimit >= 0 ? opts.scanLimit : null;
+
+  const counts = { model_completed: 0, model_failed: 0, subagent_usage: 0, cache_hit: 0, cache_miss: 0 };
+  const modelBySource = emptyBySource();
+  const subagentBySource = emptyBySource();
+  const categories = {};
+  for (const key of USAGE_CATEGORY_KEYS) {
+    categories[key] = { by_source: emptyBySource(), unknown_records: 0 };
+  }
+  const flags = { providerAggregate: false, numericNonProvider: false, usageRecord: false };
+  let providerCategory = false;
+
+  let recordsScanned = 0;
+  let truncated = opts.truncated === true;
+
+  const visit = (record) => {
+    if (!isPlainObject(record)) return;
+    const name = typeof record.semantic_name === 'string' ? record.semantic_name : '';
+    if (
+      name !== 'model.completed' &&
+      name !== 'model.failed' &&
+      name !== 'subagent.usage' &&
+      name !== 'cache.hit' &&
+      name !== 'cache.miss'
+    ) {
+      return;
+    }
+    const payload = isPlainObject(record.payload) ? record.payload : {};
+
+    if (name === 'cache.hit') {
+      counts.cache_hit += 1;
+      return;
+    }
+    if (name === 'cache.miss') {
+      counts.cache_miss += 1;
+      return;
+    }
+
+    const isModel = name === 'model.completed' || name === 'model.failed';
+    if (isModel) {
+      if (name === 'model.completed') counts.model_completed += 1;
+      else counts.model_failed += 1;
+    } else {
+      counts.subagent_usage += 1;
+    }
+
+    if (isModel && isPlainObject(payload.usage_categories)) {
+      let hasEntries = false;
+      for (const key of USAGE_CATEGORY_KEYS) {
+        const entry = payload.usage_categories[key];
+        if (!isPlainObject(entry)) continue;
+        hasEntries = true;
+        addUsageEntry(categories[key].by_source, flags, entry);
+        if (entry.source === 'UNKNOWN') categories[key].unknown_records += 1;
+        if (entry.source === 'PROVIDER' && isNumeric(entry.value)) providerCategory = true;
+      }
+      if (hasEntries) flags.usageRecord = true;
+    }
+
+    if (isPlainObject(payload.resource)) {
+      flags.usageRecord = true;
+      addUsageEntry(isModel ? modelBySource : subagentBySource, flags, payload.resource);
+    }
+  };
+
+  if (records !== null && records !== undefined && typeof records[Symbol.iterator] === 'function') {
+    for (const record of records) {
+      if (scanLimit !== null && recordsScanned === scanLimit) {
+        truncated = true;
+        break;
+      }
+      recordsScanned += 1;
+      visit(record);
+    }
+  }
+
+  let verdict;
+  let verdict_reason;
+  if (providerCategory) {
+    verdict = 'categorized';
+    verdict_reason = 'provider_categories';
+  } else if (flags.providerAggregate) {
+    verdict = 'aggregate_only';
+    verdict_reason = 'provider_aggregate';
+  } else {
+    verdict = 'no_data';
+    if (flags.numericNonProvider) verdict_reason = 'estimated_only';
+    else if (flags.usageRecord) verdict_reason = 'unknown_only';
+    else verdict_reason = 'no_usage_records';
+  }
+
+  const coverage = capCoverage(opts.coverage, maxDegraded);
+  const modelRecords = counts.model_completed + counts.model_failed;
+  const providerAggregatePresent =
+    modelBySource.PROVIDER.total !== null || subagentBySource.PROVIDER.total !== null;
+  const providerCategoryKeys = USAGE_CATEGORY_KEYS.filter(
+    (key) => categories[key].by_source.PROVIDER.total !== null,
+  );
+
+  const gaps = [];
+  if (modelRecords === 0) gaps.push(GAP_NO_MODEL_RECORDS);
+  if (!providerAggregatePresent) gaps.push(GAP_NO_PROVIDER_AGGREGATE);
+  if (providerCategoryKeys.length === 0) gaps.push(GAP_NO_PROVIDER_CATEGORIES);
+  else if (providerCategoryKeys.length < USAGE_CATEGORY_KEYS.length) gaps.push(GAP_CATEGORIES_PARTIAL);
+  if (counts.subagent_usage === 0) gaps.push(GAP_SUBAGENT_USAGE_ABSENT);
+  gaps.push(GAP_PROVIDER_IDENTITY);
+  gaps.push(GAP_PROMPT_CACHE);
+  if (coverageIsDegraded(coverage)) gaps.push(GAP_STORE_COVERAGE);
+  if (truncated) gaps.push(GAP_SCAN_TRUNCATED);
+
+  return {
+    report_version: USAGE_REPORT_VERSION,
+    verdict,
+    verdict_reason,
+    records_scanned: recordsScanned,
+    truncated,
+    scan_limit: scanLimit,
+    counts,
+    aggregate: { model: { by_source: modelBySource }, subagent: { by_source: subagentBySource } },
+    categories,
+    internal_cache: { hit: counts.cache_hit, miss: counts.cache_miss, scope: INTERNAL_CACHE_SCOPE },
+    gaps,
+    notes: NOTES.slice(),
+    coverage,
+  };
+}
+
+function renderBySource(indent, bySource) {
+  const lines = [];
+  for (const source of SOURCE_KEYS) {
+    const bucket = bySource[source];
+    const total = isNumeric(bucket.total) ? String(bucket.total) : 'unknown';
+    lines.push(`${indent}${source}: ${total} (${bucket.records} records)`);
+  }
+  return lines;
+}
+
+function renderCoverage(coverage) {
+  const lines = [];
+  if (!isPlainObject(coverage)) {
+    lines.push('  (no coverage information)');
+    return lines;
+  }
+  const counters = [
+    'records_read',
+    'corrupt_lines',
+    'quarantined_segments',
+    'expired_segments',
+    'unquarantined_segments',
+    'partial_tail_bytes',
+  ];
+  for (const key of counters) {
+    if (isNumeric(coverage[key])) lines.push(`  ${key}: ${coverage[key]}`);
+  }
+  if (isNumeric(coverage.degraded_truncated)) lines.push(`  degraded_truncated: ${coverage.degraded_truncated}`);
+  const degraded = Array.isArray(coverage.degraded) ? coverage.degraded : [];
+  if (degraded.length === 0) lines.push('  degraded: (none)');
+  else {
+    lines.push('  degraded:');
+    for (const entry of degraded) lines.push(`    - ${entry.reason} ${entry.file}`);
+  }
+  return lines;
+}
+
+export function renderUsageReadiness(report) {
+  const lines = [];
+  lines.push('Usage readiness (local, read-only)');
+  lines.push(`Readiness: ${report.verdict} (${report.verdict_reason})`);
+  let scannedLine = `Records scanned: ${report.records_scanned}`;
+  if (report.truncated && Number.isInteger(report.scan_limit)) {
+    scannedLine += ` (truncated at ${report.scan_limit})`;
+  }
+  lines.push(scannedLine);
+
+  lines.push('Aggregate usage (provider-reported only, per source)');
+  for (const family of ['model', 'subagent']) {
+    lines.push(`  ${family}:`);
+    lines.push(...renderBySource('    ', report.aggregate[family].by_source));
+  }
+
+  lines.push('Categories');
+  for (const key of USAGE_CATEGORY_KEYS) {
+    lines.push(`  ${key}:`);
+    lines.push(...renderBySource('    ', report.categories[key].by_source));
+    lines.push(`    unknown records: ${report.categories[key].unknown_records}`);
+  }
+
+  const cache = report.internal_cache;
+  const hit = isNumeric(cache.hit) ? String(cache.hit) : 'unknown';
+  const miss = isNumeric(cache.miss) ? String(cache.miss) : 'unknown';
+  lines.push(`Internal UKit cache lookups (not a provider prompt cache): hit ${hit} miss ${miss}`);
+
+  lines.push('Gaps:');
+  if (report.gaps.length === 0) lines.push('  (none)');
+  else for (const gap of report.gaps) lines.push(`  - ${gap}`);
+
+  lines.push('Store coverage:');
+  lines.push(...renderCoverage(report.coverage));
+
+  lines.push('Notes:');
+  for (const note of report.notes) lines.push(`  - ${note}`);
+
+  return lines.join('\n');
+}

package/src/core/observability/privacy/allowlist.js

@@ -11,7 +11,7 @@
  * redaction rules change so downstream readers can detect stale records.
  */
 
-export const REDACTION_VERSION = 'df-redact-6';
+export const REDACTION_VERSION = 'df-redact-7';
 
 /**
  * Envelope fields permitted on a persisted record (DF-FR01) plus the
@@ -188,6 +188,15 @@ export const ALLOWED_SUPPORT_PAYLOAD_FIELDS = new Set([
   'tokens',
   'parent_run_ref',
   'telemetry_complete',
+  // C98 FR-03 (TASK-C98-003): model token usage provenance — the closed
+  // usage_categories map and its five category keys. Counts and source tags
+  // only, never content.
+  'usage_categories',
+  'fresh_input_tokens',
+  'cache_read_tokens',
+  'cache_write_tokens',
+  'output_tokens',
+  'reasoning_tokens',
 ]);
 
 // --- size caps (prototype defaults per SPEC §14; freeze after measurement) ---

package/src/core/observability/schema/constants.js

@@ -61,6 +61,19 @@ export const IMPORTANCE_LEVELS = Object.freeze(['low', 'normal', 'high', 'critic
 // never 0.
 export const RESOURCE_SOURCES = Object.freeze(['PROVIDER', 'LOCAL_TOKENIZER', 'ESTIMATED', 'UNKNOWN']);
 
+// Schema v1.3 additive payload extension (C98 FR-02): typed per-category
+// token resources on model.completed|failed. Closed key set; an absent
+// category reads as unknown — never 0.
+export const USAGE_CATEGORY_KEYS = Object.freeze([
+  'fresh_input_tokens',
+  'cache_read_tokens',
+  'cache_write_tokens',
+  'output_tokens',
+  'reasoning_tokens',
+]);
+
+export const USAGE_CATEGORIES_FIELD = 'usage_categories';
+
 // SPEC §7: confidence is always typed and never an instruction authority.
 export const CONFIDENCE_TYPES = Object.freeze([
   'HEURISTIC',

package/src/core/observability/schema/registry.js

@@ -16,6 +16,7 @@ import { PRIVACY_CLASSES } from './constants.js';
 const V1 = '1';
 const V1_1 = '1.1';
 const V1_2 = '1.2';
+const V1_3 = '1.3';
 
 export const SEMANTIC_REGISTRY = Object.freeze({
   'execution.started': {
@@ -387,3 +388,22 @@ for (const [name, entry] of Object.entries(SEMANTIC_REGISTRY)) {
     throw new Error(`semantic registry entry ${name} has unknown privacy_class ${entry.privacy_class}`);
   }
 }
+
+// Schema v1.3 additive payload extensions (TASK-C98-002, SPEC §5 FR-02).
+// The semantic registry is append-only and model.completed|failed cannot be
+// re-defined in place, so the typed per-category usage field is versioned
+// here instead: a record either carries usage_categories (validated per
+// FR-02) or not — absence reads as unknown, never 0.
+const USAGE_CATEGORIES_EXTENSION = Object.freeze({
+  introduced_version: V1_3,
+  definition:
+    'Typed per-category token usage: five optional {value, source} resources '
+    + '(fresh_input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reasoning_tokens); '
+    + 'value is a non-negative finite integer or null; null only with source UNKNOWN — '
+    + 'an absent category or UNKNOWN stays unknown, never 0.',
+});
+
+export const PAYLOAD_EXTENSIONS = Object.freeze({
+  'model.completed': Object.freeze({ usage_categories: USAGE_CATEGORIES_EXTENSION }),
+  'model.failed': Object.freeze({ usage_categories: USAGE_CATEGORIES_EXTENSION }),
+});

package/src/core/observability/schema/validate.js

@@ -25,6 +25,8 @@ import {
   IMPORTANCE_LEVELS,
   RESOURCE_SOURCES,
   CONFIDENCE_TYPES,
+  USAGE_CATEGORY_KEYS,
+  USAGE_CATEGORIES_FIELD,
 } from './constants.js';
 import { SEMANTIC_REGISTRY, REASON_CODES } from './registry.js';
 
@@ -100,7 +102,48 @@ function validateOrigin(record, errors) {
   }
 }
 
-function validatePayload(payload, errors) {
+// Schema v1.3 additive payload extension (SPEC §5 FR-02): typed per-category
+// token usage on model.completed|failed. Additive — records without the
+// field validate exactly as before; an absent category reads as unknown.
+function validateUsageCategories(payload, semanticName, errors) {
+  const categories = payload[USAGE_CATEGORIES_FIELD];
+  if (categories === undefined || categories === null) return;
+
+  if (semanticName !== 'model.completed' && semanticName !== 'model.failed') {
+    errors.push(`payload.${USAGE_CATEGORIES_FIELD} is only valid on model.completed|model.failed`);
+    return;
+  }
+  if (!isPlainObject(categories)) {
+    errors.push(`payload.${USAGE_CATEGORIES_FIELD} must be an object keyed by usage category`);
+    return;
+  }
+  for (const [key, category] of Object.entries(categories)) {
+    if (!USAGE_CATEGORY_KEYS.includes(key)) {
+      errors.push(`payload.${USAGE_CATEGORIES_FIELD}.${key} is not a known usage category (allowed: ${USAGE_CATEGORY_KEYS.join('|')})`);
+      continue;
+    }
+    if (!isPlainObject(category)) {
+      errors.push(`payload.${USAGE_CATEGORIES_FIELD}.${key} must be an object {value, source}`);
+      continue;
+    }
+    if (!RESOURCE_SOURCES.includes(category.source)) {
+      errors.push(`payload.${USAGE_CATEGORIES_FIELD}.${key}.source must be one of ${RESOURCE_SOURCES.join('|')} (got ${JSON.stringify(category.source)})`);
+    }
+    const { value } = category;
+    if (value === null) {
+      if (category.source !== 'UNKNOWN') {
+        errors.push(`payload.${USAGE_CATEGORIES_FIELD}.${key}.value must be null only when source is UNKNOWN`);
+      }
+    } else if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+      errors.push(`payload.${USAGE_CATEGORIES_FIELD}.${key}.value must be a finite integer >= 0 or null`);
+    } else if (category.source === 'UNKNOWN') {
+      errors.push(`payload.${USAGE_CATEGORIES_FIELD}.${key}.value must be null when source is UNKNOWN — unknown is never 0`);
+    }
+  }
+}
+
+function validatePayload(payload, semanticName, errors) {
+  validateUsageCategories(payload, semanticName, errors);
   if (payload.resource !== undefined && payload.resource !== null) {
     const resource = payload.resource;
     if (!isPlainObject(resource)) {
@@ -264,7 +307,7 @@ export function validateSemanticRecord(record, context) {
   if (!isPlainObject(record.payload)) {
     errors.push('payload must be an object');
   } else {
-    validatePayload(record.payload, errors);
+    validatePayload(record.payload, record.semantic_name, errors);
   }
 
   if (RECORD_TYPES.includes(record.record_type)) {

package/src/core/observability/segments/readSegments.js

@@ -3,7 +3,7 @@
  *
  * Typed reader over the append-only JSONL segment store.
  *
- *   readSegments(root, { fromInclusive?, limit?, ioTimeoutMs?, zlib? })
+ *   readSegments(root, { fromInclusive?, limit?, ioTimeoutMs?, zlib?, readOnly? })
  *     → AsyncIterable<record> with attached:
  *         .coverage → { expired_segments, quarantined_segments, corrupt_lines,
  *                       partial_tail_bytes, cursor_missed, degraded[] }
@@ -26,6 +26,14 @@
  * A partial tail row in the active segment (hard-kill artifact) is skipped
  * and counted, never parsed. Symlinked or foreign-named files are never
  * followed or read.
+ *
+ * Read-only mode (`readOnly: true`, strict boolean): the reader never mutates
+ * the store — corrupt sealed segments (checksum mismatch, missing meta, failed
+ * decode) are skipped and reported with `checksum_mismatch_unquarantined` /
+ * `decode_failed_unquarantined` degraded entries plus a
+ * `coverage.unquarantined_segments` counter (that key exists only in
+ * read-only mode) instead of being quarantined. Non-boolean values are
+ * treated as false. Default mode is unchanged and still quarantines.
  */
 
 import fs from 'node:fs';
@@ -49,6 +57,9 @@ export function readSegments(root, opts = {}) {
   const timeoutMs = Number.isFinite(opts.ioTimeoutMs) ? opts.ioTimeoutMs : DEFAULT_IO_TIMEOUT_MS;
   const fromInclusive = typeof opts.fromInclusive === 'string' ? opts.fromInclusive : null;
   const limit = Number.isInteger(opts.limit) && opts.limit >= 0 ? opts.limit : null;
+  // Strict boolean gate: truthy non-boolean values ('yes', 1) must NOT enable
+  // read-only mode — the caller opted into quarantine-free reads explicitly.
+  const readOnly = opts.readOnly === true;
 
   const coverage = {
     expired_segments: 0,
@@ -58,6 +69,7 @@ export function readSegments(root, opts = {}) {
     cursor_missed: false,
     degraded: [],
   };
+  if (readOnly) coverage.unquarantined_segments = 0;
   const degrade = (reason, file) => {
     coverage.degraded.push({ reason, file });
   };
@@ -123,6 +135,11 @@ export function readSegments(root, opts = {}) {
       const checksumOk =
         meta && typeof meta.checksum === 'string' && meta.checksum === sha256Buffer(buf);
       if (!checksumOk) {
+        if (readOnly) {
+          coverage.unquarantined_segments += 1;
+          degrade('checksum_mismatch_unquarantined', seg.name);
+          continue;
+        }
         const q = await quarantineSegment(resolved.root, seg.name, 'checksum_mismatch', {
           ioTimeoutMs: timeoutMs,
         });
@@ -150,6 +167,11 @@ export function readSegments(root, opts = {}) {
         if (decoded === null || decoded.length === 0) {
           // Stored bytes verified but not a valid/non-empty zstd stream →
           // corrupt like a bad .jsonl: quarantine, never falsify rows.
+          if (readOnly) {
+            coverage.unquarantined_segments += 1;
+            degrade('decode_failed_unquarantined', seg.name);
+            continue;
+          }
           const q = await quarantineSegment(resolved.root, seg.name, 'decode_failed', {
             ioTimeoutMs: timeoutMs,
           });

package/template_project/.claude/ukit/runtime/observability-emit.mjs

@@ -143,9 +143,19 @@ const REASON_CODES = new Set([
 
 const RESOURCE_SEMANTICS = new Set(['model.completed', 'model.failed']);
 
+// Schema v1.3 additive payload extension (C98 FR-02): closed usage-category
+// key set — mirror of USAGE_CATEGORY_KEYS in schema/constants.js.
+const USAGE_CATEGORY_KEYS = Object.freeze([
+  'fresh_input_tokens',
+  'cache_read_tokens',
+  'cache_write_tokens',
+  'output_tokens',
+  'reasoning_tokens',
+]);
+
 // --- frozen privacy contract (mirror of privacy/allowlist.js) ------------
 
-const REDACTION_VERSION = 'df-redact-6';
+const REDACTION_VERSION = 'df-redact-7';
 const ALLOWED_FIELDS = ENVELOPE_FIELDS;
 const DENIED_PAYLOAD_FIELDS = new Set([
   'error_detail', 'feedback', 'comment', 'note_text', 'prompt', 'prompt_text',
@@ -299,7 +309,47 @@ function sanitizeObserved(record, options = {}) {
 
 // --- validate (mirror of schema/validate.js, sans cross-record context) ----
 
-function validateSemanticRecord(record) {
+// Schema v1.3 additive payload extension (SPEC FR-02): typed per-category
+// token usage on model.completed|failed. Accept/reject outcomes must match
+// validateUsageCategories in src/core/observability/schema/validate.js
+// exactly; message texts may differ.
+function validateUsageCategories(payload, semanticName, errors) {
+  const categories = payload.usage_categories;
+  if (categories === undefined || categories === null) return;
+  if (semanticName !== 'model.completed' && semanticName !== 'model.failed') {
+    errors.push('payload.usage_categories is only valid on model.completed|model.failed');
+    return;
+  }
+  if (!isPlainObject(categories)) {
+    errors.push('payload.usage_categories must be an object keyed by usage category');
+    return;
+  }
+  for (const [key, category] of Object.entries(categories)) {
+    if (!USAGE_CATEGORY_KEYS.includes(key)) {
+      errors.push(`payload.usage_categories.${key} is not a known usage category`);
+      continue;
+    }
+    if (!isPlainObject(category)) {
+      errors.push(`payload.usage_categories.${key} must be an object {value, source}`);
+      continue;
+    }
+    if (!RESOURCE_SOURCES.has(category.source)) {
+      errors.push(`payload.usage_categories.${key}.source invalid`);
+    }
+    const { value } = category;
+    if (value === null) {
+      if (category.source !== 'UNKNOWN') {
+        errors.push(`payload.usage_categories.${key}.value must be null only when source is UNKNOWN`);
+      }
+    } else if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+      errors.push(`payload.usage_categories.${key}.value must be a finite integer >= 0 or null`);
+    } else if (category.source === 'UNKNOWN') {
+      errors.push(`payload.usage_categories.${key}.value must be null when source is UNKNOWN`);
+    }
+  }
+}
+
+export function validateSemanticRecord(record) {
   const errors = [];
   if (!isPlainObject(record)) return { ok: false, errors: ['record must be a plain object'] };
 
@@ -385,6 +435,7 @@ function validateSemanticRecord(record) {
     errors.push('payload must be an object');
   } else {
     const payload = record.payload;
+    validateUsageCategories(payload, record.semantic_name, errors);
     if (payload.resource !== undefined && payload.resource !== null) {
       const resource = payload.resource;
       if (!isPlainObject(resource)) {