@ngockhoale/ukit 3.0.5 → 3.0.7

package/src/core/observability/evaluation/scorecard.js

@@ -0,0 +1,445 @@
+/**
+ * scorecard.js (TASK-014, SPEC §5 DF-FR08/DF-FR09/DF-FR12, §8) — golden
+ * corpus evaluation.
+ *
+ *   evaluateVariant(corpus, policy): Scorecard
+ *   compareScorecards(a, b, { seed }): Comparison
+ *
+ * Quality is the guardrail: golden cases declare evidence / actions /
+ * forbidden mistakes / verification / acceptable bounds — never exact-prose
+ * equality. Correction semantics are honest: when a user correction follows
+ * the first answer, that first answer is NOT an accepted success and
+ * tokens_to_success / time_to_success_ms include the observed retry.
+ *
+ * Scorecard (SPEC §8):
+ *   { metric_version, policy, quality: { met, unmet, violated, unknown,
+ *     forbidden, bounds }, latency, tokens, cohorts, provenance, fidelity,
+ *     sample_size, verdict: 'pass'|'fail'|'inconclusive', cases }
+ *
+ * Everything here is offline and deterministic — no model calls, no IO, no
+ * clock, no randomness outside the seeded bootstrap in compareScorecards.
+ */
+
+import { summarizeTrace, percentileOf } from '../analytics/summary.js';
+import { replayCase } from './replay.js';
+
+export const EVALUATION_METRIC_VERSION = 'df-m1';
+export const DEFAULT_MIN_COHORT_N = 3;
+const BOOTSTRAP_RESAMPLES = 2000;
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+/** Deterministic PRNG (mulberry32) — same construction as the bench corpus. */
+function mulberry32(seed) {
+  let a = seed >>> 0;
+  return () => {
+    a |= 0;
+    a = (a + 0x6d2b79f5) | 0;
+    let t = Math.imul(a ^ (a >>> 15), 1 | a);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function resolvePolicy(corpus, policy) {
+  if (typeof policy === 'string') {
+    const registered = isPlainObject(corpus.pre_registered)
+      && isPlainObject(corpus.pre_registered.policies)
+      ? corpus.pre_registered.policies[policy]
+      : null;
+    return { id: policy, ...(isPlainObject(registered) ? registered : {}) };
+  }
+  if (isPlainObject(policy)) {
+    return { id: typeof policy.id === 'string' ? policy.id : 'custom', ...policy };
+  }
+  return { id: 'custom' };
+}
+
+function semanticNames(records) {
+  const names = new Set();
+  for (const r of records) {
+    if (isPlainObject(r) && typeof r.semantic_name === 'string') names.add(r.semantic_name);
+  }
+  return names;
+}
+
+/** Human-reviewable action labels: operation/tool names on span records. */
+function actionLabels(records) {
+  const actions = new Set();
+  for (const r of records) {
+    if (!isPlainObject(r) || !isPlainObject(r.payload)) continue;
+    if (typeof r.payload.operation === 'string') actions.add(r.payload.operation);
+    if (typeof r.payload.tool === 'string') actions.add(r.payload.tool);
+  }
+  return actions;
+}
+
+function executionOutcome(records) {
+  for (const r of records) {
+    if (!isPlainObject(r) || !isPlainObject(r.payload)) continue;
+    if ((r.semantic_name === 'execution.completed' || r.semantic_name === 'execution.failed'
+      || r.semantic_name === 'execution.blocked') && typeof r.payload.outcome === 'string') {
+      return r.payload.outcome;
+    }
+  }
+  return null;
+}
+
+function correctionObserved(records) {
+  return records.some((r) => isPlainObject(r) && isPlainObject(r.payload)
+    && (r.payload.correction_observed === true || r.payload.kind === 'user_correction'));
+}
+
+const BOUND_METRICS = Object.freeze(['tokens_to_success', 'time_to_success_ms']);
+
+function checkBounds(bounds, metrics) {
+  const result = { checked: 0, violated: 0, unknown: 0, detail: {} };
+  if (!isPlainObject(bounds)) return result;
+  for (const metric of BOUND_METRICS) {
+    const bound = bounds[metric];
+    if (!isPlainObject(bound)) continue;
+    const value = metrics[metric];
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+      result.unknown += 1;
+      result.detail[metric] = 'unknown';
+      continue;
+    }
+    result.checked += 1;
+    const overMax = typeof bound.max === 'number' && value > bound.max;
+    const underMin = typeof bound.min === 'number' && value < bound.min;
+    if (overMax || underMin) {
+      result.violated += 1;
+      result.detail[metric] = 'violated';
+    } else {
+      result.detail[metric] = 'met';
+    }
+  }
+  return result;
+}
+
+/**
+ * Evaluate one replayed case against its rubric. Quality precedence:
+ * violated > unmet > unknown > met. SIMULATED replays can never claim a
+ * quality label — user acceptance is not observable.
+ */
+function evaluateCase(goldenCase, policyId) {
+  const replay = replayCase(goldenCase, { policyId });
+  const base = {
+    case_id: replay.case_id,
+    cohort: typeof goldenCase.cohort === 'string' ? goldenCase.cohort : 'default',
+    fidelity: { tag: replay.tag, limitations: replay.limitations },
+  };
+  if (!replay.ok) {
+    return { ...base, ok: false, error: replay.reason, quality: 'unknown', metrics: {} };
+  }
+
+  const records = replay.records;
+  const summary = summarizeTrace(records);
+  const names = semanticNames(records);
+  const actions = actionLabels(records);
+  const expect = isPlainObject(goldenCase.expect) ? goldenCase.expect : {};
+
+  const evidenceRequired = Array.isArray(expect.evidence) ? expect.evidence : [];
+  const evidenceMissing = evidenceRequired.filter((name) => !names.has(name));
+  const actionsRequired = Array.isArray(expect.actions) ? expect.actions : [];
+  const actionsMissing = actionsRequired.filter((name) => !actions.has(name));
+  const forbiddenList = Array.isArray(expect.forbidden_mistakes) ? expect.forbidden_mistakes : [];
+  const forbiddenHits = forbiddenList.filter((name) => names.has(name));
+
+  const outcome = executionOutcome(records);
+  const expectedOutcome = isPlainObject(expect.verification) && typeof expect.verification.outcome === 'string'
+    ? expect.verification.outcome
+    : null;
+  const verification = expectedOutcome === null
+    ? (outcome === null ? 'unknown' : 'met')
+    : (outcome === null ? 'unknown' : (outcome === expectedOutcome ? 'met' : 'violated'));
+
+  const succeeded = outcome === 'success'
+    || (outcome === null && summary.coverage.open_spans === 0 && summary.retries.failed_spans === 0);
+  const corrected = correctionObserved(records);
+  const metrics = {
+    tokens_to_success: succeeded ? summary.resource.total_value : null,
+    time_to_success_ms: succeeded ? summary.critical_path_ms : null,
+    model_attempts: summary.retries.model_attempts,
+    correction_observed: corrected,
+    // one accepted success per successful episode; a correction means the
+    // first answer was NOT it.
+    accepted_success: succeeded ? 1 : 0,
+    first_answer_accepted: succeeded && !corrected && summary.retries.model_attempts <= 1,
+    succeeded,
+  };
+  const bounds = checkBounds(expect.bounds, metrics);
+
+  let quality;
+  if (replay.tag === 'SIMULATED') {
+    quality = 'unknown';
+  } else if (forbiddenHits.length > 0 || verification === 'violated' || bounds.violated > 0) {
+    quality = 'violated';
+  } else if (evidenceMissing.length > 0 || actionsMissing.length > 0) {
+    quality = 'unmet';
+  } else if (verification === 'unknown' || bounds.unknown > 0) {
+    quality = 'unknown';
+  } else {
+    quality = 'met';
+  }
+
+  return {
+    ...base,
+    ok: true,
+    quality,
+    evidence: { required: evidenceRequired, missing: evidenceMissing },
+    actions: { required: actionsRequired, missing: actionsMissing },
+    forbidden_mistakes: { forbidden: forbiddenList, hits: forbiddenHits },
+    verification: { expected: expectedOutcome, observed: outcome, status: verification },
+    bounds,
+    metrics,
+    provenance: replay.provenance,
+  };
+}
+
+function meanOf(values) {
+  if (values.length === 0) return null;
+  return values.reduce((a, b) => a + b, 0) / values.length;
+}
+
+function distribution(values) {
+  const sorted = values.filter((v) => typeof v === 'number' && Number.isFinite(v)).sort((a, b) => a - b);
+  return {
+    n: sorted.length,
+    mean: meanOf(sorted),
+    p50: percentileOf(sorted, 50),
+    p95: percentileOf(sorted, 95),
+  };
+}
+
+function emptyQuality() {
+  return {
+    met: 0, unmet: 0, violated: 0, unknown: 0, forbidden: 0,
+    bounds: { checked: 0, violated: 0, unknown: 0 },
+  };
+}
+
+function accumulateQuality(bucket, result) {
+  if (result.quality === 'met') bucket.met += 1;
+  else if (result.quality === 'unmet') bucket.unmet += 1;
+  else if (result.quality === 'violated') bucket.violated += 1;
+  else bucket.unknown += 1;
+  if (result.forbidden_mistakes) bucket.forbidden += result.forbidden_mistakes.hits.length;
+  if (result.bounds) {
+    bucket.bounds.checked += result.bounds.checked;
+    bucket.bounds.violated += result.bounds.violated;
+    bucket.bounds.unknown += result.bounds.unknown;
+  }
+}
+
+function cohortVerdict(results, minN) {
+  if (results.length < minN) return 'inconclusive';
+  if (results.some((r) => r.quality === 'violated' || r.quality === 'unmet')) return 'fail';
+  if (results.every((r) => r.quality === 'unknown')) return 'inconclusive';
+  return 'pass';
+}
+
+/**
+ * @param {object} corpus — `{ cases: GoldenCase[], pre_registered?, metric_version? }`.
+ * @param {string|object} policy — policy id (resolved against
+ *   `corpus.pre_registered.policies`) or an inline policy object.
+ * @returns {object} Scorecard
+ */
+export function evaluateVariant(corpus, policy) {
+  const source = isPlainObject(corpus) ? corpus : {};
+  const cases = Array.isArray(source.cases) ? source.cases : [];
+  const pre = isPlainObject(source.pre_registered) ? source.pre_registered : {};
+  const minN = Number.isInteger(pre.min_cohort_n) && pre.min_cohort_n > 0
+    ? pre.min_cohort_n
+    : DEFAULT_MIN_COHORT_N;
+  const policyInfo = resolvePolicy(source, policy);
+
+  const results = cases.map((c) => evaluateCase(isPlainObject(c) ? c : {}, policyInfo.id));
+
+  const quality = emptyQuality();
+  const fidelityTags = {};
+  const fidelityLimitations = new Set();
+  const cohorts = new Map();
+  const latencyValues = [];
+  const tokenValues = [];
+
+  for (const result of results) {
+    accumulateQuality(quality, result);
+    fidelityTags[result.fidelity.tag] = (fidelityTags[result.fidelity.tag] || 0) + 1;
+    for (const lim of result.fidelity.limitations) fidelityLimitations.add(lim);
+    if (typeof result.metrics.time_to_success_ms === 'number') {
+      latencyValues.push(result.metrics.time_to_success_ms);
+    }
+    if (typeof result.metrics.tokens_to_success === 'number') {
+      tokenValues.push(result.metrics.tokens_to_success);
+    }
+    if (!cohorts.has(result.cohort)) cohorts.set(result.cohort, []);
+    cohorts.get(result.cohort).push(result);
+  }
+
+  const cohortReport = {};
+  for (const [key, members] of [...cohorts.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+    const q = emptyQuality();
+    for (const m of members) accumulateQuality(q, m);
+    cohortReport[key] = {
+      n: members.length,
+      min_n: minN,
+      verdict: cohortVerdict(members, minN),
+      quality: q,
+      latency: distribution(members.map((m) => m.metrics.time_to_success_ms)),
+      tokens: distribution(members.map((m) => m.metrics.tokens_to_success)),
+      cases: members.map((m) => m.case_id),
+    };
+  }
+
+  const cohortVerdicts = Object.values(cohortReport).map((c) => c.verdict);
+  const verdict = cohortVerdicts.includes('fail')
+    ? 'fail'
+    : (cohortVerdicts.length === 0 || cohortVerdicts.includes('inconclusive') ? 'inconclusive' : 'pass');
+
+  return {
+    metric_version: typeof source.metric_version === 'string'
+      ? source.metric_version
+      : EVALUATION_METRIC_VERSION,
+    policy: policyInfo,
+    quality,
+    latency: distribution(latencyValues),
+    tokens: distribution(tokenValues),
+    cohorts: cohortReport,
+    provenance: {
+      policy: policyInfo,
+      versions: {
+        metric_version: typeof source.metric_version === 'string'
+          ? source.metric_version
+          : EVALUATION_METRIC_VERSION,
+        ...(isPlainObject(pre.versions) ? pre.versions : {}),
+      },
+      pre_registered: {
+        min_cohort_n: minN,
+        cohorts: Array.isArray(pre.cohorts) ? pre.cohorts : Object.keys(cohortReport),
+        policies: isPlainObject(pre.policies) ? Object.keys(pre.policies).sort() : [policyInfo.id],
+      },
+      corpus_cases: cases.length,
+    },
+    fidelity: {
+      tags: fidelityTags,
+      limitations: [...fidelityLimitations].sort(),
+    },
+    sample_size: results.length,
+    verdict,
+    cases: results,
+  };
+}
+
+/**
+ * Paired bootstrap CI for per-case deltas (fixed seed → deterministic).
+ * Returns null when the pair count cannot support an interval.
+ */
+function bootstrapCi95(deltas, seed) {
+  if (deltas.length < 2) return null;
+  const rand = mulberry32(seed);
+  const means = [];
+  for (let i = 0; i < BOOTSTRAP_RESAMPLES; i += 1) {
+    let sum = 0;
+    for (let j = 0; j < deltas.length; j += 1) {
+      sum += deltas[Math.floor(rand() * deltas.length)];
+    }
+    means.push(sum / deltas.length);
+  }
+  means.sort((a, b) => a - b);
+  return [percentileOf(means, 2.5), percentileOf(means, 97.5)];
+}
+
+function metricDelta(pairs, key, seed) {
+  const deltas = [];
+  let skipped = 0;
+  for (const [a, b] of pairs) {
+    const va = a.metrics[key];
+    const vb = b.metrics[key];
+    if (typeof va === 'number' && Number.isFinite(va)
+      && typeof vb === 'number' && Number.isFinite(vb)) {
+      deltas.push(vb - va);
+    } else {
+      skipped += 1;
+    }
+  }
+  return {
+    n_paired: deltas.length,
+    skipped,
+    mean_delta: meanOf(deltas),
+    ci95: bootstrapCi95(deltas, seed),
+  };
+}
+
+function directionOf(metric) {
+  if (!metric.ci95) return 'inconclusive';
+  if (metric.ci95[1] < 0) return 'improvement';
+  if (metric.ci95[0] > 0) return 'regression';
+  return 'no_difference';
+}
+
+/**
+ * Compare two scorecards over the same golden case set. Paired by case_id
+ * within each cohort; differences carry a deterministic bootstrap
+ * confidence band. Token/latency deltas are proxy evidence — the report
+ * never upgrades a proxy-only delta into a user-visible improvement claim.
+ */
+export function compareScorecards(a, b, { seed = 42 } = {}) {
+  const aById = new Map((a.cases || []).map((c) => [c.case_id, c]));
+  const bById = new Map((b.cases || []).map((c) => [c.case_id, c]));
+  const cohortKeys = new Set([...Object.keys(a.cohorts || {}), ...Object.keys(b.cohorts || {})]);
+
+  const cohorts = {};
+  for (const key of [...cohortKeys].sort()) {
+    const pairs = [];
+    for (const [id, ca] of aById) {
+      const cb = bById.get(id);
+      if (cb && ca.cohort === key && cb.cohort === key && ca.ok && cb.ok) pairs.push([ca, cb]);
+    }
+    const tokens = metricDelta(pairs, 'tokens_to_success', seed);
+    const time = metricDelta(pairs, 'time_to_success_ms', seed + 1);
+    const directions = [directionOf(tokens), directionOf(time)];
+    const verdict = directions.includes('regression')
+      ? 'regression'
+      : directions.includes('improvement')
+        ? 'improvement'
+        : directions.every((d) => d === 'inconclusive') ? 'inconclusive' : 'no_difference';
+    cohorts[key] = {
+      n_paired: pairs.length,
+      tokens_to_success: tokens,
+      time_to_success_ms: time,
+      quality: {
+        met_delta: (b.cohorts?.[key]?.quality.met ?? 0) - (a.cohorts?.[key]?.quality.met ?? 0),
+        violated_delta: (b.cohorts?.[key]?.quality.violated ?? 0)
+          - (a.cohorts?.[key]?.quality.violated ?? 0),
+      },
+      verdict,
+    };
+  }
+
+  const verdicts = Object.values(cohorts).map((c) => c.verdict);
+  const verdict = verdicts.includes('regression')
+    ? 'regression'
+    : verdicts.includes('improvement')
+      ? 'improvement'
+      : verdicts.includes('inconclusive') ? 'inconclusive' : 'no_difference';
+
+  return {
+    metric_version: EVALUATION_METRIC_VERSION,
+    seed,
+    baseline: a.policy?.id ?? 'a',
+    variant: b.policy?.id ?? 'b',
+    cohorts,
+    verdict,
+    basis: 'proxy_metrics_plus_quality_labels',
+    notes: [
+      'tokens_to_success and time_to_success_ms are proxy metrics: a proxy-only '
+        + 'delta does not claim user-visible improvement — human-reviewable '
+        + 'quality labels (met/unmet/violated) are reported separately.',
+      'confidence bands are paired bootstrap intervals at a fixed seed; '
+        + 'cohorts below the pre-registered minimum are inconclusive, never failed.',
+    ],
+  };
+}

package/src/core/observability/privacy/allowlist.js

@@ -0,0 +1,185 @@
+/**
+ * allowlist.js (TASK-004, SPEC §5 DF-FR06 / §8)
+ *
+ * Canonical privacy-gate constants for the observability pipeline. The
+ * allowlist — not the secret scanner — is the persistence boundary: only
+ * fields named here may appear on a record that reaches disk, and only
+ * ALLOWED_SUPPORT_PAYLOAD_FIELDS may appear in a record projected to the
+ * user-facing support view.
+ *
+ * Bump REDACTION_VERSION whenever the allowlist, deny list, caps, or
+ * redaction rules change so downstream readers can detect stale records.
+ */
+
+export const REDACTION_VERSION = 'df-redact-3';
+
+/**
+ * Envelope fields permitted on a persisted record (DF-FR01) plus the
+ * redaction stamp this module adds. Anything else is dropped by
+ * sanitizeObserved and rejected by sanitizeForSupport.
+ */
+export const ALLOWED_FIELDS = new Set([
+  'record_type',
+  'semantic_name',
+  'schema_version',
+  'record_id',
+  'trace_id',
+  'span_id',
+  'parent_span_id',
+  'execution_id',
+  'session_id',
+  'project_ref',
+  'boot_id',
+  'writer_id',
+  'sequence',
+  'wall_time_utc',
+  'monotonic_ns',
+  'importance',
+  'privacy_class',
+  'payload',
+  'origin',
+  'redaction_version',
+]);
+
+/**
+ * Payload keys whose values are free-form content by definition and are
+ * never persisted in any form — user-entered feedback is excluded by
+ * default per SPEC §14/planner note, and raw prompts/commands/output are
+ * content, not metadata. Dropped silently by sanitizeObserved.
+ */
+export const DENIED_PAYLOAD_FIELDS = new Set([
+  'error_detail',
+  'feedback',
+  'comment',
+  'note_text',
+  'prompt',
+  'prompt_text',
+  'system_prompt',
+  'messages',
+  'command',
+  'cmd',
+  'argv',
+  'stdin',
+  'stdout',
+  'stderr',
+  'output',
+  'raw_output',
+  'body',
+  'request_body',
+  'response_body',
+  'content',
+  'text',
+  'diff',
+  'patch',
+  'snippet',
+  'env',
+  'dotenv',
+  'secret',
+  'token',
+  'api_key',
+  'apikey',
+  'password',
+  'passwd',
+  'credential',
+  'credentials',
+  'auth',
+  'authorization',
+  'cookie',
+  'cookies',
+  'session_token',
+  'private_key',
+  'email',
+  'username',
+  'hostname',
+  'host',
+  'ip',
+  'ip_address',
+  'url',
+  'uri',
+  'path',
+  'file_path',
+  'filepath',
+  'abs_path',
+  'cwd',
+  'home',
+  'home_dir',
+  'stack',
+  'stack_trace',
+  'traceback',
+]);
+
+/**
+ * Payload keys permitted in the support projection — the second,
+ * independent, default-deny boundary. Deliberately narrower than "anything
+ * the observed gate kept": codes, counts, timings, statuses, and typed
+ * resource usage only. Applied recursively to every nested object.
+ */
+export const ALLOWED_SUPPORT_PAYLOAD_FIELDS = new Set([
+  'operation',
+  'status',
+  'outcome',
+  'phase',
+  'stage',
+  'kind',
+  'type',
+  'name',
+  'tool_name',
+  'error_code',
+  'reason_code',
+  'wait_code',
+  'exit_code',
+  'signal',
+  'attempt',
+  'attempts',
+  'retry_count',
+  'duration_ms',
+  'elapsed_ms',
+  'latency_ms',
+  'queue_ms',
+  'sequence',
+  'count',
+  'total',
+  'dropped',
+  'sampled',
+  'truncated',
+  'cache',
+  'hit',
+  'miss',
+  'resource',
+  'value',
+  'unit',
+  'source',
+  'confidence',
+  'schema_version',
+  'metric_version',
+  'fingerprint',
+  'p50',
+  'p95',
+  'p99',
+  'min',
+  'max',
+  'mean',
+]);
+
+// --- size caps (prototype defaults per SPEC §14; freeze after measurement) ---
+
+/** Max serialized bytes for a sanitized record; larger records are rejected. */
+export const MAX_RECORD_BYTES = 64 * 1024;
+
+/** Max characters kept per string value; longer strings are truncated. */
+export const MAX_STRING_CHARS = 8 * 1024;
+
+/** Max serialized bytes for a record admitted to the support view. */
+export const MAX_SUPPORT_RECORD_BYTES = 16 * 1024;
+
+/** Max characters per string value in the support view. */
+export const MAX_SUPPORT_STRING_CHARS = 1024;
+
+/** Max keys kept per object at any depth. */
+export const MAX_OBJECT_KEYS = 64;
+
+/** Max items kept per array. */
+export const MAX_ARRAY_ITEMS = 128;
+
+/** Max nesting depth; deeper values are dropped. */
+export const MAX_DEPTH = 8;