@ngockhoale/ukit 3.3.2 → 3.4.0

package/src/index/tierSelection.js

@@ -0,0 +1,308 @@
+// tierSelection.js — measured reliability → tier selection (C86 BL-019,
+// SPEC FR-001 §5/§7/§8/§14).
+//
+// modelTier stops being purely static: the route-audit ↔ exec-ledger join
+// (BL-003/004/007) produces measured per-(execution-mode, tier) reliability,
+// every route logs the measured pick vs the static pick as a `tier-selection`
+// shadow receipt, and the measured map becomes authoritative ONLY when the
+// promotion gates pass (≥50 joined routes, ≥60% agreement, disagreements skew
+// cheaper, no verdict regression). Thin data → the static map stays.
+//
+// Boundary rules (SPEC §14, ARCH §Safety Boundary):
+//   * Tier names only — 'lite'|'code'|'smart'. A model ID in the artifact is
+//     dropped at read time so it can never reach a route field or receipt.
+//   * `modelRoles` config is never written — the measured map is derived state
+//     under .ukit/storage/cache/, not user configuration.
+//   * handoff-model-guard.sh enforcement is unchanged (not touched here).
+//
+// Mirror parity: template_project/.claude/ukit/index/tier-selection.mjs
+// carries the identical logic (the mirror cannot import src/). Both twins are
+// locked by tests/consistency/tierSelectionParity.test.js.
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+// Provider-neutral cost-tier vocabulary — literal mirror of the
+// lite→code→smart ladder in src/core/executionContracts.js (MODEL_TIER_ORDER).
+export const MEASURED_TIER_ORDER = Object.freeze(['lite', 'code', 'smart']);
+
+// Derived cache artifact (SPEC §7): metrics/diagnostics writes, router reads.
+export const TIER_MAP_PATH_REL = path.join('.ukit', 'storage', 'cache', 'tier-map.json');
+
+// Executor-decided defaults (task Discussion): a conservative reliability
+// floor so a cheap-but-flaky tier never displaces a reliable one, and a 24h
+// artifact TTL so a stale promote cannot silently own routing forever.
+const DEFAULT_RELIABILITY_FLOOR = 0.8;
+const TIER_MAP_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+const PROMOTION_REASONS = Object.freeze({
+  thin: 'joined<50',
+  agreement: 'agreement<0.6',
+  skew: 'disagreements-not-cheaper',
+});
+
+function tierName(value) {
+  return MEASURED_TIER_ORDER.includes(value) ? value : null;
+}
+
+// One joined row is either the {audit, ledger} pair collectRouteOutcomes
+// emits or a flat {executionMode, modelTier, writeSucceeded} row — both are
+function normalizeJoinedRow(row) {
+  if (!row || typeof row !== 'object' || Array.isArray(row)) return null;
+  const audit = row.audit && typeof row.audit === 'object' ? row.audit : row;
+  const ledger = row.ledger && typeof row.ledger === 'object' ? row.ledger : row;
+  // An empty object is not a joined row — nothing identifiable to count.
+  if (Object.keys(audit).length === 0) return null;
+  return { audit, ledger };
+}
+
+// SPEC §8: deriveTierStats(joinedRows) → { byModeTier, joinedRoutes }.
+// byModeTier is "<mode>|<tier>"-keyed; `routes` counts joined rows where the
+// route ran on that tier, `successes` counts writeSucceeded === true (the
+// ledger outcome the existing byMode buckets already count), `reliability`
+// is successes/routes. Rows without a provider-neutral tier name are counted
+// in joinedRoutes (coverage) but never land in a tier bucket — a model ID
+// must not create a pseudo-tier.
+export function deriveTierStats(joinedRows = []) {
+  const byModeTier = {};
+  const rows = Array.isArray(joinedRows) ? joinedRows : [];
+  let joinedRoutes = 0;
+  for (const raw of rows) {
+    const normalized = normalizeJoinedRow(raw);
+    if (!normalized) continue;
+    joinedRoutes += 1;
+    const mode = typeof normalized.audit.executionMode === 'string'
+      ? normalized.audit.executionMode
+      : null;
+    const tier = tierName(normalized.audit.modelTier);
+    if (!mode || !tier) continue;
+    const key = `${mode}|${tier}`;
+    const bucket = byModeTier[key] ??= { routes: 0, successes: 0, reliability: 0 };
+    bucket.routes += 1;
+    if (normalized.ledger.writeSucceeded === true) bucket.successes += 1;
+    bucket.reliability = bucket.successes / bucket.routes;
+  }
+  return { byModeTier, joinedRoutes };
+}
+
+// SPEC §8: resolveMeasuredTier({executionMode, staticTier, stats,
+// reliabilityFloor?}) → tier name | null. Cheapest tier whose measured
+// reliability for that mode meets the floor; null when the data is thin or
+// absent. `staticTier` rides the signature per SPEC but does not bias the
+// pick — the measured table decides on its own.
+export function resolveMeasuredTier({
+  executionMode = null,
+  staticTier = null,
+  stats = null,
+  reliabilityFloor = DEFAULT_RELIABILITY_FLOOR,
+} = {}) {
+  if (!executionMode || !stats || typeof stats !== 'object') return null;
+  const floor = Number.isFinite(reliabilityFloor)
+    ? reliabilityFloor
+    : DEFAULT_RELIABILITY_FLOOR;
+  const byModeTier = stats.byModeTier ?? {};
+  for (const tier of MEASURED_TIER_ORDER) {
+    const bucket = byModeTier[`${executionMode}|${tier}`];
+    if (bucket && bucket.routes >= 1 && bucket.reliability >= floor) {
+      return tier;
+    }
+  }
+  return null;
+}
+
+// SPEC §8/§14: evaluateTierPromotion({stats, staticMap, minJoined?=50,
+// minAgreement?=0.60}) → {decision, agreement, cheaperShare, regressionFree,
+// reasons[], byMode, joinedRoutes}.
+//
+// Promote only when ALL gates pass:
+//   * joined ≥ minJoined (default 50 joined route↔outcome rows);
+//   * agreement ≥ minAgreement — the measured pick equals the static pick for
+//     ≥60% of modes where a pick exists;
+//   * disagreements skew cheaper — measured picks that differ must be a
+//     lower rung (cheaperShare = cheaper/disagreements > 0.5; zero
+//     disagreements passes as perfectly agreeing);
+//   * regressionFree — no mode where the measured pick's measured
+//     reliability is lower than the static pick's measured reliability
+//     (a tier-verdict regression, verdict = writeSucceeded reliability).
+// `byMode` carries the measured pick per mode for the artifact; modes with
+// no pick fall back to static at read time.
+export function evaluateTierPromotion({
+  stats = null,
+  staticMap = {},
+  minJoined = 50,
+  minAgreement = 0.60,
+  reliabilityFloor = DEFAULT_RELIABILITY_FLOOR,
+} = {}) {
+  const byModeTier = stats?.byModeTier ?? {};
+  const joinedRoutes = Number.isFinite(stats?.joinedRoutes) ? stats.joinedRoutes : 0;
+  const modes = Object.keys(staticMap && typeof staticMap === 'object' ? staticMap : {});
+  const byMode = {};
+
+  let compared = 0;
+  let agreed = 0;
+  let disagreements = 0;
+  let cheaper = 0;
+  const regressedModes = [];
+
+  for (const mode of modes) {
+    const staticTier = staticMap[mode];
+    const measured = resolveMeasuredTier({
+      executionMode: mode,
+      staticTier,
+      stats,
+      reliabilityFloor,
+    });
+    if (measured !== null) byMode[mode] = measured;
+    if (!tierName(staticTier) || measured === null) continue;
+    compared += 1;
+    if (measured === staticTier) {
+      agreed += 1;
+      continue;
+    }
+    disagreements += 1;
+    const measuredIdx = MEASURED_TIER_ORDER.indexOf(measured);
+    const staticIdx = MEASURED_TIER_ORDER.indexOf(staticTier);
+    if (measuredIdx !== -1 && staticIdx !== -1 && measuredIdx < staticIdx) {
+      cheaper += 1;
+    }
+    const measuredRel = byModeTier[`${mode}|${measured}`]?.reliability;
+    const staticRel = byModeTier[`${mode}|${staticTier}`]?.reliability;
+    if (Number.isFinite(measuredRel) && Number.isFinite(staticRel) && measuredRel < staticRel) {
+      regressedModes.push(mode);
+    }
+  }
+
+  const agreement = compared > 0 ? agreed / compared : 0;
+  const cheaperShare = disagreements > 0 ? cheaper / disagreements : 1;
+  const regressionFree = regressedModes.length === 0;
+
+  const reasons = [];
+  if (!(joinedRoutes >= minJoined)) reasons.push(PROMOTION_REASONS.thin);
+  if (!(agreement >= minAgreement)) reasons.push(PROMOTION_REASONS.agreement);
+  if (!(cheaperShare > 0.5)) reasons.push(PROMOTION_REASONS.skew);
+  for (const mode of regressedModes) reasons.push(`regression:${mode}`);
+
+  return {
+    decision: reasons.length === 0 ? 'promote' : 'hold',
+    agreement,
+    cheaperShare,
+    regressionFree,
+    reasons,
+    byMode,
+    joinedRoutes,
+  };
+}
+
+// SPEC §7: { version: 1, derivedAt, joinedRoutes, agreement, decision,
+// byMode: {<mode>: <tier>} } — plus cheaperShare/reasons for the metrics
+// rollup (additive, bounded enums/numbers only).
+export function buildTierMapArtifact({
+  stats = null,
+  staticMap = {},
+  minJoined = 50,
+  minAgreement = 0.60,
+  now = () => new Date(),
+} = {}) {
+  const evaluation = evaluateTierPromotion({ stats, staticMap, minJoined, minAgreement });
+  return {
+    version: 1,
+    derivedAt: now().toISOString(),
+    joinedRoutes: evaluation.joinedRoutes,
+    agreement: evaluation.agreement,
+    decision: evaluation.decision,
+    byMode: evaluation.byMode,
+    cheaperShare: evaluation.cheaperShare,
+    regressionFree: evaluation.regressionFree,
+    reasons: evaluation.reasons,
+  };
+}
+
+// Atomic tmp+rename write — same discipline as route-audit/state writes so a
+// reader never sees a torn artifact. Advisory: a write failure returns
+// {written:false}, never throws (metrics must stay read-mostly-safe).
+export async function writeTierMap(projectRoot, artifact) {
+  try {
+    const filePath = path.join(projectRoot, TIER_MAP_PATH_REL);
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomBytes(6).toString('hex')}`;
+    try {
+      await fs.writeFile(tempPath, JSON.stringify(artifact, null, 2), 'utf8');
+      await fs.rename(tempPath, filePath);
+    } catch (error) {
+      try { await fs.rm(tempPath, { force: true }); } catch { /* best effort */ }
+      throw error;
+    }
+    return { written: true, path: filePath };
+  } catch (error) {
+    return { written: false, error: error?.message ?? String(error) };
+  }
+}
+
+// SPEC §8: loadTierMap(projectRoot) → TierMap|null. Absent, corrupt, wrong
+// version, invalid decision, or stale (derivedAt older than the 24h TTL) →
+// null so the static map stays authoritative. byMode values are filtered to
+// the tier vocabulary — anything else (e.g. a model ID) is dropped so it can
+// never reach a route field or receipt.
+export async function loadTierMap(projectRoot, { now = () => Date.now() } = {}) {
+  try {
+    const filePath = path.join(projectRoot, TIER_MAP_PATH_REL);
+    const doc = JSON.parse(await fs.readFile(filePath, 'utf8'));
+    if (!doc || typeof doc !== 'object' || doc.version !== 1) return null;
+    if (doc.decision !== 'promote' && doc.decision !== 'hold') return null;
+    const derivedAtMs = Date.parse(doc.derivedAt ?? '');
+    if (!Number.isFinite(derivedAtMs)) return null;
+    if (now() - derivedAtMs > TIER_MAP_MAX_AGE_MS) return null;
+    const byMode = {};
+    if (doc.byMode && typeof doc.byMode === 'object') {
+      for (const [mode, tier] of Object.entries(doc.byMode)) {
+        const name = tierName(tier);
+        if (name) byMode[mode] = name;
+      }
+    }
+    return {
+      version: 1,
+      derivedAt: doc.derivedAt,
+      joinedRoutes: Number.isFinite(doc.joinedRoutes) ? doc.joinedRoutes : 0,
+      agreement: Number.isFinite(doc.agreement) ? doc.agreement : 0,
+      decision: doc.decision,
+      byMode,
+      ...(Number.isFinite(doc.cheaperShare) ? { cheaperShare: doc.cheaperShare } : {}),
+      ...(Array.isArray(doc.reasons) ? { reasons: doc.reasons.filter((r) => typeof r === 'string') } : {}),
+    };
+  } catch {
+    return null;
+  }
+}
+
+// Route-side emission (SPEC §5 FR-001): folds the persisted measured map into
+// a route summary. Sets the additive `measuredTier` field (nullable — null
+// when data is thin or promotion is off) and, ONLY when the persisted map
+// says 'promote' AND a pick exists for this mode, overrides the authoritative
+// executionContract.modelTier with the measured tier. Never throws; absent or
+// invalid input leaves the static map authoritative.
+//
+// Returns { map, staticTier, measuredPick, promoted } — `measuredPick` is the
+// raw map pick (logged on the A/B receipt even on hold), `promoted` whether
+// the measured pick took over modelTier.
+export async function applyMeasuredTier({
+  projectRoot = process.cwd(),
+  routeSummary = null,
+  tierMap = undefined,
+} = {}) {
+  const map = tierMap !== undefined ? tierMap : await loadTierMap(projectRoot);
+  if (!routeSummary || typeof routeSummary !== 'object') {
+    return { map, staticTier: null, measuredPick: null, promoted: false };
+  }
+  const executionMode = routeSummary.executionMode ?? null;
+  const staticTier = routeSummary.executionContract?.modelTier ?? null;
+  const pick = map?.byMode?.[executionMode] ?? null;
+  const measuredPick = tierName(pick);
+  const promoted = map?.decision === 'promote' && measuredPick !== null;
+
+  routeSummary.measuredTier = promoted ? measuredPick : null;
+  if (promoted && routeSummary.executionContract && typeof routeSummary.executionContract === 'object') {
+    routeSummary.executionContract.modelTier = measuredPick;
+  }
+  return { map, staticTier, measuredPick, promoted };
+}