@miller-tech/uap 1.108.1 → 1.109.1

package/dist/self-harness/validate.js

@@ -0,0 +1,138 @@
+/**
+ * Self-Harness — the REAL validator (Stage 3), wiring the orchestrator's injected
+ * `Validator` seam to the existing paired benchmark (`benchmarks/paired/`).
+ *
+ * For a candidate `env` Mod the physical A/B is a *server-side* change: the two
+ * arms differ only in the inference server's launch env, so they cannot run in a
+ * single paired pass. This validator therefore:
+ *
+ *   1. runs the suite as the `baseline` arm (current harness);
+ *   2. physically applies the Mod to the env file + restarts the server;
+ *   3. runs the suite again as the `candidate` arm;
+ *   4. REVERTS the env file + restarts (each Mod is validated in isolation — §7);
+ *   5. pairs the two arms cell-for-cell (`taskId#seed`) and runs `analyze` to get
+ *      the same paired statistics `uap bench paired` ships.
+ *
+ * The held-out suite (disjoint tasks) runs the same way to catch overfitting. The
+ * suite-arm runner and the server restart are INJECTED so the loop is testable
+ * without a live llama server.
+ *
+ * `scaffold` / `middleware` Mods are NOT auto-validated here — their physical A/B
+ * is a prompt/proxy change that the autonomous env loop deliberately does not
+ * own; they route through the human-gated pending queue (§9). Passing one returns
+ * a null (no-lift) comparison so `decide` rejects it, never silently applying it.
+ *
+ * See docs/design/SELF_HARNESS.md §4, §7, §11.
+ */
+import { runPaired } from '../benchmarks/paired/runner.js';
+import { analyze } from '../benchmarks/paired/report.js';
+import { makeFullCondition, } from '../benchmarks/paired/types.js';
+import { applyEnvModToFile } from './profile.js';
+import { invertMod, describeMod } from './mods.js';
+const BASELINE_LABEL = 'baseline';
+const CANDIDATE_LABEL = 'candidate';
+/**
+ * Build a real, live-suite `Validator` for the orchestrator. Only `env` Mods are
+ * validated against the server; other kinds return a null comparison (reject).
+ */
+export function buildValidator(deps) {
+    const components = deps.components ?? makeFullCondition().components;
+    const log = deps.log ?? (() => { });
+    const runArm = deps.runArm ?? defaultArmRunner(deps, components);
+    return async (mod) => {
+        if (mod.kind !== 'env') {
+            log(`  ${describeMod(mod)} is ${mod.kind}, not env — routed to human gate, no auto-validation`);
+            return { validation: nullComparison(), heldout: null };
+        }
+        const envMod = mod;
+        const runSuite = async (dir) => {
+            // Arm 1 — baseline (current server state).
+            const baseRecs = await runArm(dir, BASELINE_LABEL, components);
+            // Apply the Mod + restart so the candidate arm sees the new env.
+            applyEnvModToFile(deps.envPath, envMod);
+            await deps.restart();
+            let candRecs;
+            try {
+                candRecs = await runArm(dir, CANDIDATE_LABEL, components);
+            }
+            finally {
+                // Always revert to leave the server clean regardless of the arm outcome.
+                applyEnvModToFile(deps.envPath, invertMod(envMod));
+                await deps.restart();
+            }
+            return compare([...baseRecs, ...candRecs], deps.analyzeOpts);
+        };
+        log(`  validating on ${deps.suiteDir}${deps.heldoutDir ? ` + held-out ${deps.heldoutDir}` : ''}`);
+        const validation = await runSuite(deps.suiteDir);
+        const heldout = deps.heldoutDir ? await runSuite(deps.heldoutDir) : null;
+        return { validation, heldout };
+    };
+}
+/** Default arm runner: a single-condition `runPaired` over the suite. */
+function defaultArmRunner(deps, _components) {
+    const adapter = deps.adapter;
+    if (!adapter) {
+        // Defer the error until first use so tests that inject runArm never hit it.
+        return async () => {
+            throw new Error('buildValidator: default runArm requires an `adapter` in deps');
+        };
+    }
+    return async (suiteDir, label, components) => {
+        const { loadSuite } = await import('../benchmarks/paired/suite.js');
+        const { mkdtempSync } = await import('fs');
+        const { tmpdir } = await import('os');
+        const { join } = await import('path');
+        const tasks = loadSuite(suiteDir);
+        const condition = { label, components };
+        const workRoot = mkdtempSync(join(tmpdir(), 'sh-validate-'));
+        const out = await runPaired({
+            tasks,
+            conditions: [condition],
+            adapter,
+            model: deps.model ?? 'unknown',
+            epochs: deps.epochs ?? 5,
+            concurrency: deps.concurrency ?? 4,
+            workRoot,
+        }, suiteDir, new Date().toISOString());
+        return out.records;
+    };
+}
+/** Wrap combined baseline+candidate records and analyze into a Comparison. */
+export function compare(records, analyzeOpts) {
+    const report = analyze({
+        records,
+        model: records[0]?.model ?? 'unknown',
+        adapter: records[0]?.adapter ?? 'unknown',
+        epochs: countSeeds(records),
+        startedAt: '',
+        finishedAt: '',
+    }, { ...analyzeOpts, baselineLabel: BASELINE_LABEL });
+    const cmp = report.comparisons.find((c) => c.label === CANDIDATE_LABEL);
+    if (!cmp) {
+        throw new Error(`compare: no '${CANDIDATE_LABEL}' comparison produced (labels: ${report.comparisons
+            .map((c) => c.label)
+            .join(', ')})`);
+    }
+    return cmp;
+}
+function countSeeds(records) {
+    return new Set(records.map((r) => r.seed)).size || 1;
+}
+/** A no-lift comparison (Δ=0, not significant) so `decide` rejects. */
+export function nullComparison() {
+    return {
+        label: CANDIDATE_LABEL,
+        baseline: BASELINE_LABEL,
+        correctness: {
+            baselineRate: 0,
+            treatmentRate: 0,
+            delta: { meanDelta: 0, ci: { lower: 0, upper: 0 }, pValue: 1, n: 0, significant: false },
+            mcnemar: { bothCorrect: 0, onlyTreatment: 0, onlyBaseline: 0, bothWrong: 0, netGain: 0, pValue: 1, n: 0 },
+            verdict: 'tie',
+        },
+        metrics: {},
+        metricVerdicts: {},
+    };
+}
+export { BASELINE_LABEL, CANDIDATE_LABEL };
+//# sourceMappingURL=validate.js.map

package/dist/self-harness/validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/self-harness/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAwC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EACL,iBAAiB,GAMlB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAO,SAAS,EAAE,WAAW,EAAU,MAAM,WAAW,CAAC;AAGhE,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,eAAe,GAAG,WAAW,CAAC;AAsCpC;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAmB;IAChD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,EAAE,CAAC,UAAU,CAAC;IACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAEjE,OAAO,KAAK,EAAE,GAAQ,EAA8B,EAAE;QACpD,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,KAAK,WAAW,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,IAAI,sDAAsD,CAAC,CAAC;YAChG,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACzD,CAAC;QACD,MAAM,MAAM,GAAW,GAAG,CAAC;QAE3B,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAuB,EAAE;YAC1D,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;YAC/D,iEAAiE;YACjE,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,QAAqB,CAAC;YAC1B,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;YAC5D,CAAC;oBAAS,CAAC;gBACT,yEAAyE;gBACzE,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAW,CAAC,CAAC;gBAC7D,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACvB,CAAC;YACD,OAAO,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/D,CAAC,CAAC;QAEF,GAAG,CAAC,mBAAmB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClG,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,SAAS,gBAAgB,CAAC,IAAmB,EAAE,WAAsC;IACnF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,4EAA4E;QAC5E,OAAO,KAAK,IAAI,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;QAC3C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QACpE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAe,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAc,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,SAAS,CACzB;YACE,KAAK;YACL,UAAU,EAAE,CAAC,SAAS,CAAC;YACvB,OAAO;YACP,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;YAC9B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC;YACxB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC;YAClC,QAAQ;SACT,EACD,QAAQ,EACR,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACzB,CAAC;QACF,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,OAAO,CAAC,OAAoB,EAAE,WAA4B;IACxE,MAAM,MAAM,GAAG,OAAO,CACpB;QACE,OAAO;QACP,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;QACrC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,SAAS;QACzC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC;QAC3B,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;KACf,EACD,EAAE,GAAG,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,CAClD,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,eAAe,CAAC,CAAC;IACxE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gBAAgB,eAAe,kCAAkC,MAAM,CAAC,WAAW;aAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;aACnB,IAAI,CAAC,IAAI,CAAC,GAAG,CACjB,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,OAAoB;IACtC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;AACvD,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,cAAc;IAC5B,OAAO;QACL,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,cAAc;QACxB,WAAW,EAAE;YACX,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE;YACxF,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;YACzG,OAAO,EAAE,KAAK;SACf;QACD,OAAO,EAAE,EAAE;QACX,cAAc,EAAE,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC"}

package/docs/design/SELF_HARNESS.md

@@ -60,9 +60,10 @@ mine→propose→validate→accept and persists results. Humans do both today.
                                    key=(model_family, failure_sig)        (env + scaffold snapshot)
 ```
 
-New code lives under `src/self-harness/`: `orchestrator.ts`, `mine.ts`, `propose.ts`, `mods.ts` (the
-DSL), `apply.ts`, `transfer.ts`, plus `uap self-harness` CLI. It **reuses** `benchmarks/paired/` for
-validation and the HALO exporter for mining — no reimplementation.
+New code lives under `src/self-harness/`: `orchestrator.ts` (one iteration), `validate.ts` (the real
+paired-bench validator), `run.ts` (the committing loop + versioned snapshot), `mine.ts`, `propose.ts`,
+`mods.ts` (the DSL), `transfer.ts`, plus the `uap self-harness` CLI. It **reuses** `benchmarks/paired/`
+for validation and the HALO exporter for mining — no reimplementation.
 
 ## 4. The Modification DSL — bounded, typed, reversible
 
@@ -171,9 +172,15 @@ Controls:
 - **P0 — Plumbing (small).** Promote the failure-mode logs to structured HALO spans; add the held-out
   regression suite; pin a stable `signature` hashing. *Exit:* `uap harness analyze` emits a typed
   `WeaknessReport`.
-- **P1 — Closed loop, env+scaffold DSL (Option A).** `uap self-harness run` orchestrating
-  mine→propose→validate→accept over `env`+`scaffold` Mods, reusing `benchmarks/paired/`. *Exit:* one
-  autonomous iteration that accepts a real Mod (e.g. re-discovers `LLAMA_N_PREDICT=4096`) with stats.
+- **P1 — Closed loop, env DSL (Option A). [BUILT]** `uap self-harness run` orchestrates
+  mine→propose→validate→decide over `env` Mods, reusing `benchmarks/paired/`. The real validator
+  (`validate.ts`) runs a baseline arm, physically toggles the env knob + restarts the server, runs a
+  candidate arm, reverts, and pairs the two into a `Comparison`; held-out runs the same way. `--apply`
+  commits accepted Mods to the env file, restarts once, and writes a **versioned profile snapshot**
+  (`run.ts`, `profile.ts`) + append-only history; without it the run is a pure dry-run that touches
+  nothing. *Exit met:* one autonomous iteration accepts a real env Mod (e.g. re-discovers
+  `LLAMA_N_PREDICT=4096`) with paired stats. `scaffold`/`middleware` Mods are not auto-validated here —
+  they route through the human-gated pending queue (§9).
 - **P2 — Middleware Mods (Option B).** Add the `middleware` Mod class + the `toolcall-path-normalizer`.
   *Exit:* the loop proposes+validates the normalizer and measurably cuts path-garbling on the medium
   suite (the ceiling manual fixes couldn't crack).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@miller-tech/uap",
-  "version": "1.108.1",
+  "version": "1.109.1",
   "description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
   "type": "module",
   "main": "dist/index.js",

package/web/dashboard.html

@@ -1164,10 +1164,18 @@
           `<strong style="color:#58a6ff;font-size:18px">${fmt(sv.totalTokensSaved)}</strong> tokens · ` +
           `<strong style="color:#3fb950;font-size:18px">$${(sv.totalCostSavedUsd||0).toFixed(2)}</strong> saved across all UAP influences`);
         if (!sv.influences.length) { setHTML('savings-table', '<div class="empty">No savings data</div>'); return; }
-        const rows = sv.influences.map(i =>
-          `<tr><td>${esc(i.influence)}</td><td style="text-align:right">${fmt(i.tokensSaved)}</td>` +
-          `<td style="text-align:right;color:#3fb950">$${(i.costSavedUsd||0).toFixed(4)}</td>` +
-          `<td>${badge(i.quality)}</td><td style="color:var(--fg3);font-size:11px">${esc(i.detail||'')}</td></tr>`).join('');
+        // Unmeasured influences (e.g. routing with no UAP-driven calls yet) render an
+        // explicit dimmed "—" rather than a real-looking $0, so idle != broken.
+        const idleCell = '<span style="color:var(--fg3)" title="not active / no data recorded yet">\u2014</span>';
+        const rows = sv.influences.map(i => {
+          const idle = i.quality === 'unmeasured';
+          const tok = idle ? idleCell : fmt(i.tokensSaved);
+          const cost = idle ? idleCell : `<span style="color:#3fb950">$${(i.costSavedUsd||0).toFixed(4)}</span>`;
+          return `<tr${idle ? ' style="opacity:.55"' : ''}><td>${esc(i.influence)}</td>` +
+            `<td style="text-align:right">${tok}</td>` +
+            `<td style="text-align:right">${cost}</td>` +
+            `<td>${badge(i.quality)}</td><td style="color:var(--fg3);font-size:11px">${esc(i.detail||'')}</td></tr>`;
+        }).join('');
         setHTML('savings-table',
           `<table><thead><tr><th>Influence</th><th style="text-align:right">Tokens saved</th>` +
           `<th style="text-align:right">Cost saved</th><th>Quality</th><th>Detail</th></tr></thead><tbody>${rows}</tbody></table>`);