@miller-tech/uap 1.175.13 → 1.178.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/benchmarks/harness-card.d.ts +86 -0
- package/dist/benchmarks/harness-card.d.ts.map +1 -0
- package/dist/benchmarks/harness-card.js +153 -0
- package/dist/benchmarks/harness-card.js.map +1 -0
- package/dist/benchmarks/paired/report.d.ts +15 -0
- package/dist/benchmarks/paired/report.d.ts.map +1 -1
- package/dist/benchmarks/paired/report.js +12 -0
- package/dist/benchmarks/paired/report.js.map +1 -1
- package/dist/benchmarks/paired/types.d.ts +1 -1
- package/dist/bin/cli.js +27 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/cli/bench.d.ts.map +1 -1
- package/dist/cli/bench.js +64 -3
- package/dist/cli/bench.js.map +1 -1
- package/dist/cli/harness.d.ts +3 -2
- package/dist/cli/harness.d.ts.map +1 -1
- package/dist/cli/harness.js +81 -0
- package/dist/cli/harness.js.map +1 -1
- package/dist/cli/memory.d.ts +7 -1
- package/dist/cli/memory.d.ts.map +1 -1
- package/dist/cli/memory.js +165 -0
- package/dist/cli/memory.js.map +1 -1
- package/dist/delivery/agentic-executor.d.ts +76 -2
- package/dist/delivery/agentic-executor.d.ts.map +1 -1
- package/dist/delivery/agentic-executor.js +231 -29
- package/dist/delivery/agentic-executor.js.map +1 -1
- package/dist/delivery/edit-match.d.ts +134 -0
- package/dist/delivery/edit-match.d.ts.map +1 -0
- package/dist/delivery/edit-match.js +331 -0
- package/dist/delivery/edit-match.js.map +1 -0
- package/dist/memory/dynamic-retrieval.d.ts.map +1 -1
- package/dist/memory/dynamic-retrieval.js +57 -0
- package/dist/memory/dynamic-retrieval.js.map +1 -1
- package/dist/memory/reconstruct-ingest.d.ts +88 -0
- package/dist/memory/reconstruct-ingest.d.ts.map +1 -0
- package/dist/memory/reconstruct-ingest.js +165 -0
- package/dist/memory/reconstruct-ingest.js.map +1 -0
- package/dist/memory/reconstruct-store.d.ts +143 -0
- package/dist/memory/reconstruct-store.d.ts.map +1 -0
- package/dist/memory/reconstruct-store.js +312 -0
- package/dist/memory/reconstruct-store.js.map +1 -0
- package/dist/memory/reconstruct.d.ts +272 -0
- package/dist/memory/reconstruct.d.ts.map +1 -0
- package/dist/memory/reconstruct.js +552 -0
- package/dist/memory/reconstruct.js.map +1 -0
- package/dist/policies/schemas/policy.d.ts +2 -2
- package/dist/self-harness/evidence-proposer.d.ts +58 -0
- package/dist/self-harness/evidence-proposer.d.ts.map +1 -0
- package/dist/self-harness/evidence-proposer.js +132 -0
- package/dist/self-harness/evidence-proposer.js.map +1 -0
- package/dist/self-harness/manifest.d.ts +130 -0
- package/dist/self-harness/manifest.d.ts.map +1 -0
- package/dist/self-harness/manifest.js +247 -0
- package/dist/self-harness/manifest.js.map +1 -0
- package/dist/self-harness/mods.d.ts +57 -3
- package/dist/self-harness/mods.d.ts.map +1 -1
- package/dist/self-harness/mods.js +72 -0
- package/dist/self-harness/mods.js.map +1 -1
- package/dist/self-harness/orchestrator.d.ts +27 -0
- package/dist/self-harness/orchestrator.d.ts.map +1 -1
- package/dist/self-harness/orchestrator.js +47 -4
- package/dist/self-harness/orchestrator.js.map +1 -1
- package/dist/self-harness/pending.d.ts +17 -3
- package/dist/self-harness/pending.d.ts.map +1 -1
- package/dist/self-harness/pending.js +26 -4
- package/dist/self-harness/pending.js.map +1 -1
- package/dist/self-harness/profile.d.ts +17 -1
- package/dist/self-harness/profile.d.ts.map +1 -1
- package/dist/self-harness/profile.js +11 -4
- package/dist/self-harness/profile.js.map +1 -1
- package/dist/self-harness/run.d.ts +19 -0
- package/dist/self-harness/run.d.ts.map +1 -1
- package/dist/self-harness/run.js +43 -4
- package/dist/self-harness/run.js.map +1 -1
- package/dist/self-harness/transfer.d.ts.map +1 -1
- package/dist/self-harness/transfer.js +2 -0
- package/dist/self-harness/transfer.js.map +1 -1
- package/dist/self-harness/validate.d.ts +17 -6
- package/dist/self-harness/validate.d.ts.map +1 -1
- package/dist/self-harness/validate.js +73 -8
- package/dist/self-harness/validate.js.map +1 -1
- package/dist/telemetry/tool-calls.d.ts +82 -0
- package/dist/telemetry/tool-calls.d.ts.map +1 -0
- package/dist/telemetry/tool-calls.js +280 -0
- package/dist/telemetry/tool-calls.js.map +1 -0
- package/dist/telemetry/tool-failure.d.ts +71 -0
- package/dist/telemetry/tool-failure.d.ts.map +1 -0
- package/dist/telemetry/tool-failure.js +123 -0
- package/dist/telemetry/tool-failure.js.map +1 -0
- package/docs/plans/harness-engineering-uplift-2026-07-31.md +280 -0
- package/package.json +1 -1
- package/src/policies/enforcers/__pycache__/_common.cpython-312.pyc +0 -0
- package/templates/hooks/__pycache__/deliver_autoroute.cpython-312.pyc +0 -0
- package/tools/agents/scripts/__pycache__/toolcall_path_normalizer.cpython-312.pyc +0 -0
package/dist/self-harness/run.js
CHANGED
|
@@ -18,7 +18,9 @@ import { appendFileSync, mkdirSync } from 'fs';
|
|
|
18
18
|
import { dirname } from 'path';
|
|
19
19
|
import { describeMod } from './mods.js';
|
|
20
20
|
import { runIteration } from './orchestrator.js';
|
|
21
|
+
import { upsertEnvValue } from './profile.js';
|
|
21
22
|
import { applyEnvModToFile, loadProfileSnapshot, saveProfileSnapshot, } from './profile.js';
|
|
23
|
+
import { middlewareEnvKey } from './validate.js';
|
|
22
24
|
export async function runSelfHarnessLoop(deps) {
|
|
23
25
|
const log = deps.log ?? (() => { });
|
|
24
26
|
const iteration = await runIteration({
|
|
@@ -32,6 +34,11 @@ export async function runSelfHarnessLoop(deps) {
|
|
|
32
34
|
transferStore: deps.transferStore,
|
|
33
35
|
validatedAt: deps.now,
|
|
34
36
|
provenance: deps.provenance,
|
|
37
|
+
// Harness plan C: without these the manifest machinery exists but never
|
|
38
|
+
// runs, and "acceptance is provisional" is a claim with no mechanism.
|
|
39
|
+
manifests: deps.manifests,
|
|
40
|
+
priorRecords: deps.priorRecords,
|
|
41
|
+
manifestPolicy: deps.manifestPolicy,
|
|
35
42
|
log,
|
|
36
43
|
});
|
|
37
44
|
const accepted = iteration.accepted;
|
|
@@ -39,9 +46,17 @@ export async function runSelfHarnessLoop(deps) {
|
|
|
39
46
|
if (!deps.apply && accepted.length > 0) {
|
|
40
47
|
log(`dry-run: ${accepted.length} Mod(s) would be committed — pass --apply to persist`);
|
|
41
48
|
}
|
|
42
|
-
return { iteration, committed: [], persisted: false, snapshot: null };
|
|
49
|
+
return { iteration, committed: [], unapplied: accepted, persisted: false, snapshot: null };
|
|
43
50
|
}
|
|
44
|
-
// --- Commit: physically apply accepted
|
|
51
|
+
// --- Commit: physically apply accepted Mods, restart, persist. ---
|
|
52
|
+
//
|
|
53
|
+
// This used to filter to `env` and then report ALL accepted Mods as
|
|
54
|
+
// `committed`, so an accepted tool/middleware Mod was announced as applied,
|
|
55
|
+
// written into the snapshot, and had zero effect on the running system. Each
|
|
56
|
+
// kind now gets an explicit disposition, and only the ones actually applied
|
|
57
|
+
// are reported.
|
|
58
|
+
const committed = [];
|
|
59
|
+
const unapplied = [];
|
|
45
60
|
const envMods = accepted.filter((m) => m.kind === 'env');
|
|
46
61
|
if (envMods.length > 0) {
|
|
47
62
|
if (!deps.envPath)
|
|
@@ -49,10 +64,34 @@ export async function runSelfHarnessLoop(deps) {
|
|
|
49
64
|
for (const mod of envMods) {
|
|
50
65
|
applyEnvModToFile(deps.envPath, mod);
|
|
51
66
|
log(`committed: ${describeMod(mod)} -> ${deps.envPath}`);
|
|
67
|
+
committed.push(mod);
|
|
52
68
|
}
|
|
53
69
|
if (deps.restart)
|
|
54
70
|
await deps.restart();
|
|
55
71
|
}
|
|
72
|
+
// Tool + middleware knobs are executor/proxy env. `runtimeEnvPath` persists
|
|
73
|
+
// them where the next run will read them; without it they cannot be committed,
|
|
74
|
+
// and saying so is better than pretending.
|
|
75
|
+
const runtimeMods = accepted.filter((m) => m.kind === 'tool' || m.kind === 'middleware');
|
|
76
|
+
for (const mod of runtimeMods) {
|
|
77
|
+
const key = mod.kind === 'tool' ? mod.key : middlewareEnvKey(mod.id);
|
|
78
|
+
const value = mod.kind === 'tool' ? mod.to : mod.params.enabled === false ? '0' : '1';
|
|
79
|
+
if (!deps.runtimeEnvPath) {
|
|
80
|
+
log(`NOT committed (no runtimeEnvPath): ${describeMod(mod)}`);
|
|
81
|
+
unapplied.push(mod);
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
upsertEnvValue(deps.runtimeEnvPath, key, value);
|
|
85
|
+
process.env[key] = value;
|
|
86
|
+
log(`committed: ${describeMod(mod)} -> ${deps.runtimeEnvPath}`);
|
|
87
|
+
committed.push(mod);
|
|
88
|
+
}
|
|
89
|
+
for (const mod of accepted) {
|
|
90
|
+
if (!committed.includes(mod) && !unapplied.includes(mod)) {
|
|
91
|
+
log(`NOT committed (${mod.kind} Mods are human-gated): ${describeMod(mod)}`);
|
|
92
|
+
unapplied.push(mod);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
56
95
|
const prev = deps.snapshotPath ? loadProfileSnapshot(deps.snapshotPath) : null;
|
|
57
96
|
const version = (prev?.version ?? 0) + 1;
|
|
58
97
|
const snapshot = {
|
|
@@ -60,7 +99,7 @@ export async function runSelfHarnessLoop(deps) {
|
|
|
60
99
|
updatedAt: deps.now,
|
|
61
100
|
model: deps.model,
|
|
62
101
|
profile: iteration.profile,
|
|
63
|
-
accepted:
|
|
102
|
+
accepted: committed.map(describeMod),
|
|
64
103
|
provenance: deps.provenance ?? 'self-harness run',
|
|
65
104
|
};
|
|
66
105
|
if (deps.snapshotPath) {
|
|
@@ -78,6 +117,6 @@ export async function runSelfHarnessLoop(deps) {
|
|
|
78
117
|
mkdirSync(dirname(deps.historyPath), { recursive: true });
|
|
79
118
|
appendFileSync(deps.historyPath, JSON.stringify(entry) + '\n', 'utf-8');
|
|
80
119
|
}
|
|
81
|
-
return { iteration, committed
|
|
120
|
+
return { iteration, committed, unapplied, persisted: true, snapshot };
|
|
82
121
|
}
|
|
83
122
|
//# sourceMappingURL=run.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/self-harness/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,OAAO,EAAO,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAwC,MAAM,mBAAmB,CAAC;AACvF,OAAO,EAEL,iBAAiB,EACjB,mBAAmB,EACnB,mBAAmB,GAEpB,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/self-harness/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAE/B,OAAO,EAAO,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAwC,MAAM,mBAAmB,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAEL,iBAAiB,EACjB,mBAAmB,EACnB,mBAAmB,GAEpB,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAsEjD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAiB;IACxD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC;QACnC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,WAAW,EAAE,IAAI,CAAC,GAAG;QACrB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,wEAAwE;QACxE,sEAAsE;QACtE,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,GAAG;KACJ,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;IACpC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,sDAAsD,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC7F,CAAC;IAED,oEAAoE;IACpE,EAAE;IACF,oEAAoE;IACpE,4EAA4E;IAC5E,6EAA6E;IAC7E,4EAA4E;IAC5E,gBAAgB;IAChB,MAAM,SAAS,GAAU,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAU,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAsC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IAC7F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QACjG,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,cAAc,WAAW,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACzD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;IACD,4EAA4E;IAC5E,+EAA+E;IAC/E,2CAA2C;IAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;IACzF,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACtF,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,GAAG,CAAC,sCAAsC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9D,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACzB,GAAG,CAAC,cAAc,WAAW,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,kBAAkB,GAAG,CAAC,IAAI,2BAA2B,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7E,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/E,MAAM,OAAO,GAAG,CAAC,IAAI,EAAE,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAoB;QAChC,OAAO;QACP,SAAS,EAAE,IAAI,CAAC,GAAG;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,kBAAkB;KAClD,CAAC;IAEF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,mBAAmB,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACjD,GAAG,CAAC,qBAAqB,OAAO,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,MAAM,KAAK,GAAiB;YAC1B,EAAE,EAAE,IAAI,CAAC,GAAG;YACZ,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO;YACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;SAChC,CAAC;QACF,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AACxE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../../src/self-harness/transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,EAAE,GAAG,EAA4B,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAkC,MAAM,eAAe,CAAC;AAC5E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAG7C,MAAM,WAAW,aAAa;IAC5B,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,GAAG,CAAC;IACT,8EAA8E;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uEAAuE;AACvE,wBAAgB,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../../src/self-harness/transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,EAAE,GAAG,EAA4B,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAkC,MAAM,eAAe,CAAC;AAC5E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAG7C,MAAM,WAAW,aAAa;IAC5B,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,GAAG,CAAC;IACT,8EAA8E;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uEAAuE;AACvE,wBAAgB,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAavC;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,GAAG,EAAE,CAW7C;AAED,MAAM,WAAW,oBAAoB;IACnC,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B;;iEAE6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IACnC,2EAA2E;IAC3E,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,aAAa,EAAE,CAAC;IACvE,GAAG,IAAI,aAAa,EAAE,CAAC;IACvB,iEAAiE;IACjE,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;CACzC;AAED,+CAA+C;AAC/C,qBAAa,mBAAoB,YAAW,aAAa;IACvD,SAAS,CAAC,OAAO,EAAE,aAAa,EAAE,CAAM;IAExC,MAAM,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAOlC,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,GAAE,oBAAyB,GAAG,aAAa,EAAE;IAU1E,GAAG,IAAI,aAAa,EAAE;IAItB,KAAK,CAAC,IAAI,GAAE,YAAiB,GAAG,WAAW;CAiB5C;AAED,uFAAuF;AACvF,qBAAa,iBAAkB,SAAQ,mBAAmB;IAC5C,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,EAAE,MAAM;IAWhC,MAAM,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAKlC,KAAK,CAAC,IAAI,GAAE,YAAiB,GAAG,WAAW;IAMpD,OAAO,CAAC,OAAO;CAIhB;AAED,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,aAAa,CAEhB;AAED,wFAAwF;AACxF,wBAAgB,aAAa,CAAC,CAAC,EAAE,aAAa,GAAG,MAAM,CAGtD;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAanF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../src/self-harness/transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAO,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAe,cAAc,EAAkB,MAAM,eAAe,CAAC;AAoB5E,uEAAuE;AACvE,MAAM,UAAU,MAAM,CAAC,GAAQ;IAC7B,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,KAAK;YACR,OAAO,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,UAAU;YACb,OAAO,YAAY,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3D,KAAK,YAAY;YACf,OAAO,MAAM,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACtD,KAAK,QAAQ;YACX,OAAO,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../src/self-harness/transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAO,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAe,cAAc,EAAkB,MAAM,eAAe,CAAC;AAoB5E,uEAAuE;AACvE,MAAM,UAAU,MAAM,CAAC,GAAQ;IAC7B,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;QACjB,KAAK,KAAK;YACR,OAAO,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,UAAU;YACb,OAAO,YAAY,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3D,KAAK,YAAY;YACf,OAAO,MAAM,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACtD,KAAK,QAAQ;YACX,OAAO,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,MAAM;YACT,OAAO,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;IACvC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAW;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAkCD,+CAA+C;AAC/C,MAAM,OAAO,mBAAmB;IACpB,OAAO,GAAoB,EAAE,CAAC;IAExC,MAAM,CAAC,KAAoB;QACzB,+EAA+E;QAC/E,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAC/E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,IAAiB,EAAE,OAA6B,EAAE;QACtD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7E,OAAO,IAAI,CAAC,OAAO;aAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;aAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACrE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,GAAG;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAqB,EAAE;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,OAAO,GAAoB,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACvC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,GAAG,GAAG,QAAQ,CAAC;YAC7B,MAAM,YAAY,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC;YACvD,IAAI,KAAK,IAAI,YAAY,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChB,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAChD,CAAC;CACF;AAED,uFAAuF;AACvF,MAAM,OAAO,iBAAkB,SAAQ,mBAAmB;IAC3B;IAA7B,YAA6B,IAAY;QACvC,KAAK,EAAE,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAQ;QAEvC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YACzD,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEQ,MAAM,CAAC,KAAoB;QAClC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAEQ,KAAK,CAAC,OAAqB,EAAE;QACpC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM;YAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QACvC,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,OAAO;QACb,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC3E,CAAC;CACF;AAED,4EAA4E;AAC5E,MAAM,UAAU,SAAS,CAAC,IASzB;IACC,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;AACxC,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,aAAa,CAAC,CAAgB;IAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/D,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,UAAU,GAAG,CAAC;AACtF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAoB,EAAE,IAAc;IACvE,OAAO;QACL,EAAE,EAAE,YAAY,IAAI,CAAC,EAAE,EAAE;QACzB,OAAO,CAAC,UAA4B,EAAE,OAAuB;YAC3D,MAAM,KAAK,GAAU,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;oBAC/D,IAAI,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE;wBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;YACD,OAAO,UAAU,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -17,10 +17,14 @@
|
|
|
17
17
|
* suite-arm runner and the server restart are INJECTED so the loop is testable
|
|
18
18
|
* without a live llama server.
|
|
19
19
|
*
|
|
20
|
-
* `
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
20
|
+
* `tool` and `middleware` Mods ARE auto-validated (harness plan B), but by a
|
|
21
|
+
* different physical route: they are CLIENT-side, so the arms differ by a
|
|
22
|
+
* process env var and no server restart is needed.
|
|
23
|
+
*
|
|
24
|
+
* `scaffold` / `config` Mods are NOT auto-validated here — free-form prompt text
|
|
25
|
+
* is unbounded, and prompt-only edits measured NEGATIVE (-2.3pp, arXiv
|
|
26
|
+
* 2604.25850). They route through the human-gated pending queue (§9). Passing one
|
|
27
|
+
* returns a null (no-lift) comparison so `decide` rejects it.
|
|
24
28
|
*
|
|
25
29
|
* See docs/design/SELF_HARNESS.md §4, §7, §11.
|
|
26
30
|
*/
|
|
@@ -60,10 +64,17 @@ export interface ValidatorDeps {
|
|
|
60
64
|
log?: (msg: string) => void;
|
|
61
65
|
}
|
|
62
66
|
/**
|
|
63
|
-
* Build a real, live-suite `Validator` for the orchestrator.
|
|
64
|
-
* validated against the server;
|
|
67
|
+
* Build a real, live-suite `Validator` for the orchestrator. `env` Mods are
|
|
68
|
+
* validated against the server; `tool`/`middleware` Mods against a process-env
|
|
69
|
+
* flip; everything else returns a null comparison (reject).
|
|
65
70
|
*/
|
|
66
71
|
export declare function buildValidator(deps: ValidatorDeps): Validator;
|
|
72
|
+
/**
|
|
73
|
+
* Env var a middleware toggle maps to, e.g. `toolcall-path-normalizer` ->
|
|
74
|
+
* `UAP_MW_TOOLCALL_PATH_NORMALIZER`. Shared with the proxy so a validated
|
|
75
|
+
* toggle means the same thing in both processes.
|
|
76
|
+
*/
|
|
77
|
+
export declare function middlewareEnvKey(id: string): string;
|
|
67
78
|
/** Wrap combined baseline+candidate records and analyze into a Comparison. */
|
|
68
79
|
export declare function compare(records: RunRecord[], analyzeOpts?: AnalyzeOptions): Comparison;
|
|
69
80
|
/** A no-lift comparison (Δ=0, not significant) so `decide` rejects. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/self-harness/validate.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/self-harness/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAGH,OAAO,EAAW,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAEL,KAAK,YAAY,EAEjB,KAAK,SAAS,EAEd,KAAK,YAAY,EAClB,MAAM,+BAA+B,CAAC;AAGvC,OAAO,KAAK,EAAE,SAAS,EAAqB,MAAM,mBAAmB,CAAC;AAEtE,QAAA,MAAM,cAAc,aAAa,CAAC;AAClC,QAAA,MAAM,eAAe,cAAc,CAAC;AAEpC,oFAAoF;AACpF,MAAM,MAAM,SAAS,GAAG,CACtB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,KAClC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;AAE1B,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B;;;OAGG;IACH,UAAU,CAAC,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;IACvC,oFAAoF;IACpF,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,wDAAwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oFAAoF;IACpF,WAAW,CAAC,EAAE,cAAc,CAAC;IAC7B,8EAA8E;IAC9E,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,CAoF7D;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAEnD;AAqCD,8EAA8E;AAC9E,wBAAgB,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,WAAW,CAAC,EAAE,cAAc,GAAG,UAAU,CAqBtF;AAMD,uEAAuE;AACvE,wBAAgB,cAAc,IAAI,UAAU,CAc3C;AAED,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC"}
|
|
@@ -17,10 +17,14 @@
|
|
|
17
17
|
* suite-arm runner and the server restart are INJECTED so the loop is testable
|
|
18
18
|
* without a live llama server.
|
|
19
19
|
*
|
|
20
|
-
* `
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
20
|
+
* `tool` and `middleware` Mods ARE auto-validated (harness plan B), but by a
|
|
21
|
+
* different physical route: they are CLIENT-side, so the arms differ by a
|
|
22
|
+
* process env var and no server restart is needed.
|
|
23
|
+
*
|
|
24
|
+
* `scaffold` / `config` Mods are NOT auto-validated here — free-form prompt text
|
|
25
|
+
* is unbounded, and prompt-only edits measured NEGATIVE (-2.3pp, arXiv
|
|
26
|
+
* 2604.25850). They route through the human-gated pending queue (§9). Passing one
|
|
27
|
+
* returns a null (no-lift) comparison so `decide` rejects it.
|
|
24
28
|
*
|
|
25
29
|
* See docs/design/SELF_HARNESS.md §4, §7, §11.
|
|
26
30
|
*/
|
|
@@ -28,20 +32,73 @@ import { runPaired } from '../benchmarks/paired/runner.js';
|
|
|
28
32
|
import { analyze } from '../benchmarks/paired/report.js';
|
|
29
33
|
import { makeFullCondition, } from '../benchmarks/paired/types.js';
|
|
30
34
|
import { applyEnvModToFile } from './profile.js';
|
|
31
|
-
import { invertMod, describeMod } from './mods.js';
|
|
35
|
+
import { invertMod, describeMod, validateMod } from './mods.js';
|
|
32
36
|
const BASELINE_LABEL = 'baseline';
|
|
33
37
|
const CANDIDATE_LABEL = 'candidate';
|
|
34
38
|
/**
|
|
35
|
-
* Build a real, live-suite `Validator` for the orchestrator.
|
|
36
|
-
* validated against the server;
|
|
39
|
+
* Build a real, live-suite `Validator` for the orchestrator. `env` Mods are
|
|
40
|
+
* validated against the server; `tool`/`middleware` Mods against a process-env
|
|
41
|
+
* flip; everything else returns a null comparison (reject).
|
|
37
42
|
*/
|
|
38
43
|
export function buildValidator(deps) {
|
|
39
44
|
const components = deps.components ?? makeFullCondition().components;
|
|
40
45
|
const log = deps.log ?? (() => { });
|
|
41
46
|
const runArm = deps.runArm ?? defaultArmRunner(deps, components);
|
|
42
47
|
return async (mod) => {
|
|
48
|
+
// Harness plan B: `tool` and `middleware` Mods are now auto-validated too.
|
|
49
|
+
// They are CLIENT-side (executor env / proxy config), so their A/B needs a
|
|
50
|
+
// process-env flip on the candidate arm rather than a server restart — which
|
|
51
|
+
// is exactly why they were reachable all along and simply were not wired.
|
|
52
|
+
// `scaffold`/`config` still route to the human gate: free-form prompt text is
|
|
53
|
+
// unbounded, and prompt-only edits measured NEGATIVE (arXiv 2604.25850).
|
|
54
|
+
if (mod.kind === 'tool' || mod.kind === 'middleware') {
|
|
55
|
+
// A Mod reaching this far sets a process env var, which is the only
|
|
56
|
+
// env-injection primitive in the codebase (NODE_OPTIONS, ALLOW_STUBS,
|
|
57
|
+
// ALLOW_BASH...). `buildValidator` is exported public API and the Proposer
|
|
58
|
+
// interface is explicitly "the seam an LLM proposer drops into", so the
|
|
59
|
+
// allow-list check cannot be left to the proposers' good manners.
|
|
60
|
+
const check = validateMod(mod);
|
|
61
|
+
if (!check.ok) {
|
|
62
|
+
log(` refusing ${describeMod(mod)}: ${check.reason}`);
|
|
63
|
+
return { validation: nullComparison(), heldout: null };
|
|
64
|
+
}
|
|
65
|
+
const key = mod.kind === 'tool' ? mod.key : middlewareEnvKey(mod.id);
|
|
66
|
+
const value = mod.kind === 'tool' ? mod.to : mod.params.enabled === false ? '0' : '1';
|
|
67
|
+
// Capture the OBSERVED prior, not the Mod's claimed `from`: a proposer that
|
|
68
|
+
// mis-states `from` would otherwise leave the wrong value set for every
|
|
69
|
+
// later candidate in this process, silently degrading their baselines.
|
|
70
|
+
let prior;
|
|
71
|
+
const apply = () => {
|
|
72
|
+
prior = process.env[key];
|
|
73
|
+
process.env[key] = value;
|
|
74
|
+
};
|
|
75
|
+
const restore = () => {
|
|
76
|
+
if (prior === undefined)
|
|
77
|
+
delete process.env[key];
|
|
78
|
+
else
|
|
79
|
+
process.env[key] = prior;
|
|
80
|
+
};
|
|
81
|
+
const runClientSuite = async (dir) => {
|
|
82
|
+
const baseRecs = await runArm(dir, BASELINE_LABEL, components);
|
|
83
|
+
apply();
|
|
84
|
+
let candRecs;
|
|
85
|
+
try {
|
|
86
|
+
candRecs = await runArm(dir, CANDIDATE_LABEL, components);
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
// Always restore: a leaked env var would silently contaminate every
|
|
90
|
+
// later candidate in the same process (§7 isolation).
|
|
91
|
+
restore();
|
|
92
|
+
}
|
|
93
|
+
return compare([...baseRecs, ...candRecs], deps.analyzeOpts);
|
|
94
|
+
};
|
|
95
|
+
log(` auto-validating ${mod.kind} Mod: ${describeMod(mod)}`);
|
|
96
|
+
const validation = await runClientSuite(deps.suiteDir);
|
|
97
|
+
const heldout = deps.heldoutDir ? await runClientSuite(deps.heldoutDir) : null;
|
|
98
|
+
return { validation, heldout };
|
|
99
|
+
}
|
|
43
100
|
if (mod.kind !== 'env') {
|
|
44
|
-
log(` ${describeMod(mod)} is ${mod.kind}
|
|
101
|
+
log(` ${describeMod(mod)} is ${mod.kind} — routed to human gate, no auto-validation`);
|
|
45
102
|
return { validation: nullComparison(), heldout: null };
|
|
46
103
|
}
|
|
47
104
|
const envMod = mod;
|
|
@@ -68,6 +125,14 @@ export function buildValidator(deps) {
|
|
|
68
125
|
return { validation, heldout };
|
|
69
126
|
};
|
|
70
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* Env var a middleware toggle maps to, e.g. `toolcall-path-normalizer` ->
|
|
130
|
+
* `UAP_MW_TOOLCALL_PATH_NORMALIZER`. Shared with the proxy so a validated
|
|
131
|
+
* toggle means the same thing in both processes.
|
|
132
|
+
*/
|
|
133
|
+
export function middlewareEnvKey(id) {
|
|
134
|
+
return `UAP_MW_${id.toUpperCase().replace(/[^A-Z0-9]+/g, '_')}`;
|
|
135
|
+
}
|
|
71
136
|
/** Default arm runner: a single-condition `runPaired` over the suite. */
|
|
72
137
|
function defaultArmRunner(deps, _components) {
|
|
73
138
|
const adapter = deps.adapter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/self-harness/validate.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/self-harness/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAwC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EACL,iBAAiB,GAMlB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAO,SAAS,EAAE,WAAW,EAAE,WAAW,EAAU,MAAM,WAAW,CAAC;AAG7E,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,eAAe,GAAG,WAAW,CAAC;AAsCpC;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAmB;IAChD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,EAAE,CAAC,UAAU,CAAC;IACrE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAEjE,OAAO,KAAK,EAAE,GAAQ,EAA8B,EAAE;QACpD,2EAA2E;QAC3E,2EAA2E;QAC3E,6EAA6E;QAC7E,0EAA0E;QAC1E,8EAA8E;QAC9E,yEAAyE;QACzE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACrD,oEAAoE;YACpE,sEAAsE;YACtE,2EAA2E;YAC3E,wEAAwE;YACxE,kEAAkE;YAClE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;gBACd,GAAG,CAAC,cAAc,WAAW,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBACvD,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACzD,CAAC;YACD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACtF,4EAA4E;YAC5E,wEAAwE;YACxE,uEAAuE;YACvE,IAAI,KAAyB,CAAC;YAC9B,MAAM,KAAK,GAAG,GAAG,EAAE;gBACjB,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC3B,CAAC,CAAC;YACF,MAAM,OAAO,GAAG,GAAG,EAAE;gBACnB,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;;oBAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAChC,CAAC,CAAC;YACF,MAAM,cAAc,GAAG,KAAK,EAAE,GAAW,EAAuB,EAAE;gBAChE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;gBAC/D,KAAK,EAAE,CAAC;gBACR,IAAI,QAAqB,CAAC;gBAC1B,IAAI,CAAC;oBACH,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;gBAC5D,CAAC;wBAAS,CAAC;oBACT,oEAAoE;oBACpE,sDAAsD;oBACtD,OAAO,EAAE,CAAC;gBACZ,CAAC;gBACD,OAAO,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC/D,CAAC,CAAC;YACF,GAAG,CAAC,qBAAqB,GAAG,CAAC,IAAI,SAAS,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/E,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QACjC,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,KAAK,WAAW,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,IAAI,6CAA6C,CAAC,CAAC;YACvF,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACzD,CAAC;QACD,MAAM,MAAM,GAAW,GAAG,CAAC;QAE3B,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAuB,EAAE;YAC1D,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;YAC/D,iEAAiE;YACjE,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,QAAqB,CAAC;YAC1B,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;YAC5D,CAAC;oBAAS,CAAC;gBACT,yEAAyE;gBACzE,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAW,CAAC,CAAC;gBAC7D,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACvB,CAAC;YACD,OAAO,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/D,CAAC,CAAC;QAEF,GAAG,CAAC,mBAAmB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClG,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IACjC,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAU;IACzC,OAAO,UAAU,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,yEAAyE;AACzE,SAAS,gBAAgB,CAAC,IAAmB,EAAE,WAAsC;IACnF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,4EAA4E;QAC5E,OAAO,KAAK,IAAI,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;QAC3C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QACpE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAe,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAc,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,SAAS,CACzB;YACE,KAAK;YACL,UAAU,EAAE,CAAC,SAAS,CAAC;YACvB,OAAO;YACP,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;YAC9B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC;YACxB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC;YAClC,QAAQ;SACT,EACD,QAAQ,EACR,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACzB,CAAC;QACF,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,OAAO,CAAC,OAAoB,EAAE,WAA4B;IACxE,MAAM,MAAM,GAAG,OAAO,CACpB;QACE,OAAO;QACP,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;QACrC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,SAAS;QACzC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC;QAC3B,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;KACf,EACD,EAAE,GAAG,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,CAClD,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,eAAe,CAAC,CAAC;IACxE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gBAAgB,eAAe,kCAAkC,MAAM,CAAC,WAAW;aAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;aACnB,IAAI,CAAC,IAAI,CAAC,GAAG,CACjB,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,OAAoB;IACtC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;AACvD,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,cAAc;IAC5B,OAAO;QACL,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,cAAc;QACxB,WAAW,EAAE;YACX,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE;YACxF,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;YACzG,OAAO,EAAE,KAAK;SACf;QACD,OAAO,EAAE,EAAE;QACX,cAAc,EAAE,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { type HarnessComponent, type ToolOutcomeClass } from './tool-failure.js';
|
|
2
|
+
/**
|
|
3
|
+
* Where the corpus lives.
|
|
4
|
+
*
|
|
5
|
+
* NOT under the target project. Two reasons, both found in review:
|
|
6
|
+
*
|
|
7
|
+
* - A paired bench runs every cell in a throwaway temp dir, so a project-rooted
|
|
8
|
+
* corpus is written into directories that are deleted moments later. The
|
|
9
|
+
* evidence from the exact workload the loop is meant to learn from vanished,
|
|
10
|
+
* and `uap harness evidence` read a different path again (`process.cwd()`).
|
|
11
|
+
* - On a customer repo it created `agents/data/memory/telemetry.db` inside
|
|
12
|
+
* their tree — untracked noise at best, committed by an `add -A` at worst.
|
|
13
|
+
*
|
|
14
|
+
* A single per-user store keeps the corpus continuous across runs, which is what
|
|
15
|
+
* cross-run failure ranking needs. `UAP_TOOL_CORPUS_DIR` overrides it (tests).
|
|
16
|
+
*/
|
|
17
|
+
export declare function toolCorpusRoot(): string;
|
|
18
|
+
export interface ToolCallRecord {
|
|
19
|
+
runId?: string;
|
|
20
|
+
taskId?: string;
|
|
21
|
+
tool: string;
|
|
22
|
+
/** Raw result string handed back to the model — classified, then discarded. */
|
|
23
|
+
result: string;
|
|
24
|
+
turn?: number;
|
|
25
|
+
path?: string;
|
|
26
|
+
latencyMs?: number;
|
|
27
|
+
bytes?: number;
|
|
28
|
+
/** Override the derived component (e.g. a middleware-attributed call). */
|
|
29
|
+
component?: HarnessComponent;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Record one tool call. Returns the class it was filed under so callers can
|
|
33
|
+
* assert on it in tests without reading the DB back.
|
|
34
|
+
*/
|
|
35
|
+
export declare function recordToolCall(rec: ToolCallRecord, root?: string): ToolOutcomeClass;
|
|
36
|
+
export interface ToolFailureRow {
|
|
37
|
+
component: HarnessComponent | string;
|
|
38
|
+
tool: string;
|
|
39
|
+
outcome: ToolOutcomeClass | string;
|
|
40
|
+
count: number;
|
|
41
|
+
}
|
|
42
|
+
export interface EvidenceSummary {
|
|
43
|
+
totalCalls: number;
|
|
44
|
+
failedCalls: number;
|
|
45
|
+
/** Share of calls that failed, 0..1. The headline harness-health number. */
|
|
46
|
+
failureRate: number;
|
|
47
|
+
/** Failure counts grouped by component, most-costly first. */
|
|
48
|
+
byComponent: Array<{
|
|
49
|
+
component: string;
|
|
50
|
+
failures: number;
|
|
51
|
+
calls: number;
|
|
52
|
+
}>;
|
|
53
|
+
/** Failure classes, most frequent first — the evolve stage's work queue. */
|
|
54
|
+
topFailures: ToolFailureRow[];
|
|
55
|
+
/** Edit-tool specific health, because it is the highest-leverage surface. */
|
|
56
|
+
editHealth: {
|
|
57
|
+
attempts: number;
|
|
58
|
+
exact: number;
|
|
59
|
+
tolerant: number;
|
|
60
|
+
misses: number;
|
|
61
|
+
ambiguous: number;
|
|
62
|
+
/** Share of edit attempts that landed at all, 0..1. */
|
|
63
|
+
successRate: number;
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Distil the corpus into the layered view the propose stage consumes: an
|
|
68
|
+
* overall health number, per-component attribution, a ranked failure queue, and
|
|
69
|
+
* the edit-tool detail.
|
|
70
|
+
*
|
|
71
|
+
* `runId` scopes the summary to a single run; omit it for the whole corpus.
|
|
72
|
+
*/
|
|
73
|
+
export declare function summarizeToolCalls(runId?: string, root?: string): EvidenceSummary;
|
|
74
|
+
/**
|
|
75
|
+
* Render the summary as the prose block the propose stage puts in front of a
|
|
76
|
+
* model. Kept here (not in the prompt) so evidence and its formatting version
|
|
77
|
+
* together, and so a human can read the same text with `uap harness evidence`.
|
|
78
|
+
*/
|
|
79
|
+
export declare function renderEvidence(summary: EvidenceSummary): string;
|
|
80
|
+
/** Test-only: close cached handles so temp-dir fixtures do not leak. */
|
|
81
|
+
export declare function closeToolCallStores(root?: string): void;
|
|
82
|
+
//# sourceMappingURL=tool-calls.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../../src/telemetry/tool-calls.ts"],"names":[],"mappings":"AAsBA,OAAO,EAIL,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACtB,MAAM,mBAAmB,CAAC;AAc3B;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAmED,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,SAAS,CAAC,EAAE,gBAAgB,CAAC;CAC9B;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,IAAI,SAAmB,GAAG,gBAAgB,CAwC7F;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,gBAAgB,GAAG,MAAM,CAAC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,WAAW,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,4EAA4E;IAC5E,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,6EAA6E;IAC7E,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,uDAAuD;QACvD,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAWD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,SAAmB,GAAG,eAAe,CAmE3F;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CA8B/D;AAED,wEAAwE;AACxE,wBAAgB,mBAAmB,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAqBvD"}
|