harness-everything 0.22.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +6 -0
- package/README.md +30 -0
- package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/README.md +87 -0
- package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/paired-latency.json +107 -0
- package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/report.json +856 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/README.md +92 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/haiku-spec-conformance.md +241 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/luna-adversarial.md +212 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/skill-top1-probe.js +83 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/skill-top1-probe.json +7700 -0
- package/ci/fixtures/system-one-worker-probe.js +15 -0
- package/ci/mechanism-system-one-resident.test.js +114 -3
- package/ci/mechanism-system-one-router.test.js +25 -0
- package/docs/platform-capabilities.md +7 -2
- package/docs/system-one-routing.md +23 -4
- package/evals/pilot-router-nonsoftware/graders/no-tier-language.md +6 -0
- package/evals/pilot-router-nonsoftware/graders/skill-not-fired.md +9 -0
- package/evals/pilot-router-nonsoftware/prompt.md +8 -0
- package/evals/pilot-router-tier3/graders/skill-fired.md +6 -0
- package/evals/pilot-router-tier3/prompt.md +8 -0
- package/evals/pilot-tdd-bugfix/graders/skill-fired.md +6 -0
- package/evals/pilot-tdd-bugfix/prompt.md +8 -0
- package/evals/pilot-tdd-trivia/graders/no-red-phase-claim.md +7 -0
- package/evals/pilot-tdd-trivia/graders/skill-not-fired.md +9 -0
- package/evals/pilot-tdd-trivia/prompt.md +8 -0
- package/evals/pilot-verify-discussion/graders/no-verification-claim.md +7 -0
- package/evals/pilot-verify-discussion/graders/skill-not-fired.md +9 -0
- package/evals/pilot-verify-discussion/prompt.md +8 -0
- package/evals/pilot-verify-preship/graders/skill-fired.md +6 -0
- package/evals/pilot-verify-preship/prompt.md +8 -0
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/system-one/provider.js +14 -2
- package/harness-everything/scripts/system-one/resident.js +26 -2
- package/harness-everything/scripts/system-one/router.js +17 -1
- package/harness-everything/scripts/tier-router.js +13 -4
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/provider.js +14 -2
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js +26 -2
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js +17 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +13 -4
- package/sbom.cdx.json +7 -7
- package/scripts/evaluate-system-one.js +6 -6
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Preload (NODE_OPTIONS --require) that proves which System One client path ran.
|
|
3
|
+
// HARNESS_TEST_WORKER_MODE=deny makes every worker thread fail to start; =count records
|
|
4
|
+
// how many were started into HARNESS_TEST_WORKER_COUNT_FILE when the process exits.
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const threads = require('node:worker_threads');
|
|
7
|
+
const mode = process.env.HARNESS_TEST_WORKER_MODE;
|
|
8
|
+
const Original = threads.Worker;
|
|
9
|
+
let count = 0;
|
|
10
|
+
if (mode === 'deny') {
|
|
11
|
+
threads.Worker = class { constructor() { throw new Error('worker-disabled-by-test'); } };
|
|
12
|
+
} else if (mode === 'count') {
|
|
13
|
+
threads.Worker = class extends Original { constructor(...args) { count += 1; super(...args); } };
|
|
14
|
+
process.on('exit', () => fs.writeFileSync(process.env.HARNESS_TEST_WORKER_COUNT_FILE, String(count)));
|
|
15
|
+
}
|
|
@@ -33,6 +33,16 @@ function cleanup(f) {
|
|
|
33
33
|
try { resident.stop(f.manifest, f.manifestPath); } catch (_) { /* best effort */ }
|
|
34
34
|
fs.rmSync(f.dir, { recursive: true, force: true });
|
|
35
35
|
}
|
|
36
|
+
// A client view whose state file names the live server with a wrong token. The server polls only its own
|
|
37
|
+
// state file and exits once that file stops naming it, so rewriting it would race the wrong-token request.
|
|
38
|
+
function wrongTokenView(f) {
|
|
39
|
+
const good = JSON.parse(fs.readFileSync(resident.stateFiles(f.manifest, f.manifestPath).state, 'utf8'));
|
|
40
|
+
const viewDir = fs.mkdtempSync(path.join(f.dir, 'client-view-'));
|
|
41
|
+
const manifestPath = path.join(viewDir, 'manifest.json');
|
|
42
|
+
fs.writeFileSync(manifestPath, JSON.stringify(f.manifest));
|
|
43
|
+
fs.writeFileSync(resident.stateFiles(f.manifest, manifestPath).state, JSON.stringify({ ...good, token: 'b'.repeat(64) }));
|
|
44
|
+
return manifestPath;
|
|
45
|
+
}
|
|
36
46
|
|
|
37
47
|
test('S1-RS01 manifest transport fields and digest-bound file names', () => {
|
|
38
48
|
const f = fixture();
|
|
@@ -80,10 +90,9 @@ test('S1-RS03 stale state is removed and replaced; a wrong token is rejected wit
|
|
|
80
90
|
assert.equal(f.spawns.length, 1);
|
|
81
91
|
assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
|
|
82
92
|
const good = JSON.parse(fs.readFileSync(state, 'utf8'));
|
|
83
|
-
|
|
84
|
-
assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
|
|
93
|
+
assert.deepEqual(resident.score(request, f.manifest, wrongTokenView(f), f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
|
|
85
94
|
assert.equal(f.spawns.length, 1);
|
|
86
|
-
|
|
95
|
+
assert.equal(resident.status(f.manifest, f.manifestPath).running, true, 'a wrong token does not stop the server');
|
|
87
96
|
for (const bad of ['not-json', JSON.stringify({ ...good, extra: 1 }), JSON.stringify({ ...good, token: 'short' })]) {
|
|
88
97
|
fs.writeFileSync(state, bad);
|
|
89
98
|
assert.equal(resident.status(f.manifest, f.manifestPath).running, false);
|
|
@@ -183,3 +192,105 @@ test('S1-RS09 a corrupted state file never leaves an orphan server behind', () =
|
|
|
183
192
|
assert.equal(resident.status(f.manifest, f.manifestPath).pid, fresh.pid);
|
|
184
193
|
} finally { cleanup(f); }
|
|
185
194
|
});
|
|
195
|
+
|
|
196
|
+
test('S1-RS10 scoreAsync matches the sync client and keeps every start/failure mapping', async () => {
|
|
197
|
+
const f = fixture();
|
|
198
|
+
try {
|
|
199
|
+
assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
|
|
200
|
+
assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
|
|
201
|
+
assert.deepEqual(f.spawns, ['stub']);
|
|
202
|
+
assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
|
|
203
|
+
const sync = resident.score(request, f.manifest, f.manifestPath, f.deps);
|
|
204
|
+
assert.equal(sync.status, 'scored');
|
|
205
|
+
assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), sync);
|
|
206
|
+
assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), sync);
|
|
207
|
+
assert.deepEqual(await resident.scoreAsync(request, f.manifest, wrongTokenView(f), f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
|
|
208
|
+
await assert.rejects(resident.scoreAsync({ ...request, extra: 1 }, f.manifest, f.manifestPath, f.deps));
|
|
209
|
+
assert.equal(f.spawns.length, 1);
|
|
210
|
+
} finally { cleanup(f); }
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test('S1-RS10b scoreAsync removes refused state, restarts once, and times out slow servers', async () => {
|
|
214
|
+
const f = fixture();
|
|
215
|
+
// A live PID that is not this test process: cleanup may kill whatever the state file names.
|
|
216
|
+
const sleeper = require('node:child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 60000)'], { stdio: 'ignore' });
|
|
217
|
+
try {
|
|
218
|
+
const net = require('node:net');
|
|
219
|
+
const closed = net.createServer();
|
|
220
|
+
await new Promise(resolve => closed.listen(0, '127.0.0.1', resolve));
|
|
221
|
+
const port = closed.address().port;
|
|
222
|
+
await new Promise(resolve => closed.close(resolve));
|
|
223
|
+
const { state } = resident.stateFiles(f.manifest, f.manifestPath);
|
|
224
|
+
fs.writeFileSync(state, JSON.stringify({ schemaVersion: 1, pid: sleeper.pid, port, token: 'a'.repeat(64), startedAt: 1 }));
|
|
225
|
+
assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
|
|
226
|
+
assert.deepEqual(f.spawns, ['stub']);
|
|
227
|
+
} finally { sleeper.kill(); cleanup(f); }
|
|
228
|
+
const slow = fixture('stub-slow');
|
|
229
|
+
try {
|
|
230
|
+
assert.equal(resident.ensureReady(slow.manifest, slow.manifestPath, 20000, slow.deps), true);
|
|
231
|
+
const start = Date.now();
|
|
232
|
+
assert.deepEqual(await resident.scoreAsync(request, slow.manifest, slow.manifestPath, slow.deps), { status: 'unavailable', reason: 'provider-timeout' });
|
|
233
|
+
assert.ok(Date.now() - start < 2500);
|
|
234
|
+
assert.equal(slow.spawns.length, 1);
|
|
235
|
+
} finally { cleanup(slow); }
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
test('S1-RS10c provider.scoreAsync dispatches by transport with the sync result shapes', async () => {
|
|
239
|
+
const f = fixture();
|
|
240
|
+
try {
|
|
241
|
+
assert.deepEqual(await provider.scoreAsync(request, path.join(f.dir, 'missing.json')), { status: 'unavailable', reason: 'provider-config' });
|
|
242
|
+
assert.deepEqual(await provider.scoreAsync(request, 'relative.json'), { status: 'unavailable', reason: 'provider-config' });
|
|
243
|
+
assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
|
|
244
|
+
assert.deepEqual(await provider.scoreAsync(request, f.manifestPath), provider.score(request, f.manifestPath));
|
|
245
|
+
const oneshot = path.join(f.dir, 'oneshot.json');
|
|
246
|
+
fs.writeFileSync(oneshot, JSON.stringify({ ...f.manifest, transport: 'oneshot' }));
|
|
247
|
+
assert.deepEqual(await provider.scoreAsync(request, oneshot), provider.score(request, oneshot));
|
|
248
|
+
} finally { cleanup(f); }
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
// The probe preload proves which client path ran inside a real child process.
|
|
252
|
+
const probeEnv = (mode, extra = {}) => ({ ...process.env,
|
|
253
|
+
NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --require ./ci/fixtures/system-one-worker-probe.js`.trim(),
|
|
254
|
+
HARNESS_TEST_WORKER_MODE: mode, ...extra });
|
|
255
|
+
|
|
256
|
+
test('S1-RS11 the hook entry pre-scores over async IPC: it still scores when worker threads cannot start', () => {
|
|
257
|
+
const f = fixture();
|
|
258
|
+
const root = path.resolve(__dirname, '..');
|
|
259
|
+
try {
|
|
260
|
+
assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
|
|
261
|
+
const env = probeEnv('deny', { HARNESS_SYSTEM_ONE_MODE: 'shadow', HARNESS_SYSTEM_ONE_CONFIG: f.manifestPath });
|
|
262
|
+
const kernel = path.join(root, 'harness-everything/scripts/kernel-router-core.js');
|
|
263
|
+
const diagnostic = out => JSON.parse(out.match(/=> SYSTEM ONE: (.*)/)[1]);
|
|
264
|
+
const argv = spawnSync(process.execPath, [kernel, 'fix a typo in README'], { cwd: root, encoding: 'utf8', env });
|
|
265
|
+
assert.equal(argv.status, 0, argv.stderr);
|
|
266
|
+
assert.equal(diagnostic(argv.stdout).status, 'accepted');
|
|
267
|
+
const hook = spawnSync(process.execPath, [kernel], { cwd: root, encoding: 'utf8', env, input: JSON.stringify({ prompt: 'fix a typo in README' }) });
|
|
268
|
+
assert.equal(hook.status, 0, hook.stderr);
|
|
269
|
+
assert.equal(diagnostic(hook.stdout).status, 'accepted');
|
|
270
|
+
const off = spawnSync(process.execPath, [kernel, 'fix a typo in README'], { cwd: root, encoding: 'utf8', env: { ...env, HARNESS_SYSTEM_ONE_MODE: 'off' } });
|
|
271
|
+
assert.equal(off.status, 0, off.stderr);
|
|
272
|
+
assert.doesNotMatch(off.stdout, /SYSTEM ONE/);
|
|
273
|
+
assert.equal(f.spawns.length, 1);
|
|
274
|
+
} finally { cleanup(f); }
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test('S1-RS12 the evaluator times the async path: no worker thread per warm score', () => {
|
|
278
|
+
const f = fixture();
|
|
279
|
+
const root = path.resolve(__dirname, '..');
|
|
280
|
+
try {
|
|
281
|
+
assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
|
|
282
|
+
const countFile = path.join(f.dir, 'workers.txt');
|
|
283
|
+
const output = path.join(f.dir, 'report.json');
|
|
284
|
+
const corpus = path.join(root, 'benchmarks/fixtures/system-one-routing.json');
|
|
285
|
+
const child = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, f.manifestPath, output],
|
|
286
|
+
{ cwd: root, encoding: 'utf8', env: probeEnv('count', { HARNESS_TEST_WORKER_COUNT_FILE: countFile }) });
|
|
287
|
+
assert.equal(child.status, 0, child.stderr);
|
|
288
|
+
const report = JSON.parse(fs.readFileSync(output, 'utf8'));
|
|
289
|
+
assert.equal(report.evidence.residentReady, true);
|
|
290
|
+
// Every sample reached the warm server (the seed's one over-limit context is a declined provider-exit).
|
|
291
|
+
const runs = report.records.flatMap(r => r.runs);
|
|
292
|
+
assert.ok(runs.every(run => run.coldStart === false && ['accepted', 'provider-exit'].includes(run.decision.status === 'accepted' ? 'accepted' : run.decision.reason)));
|
|
293
|
+
assert.ok(runs.filter(run => run.decision.status === 'accepted').length >= runs.length - 2);
|
|
294
|
+
assert.ok(Number(fs.readFileSync(countFile, 'utf8')) < report.records.length * 2, 'warm scores must not start a worker thread each');
|
|
295
|
+
} finally { cleanup(f); }
|
|
296
|
+
});
|
|
@@ -99,3 +99,28 @@ test('S1-R05 structural floor: a scorer can raise but never lower a deterministi
|
|
|
99
99
|
assert.equal(plan.verification.independent, true);
|
|
100
100
|
} finally { provider.score = original; console.log = log; if (mode === undefined) delete process.env.HARNESS_SYSTEM_ONE_MODE; else process.env.HARNESS_SYSTEM_ONE_MODE = mode; }
|
|
101
101
|
});
|
|
102
|
+
test('S1-R06 prescoreTier: off loads nothing; a matching request reuses the async result; mismatch and failure keep sync semantics', async () => {
|
|
103
|
+
const { prescoreTier } = require('../harness-everything/scripts/system-one/router');
|
|
104
|
+
const provider = require('../harness-everything/scripts/system-one/provider');
|
|
105
|
+
const forbidden = () => { throw new Error('must not invoke'); };
|
|
106
|
+
for (const env of [{}, { HARNESS_SYSTEM_ONE_MODE: 'off' }, { HARNESS_SYSTEM_ONE_MODE: 'bogus' }]) assert.equal(await prescoreTier('實作功能', env, forbidden), null);
|
|
107
|
+
assert.equal(await prescoreTier('', { HARNESS_SYSTEM_ONE_MODE: 'shadow' }, forbidden), null);
|
|
108
|
+
const env = { HARNESS_SYSTEM_ONE_MODE: 'prefer', HARNESS_SYSTEM_ONE_CONFIG: '/abs/manifest.json' };
|
|
109
|
+
const input = { prompt: '實作功能', tier: 'Unclassified' };
|
|
110
|
+
const configs = [];
|
|
111
|
+
const pre = await prescoreTier(input.prompt, env, async (req, config) => { configs.push(config); return scorer('tier2')(req); });
|
|
112
|
+
assert.deepEqual(configs, ['/abs/manifest.json']);
|
|
113
|
+
assert.deepEqual(selectTier(input, env, pre), selectTier(input, env, scorer('tier2')));
|
|
114
|
+
const original = provider.score;
|
|
115
|
+
try {
|
|
116
|
+
let used = 0;
|
|
117
|
+
provider.score = req => { used += 1; return scorer('tier3')(req); };
|
|
118
|
+
assert.equal(selectTier({ ...input, prompt: 'different prompt' }, env, pre).tier, 'Tier 3 (Macro Task)');
|
|
119
|
+
assert.equal(used, 1);
|
|
120
|
+
} finally { provider.score = original; }
|
|
121
|
+
const failing = await prescoreTier(input.prompt, env, async () => { throw new Error('secret'); });
|
|
122
|
+
const failed = selectTier(input, env, failing);
|
|
123
|
+
assert.deepEqual(failed, selectTier(input, env, () => { throw new Error('secret'); }));
|
|
124
|
+
assert.equal(failed.diagnostic.reason, 'provider-unavailable');
|
|
125
|
+
assert.ok(!JSON.stringify(failed).includes('secret'));
|
|
126
|
+
});
|
|
@@ -130,7 +130,12 @@ contract; this establishes no new live-host capability. Python, PyTorch and a
|
|
|
130
130
|
compatible local checkpoint are optional external dependencies, not npm payloads;
|
|
131
131
|
the explicit `npm run system-one:install` sets them up and hooks never run it.
|
|
132
132
|
With `transport: "resident"`, an opted-in router hook may spawn one detached
|
|
133
|
-
loopback-only Python server per manifest
|
|
134
|
-
|
|
133
|
+
loopback-only Python server per manifest. One retained Windows + Claude Code
|
|
134
|
+
headless session showed the hook-spawned server outliving the host; macOS/Linux
|
|
135
|
+
and Codex hosts are not live-verified, and this does not change the matrix.
|
|
136
|
+
Hooks read the mode from the host process environment: Claude Code settings
|
|
137
|
+
`env` reaches hooks and every agent-run command, while a Codex started from an
|
|
138
|
+
ordinary terminal sees it only when set at user/OS level. Keep the mode `off`
|
|
139
|
+
until a Harness-trained checkpoint passes the rollout gates.
|
|
135
140
|
Public Skills-only artifacts do not acquire lifecycle hooks from this feature.
|
|
136
141
|
See [System One routing](system-one-routing.md) for limits, rollback and evidence.
|
|
@@ -148,9 +148,21 @@ format, which Phase 0 validation still checks. Any other failure returns
|
|
|
148
148
|
`{ok: false, reason}` with a bounded code; no prompt, traceback, path or score is
|
|
149
149
|
echoed in errors. The server never executes actions and is not given a driver.
|
|
150
150
|
|
|
151
|
-
**Client.** The
|
|
152
|
-
|
|
153
|
-
|
|
151
|
+
**Client.** The hook entry pre-scores asynchronously. When `tier-router.js` runs
|
|
152
|
+
as a script (the kernel and packaged hooks spawn it) in `shadow` or `prefer`
|
|
153
|
+
mode, it builds the fixed-catalog tier request before routing and makes one
|
|
154
|
+
direct async loopback round trip (`scoreAsync`), with no worker thread. Start,
|
|
155
|
+
lock, stale-state, deadline (1000 ms) and reply-limit (1 MiB) rules are the same
|
|
156
|
+
as the synchronous path. The router uses the pre-scored result only when its
|
|
157
|
+
request hash equals the request the router builds; otherwise it scores
|
|
158
|
+
synchronously. A pre-score error is reported exactly as a synchronous scorer
|
|
159
|
+
error would be. In `off` mode the entry neither pre-scores nor contacts a
|
|
160
|
+
provider. The synchronous `score` API remains for programmatic callers: it
|
|
161
|
+
makes the same round trip on a worker thread and waits with `Atomics.wait`.
|
|
162
|
+
Starting that worker costs 29–41 ms per call on the reference host, which is why
|
|
163
|
+
the hook avoids it. The paired measurements are in
|
|
164
|
+
[cua-s1-forms-resident-async-cpu-2026-09-23](../benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/README.md).
|
|
165
|
+
Both paths map results to the existing reasons: timeout → `provider-timeout` (no respawn);
|
|
154
166
|
refused connection or dead PID → stale state removed and a new start
|
|
155
167
|
(`provider-starting`); `unauthorized` → `provider-unavailable`; any other
|
|
156
168
|
server-declined request → `provider-exit`; malformed replies → `provider-json`.
|
|
@@ -158,7 +170,9 @@ server-declined request → `provider-exit`; malformed replies → `provider-jso
|
|
|
158
170
|
**Operators.** `node harness-everything/scripts/system-one/resident.js
|
|
159
171
|
start|status|stop <absolute manifest>` controls a server explicitly; `start`
|
|
160
172
|
waits up to 60 s for readiness. The evaluation CLI waits for readiness before
|
|
161
|
-
measuring, so resident runs are recorded with `coldStart: false`.
|
|
173
|
+
measuring, so resident runs are recorded with `coldStart: false`. It times
|
|
174
|
+
`scoreAsync`, the path the hook uses, so `warmLatency` measures what a prompt
|
|
175
|
+
actually pays. The installer
|
|
162
176
|
writes `transport: "resident"`, verifies one resident score, and stops the server
|
|
163
177
|
it started. Repository tests use a stub scorer behind the real server/protocol
|
|
164
178
|
code; they prove the mechanism, not host survival of detached processes or model
|
|
@@ -195,6 +209,11 @@ selected candidate, model identity, confidence and margin; it omits prompt text,
|
|
|
195
209
|
score vectors and artifact paths. The existing router contract schema is unchanged.
|
|
196
210
|
Kernel/packaged entry points inherit these host environment variables. Prompts
|
|
197
211
|
and hook payload fields cannot set model configuration or launch executables.
|
|
212
|
+
Claude Code settings `env` is hot-reloaded and reaches hooks and every command
|
|
213
|
+
the agent runs; Codex hooks inherit the `codex` process environment, so a Codex
|
|
214
|
+
started from an ordinary terminal needs user/OS-level variables (which then apply
|
|
215
|
+
to every program the user starts). The recommended setting is `off` until a
|
|
216
|
+
`harness-routing-v1` checkpoint passes the rollout gates.
|
|
198
217
|
Repository tests establish mechanism/package evidence only; actual host loading
|
|
199
218
|
and semantic quality remain unverified until retained sessions demonstrate them.
|
|
200
219
|
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
---
|
|
2
|
+
max_turns: 15
|
|
3
|
+
allowed_tools: [Read, Glob, Grep, Skill]
|
|
4
|
+
tags: [pilot, positive-trigger, tier-3]
|
|
5
|
+
description: "Architectural, repo-wide request should route through the harness-everything router and land on Tier 3."
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Route this coding request: split our monolith into two services (orders and billing) with an event bus between them; this touches deployment, data migration, and every team's code.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
---
|
|
2
|
+
max_turns: 15
|
|
3
|
+
allowed_tools: [Read, Glob, Grep, Skill]
|
|
4
|
+
tags: [pilot, positive-trigger]
|
|
5
|
+
description: "A bug-fix request that asks for a regression test should engage the tdd skill's RED/GREEN loop."
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Fix a bug with a regression test: the discount calculation rounds wrong for prices ending in .99.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
---
|
|
2
|
+
max_turns: 10
|
|
3
|
+
allowed_tools: [Read, Glob, Grep, Skill]
|
|
4
|
+
tags: [pilot, negative-trigger]
|
|
5
|
+
description: "A one-line trivia question about TDD terminology must not start a RED/GREEN test-writing loop."
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
In one sentence, what does the RED phase of test-driven development mean?
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
---
|
|
2
|
+
max_turns: 10
|
|
3
|
+
allowed_tools: [Read, Glob, Grep, Skill]
|
|
4
|
+
tags: [pilot, negative-trigger]
|
|
5
|
+
description: "A design discussion with no completed work must not launch the verification-loop skill."
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
Sketch two possible approaches for caching API responses and list trade-offs. Don't write any code yet.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
---
|
|
2
|
+
max_turns: 15
|
|
3
|
+
allowed_tools: [Read, Glob, Grep, Skill]
|
|
4
|
+
tags: [pilot, positive-trigger]
|
|
5
|
+
description: "A pre-completion request to prove work is done should trigger the verification-loop skill."
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You said the refactor is finished. Run the full verification loop and show me the evidence before we call this done.
|
|
@@ -51,7 +51,19 @@ function score(request, manifestPath) {
|
|
|
51
51
|
let manifest;
|
|
52
52
|
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
53
53
|
if (manifest.transport === 'resident') return require('./resident').score({ ...request }, manifest, manifestPath);
|
|
54
|
-
|
|
54
|
+
return oneShot(request, manifest);
|
|
55
|
+
}
|
|
56
|
+
// Async callers (the hook entry) reach a resident server without the sync worker bridge.
|
|
57
|
+
// One-shot has no server to overlap with, so it runs the same child synchronously.
|
|
58
|
+
async function scoreAsync(request, manifestPath) {
|
|
59
|
+
validateRequest(request);
|
|
60
|
+
let manifest;
|
|
61
|
+
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
62
|
+
if (manifest.transport === 'resident') return require('./resident').scoreAsync({ ...request }, manifest, manifestPath);
|
|
63
|
+
return oneShot(request, manifest);
|
|
64
|
+
}
|
|
65
|
+
// Send the exact validated manifest to the child on stdin: no config reread race.
|
|
66
|
+
function oneShot(request, manifest) {
|
|
55
67
|
return runProvider({ ...request }, manifest.python, [ADAPTER, JSON.stringify(manifest)], manifest.timeoutMs || 2000);
|
|
56
68
|
}
|
|
57
69
|
// The probe reports what is installed; only a PEP 610 VCS commit counts as a source revision.
|
|
@@ -76,4 +88,4 @@ function provenance(manifestPath) {
|
|
|
76
88
|
try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
|
|
77
89
|
catch (_) { return unavailable('provider-json'); }
|
|
78
90
|
}
|
|
79
|
-
module.exports = { score, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
91
|
+
module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
const fs = require('node:fs');
|
|
6
6
|
const path = require('node:path');
|
|
7
7
|
const { createHash } = require('node:crypto');
|
|
8
|
+
const net = require('node:net');
|
|
8
9
|
const { spawn } = require('node:child_process');
|
|
9
10
|
const { Worker } = require('node:worker_threads');
|
|
10
11
|
const { validateRequest } = require('./contract');
|
|
@@ -81,6 +82,20 @@ function callSync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
|
|
|
81
82
|
try { return { reply: JSON.parse(Buffer.from(out, 0, Atomics.load(signal, 1)).toString('utf8')) }; }
|
|
82
83
|
catch (_) { return { error: 'json' }; }
|
|
83
84
|
}
|
|
85
|
+
// The same round trip for async callers (the hook entry): no worker thread to bootstrap.
|
|
86
|
+
function callAsync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
|
|
87
|
+
return new Promise(resolve => {
|
|
88
|
+
const chunks = []; let size = 0; let done = false; let socket = null;
|
|
89
|
+
const finish = value => { if (done) return; done = true; clearTimeout(timer); if (socket) socket.destroy(); resolve(value); };
|
|
90
|
+
const timer = setTimeout(() => finish({ error: 'timeout' }), timeoutMs);
|
|
91
|
+
socket = net.connect({ port, host: '127.0.0.1' }, () => socket.end(`${JSON.stringify(message)}\n`));
|
|
92
|
+
socket.on('data', c => { size += c.length; if (size > MAX_REPLY_BYTES) finish({ error: 'limit' }); else chunks.push(c); });
|
|
93
|
+
socket.on('end', () => {
|
|
94
|
+
try { finish({ reply: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); } catch (_) { finish({ error: 'json' }); }
|
|
95
|
+
});
|
|
96
|
+
socket.on('error', e => finish({ error: e.code === 'ECONNREFUSED' ? 'refused' : 'io' }));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
84
99
|
|
|
85
100
|
function start(manifest, files, deps) {
|
|
86
101
|
const lock = lockInfo(files.lock);
|
|
@@ -114,7 +129,16 @@ function score(request, manifest, manifestPath, deps = {}) {
|
|
|
114
129
|
const files = stateFiles(manifest, manifestPath);
|
|
115
130
|
const state = liveState(files);
|
|
116
131
|
if (!state) return start(manifest, files, deps);
|
|
117
|
-
|
|
132
|
+
return outcome(callSync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
|
|
133
|
+
}
|
|
134
|
+
async function scoreAsync(request, manifest, manifestPath, deps = {}) {
|
|
135
|
+
validateRequest(request);
|
|
136
|
+
const files = stateFiles(manifest, manifestPath);
|
|
137
|
+
const state = liveState(files);
|
|
138
|
+
if (!state) return start(manifest, files, deps);
|
|
139
|
+
return outcome(await callAsync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
|
|
140
|
+
}
|
|
141
|
+
function outcome(result, files, state, manifest, deps) {
|
|
118
142
|
if (result.reply) {
|
|
119
143
|
if (result.reply.ok === true && Object.hasOwn(result.reply, 'response')) return { status: 'scored', response: result.reply.response };
|
|
120
144
|
if (result.reply.ok === false) return unavailable(result.reply.reason === 'unauthorized' ? 'provider-unavailable' : 'provider-exit');
|
|
@@ -173,4 +197,4 @@ if (require.main === module) {
|
|
|
173
197
|
if (op === 'stop') stop(manifest, manifestPath);
|
|
174
198
|
process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
|
|
175
199
|
}
|
|
176
|
-
module.exports = { stateFiles, readState, callSync, score, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
200
|
+
module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
@@ -34,4 +34,20 @@ function selectTier(input, env = process.env, scorer = provider.score) {
|
|
|
34
34
|
const reason = input.explicit && accepted ? 'explicit-precedence' : belowFloor ? 'structural-floor' : decision.reason;
|
|
35
35
|
return { tier: applied ? LABELS[decision.selectedId] : input.tier, diagnostic: { mode, ...decision, applied, reason } };
|
|
36
36
|
}
|
|
37
|
-
|
|
37
|
+
// Hook entry: score the same fixed-catalog request asynchronously before routing (no sync worker bridge).
|
|
38
|
+
// Returns a scorer for selectTier, or null when System One is off/invalid; nothing is called then.
|
|
39
|
+
async function prescoreTier(prompt, env = process.env, scoreAsync = provider.scoreAsync) {
|
|
40
|
+
if (!['shadow', 'prefer'].includes(env.HARNESS_SYSTEM_ONE_MODE)) return null;
|
|
41
|
+
let request;
|
|
42
|
+
try { request = createRequest('tier', prompt, TIER_OPTIONS); }
|
|
43
|
+
catch (_) { return null; }
|
|
44
|
+
let result; let failed = false;
|
|
45
|
+
try { result = await scoreAsync(request, env.HARNESS_SYSTEM_ONE_CONFIG); }
|
|
46
|
+
catch (_) { failed = true; }
|
|
47
|
+
return (req, config) => {
|
|
48
|
+
if (req.requestHash !== request.requestHash) return provider.score(req, config);
|
|
49
|
+
if (failed) throw new Error('provider-unavailable');
|
|
50
|
+
return result;
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
module.exports = { selectTier, prescoreTier, TIER_OPTIONS };
|
|
@@ -231,7 +231,7 @@ function emitDynamicSkills(promptLower, context, recommendedGuides) {
|
|
|
231
231
|
}
|
|
232
232
|
}
|
|
233
233
|
|
|
234
|
-
function run(userPrompt, context) {
|
|
234
|
+
function run(userPrompt, context, options = {}) {
|
|
235
235
|
console.log(`[Tier Routing Pre-check]`);
|
|
236
236
|
|
|
237
237
|
const promptLower = userPrompt.toLowerCase();
|
|
@@ -317,7 +317,7 @@ function run(userPrompt, context) {
|
|
|
317
317
|
const structuralFloor = hasMacroSignal ? 'tier3'
|
|
318
318
|
: !isTrivialDocsEdit && hasMultipleTasks && hasMultipleSentences ? 'tier2' : null;
|
|
319
319
|
const semantic = selectTier({ prompt: userPrompt, tier: recommendedTier, floor: structuralFloor,
|
|
320
|
-
explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) });
|
|
320
|
+
explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) }, process.env, options.systemOneScorer || undefined);
|
|
321
321
|
if (semantic.diagnostic) console.log(`\n=> SYSTEM ONE: ${JSON.stringify(semantic.diagnostic)}`);
|
|
322
322
|
if (semantic.diagnostic?.applied) {
|
|
323
323
|
recommendedTier = semantic.tier;
|
|
@@ -460,12 +460,21 @@ function run(userPrompt, context) {
|
|
|
460
460
|
return contract;
|
|
461
461
|
}
|
|
462
462
|
|
|
463
|
+
// Entry point: pre-score System One asynchronously so routing never needs the sync worker bridge.
|
|
464
|
+
async function main(prompt, context) {
|
|
465
|
+
let systemOneScorer = null;
|
|
466
|
+
if (['shadow', 'prefer'].includes(process.env.HARNESS_SYSTEM_ONE_MODE)) {
|
|
467
|
+
try { systemOneScorer = await require('./system-one/router').prescoreTier(prompt); } catch (_) { /* the sync path still scores */ }
|
|
468
|
+
}
|
|
469
|
+
run(prompt, context, { systemOneScorer });
|
|
470
|
+
}
|
|
471
|
+
|
|
463
472
|
if (require.main === module) {
|
|
464
473
|
let userPrompt = process.argv[2] || '';
|
|
465
474
|
let hookContext = null;
|
|
466
475
|
|
|
467
476
|
if (process.argv[2]) {
|
|
468
|
-
|
|
477
|
+
main(userPrompt);
|
|
469
478
|
} else if (process.stdin.isTTY) {
|
|
470
479
|
run('');
|
|
471
480
|
} else {
|
|
@@ -482,7 +491,7 @@ if (process.argv[2]) {
|
|
|
482
491
|
} catch (err) {
|
|
483
492
|
// Not valid JSON on stdin - fall back to argv/manual behavior.
|
|
484
493
|
}
|
|
485
|
-
|
|
494
|
+
main(userPrompt, hookContext);
|
|
486
495
|
});
|
|
487
496
|
}
|
|
488
497
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "harness-everything",
|
|
4
|
-
"version": "0.22.
|
|
4
|
+
"version": "0.22.1",
|
|
5
5
|
"description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Miya Daniel",
|
|
@@ -51,7 +51,19 @@ function score(request, manifestPath) {
|
|
|
51
51
|
let manifest;
|
|
52
52
|
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
53
53
|
if (manifest.transport === 'resident') return require('./resident').score({ ...request }, manifest, manifestPath);
|
|
54
|
-
|
|
54
|
+
return oneShot(request, manifest);
|
|
55
|
+
}
|
|
56
|
+
// Async callers (the hook entry) reach a resident server without the sync worker bridge.
|
|
57
|
+
// One-shot has no server to overlap with, so it runs the same child synchronously.
|
|
58
|
+
async function scoreAsync(request, manifestPath) {
|
|
59
|
+
validateRequest(request);
|
|
60
|
+
let manifest;
|
|
61
|
+
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
62
|
+
if (manifest.transport === 'resident') return require('./resident').scoreAsync({ ...request }, manifest, manifestPath);
|
|
63
|
+
return oneShot(request, manifest);
|
|
64
|
+
}
|
|
65
|
+
// Send the exact validated manifest to the child on stdin: no config reread race.
|
|
66
|
+
function oneShot(request, manifest) {
|
|
55
67
|
return runProvider({ ...request }, manifest.python, [ADAPTER, JSON.stringify(manifest)], manifest.timeoutMs || 2000);
|
|
56
68
|
}
|
|
57
69
|
// The probe reports what is installed; only a PEP 610 VCS commit counts as a source revision.
|
|
@@ -76,4 +88,4 @@ function provenance(manifestPath) {
|
|
|
76
88
|
try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
|
|
77
89
|
catch (_) { return unavailable('provider-json'); }
|
|
78
90
|
}
|
|
79
|
-
module.exports = { score, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
91
|
+
module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|