harness-everything 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +12 -0
- package/README.md +30 -0
- package/benchmarks/fixtures/system-one-holdout-draft.json +2039 -0
- package/benchmarks/fixtures/system-one-holdout-reviews.json +1587 -0
- package/benchmarks/fixtures/system-one-holdout.json +7315 -0
- package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/README.md +87 -0
- package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/paired-latency.json +107 -0
- package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/report.json +856 -0
- package/benchmarks/results/system-one/lexical-baseline-reviewed-holdout-2026-09-23/README.md +47 -0
- package/benchmarks/results/system-one/lexical-baseline-reviewed-holdout-2026-09-23/report.json +5222 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/README.md +92 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/haiku-spec-conformance.md +241 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/luna-adversarial.md +212 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/skill-top1-probe.js +83 -0
- package/benchmarks/results/system-one/pre-merge-review-2026-09-23/skill-top1-probe.json +7700 -0
- package/ci/fixtures/system-one-worker-probe.js +15 -0
- package/ci/mechanism-system-one-corpus.test.js +154 -0
- package/ci/mechanism-system-one-resident.test.js +114 -3
- package/ci/mechanism-system-one-router.test.js +25 -0
- package/docs/platform-capabilities.md +7 -2
- package/docs/system-one-corpus.md +103 -0
- package/docs/system-one-routing.md +25 -5
- package/evals/pilot-router-nonsoftware/graders/no-tier-language.md +6 -0
- package/evals/pilot-router-nonsoftware/graders/skill-not-fired.md +9 -0
- package/evals/pilot-router-nonsoftware/prompt.md +8 -0
- package/evals/pilot-router-tier3/graders/skill-fired.md +6 -0
- package/evals/pilot-router-tier3/prompt.md +8 -0
- package/evals/pilot-tdd-bugfix/graders/skill-fired.md +6 -0
- package/evals/pilot-tdd-bugfix/prompt.md +8 -0
- package/evals/pilot-tdd-trivia/graders/no-red-phase-claim.md +7 -0
- package/evals/pilot-tdd-trivia/graders/skill-not-fired.md +9 -0
- package/evals/pilot-tdd-trivia/prompt.md +8 -0
- package/evals/pilot-verify-discussion/graders/no-verification-claim.md +7 -0
- package/evals/pilot-verify-discussion/graders/skill-not-fired.md +9 -0
- package/evals/pilot-verify-discussion/prompt.md +8 -0
- package/evals/pilot-verify-preship/graders/skill-fired.md +6 -0
- package/evals/pilot-verify-preship/prompt.md +8 -0
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/system-one/provider.js +14 -2
- package/harness-everything/scripts/system-one/resident.js +26 -2
- package/harness-everything/scripts/system-one/router.js +17 -1
- package/harness-everything/scripts/tier-router.js +13 -4
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/provider.js +14 -2
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js +26 -2
- package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js +17 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +13 -4
- package/sbom.cdx.json +7 -7
- package/scripts/evaluate-system-one.js +6 -6
- package/scripts/system-one-corpus.js +99 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
---
|
|
2
|
+
max_turns: 15
|
|
3
|
+
allowed_tools: [Read, Glob, Grep, Skill]
|
|
4
|
+
tags: [pilot, positive-trigger]
|
|
5
|
+
description: "A pre-completion request to prove work is done should trigger the verification-loop skill."
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You said the refactor is finished. Run the full verification loop and show me the evidence before we call this done.
|
|
@@ -51,7 +51,19 @@ function score(request, manifestPath) {
|
|
|
51
51
|
let manifest;
|
|
52
52
|
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
53
53
|
if (manifest.transport === 'resident') return require('./resident').score({ ...request }, manifest, manifestPath);
|
|
54
|
-
|
|
54
|
+
return oneShot(request, manifest);
|
|
55
|
+
}
|
|
56
|
+
// Async callers (the hook entry) reach a resident server without the sync worker bridge.
|
|
57
|
+
// One-shot has no server to overlap with, so it runs the same child synchronously.
|
|
58
|
+
async function scoreAsync(request, manifestPath) {
|
|
59
|
+
validateRequest(request);
|
|
60
|
+
let manifest;
|
|
61
|
+
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
62
|
+
if (manifest.transport === 'resident') return require('./resident').scoreAsync({ ...request }, manifest, manifestPath);
|
|
63
|
+
return oneShot(request, manifest);
|
|
64
|
+
}
|
|
65
|
+
// Send the exact validated manifest to the child on stdin: no config reread race.
|
|
66
|
+
function oneShot(request, manifest) {
|
|
55
67
|
return runProvider({ ...request }, manifest.python, [ADAPTER, JSON.stringify(manifest)], manifest.timeoutMs || 2000);
|
|
56
68
|
}
|
|
57
69
|
// The probe reports what is installed; only a PEP 610 VCS commit counts as a source revision.
|
|
@@ -76,4 +88,4 @@ function provenance(manifestPath) {
|
|
|
76
88
|
try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
|
|
77
89
|
catch (_) { return unavailable('provider-json'); }
|
|
78
90
|
}
|
|
79
|
-
module.exports = { score, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
91
|
+
module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
const fs = require('node:fs');
|
|
6
6
|
const path = require('node:path');
|
|
7
7
|
const { createHash } = require('node:crypto');
|
|
8
|
+
const net = require('node:net');
|
|
8
9
|
const { spawn } = require('node:child_process');
|
|
9
10
|
const { Worker } = require('node:worker_threads');
|
|
10
11
|
const { validateRequest } = require('./contract');
|
|
@@ -81,6 +82,20 @@ function callSync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
|
|
|
81
82
|
try { return { reply: JSON.parse(Buffer.from(out, 0, Atomics.load(signal, 1)).toString('utf8')) }; }
|
|
82
83
|
catch (_) { return { error: 'json' }; }
|
|
83
84
|
}
|
|
85
|
+
// The same round trip for async callers (the hook entry): no worker thread to bootstrap.
|
|
86
|
+
function callAsync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
|
|
87
|
+
return new Promise(resolve => {
|
|
88
|
+
const chunks = []; let size = 0; let done = false; let socket = null;
|
|
89
|
+
const finish = value => { if (done) return; done = true; clearTimeout(timer); if (socket) socket.destroy(); resolve(value); };
|
|
90
|
+
const timer = setTimeout(() => finish({ error: 'timeout' }), timeoutMs);
|
|
91
|
+
socket = net.connect({ port, host: '127.0.0.1' }, () => socket.end(`${JSON.stringify(message)}\n`));
|
|
92
|
+
socket.on('data', c => { size += c.length; if (size > MAX_REPLY_BYTES) finish({ error: 'limit' }); else chunks.push(c); });
|
|
93
|
+
socket.on('end', () => {
|
|
94
|
+
try { finish({ reply: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); } catch (_) { finish({ error: 'json' }); }
|
|
95
|
+
});
|
|
96
|
+
socket.on('error', e => finish({ error: e.code === 'ECONNREFUSED' ? 'refused' : 'io' }));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
84
99
|
|
|
85
100
|
function start(manifest, files, deps) {
|
|
86
101
|
const lock = lockInfo(files.lock);
|
|
@@ -114,7 +129,16 @@ function score(request, manifest, manifestPath, deps = {}) {
|
|
|
114
129
|
const files = stateFiles(manifest, manifestPath);
|
|
115
130
|
const state = liveState(files);
|
|
116
131
|
if (!state) return start(manifest, files, deps);
|
|
117
|
-
|
|
132
|
+
return outcome(callSync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
|
|
133
|
+
}
|
|
134
|
+
async function scoreAsync(request, manifest, manifestPath, deps = {}) {
|
|
135
|
+
validateRequest(request);
|
|
136
|
+
const files = stateFiles(manifest, manifestPath);
|
|
137
|
+
const state = liveState(files);
|
|
138
|
+
if (!state) return start(manifest, files, deps);
|
|
139
|
+
return outcome(await callAsync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
|
|
140
|
+
}
|
|
141
|
+
function outcome(result, files, state, manifest, deps) {
|
|
118
142
|
if (result.reply) {
|
|
119
143
|
if (result.reply.ok === true && Object.hasOwn(result.reply, 'response')) return { status: 'scored', response: result.reply.response };
|
|
120
144
|
if (result.reply.ok === false) return unavailable(result.reply.reason === 'unauthorized' ? 'provider-unavailable' : 'provider-exit');
|
|
@@ -173,4 +197,4 @@ if (require.main === module) {
|
|
|
173
197
|
if (op === 'stop') stop(manifest, manifestPath);
|
|
174
198
|
process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
|
|
175
199
|
}
|
|
176
|
-
module.exports = { stateFiles, readState, callSync, score, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
200
|
+
module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
@@ -34,4 +34,20 @@ function selectTier(input, env = process.env, scorer = provider.score) {
|
|
|
34
34
|
const reason = input.explicit && accepted ? 'explicit-precedence' : belowFloor ? 'structural-floor' : decision.reason;
|
|
35
35
|
return { tier: applied ? LABELS[decision.selectedId] : input.tier, diagnostic: { mode, ...decision, applied, reason } };
|
|
36
36
|
}
|
|
37
|
-
|
|
37
|
+
// Hook entry: score the same fixed-catalog request asynchronously before routing (no sync worker bridge).
|
|
38
|
+
// Returns a scorer for selectTier, or null when System One is off/invalid; nothing is called then.
|
|
39
|
+
async function prescoreTier(prompt, env = process.env, scoreAsync = provider.scoreAsync) {
|
|
40
|
+
if (!['shadow', 'prefer'].includes(env.HARNESS_SYSTEM_ONE_MODE)) return null;
|
|
41
|
+
let request;
|
|
42
|
+
try { request = createRequest('tier', prompt, TIER_OPTIONS); }
|
|
43
|
+
catch (_) { return null; }
|
|
44
|
+
let result; let failed = false;
|
|
45
|
+
try { result = await scoreAsync(request, env.HARNESS_SYSTEM_ONE_CONFIG); }
|
|
46
|
+
catch (_) { failed = true; }
|
|
47
|
+
return (req, config) => {
|
|
48
|
+
if (req.requestHash !== request.requestHash) return provider.score(req, config);
|
|
49
|
+
if (failed) throw new Error('provider-unavailable');
|
|
50
|
+
return result;
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
module.exports = { selectTier, prescoreTier, TIER_OPTIONS };
|
|
@@ -231,7 +231,7 @@ function emitDynamicSkills(promptLower, context, recommendedGuides) {
|
|
|
231
231
|
}
|
|
232
232
|
}
|
|
233
233
|
|
|
234
|
-
function run(userPrompt, context) {
|
|
234
|
+
function run(userPrompt, context, options = {}) {
|
|
235
235
|
console.log(`[Tier Routing Pre-check]`);
|
|
236
236
|
|
|
237
237
|
const promptLower = userPrompt.toLowerCase();
|
|
@@ -317,7 +317,7 @@ function run(userPrompt, context) {
|
|
|
317
317
|
const structuralFloor = hasMacroSignal ? 'tier3'
|
|
318
318
|
: !isTrivialDocsEdit && hasMultipleTasks && hasMultipleSentences ? 'tier2' : null;
|
|
319
319
|
const semantic = selectTier({ prompt: userPrompt, tier: recommendedTier, floor: structuralFloor,
|
|
320
|
-
explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) });
|
|
320
|
+
explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) }, process.env, options.systemOneScorer || undefined);
|
|
321
321
|
if (semantic.diagnostic) console.log(`\n=> SYSTEM ONE: ${JSON.stringify(semantic.diagnostic)}`);
|
|
322
322
|
if (semantic.diagnostic?.applied) {
|
|
323
323
|
recommendedTier = semantic.tier;
|
|
@@ -460,12 +460,21 @@ function run(userPrompt, context) {
|
|
|
460
460
|
return contract;
|
|
461
461
|
}
|
|
462
462
|
|
|
463
|
+
// Entry point: pre-score System One asynchronously so routing never needs the sync worker bridge.
|
|
464
|
+
async function main(prompt, context) {
|
|
465
|
+
let systemOneScorer = null;
|
|
466
|
+
if (['shadow', 'prefer'].includes(process.env.HARNESS_SYSTEM_ONE_MODE)) {
|
|
467
|
+
try { systemOneScorer = await require('./system-one/router').prescoreTier(prompt); } catch (_) { /* the sync path still scores */ }
|
|
468
|
+
}
|
|
469
|
+
run(prompt, context, { systemOneScorer });
|
|
470
|
+
}
|
|
471
|
+
|
|
463
472
|
if (require.main === module) {
|
|
464
473
|
let userPrompt = process.argv[2] || '';
|
|
465
474
|
let hookContext = null;
|
|
466
475
|
|
|
467
476
|
if (process.argv[2]) {
|
|
468
|
-
|
|
477
|
+
main(userPrompt);
|
|
469
478
|
} else if (process.stdin.isTTY) {
|
|
470
479
|
run('');
|
|
471
480
|
} else {
|
|
@@ -482,7 +491,7 @@ if (process.argv[2]) {
|
|
|
482
491
|
} catch (err) {
|
|
483
492
|
// Not valid JSON on stdin - fall back to argv/manual behavior.
|
|
484
493
|
}
|
|
485
|
-
|
|
494
|
+
main(userPrompt, hookContext);
|
|
486
495
|
});
|
|
487
496
|
}
|
|
488
497
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "harness-everything",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.23.0",
|
|
5
5
|
"description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Miya Daniel",
|
|
@@ -51,7 +51,19 @@ function score(request, manifestPath) {
|
|
|
51
51
|
let manifest;
|
|
52
52
|
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
53
53
|
if (manifest.transport === 'resident') return require('./resident').score({ ...request }, manifest, manifestPath);
|
|
54
|
-
|
|
54
|
+
return oneShot(request, manifest);
|
|
55
|
+
}
|
|
56
|
+
// Async callers (the hook entry) reach a resident server without the sync worker bridge.
|
|
57
|
+
// One-shot has no server to overlap with, so it runs the same child synchronously.
|
|
58
|
+
async function scoreAsync(request, manifestPath) {
|
|
59
|
+
validateRequest(request);
|
|
60
|
+
let manifest;
|
|
61
|
+
try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
|
|
62
|
+
if (manifest.transport === 'resident') return require('./resident').scoreAsync({ ...request }, manifest, manifestPath);
|
|
63
|
+
return oneShot(request, manifest);
|
|
64
|
+
}
|
|
65
|
+
// Send the exact validated manifest to the child on stdin: no config reread race.
|
|
66
|
+
function oneShot(request, manifest) {
|
|
55
67
|
return runProvider({ ...request }, manifest.python, [ADAPTER, JSON.stringify(manifest)], manifest.timeoutMs || 2000);
|
|
56
68
|
}
|
|
57
69
|
// The probe reports what is installed; only a PEP 610 VCS commit counts as a source revision.
|
|
@@ -76,4 +88,4 @@ function provenance(manifestPath) {
|
|
|
76
88
|
try { return { status: 'recorded', provenance: validateProvenance(result.value) }; }
|
|
77
89
|
catch (_) { return unavailable('provider-json'); }
|
|
78
90
|
}
|
|
79
|
-
module.exports = { score, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
91
|
+
module.exports = { score, scoreAsync, runProvider, validateManifest, readManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION };
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
const fs = require('node:fs');
|
|
6
6
|
const path = require('node:path');
|
|
7
7
|
const { createHash } = require('node:crypto');
|
|
8
|
+
const net = require('node:net');
|
|
8
9
|
const { spawn } = require('node:child_process');
|
|
9
10
|
const { Worker } = require('node:worker_threads');
|
|
10
11
|
const { validateRequest } = require('./contract');
|
|
@@ -81,6 +82,20 @@ function callSync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
|
|
|
81
82
|
try { return { reply: JSON.parse(Buffer.from(out, 0, Atomics.load(signal, 1)).toString('utf8')) }; }
|
|
82
83
|
catch (_) { return { error: 'json' }; }
|
|
83
84
|
}
|
|
85
|
+
// The same round trip for async callers (the hook entry): no worker thread to bootstrap.
|
|
86
|
+
function callAsync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
|
|
87
|
+
return new Promise(resolve => {
|
|
88
|
+
const chunks = []; let size = 0; let done = false; let socket = null;
|
|
89
|
+
const finish = value => { if (done) return; done = true; clearTimeout(timer); if (socket) socket.destroy(); resolve(value); };
|
|
90
|
+
const timer = setTimeout(() => finish({ error: 'timeout' }), timeoutMs);
|
|
91
|
+
socket = net.connect({ port, host: '127.0.0.1' }, () => socket.end(`${JSON.stringify(message)}\n`));
|
|
92
|
+
socket.on('data', c => { size += c.length; if (size > MAX_REPLY_BYTES) finish({ error: 'limit' }); else chunks.push(c); });
|
|
93
|
+
socket.on('end', () => {
|
|
94
|
+
try { finish({ reply: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); } catch (_) { finish({ error: 'json' }); }
|
|
95
|
+
});
|
|
96
|
+
socket.on('error', e => finish({ error: e.code === 'ECONNREFUSED' ? 'refused' : 'io' }));
|
|
97
|
+
});
|
|
98
|
+
}
|
|
84
99
|
|
|
85
100
|
function start(manifest, files, deps) {
|
|
86
101
|
const lock = lockInfo(files.lock);
|
|
@@ -114,7 +129,16 @@ function score(request, manifest, manifestPath, deps = {}) {
|
|
|
114
129
|
const files = stateFiles(manifest, manifestPath);
|
|
115
130
|
const state = liveState(files);
|
|
116
131
|
if (!state) return start(manifest, files, deps);
|
|
117
|
-
|
|
132
|
+
return outcome(callSync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
|
|
133
|
+
}
|
|
134
|
+
async function scoreAsync(request, manifest, manifestPath, deps = {}) {
|
|
135
|
+
validateRequest(request);
|
|
136
|
+
const files = stateFiles(manifest, manifestPath);
|
|
137
|
+
const state = liveState(files);
|
|
138
|
+
if (!state) return start(manifest, files, deps);
|
|
139
|
+
return outcome(await callAsync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
|
|
140
|
+
}
|
|
141
|
+
function outcome(result, files, state, manifest, deps) {
|
|
118
142
|
if (result.reply) {
|
|
119
143
|
if (result.reply.ok === true && Object.hasOwn(result.reply, 'response')) return { status: 'scored', response: result.reply.response };
|
|
120
144
|
if (result.reply.ok === false) return unavailable(result.reply.reason === 'unauthorized' ? 'provider-unavailable' : 'provider-exit');
|
|
@@ -173,4 +197,4 @@ if (require.main === module) {
|
|
|
173
197
|
if (op === 'stop') stop(manifest, manifestPath);
|
|
174
198
|
process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
|
|
175
199
|
}
|
|
176
|
-
module.exports = { stateFiles, readState, callSync, score, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
200
|
+
module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
|
|
@@ -34,4 +34,20 @@ function selectTier(input, env = process.env, scorer = provider.score) {
|
|
|
34
34
|
const reason = input.explicit && accepted ? 'explicit-precedence' : belowFloor ? 'structural-floor' : decision.reason;
|
|
35
35
|
return { tier: applied ? LABELS[decision.selectedId] : input.tier, diagnostic: { mode, ...decision, applied, reason } };
|
|
36
36
|
}
|
|
37
|
-
|
|
37
|
+
// Hook entry: score the same fixed-catalog request asynchronously before routing (no sync worker bridge).
|
|
38
|
+
// Returns a scorer for selectTier, or null when System One is off/invalid; nothing is called then.
|
|
39
|
+
async function prescoreTier(prompt, env = process.env, scoreAsync = provider.scoreAsync) {
|
|
40
|
+
if (!['shadow', 'prefer'].includes(env.HARNESS_SYSTEM_ONE_MODE)) return null;
|
|
41
|
+
let request;
|
|
42
|
+
try { request = createRequest('tier', prompt, TIER_OPTIONS); }
|
|
43
|
+
catch (_) { return null; }
|
|
44
|
+
let result; let failed = false;
|
|
45
|
+
try { result = await scoreAsync(request, env.HARNESS_SYSTEM_ONE_CONFIG); }
|
|
46
|
+
catch (_) { failed = true; }
|
|
47
|
+
return (req, config) => {
|
|
48
|
+
if (req.requestHash !== request.requestHash) return provider.score(req, config);
|
|
49
|
+
if (failed) throw new Error('provider-unavailable');
|
|
50
|
+
return result;
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
module.exports = { selectTier, prescoreTier, TIER_OPTIONS };
|
|
@@ -231,7 +231,7 @@ function emitDynamicSkills(promptLower, context, recommendedGuides) {
|
|
|
231
231
|
}
|
|
232
232
|
}
|
|
233
233
|
|
|
234
|
-
function run(userPrompt, context) {
|
|
234
|
+
function run(userPrompt, context, options = {}) {
|
|
235
235
|
console.log(`[Tier Routing Pre-check]`);
|
|
236
236
|
|
|
237
237
|
const promptLower = userPrompt.toLowerCase();
|
|
@@ -317,7 +317,7 @@ function run(userPrompt, context) {
|
|
|
317
317
|
const structuralFloor = hasMacroSignal ? 'tier3'
|
|
318
318
|
: !isTrivialDocsEdit && hasMultipleTasks && hasMultipleSentences ? 'tier2' : null;
|
|
319
319
|
const semantic = selectTier({ prompt: userPrompt, tier: recommendedTier, floor: structuralFloor,
|
|
320
|
-
explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) });
|
|
320
|
+
explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) }, process.env, options.systemOneScorer || undefined);
|
|
321
321
|
if (semantic.diagnostic) console.log(`\n=> SYSTEM ONE: ${JSON.stringify(semantic.diagnostic)}`);
|
|
322
322
|
if (semantic.diagnostic?.applied) {
|
|
323
323
|
recommendedTier = semantic.tier;
|
|
@@ -460,12 +460,21 @@ function run(userPrompt, context) {
|
|
|
460
460
|
return contract;
|
|
461
461
|
}
|
|
462
462
|
|
|
463
|
+
// Entry point: pre-score System One asynchronously so routing never needs the sync worker bridge.
|
|
464
|
+
async function main(prompt, context) {
|
|
465
|
+
let systemOneScorer = null;
|
|
466
|
+
if (['shadow', 'prefer'].includes(process.env.HARNESS_SYSTEM_ONE_MODE)) {
|
|
467
|
+
try { systemOneScorer = await require('./system-one/router').prescoreTier(prompt); } catch (_) { /* the sync path still scores */ }
|
|
468
|
+
}
|
|
469
|
+
run(prompt, context, { systemOneScorer });
|
|
470
|
+
}
|
|
471
|
+
|
|
463
472
|
if (require.main === module) {
|
|
464
473
|
let userPrompt = process.argv[2] || '';
|
|
465
474
|
let hookContext = null;
|
|
466
475
|
|
|
467
476
|
if (process.argv[2]) {
|
|
468
|
-
|
|
477
|
+
main(userPrompt);
|
|
469
478
|
} else if (process.stdin.isTTY) {
|
|
470
479
|
run('');
|
|
471
480
|
} else {
|
|
@@ -482,7 +491,7 @@ if (process.argv[2]) {
|
|
|
482
491
|
} catch (err) {
|
|
483
492
|
// Not valid JSON on stdin - fall back to argv/manual behavior.
|
|
484
493
|
}
|
|
485
|
-
|
|
494
|
+
main(userPrompt, hookContext);
|
|
486
495
|
});
|
|
487
496
|
}
|
|
488
497
|
}
|
package/sbom.cdx.json
CHANGED
|
@@ -2,23 +2,23 @@
|
|
|
2
2
|
"$schema": "https://cyclonedx.org/schema/bom-1.7.schema.json",
|
|
3
3
|
"bomFormat": "CycloneDX",
|
|
4
4
|
"specVersion": "1.7",
|
|
5
|
-
"serialNumber": "urn:uuid:
|
|
5
|
+
"serialNumber": "urn:uuid:a1c20a4d-4ba9-561e-92d1-aae9c14cdc08",
|
|
6
6
|
"version": 1,
|
|
7
7
|
"metadata": {
|
|
8
|
-
"timestamp": "2026-09-
|
|
8
|
+
"timestamp": "2026-09-23T11:44:30.450Z",
|
|
9
9
|
"tools": [
|
|
10
10
|
{
|
|
11
11
|
"vendor": "Harness Everything",
|
|
12
12
|
"name": "generate-sbom.js",
|
|
13
|
-
"version": "0.
|
|
13
|
+
"version": "0.23.0"
|
|
14
14
|
}
|
|
15
15
|
],
|
|
16
16
|
"component": {
|
|
17
17
|
"type": "application",
|
|
18
|
-
"bom-ref": "pkg:npm/harness-everything@0.
|
|
18
|
+
"bom-ref": "pkg:npm/harness-everything@0.23.0",
|
|
19
19
|
"name": "harness-everything",
|
|
20
|
-
"version": "0.
|
|
21
|
-
"purl": "pkg:npm/harness-everything@0.
|
|
20
|
+
"version": "0.23.0",
|
|
21
|
+
"purl": "pkg:npm/harness-everything@0.23.0"
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"components": [
|
|
@@ -1999,7 +1999,7 @@
|
|
|
1999
1999
|
]
|
|
2000
2000
|
},
|
|
2001
2001
|
{
|
|
2002
|
-
"ref": "pkg:npm/harness-everything@0.
|
|
2002
|
+
"ref": "pkg:npm/harness-everything@0.23.0",
|
|
2003
2003
|
"dependsOn": [
|
|
2004
2004
|
"pkg:npm/%40commitlint/cli@19.8.1",
|
|
2005
2005
|
"pkg:npm/%40commitlint/config-conventional@19.8.1",
|
|
@@ -4,12 +4,12 @@ const fs = require('node:fs');
|
|
|
4
4
|
const os = require('node:os');
|
|
5
5
|
const { performance } = require('node:perf_hooks');
|
|
6
6
|
const { validateCorpus, evaluate } = require('../harness-everything/scripts/system-one/evaluate');
|
|
7
|
-
const {
|
|
7
|
+
const { scoreAsync, provenance, readManifest } = require('../harness-everything/scripts/system-one/provider');
|
|
8
8
|
const resident = require('../harness-everything/scripts/system-one/resident');
|
|
9
9
|
const { decide } = require('../harness-everything/scripts/system-one/contract');
|
|
10
10
|
const { run: route } = require('../harness-everything/scripts/tier-router');
|
|
11
11
|
|
|
12
|
-
function run(corpus, manifest) {
|
|
12
|
+
async function run(corpus, manifest) {
|
|
13
13
|
validateCorpus(corpus);
|
|
14
14
|
let config = null;
|
|
15
15
|
try { config = readManifest(manifest); } catch (_) { /* Missing provider remains an explicit unavailable measurement. */ }
|
|
@@ -28,7 +28,7 @@ function run(corpus, manifest) {
|
|
|
28
28
|
const runs = [];
|
|
29
29
|
for (let i = 0; i < 2; i++) {
|
|
30
30
|
const start = performance.now();
|
|
31
|
-
const result =
|
|
31
|
+
const result = await scoreAsync(c.request, manifest);
|
|
32
32
|
const decision = result.status === 'scored' ? decide(c.request, result.response) : result;
|
|
33
33
|
if (decision.selectedId === 'unclassified') Object.assign(decision, { status: 'abstain', reason: 'unclassified', selectedId: null });
|
|
34
34
|
const scores = ['accepted', 'abstain'].includes(decision.status) ? result.response.scores.map(s => s.probability) : null;
|
|
@@ -55,12 +55,12 @@ function run(corpus, manifest) {
|
|
|
55
55
|
limitations: ['No independently verified holdout review', ...(warm ? [] : ['No warm inference measurement']), 'No live-host or policy evidence'] }, records };
|
|
56
56
|
}
|
|
57
57
|
if (require.main === module) {
|
|
58
|
-
|
|
58
|
+
(async () => {
|
|
59
59
|
const [input, manifest, output, extra] = process.argv.slice(2);
|
|
60
60
|
if (!input || !manifest || !output || extra) throw new Error('usage');
|
|
61
|
-
const report = run(JSON.parse(fs.readFileSync(input, 'utf8')), manifest);
|
|
61
|
+
const report = await run(JSON.parse(fs.readFileSync(input, 'utf8')), manifest);
|
|
62
62
|
fs.writeFileSync(output, JSON.stringify(report, null, 2) + '\n');
|
|
63
63
|
console.log(JSON.stringify({ cases: report.cases, coverage: report.model.coverage, rolloutReady: report.rolloutReady }));
|
|
64
|
-
}
|
|
64
|
+
})().catch(() => { console.error('System One evaluation failed: invalid corpus, arguments, or output path.'); process.exitCode = 1; });
|
|
65
65
|
}
|
|
66
66
|
module.exports = { run };
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
// Assembles the System One tier holdout from a draft plus human review decisions.
|
|
4
|
+
// See docs/system-one-corpus.md. A decision counts only for the exact prompt text it reviewed.
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const { createHash } = require('node:crypto');
|
|
7
|
+
const { createRequest } = require('../harness-everything/scripts/system-one/contract');
|
|
8
|
+
const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
|
|
9
|
+
const { validateCorpus } = require('../harness-everything/scripts/system-one/evaluate');
|
|
10
|
+
|
|
11
|
+
const SOURCES = ['synthetic:authored', 'derived:local-history'];
|
|
12
|
+
const LANGUAGES = ['en', 'zh-TW'];
|
|
13
|
+
const GOLD = ['tier1', 'tier2', 'tier3', null];
|
|
14
|
+
const REVIEWER = 'human:repository-owner';
|
|
15
|
+
// Drafts are committed publicly: no email addresses or user home paths.
|
|
16
|
+
const PRIVATE = [/[\w.+-]+@[\w-]+\.[\w.-]+/, /[A-Za-z]:\\Users\\/i, /\/home\/[^/\s]+/, /\/Users\/[^/\s]+/];
|
|
17
|
+
const exact = (v, keys) => v && typeof v === 'object' && !Array.isArray(v)
|
|
18
|
+
&& Object.keys(v).length === keys.length && keys.every(k => Object.hasOwn(v, k));
|
|
19
|
+
const nonempty = v => typeof v === 'string' && v.trim().length > 0;
|
|
20
|
+
const fail = code => { throw new TypeError(code); };
|
|
21
|
+
const promptHash = text => createHash('sha256').update(text, 'utf8').digest('hex');
|
|
22
|
+
|
|
23
|
+
function validateDraft(draft) {
|
|
24
|
+
if (!exact(draft, ['schemaVersion', 'cases']) || draft.schemaVersion !== 1 || !Array.isArray(draft.cases) || !draft.cases.length) fail('invalid-draft');
|
|
25
|
+
const ids = new Set(); const prompts = new Set();
|
|
26
|
+
for (const c of draft.cases) {
|
|
27
|
+
if (!exact(c, ['id', 'family', 'language', 'source', 'prompt', 'proposedGold', 'rationale'])
|
|
28
|
+
|| ![c.id, c.family, c.prompt, c.rationale].every(nonempty) || !/^[a-z0-9._-]{1,80}$/.test(c.id)
|
|
29
|
+
|| !LANGUAGES.includes(c.language) || !SOURCES.includes(c.source) || !GOLD.includes(c.proposedGold)
|
|
30
|
+
|| ids.has(c.id) || prompts.has(c.prompt) || PRIVATE.some(re => re.test(c.prompt))) fail('invalid-draft');
|
|
31
|
+
ids.add(c.id); prompts.add(c.prompt);
|
|
32
|
+
}
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function validateReviews(reviews) {
|
|
37
|
+
if (!exact(reviews, ['schemaVersion', 'decisions']) || reviews.schemaVersion !== 1 || !Array.isArray(reviews.decisions)) fail('invalid-reviews');
|
|
38
|
+
const ids = new Set();
|
|
39
|
+
for (const d of reviews.decisions) {
|
|
40
|
+
if (!exact(d, ['id', 'promptHash', 'decision', 'gold', 'reviewer']) || !nonempty(d.id) || ids.has(d.id)
|
|
41
|
+
|| typeof d.promptHash !== 'string' || !/^[0-9a-f]{64}$/.test(d.promptHash)
|
|
42
|
+
|| !['accept', 'relabel', 'reject'].includes(d.decision) || !GOLD.includes(d.gold) || d.reviewer !== REVIEWER
|
|
43
|
+
|| (d.decision === 'reject' && d.gold !== null)) fail('invalid-reviews');
|
|
44
|
+
ids.add(d.id);
|
|
45
|
+
}
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function build(draft, reviews) {
|
|
50
|
+
validateDraft(draft);
|
|
51
|
+
const byId = new Map();
|
|
52
|
+
if (reviews !== null) {
|
|
53
|
+
validateReviews(reviews);
|
|
54
|
+
for (const d of reviews.decisions) {
|
|
55
|
+
const c = draft.cases.find(item => item.id === d.id);
|
|
56
|
+
if (!c) fail('unknown-review-case');
|
|
57
|
+
// Accept keeps the proposed gold; relabel must change it. Checked against the current proposal.
|
|
58
|
+
if ((d.decision === 'accept' && d.gold !== c.proposedGold) || (d.decision === 'relabel' && d.gold === c.proposedGold)) fail('invalid-reviews');
|
|
59
|
+
byId.set(d.id, d);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const summary = { cases: 0, reviewed: 0, rejected: 0, stale: 0, unreviewed: 0,
|
|
63
|
+
byLanguage: { en: 0, 'zh-TW': 0 }, byGold: { tier1: 0, tier2: 0, tier3: 0, unclassified: 0 }, reviewedHoldoutGate: false };
|
|
64
|
+
const cases = [];
|
|
65
|
+
for (const c of draft.cases) {
|
|
66
|
+
const d = byId.get(c.id);
|
|
67
|
+
const current = d && d.promptHash === promptHash(c.prompt);
|
|
68
|
+
if (d && !current) summary.stale += 1;
|
|
69
|
+
if (current && d.decision === 'reject') { summary.rejected += 1; continue; }
|
|
70
|
+
const reviewed = Boolean(current);
|
|
71
|
+
if (reviewed) summary.reviewed += 1; else if (!d) summary.unreviewed += 1;
|
|
72
|
+
const gold = reviewed ? d.gold : c.proposedGold;
|
|
73
|
+
cases.push({ id: c.id, family: c.family, split: 'holdout', language: c.language, source: c.source, reviewed,
|
|
74
|
+
request: createRequest('tier', c.prompt, TIER_OPTIONS), gold });
|
|
75
|
+
summary.byLanguage[c.language] += 1;
|
|
76
|
+
summary.byGold[gold === null ? 'unclassified' : gold] += 1;
|
|
77
|
+
}
|
|
78
|
+
summary.cases = cases.length;
|
|
79
|
+
const corpus = { schemaVersion: 1, cases };
|
|
80
|
+
validateCorpus(corpus);
|
|
81
|
+
summary.reviewedHoldoutGate = cases.length >= 200 && cases.every(c => c.reviewed)
|
|
82
|
+
&& LANGUAGES.every(lang => summary.byLanguage[lang] >= 50);
|
|
83
|
+
return { corpus, summary };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (require.main === module) {
|
|
87
|
+
try {
|
|
88
|
+
const [op, draftPath, reviewsPath, outPath, extra] = process.argv.slice(2);
|
|
89
|
+
if (op !== 'build' || !draftPath || !reviewsPath || !outPath || extra) throw new Error('usage');
|
|
90
|
+
const read = file => JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
91
|
+
const { corpus, summary } = build(read(draftPath), reviewsPath === '-' ? null : read(reviewsPath));
|
|
92
|
+
fs.writeFileSync(outPath, `${JSON.stringify(corpus, null, 2)}\n`);
|
|
93
|
+
console.log(JSON.stringify(summary));
|
|
94
|
+
} catch (_) {
|
|
95
|
+
console.error('Usage: system-one-corpus.js build <draft.json> <reviews.json|-> <corpus.json> (invalid draft, reviews, or path)');
|
|
96
|
+
process.exitCode = 1;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
module.exports = { validateDraft, validateReviews, build, promptHash, REVIEWER };
|