harness-everything 0.22.0 → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +6 -0
  4. package/README.md +30 -0
  5. package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/README.md +87 -0
  6. package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/paired-latency.json +107 -0
  7. package/benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/report.json +856 -0
  8. package/benchmarks/results/system-one/pre-merge-review-2026-09-23/README.md +92 -0
  9. package/benchmarks/results/system-one/pre-merge-review-2026-09-23/haiku-spec-conformance.md +241 -0
  10. package/benchmarks/results/system-one/pre-merge-review-2026-09-23/luna-adversarial.md +212 -0
  11. package/benchmarks/results/system-one/pre-merge-review-2026-09-23/skill-top1-probe.js +83 -0
  12. package/benchmarks/results/system-one/pre-merge-review-2026-09-23/skill-top1-probe.json +7700 -0
  13. package/ci/fixtures/system-one-worker-probe.js +15 -0
  14. package/ci/mechanism-system-one-resident.test.js +114 -3
  15. package/ci/mechanism-system-one-router.test.js +25 -0
  16. package/docs/platform-capabilities.md +7 -2
  17. package/docs/system-one-routing.md +23 -4
  18. package/evals/pilot-router-nonsoftware/graders/no-tier-language.md +6 -0
  19. package/evals/pilot-router-nonsoftware/graders/skill-not-fired.md +9 -0
  20. package/evals/pilot-router-nonsoftware/prompt.md +8 -0
  21. package/evals/pilot-router-tier3/graders/skill-fired.md +6 -0
  22. package/evals/pilot-router-tier3/prompt.md +8 -0
  23. package/evals/pilot-tdd-bugfix/graders/skill-fired.md +6 -0
  24. package/evals/pilot-tdd-bugfix/prompt.md +8 -0
  25. package/evals/pilot-tdd-trivia/graders/no-red-phase-claim.md +7 -0
  26. package/evals/pilot-tdd-trivia/graders/skill-not-fired.md +9 -0
  27. package/evals/pilot-tdd-trivia/prompt.md +8 -0
  28. package/evals/pilot-verify-discussion/graders/no-verification-claim.md +7 -0
  29. package/evals/pilot-verify-discussion/graders/skill-not-fired.md +9 -0
  30. package/evals/pilot-verify-discussion/prompt.md +8 -0
  31. package/evals/pilot-verify-preship/graders/skill-fired.md +6 -0
  32. package/evals/pilot-verify-preship/prompt.md +8 -0
  33. package/harness-everything/SKILL.md +1 -1
  34. package/harness-everything/scripts/system-one/provider.js +14 -2
  35. package/harness-everything/scripts/system-one/resident.js +26 -2
  36. package/harness-everything/scripts/system-one/router.js +17 -1
  37. package/harness-everything/scripts/tier-router.js +13 -4
  38. package/opencode-plugin/plugin.json +1 -1
  39. package/package.json +1 -1
  40. package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
  41. package/plugins/harness-everything/plugin.json +1 -1
  42. package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
  43. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/provider.js +14 -2
  44. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js +26 -2
  45. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js +17 -1
  46. package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +13 -4
  47. package/sbom.cdx.json +7 -7
  48. package/scripts/evaluate-system-one.js +6 -6
@@ -5,6 +5,7 @@
5
5
  const fs = require('node:fs');
6
6
  const path = require('node:path');
7
7
  const { createHash } = require('node:crypto');
8
+ const net = require('node:net');
8
9
  const { spawn } = require('node:child_process');
9
10
  const { Worker } = require('node:worker_threads');
10
11
  const { validateRequest } = require('./contract');
@@ -81,6 +82,20 @@ function callSync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
81
82
  try { return { reply: JSON.parse(Buffer.from(out, 0, Atomics.load(signal, 1)).toString('utf8')) }; }
82
83
  catch (_) { return { error: 'json' }; }
83
84
  }
85
+ // The same round trip for async callers (the hook entry): no worker thread to bootstrap.
86
+ function callAsync(port, message, timeoutMs = CALL_TIMEOUT_MS) {
87
+ return new Promise(resolve => {
88
+ const chunks = []; let size = 0; let done = false; let socket = null;
89
+ const finish = value => { if (done) return; done = true; clearTimeout(timer); if (socket) socket.destroy(); resolve(value); };
90
+ const timer = setTimeout(() => finish({ error: 'timeout' }), timeoutMs);
91
+ socket = net.connect({ port, host: '127.0.0.1' }, () => socket.end(`${JSON.stringify(message)}\n`));
92
+ socket.on('data', c => { size += c.length; if (size > MAX_REPLY_BYTES) finish({ error: 'limit' }); else chunks.push(c); });
93
+ socket.on('end', () => {
94
+ try { finish({ reply: JSON.parse(Buffer.concat(chunks).toString('utf8')) }); } catch (_) { finish({ error: 'json' }); }
95
+ });
96
+ socket.on('error', e => finish({ error: e.code === 'ECONNREFUSED' ? 'refused' : 'io' }));
97
+ });
98
+ }
84
99
 
85
100
  function start(manifest, files, deps) {
86
101
  const lock = lockInfo(files.lock);
@@ -114,7 +129,16 @@ function score(request, manifest, manifestPath, deps = {}) {
114
129
  const files = stateFiles(manifest, manifestPath);
115
130
  const state = liveState(files);
116
131
  if (!state) return start(manifest, files, deps);
117
- const result = callSync(state.port, { token: state.token, op: 'score', request });
132
+ return outcome(callSync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
133
+ }
134
+ async function scoreAsync(request, manifest, manifestPath, deps = {}) {
135
+ validateRequest(request);
136
+ const files = stateFiles(manifest, manifestPath);
137
+ const state = liveState(files);
138
+ if (!state) return start(manifest, files, deps);
139
+ return outcome(await callAsync(state.port, { token: state.token, op: 'score', request }), files, state, manifest, deps);
140
+ }
141
+ function outcome(result, files, state, manifest, deps) {
118
142
  if (result.reply) {
119
143
  if (result.reply.ok === true && Object.hasOwn(result.reply, 'response')) return { status: 'scored', response: result.reply.response };
120
144
  if (result.reply.ok === false) return unavailable(result.reply.reason === 'unauthorized' ? 'provider-unavailable' : 'provider-exit');
@@ -173,4 +197,4 @@ if (require.main === module) {
173
197
  if (op === 'stop') stop(manifest, manifestPath);
174
198
  process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
175
199
  }
176
- module.exports = { stateFiles, readState, callSync, score, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
200
+ module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
@@ -34,4 +34,20 @@ function selectTier(input, env = process.env, scorer = provider.score) {
34
34
  const reason = input.explicit && accepted ? 'explicit-precedence' : belowFloor ? 'structural-floor' : decision.reason;
35
35
  return { tier: applied ? LABELS[decision.selectedId] : input.tier, diagnostic: { mode, ...decision, applied, reason } };
36
36
  }
37
- module.exports = { selectTier, TIER_OPTIONS };
37
+ // Hook entry: score the same fixed-catalog request asynchronously before routing (no sync worker bridge).
38
+ // Returns a scorer for selectTier, or null when System One is off/invalid; nothing is called then.
39
+ async function prescoreTier(prompt, env = process.env, scoreAsync = provider.scoreAsync) {
40
+ if (!['shadow', 'prefer'].includes(env.HARNESS_SYSTEM_ONE_MODE)) return null;
41
+ let request;
42
+ try { request = createRequest('tier', prompt, TIER_OPTIONS); }
43
+ catch (_) { return null; }
44
+ let result; let failed = false;
45
+ try { result = await scoreAsync(request, env.HARNESS_SYSTEM_ONE_CONFIG); }
46
+ catch (_) { failed = true; }
47
+ return (req, config) => {
48
+ if (req.requestHash !== request.requestHash) return provider.score(req, config);
49
+ if (failed) throw new Error('provider-unavailable');
50
+ return result;
51
+ };
52
+ }
53
+ module.exports = { selectTier, prescoreTier, TIER_OPTIONS };
@@ -231,7 +231,7 @@ function emitDynamicSkills(promptLower, context, recommendedGuides) {
231
231
  }
232
232
  }
233
233
 
234
- function run(userPrompt, context) {
234
+ function run(userPrompt, context, options = {}) {
235
235
  console.log(`[Tier Routing Pre-check]`);
236
236
 
237
237
  const promptLower = userPrompt.toLowerCase();
@@ -317,7 +317,7 @@ function run(userPrompt, context) {
317
317
  const structuralFloor = hasMacroSignal ? 'tier3'
318
318
  : !isTrivialDocsEdit && hasMultipleTasks && hasMultipleSentences ? 'tier2' : null;
319
319
  const semantic = selectTier({ prompt: userPrompt, tier: recommendedTier, floor: structuralFloor,
320
- explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) });
320
+ explicit: Boolean(detectExplicitStrategy(userPrompt, detectFableModel(userPrompt))) }, process.env, options.systemOneScorer || undefined);
321
321
  if (semantic.diagnostic) console.log(`\n=> SYSTEM ONE: ${JSON.stringify(semantic.diagnostic)}`);
322
322
  if (semantic.diagnostic?.applied) {
323
323
  recommendedTier = semantic.tier;
@@ -460,12 +460,21 @@ function run(userPrompt, context) {
460
460
  return contract;
461
461
  }
462
462
 
463
+ // Entry point: pre-score System One asynchronously so routing never needs the sync worker bridge.
464
+ async function main(prompt, context) {
465
+ let systemOneScorer = null;
466
+ if (['shadow', 'prefer'].includes(process.env.HARNESS_SYSTEM_ONE_MODE)) {
467
+ try { systemOneScorer = await require('./system-one/router').prescoreTier(prompt); } catch (_) { /* the sync path still scores */ }
468
+ }
469
+ run(prompt, context, { systemOneScorer });
470
+ }
471
+
463
472
  if (require.main === module) {
464
473
  let userPrompt = process.argv[2] || '';
465
474
  let hookContext = null;
466
475
 
467
476
  if (process.argv[2]) {
468
- run(userPrompt);
477
+ main(userPrompt);
469
478
  } else if (process.stdin.isTTY) {
470
479
  run('');
471
480
  } else {
@@ -482,7 +491,7 @@ if (process.argv[2]) {
482
491
  } catch (err) {
483
492
  // Not valid JSON on stdin - fall back to argv/manual behavior.
484
493
  }
485
- run(userPrompt, hookContext);
494
+ main(userPrompt, hookContext);
486
495
  });
487
496
  }
488
497
  }
package/sbom.cdx.json CHANGED
@@ -2,23 +2,23 @@
2
2
  "$schema": "https://cyclonedx.org/schema/bom-1.7.schema.json",
3
3
  "bomFormat": "CycloneDX",
4
4
  "specVersion": "1.7",
5
- "serialNumber": "urn:uuid:0f18cc7a-103c-5369-a33b-bdbe1bf12efb",
5
+ "serialNumber": "urn:uuid:3cb9e1e8-a9f0-5bc4-87ab-794acb1e79c3",
6
6
  "version": 1,
7
7
  "metadata": {
8
- "timestamp": "2026-09-23T05:03:30.779Z",
8
+ "timestamp": "2026-09-23T11:35:56.699Z",
9
9
  "tools": [
10
10
  {
11
11
  "vendor": "Harness Everything",
12
12
  "name": "generate-sbom.js",
13
- "version": "0.22.0"
13
+ "version": "0.22.1"
14
14
  }
15
15
  ],
16
16
  "component": {
17
17
  "type": "application",
18
- "bom-ref": "pkg:npm/harness-everything@0.22.0",
18
+ "bom-ref": "pkg:npm/harness-everything@0.22.1",
19
19
  "name": "harness-everything",
20
- "version": "0.22.0",
21
- "purl": "pkg:npm/harness-everything@0.22.0"
20
+ "version": "0.22.1",
21
+ "purl": "pkg:npm/harness-everything@0.22.1"
22
22
  }
23
23
  },
24
24
  "components": [
@@ -1999,7 +1999,7 @@
1999
1999
  ]
2000
2000
  },
2001
2001
  {
2002
- "ref": "pkg:npm/harness-everything@0.22.0",
2002
+ "ref": "pkg:npm/harness-everything@0.22.1",
2003
2003
  "dependsOn": [
2004
2004
  "pkg:npm/%40commitlint/cli@19.8.1",
2005
2005
  "pkg:npm/%40commitlint/config-conventional@19.8.1",
@@ -4,12 +4,12 @@ const fs = require('node:fs');
4
4
  const os = require('node:os');
5
5
  const { performance } = require('node:perf_hooks');
6
6
  const { validateCorpus, evaluate } = require('../harness-everything/scripts/system-one/evaluate');
7
- const { score, provenance, readManifest } = require('../harness-everything/scripts/system-one/provider');
7
+ const { scoreAsync, provenance, readManifest } = require('../harness-everything/scripts/system-one/provider');
8
8
  const resident = require('../harness-everything/scripts/system-one/resident');
9
9
  const { decide } = require('../harness-everything/scripts/system-one/contract');
10
10
  const { run: route } = require('../harness-everything/scripts/tier-router');
11
11
 
12
- function run(corpus, manifest) {
12
+ async function run(corpus, manifest) {
13
13
  validateCorpus(corpus);
14
14
  let config = null;
15
15
  try { config = readManifest(manifest); } catch (_) { /* Missing provider remains an explicit unavailable measurement. */ }
@@ -28,7 +28,7 @@ function run(corpus, manifest) {
28
28
  const runs = [];
29
29
  for (let i = 0; i < 2; i++) {
30
30
  const start = performance.now();
31
- const result = score(c.request, manifest);
31
+ const result = await scoreAsync(c.request, manifest);
32
32
  const decision = result.status === 'scored' ? decide(c.request, result.response) : result;
33
33
  if (decision.selectedId === 'unclassified') Object.assign(decision, { status: 'abstain', reason: 'unclassified', selectedId: null });
34
34
  const scores = ['accepted', 'abstain'].includes(decision.status) ? result.response.scores.map(s => s.probability) : null;
@@ -55,12 +55,12 @@ function run(corpus, manifest) {
55
55
  limitations: ['No independently verified holdout review', ...(warm ? [] : ['No warm inference measurement']), 'No live-host or policy evidence'] }, records };
56
56
  }
57
57
  if (require.main === module) {
58
- try {
58
+ (async () => {
59
59
  const [input, manifest, output, extra] = process.argv.slice(2);
60
60
  if (!input || !manifest || !output || extra) throw new Error('usage');
61
- const report = run(JSON.parse(fs.readFileSync(input, 'utf8')), manifest);
61
+ const report = await run(JSON.parse(fs.readFileSync(input, 'utf8')), manifest);
62
62
  fs.writeFileSync(output, JSON.stringify(report, null, 2) + '\n');
63
63
  console.log(JSON.stringify({ cases: report.cases, coverage: report.model.coverage, rolloutReady: report.rolloutReady }));
64
- } catch (_) { console.error('System One evaluation failed: invalid corpus, arguments, or output path.'); process.exitCode = 1; }
64
+ })().catch(() => { console.error('System One evaluation failed: invalid corpus, arguments, or output path.'); process.exitCode = 1; });
65
65
  }
66
66
  module.exports = { run };