harness-everything 0.20.7 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +26 -0
  4. package/benchmarks/fixtures/system-one-routing.json +549 -0
  5. package/benchmarks/results/system-one/cua-s1-forms-cpu-2026-09-23/README.md +62 -0
  6. package/benchmarks/results/system-one/cua-s1-forms-cpu-2026-09-23/report.json +856 -0
  7. package/benchmarks/results/system-one/cua-s1-forms-resident-cpu-2026-09-23/README.md +74 -0
  8. package/benchmarks/results/system-one/cua-s1-forms-resident-cpu-2026-09-23/report.json +856 -0
  9. package/ci/fixtures/system-one-provider.cjs +13 -0
  10. package/ci/fixtures/system-one-resident-stub.py +28 -0
  11. package/ci/mechanism-packaging-bytecode.test.js +61 -0
  12. package/ci/mechanism-system-one-contract.test.js +48 -0
  13. package/ci/mechanism-system-one-evaluation.test.js +87 -0
  14. package/ci/mechanism-system-one-install.test.js +106 -0
  15. package/ci/mechanism-system-one-provider.test.js +63 -0
  16. package/ci/mechanism-system-one-resident.test.js +185 -0
  17. package/ci/mechanism-system-one-router.test.js +101 -0
  18. package/ci/system-one-adapter-test.py +222 -0
  19. package/docs/platform-capabilities.md +14 -0
  20. package/docs/system-one-routing.md +271 -0
  21. package/harness-everything/SKILL.md +1 -1
  22. package/harness-everything/scripts/system-one/contract.js +45 -0
  23. package/harness-everything/scripts/system-one/cua_adapter.py +353 -0
  24. package/harness-everything/scripts/system-one/evaluate.js +98 -0
  25. package/harness-everything/scripts/system-one/install.js +161 -0
  26. package/harness-everything/scripts/system-one/provider.js +79 -0
  27. package/harness-everything/scripts/system-one/resident.js +176 -0
  28. package/harness-everything/scripts/system-one/router.js +37 -0
  29. package/harness-everything/scripts/tier-router.js +16 -0
  30. package/opencode-plugin/plugin.json +1 -1
  31. package/package.json +2 -1
  32. package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
  33. package/plugins/harness-everything/plugin.json +1 -1
  34. package/plugins/harness-everything/scripts/lib/packaging-noise.js +6 -0
  35. package/plugins/harness-everything/scripts/lib/skills.js +2 -0
  36. package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
  37. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/contract.js +45 -0
  38. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/cua_adapter.py +353 -0
  39. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/evaluate.js +98 -0
  40. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/install.js +161 -0
  41. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/provider.js +79 -0
  42. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js +176 -0
  43. package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js +37 -0
  44. package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +16 -0
  45. package/sbom.cdx.json +7 -7
  46. package/scripts/build-openai-submission.js +2 -0
  47. package/scripts/evaluate-system-one.js +66 -0
  48. package/scripts/lib/packaging-noise.js +6 -0
  49. package/scripts/lib/skills.js +2 -0
  50. package/scripts/sync-openai-plugin.js +2 -1
@@ -0,0 +1,353 @@
1
+ """Optional CPU CUA-S1 bridge. Stdout is protocol-only; never execute UI actions."""
2
+ import hashlib
3
+ import hmac
4
+ import json
5
+ import os
6
+ from pathlib import Path
7
+ import re
8
+ import secrets
9
+ import socket
10
+ import subprocess
11
+ import sys
12
+ import threading
13
+ import time
14
+
15
+
16
+ def verify_artifacts(manifest):
17
+ weights = Path(manifest['checkpoint'])
18
+ if not weights.is_absolute() or weights.suffix != '.safetensors':
19
+ raise ValueError('checkpoint-format')
20
+ for file, key in [(weights, 'weightsSha256'), (weights.with_suffix('.json'), 'configSha256')]:
21
+ with file.open('rb') as handle:
22
+ digest = hashlib.file_digest(handle, 'sha256').hexdigest()
23
+ if digest != manifest[key]:
24
+ raise ValueError('artifact-hash')
25
+ return weights
26
+
27
+
28
+ def check_limits(request, config):
29
+ if len(request['context'].encode('utf-8')) > config['context_tokens'] or any(
30
+ len(option['text'].encode('utf-8')) > config['option_tokens'] for option in request['options']
31
+ ):
32
+ raise ValueError('input-too-long')
33
+
34
+
35
+ def load_scorer(weights, manifest):
36
+ """Load the verified checkpoint once; return (score(request) -> probabilities, config)."""
37
+ import torch
38
+ from cua_s1.model import ChoiceExample, load_checkpoint
39
+
40
+ torch.set_num_threads(1)
41
+ torch.use_deterministic_algorithms(True)
42
+ model, collator, config = load_checkpoint(weights, 'cpu')
43
+
44
+ def score(request):
45
+ example = ChoiceExample(context=request['context'], options=tuple(o['text'] for o in request['options']), label=0)
46
+ with torch.inference_mode():
47
+ return model(collator([example])).softmax(-1)[0].tolist()
48
+ return score, config
49
+
50
+
51
+ def build_response(request, manifest, probabilities):
52
+ return {
53
+ 'schemaVersion': 1, 'requestHash': request['requestHash'], 'catalogHash': request['catalogHash'],
54
+ 'model': {'id': manifest['modelId'], 'revision': manifest['revision'], 'domain': manifest['domain']},
55
+ 'scores': [{'id': option['id'], 'probability': probability} for option, probability in zip(request['options'], probabilities, strict=True)],
56
+ }
57
+
58
+
59
+ def infer(request, manifest):
60
+ weights = verify_artifacts(manifest)
61
+ score, config = load_scorer(weights, manifest)
62
+ check_limits(request, config)
63
+ return build_response(request, manifest, score(request))
64
+
65
+
66
+ MAX_REQUEST_BYTES = 2 * 1024 * 1024
67
+ MAX_OPTIONS = 256
68
+ MAX_CLIENTS = 64
69
+ READ_DEADLINE_S = 1.0 # never longer than the Node client's 1000 ms call deadline
70
+ STARTUP_REASONS = ('artifact-hash', 'checkpoint-format')
71
+ ID_PATTERN = re.compile(r'[a-z0-9._-]{1,80}')
72
+ HASH_PATTERN = re.compile(r'[0-9a-f]{64}')
73
+ _USER_SID = []
74
+
75
+
76
+ def _artifact_stamp(weights):
77
+ return [(p.stat().st_size, p.stat().st_mtime_ns) for p in (weights, weights.with_suffix('.json'))]
78
+
79
+
80
+ def _restrict_to_current_user(path):
81
+ """Windows: replace the whole DACL with one protected full-control ACE for the current user.
82
+
83
+ Editing ACEs is not enough: some hosts create files with explicit (non-inherited) SYSTEM,
84
+ Administrators or OWNER RIGHTS entries that survive `icacls /inheritance:r /grant:r`.
85
+ """
86
+ if os.name != 'nt':
87
+ return
88
+ import ctypes
89
+ from ctypes import wintypes
90
+ if not _USER_SID:
91
+ tools = Path(os.environ.get('SystemRoot', 'C:/Windows')) / 'System32'
92
+ out = subprocess.run([str(tools / 'whoami.exe'), '/user', '/fo', 'csv', '/nh'],
93
+ capture_output=True, text=True, timeout=10, check=True).stdout
94
+ sid = out.strip().split(',')[-1].strip().strip('"')
95
+ if not re.fullmatch(r'S-1-[0-9-]+', sid):
96
+ raise OSError('user-sid')
97
+ _USER_SID.append(sid)
98
+ advapi32 = ctypes.WinDLL('advapi32', use_last_error=True)
99
+ kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
100
+ descriptor = ctypes.c_void_p()
101
+ if not advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW(
102
+ f'D:P(A;;FA;;;{_USER_SID[0]})', 1, ctypes.byref(descriptor), None):
103
+ raise OSError('dacl-build')
104
+ try:
105
+ present, defaulted, dacl = wintypes.BOOL(), wintypes.BOOL(), ctypes.c_void_p()
106
+ if not advapi32.GetSecurityDescriptorDacl(descriptor, ctypes.byref(present), ctypes.byref(dacl), ctypes.byref(defaulted)):
107
+ raise OSError('dacl-read')
108
+ SE_FILE_OBJECT, DACL_INFO, PROTECTED_DACL_INFO = 1, 0x4, 0x80000000
109
+ advapi32.SetNamedSecurityInfoW.argtypes = [wintypes.LPWSTR, ctypes.c_int, wintypes.DWORD,
110
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
111
+ if advapi32.SetNamedSecurityInfoW(str(path), SE_FILE_OBJECT, DACL_INFO | PROTECTED_DACL_INFO, None, None, dacl, None) != 0:
112
+ raise OSError('dacl-set')
113
+ finally:
114
+ kernel32.LocalFree(descriptor)
115
+
116
+
117
+ def _write_private_json(target, value):
118
+ """Atomic replace of an owner-only file (0600; explicit owner-only DACL on Windows) before the token is written."""
119
+ tmp = target.with_name(f'.{target.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp')
120
+ os.close(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600))
121
+ _restrict_to_current_user(tmp)
122
+ with open(tmp, 'w', encoding='utf-8') as handle:
123
+ json.dump(value, handle)
124
+ os.replace(tmp, target)
125
+
126
+
127
+ def _read_request_line(conn):
128
+ buffer = bytearray()
129
+ while b'\n' not in buffer and len(buffer) <= MAX_REQUEST_BYTES:
130
+ chunk = conn.recv(65536)
131
+ if not chunk:
132
+ break
133
+ buffer.extend(chunk)
134
+ if len(buffer.split(b'\n', 1)[0]) > MAX_REQUEST_BYTES:
135
+ # Drain to EOF (bounded by size and the read deadline) so the reply is not lost to a reset.
136
+ drained = 0
137
+ try:
138
+ while drained < 8 * MAX_REQUEST_BYTES and conn.recv(65536):
139
+ drained += 65536
140
+ except OSError:
141
+ pass
142
+ raise ValueError('request-limit')
143
+ return bytes(buffer).split(b'\n', 1)[0]
144
+
145
+
146
+ def _valid_score_request(request):
147
+ """Phase 0 request structure. Hashes are fingerprints, not authentication, so they are shape-checked only."""
148
+ if (not isinstance(request, dict)
149
+ or set(request) != {'schemaVersion', 'task', 'context', 'options', 'catalogHash', 'requestHash'}
150
+ or request['schemaVersion'] != 1 or not isinstance(request['task'], str) or not ID_PATTERN.fullmatch(request['task'])
151
+ or not isinstance(request['context'], str)
152
+ or not all(isinstance(request[k], str) and HASH_PATTERN.fullmatch(request[k]) for k in ('catalogHash', 'requestHash'))):
153
+ return False
154
+ options = request['options']
155
+ if not isinstance(options, list) or not 2 <= len(options) <= MAX_OPTIONS:
156
+ return False
157
+ if any(not isinstance(o, dict) or set(o) != {'id', 'text'} or not isinstance(o['id'], str)
158
+ or not ID_PATTERN.fullmatch(o['id']) or not isinstance(o['text'], str) for o in options):
159
+ return False
160
+ return len({o['id'] for o in options}) == len(options)
161
+
162
+
163
+ def _handle(conn, token, manifest, weights, stamp, score, config):
164
+ """Return (reply, stop). Errors carry bounded reason codes only: no prompt, path, or traceback."""
165
+ invalid = {'ok': False, 'reason': 'invalid-request'}
166
+ try:
167
+ line = _read_request_line(conn)
168
+ except ValueError:
169
+ return {'ok': False, 'reason': 'request-limit'}, False
170
+ except OSError:
171
+ return invalid, False
172
+ try:
173
+ message = json.loads(line)
174
+ except ValueError:
175
+ return invalid, False
176
+ if not isinstance(message, dict):
177
+ return invalid, False
178
+ supplied = message.get('token')
179
+ if not isinstance(supplied, str) or not hmac.compare_digest(supplied.encode('utf-8'), token.encode('utf-8')):
180
+ return {'ok': False, 'reason': 'unauthorized'}, False
181
+ op = message.get('op')
182
+ if op == 'ping':
183
+ return {'ok': True, 'pid': os.getpid(), 'model': {'id': manifest['modelId'], 'revision': manifest['revision'], 'domain': manifest['domain']}}, False
184
+ if op == 'shutdown':
185
+ return {'ok': True}, True
186
+ if op != 'score' or not _valid_score_request(message.get('request')):
187
+ return invalid, False
188
+ try:
189
+ changed = _artifact_stamp(weights) != stamp
190
+ except OSError:
191
+ changed = True
192
+ if changed:
193
+ return {'ok': False, 'reason': 'artifact-changed'}, True
194
+ request = message['request']
195
+ try:
196
+ check_limits(request, config)
197
+ except ValueError:
198
+ return {'ok': False, 'reason': 'input-too-long'}, False
199
+ try:
200
+ return {'ok': True, 'response': build_response(request, manifest, score(request))}, False
201
+ except Exception:
202
+ return {'ok': False, 'reason': 'score-failed'}, False
203
+
204
+
205
+ def serve(manifest, state_path, lock_path, scorer_factory=None, idle_timeout_s=None):
206
+ """Resident loopback scorer: verify, load once, publish state, serve until idle/shutdown/artifact change."""
207
+ state_path, lock_path = Path(state_path), Path(lock_path)
208
+ try:
209
+ weights = verify_artifacts(manifest)
210
+ stamp = _artifact_stamp(weights)
211
+ score, config = (scorer_factory or load_scorer)(weights, manifest)
212
+ except Exception as exc:
213
+ reason = str(exc) if str(exc) in STARTUP_REASONS else 'load-failed'
214
+ try:
215
+ _write_private_json(lock_path, {'failed': reason, 'at': time.time()})
216
+ except OSError:
217
+ pass
218
+ return 1
219
+ idle = idle_timeout_s if idle_timeout_s is not None else manifest.get('idleTimeoutMs', 1800000) / 1000
220
+ token = secrets.token_hex(32)
221
+ pid = os.getpid()
222
+ stop = threading.Event()
223
+ score_lock = threading.Lock() # one model, one inference at a time; connections are handled concurrently
224
+ slots = threading.BoundedSemaphore(MAX_CLIENTS)
225
+
226
+ def guarded_score(request):
227
+ with score_lock:
228
+ return score(request)
229
+
230
+ def send(conn, reply):
231
+ try:
232
+ conn.sendall((json.dumps(reply, allow_nan=False) + '\n').encode('utf-8'))
233
+ except OSError:
234
+ pass
235
+
236
+ def worker(conn):
237
+ try:
238
+ with conn:
239
+ conn.settimeout(READ_DEADLINE_S)
240
+ reply, halt = _handle(conn, token, manifest, weights, stamp, guarded_score, config)
241
+ send(conn, reply)
242
+ if halt:
243
+ stop.set()
244
+ finally:
245
+ slots.release()
246
+
247
+ server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
248
+ try:
249
+ server.bind(('127.0.0.1', 0))
250
+ server.listen(MAX_CLIENTS)
251
+ server.settimeout(0.25)
252
+ _write_private_json(state_path, {'schemaVersion': 1, 'pid': pid, 'port': server.getsockname()[1], 'token': token, 'startedAt': time.time()})
253
+ try:
254
+ lock_path.unlink()
255
+ except FileNotFoundError:
256
+ pass
257
+ last = time.monotonic()
258
+ # A server whose state file no longer names it is undiscoverable (replaced or corrupted): exit, never orphan.
259
+ while not stop.is_set() and time.monotonic() - last <= idle and _owns_state(state_path, pid, token):
260
+ try:
261
+ conn, _ = server.accept()
262
+ except socket.timeout:
263
+ continue
264
+ last = time.monotonic()
265
+ if not slots.acquire(blocking=False):
266
+ with conn:
267
+ conn.settimeout(0.2)
268
+ try:
269
+ _read_request_line(conn)
270
+ except (OSError, ValueError):
271
+ pass
272
+ send(conn, {'ok': False, 'reason': 'busy'})
273
+ continue
274
+ threading.Thread(target=worker, args=(conn,), daemon=True).start()
275
+ finally:
276
+ server.close()
277
+ if _owns_state(state_path, pid, token, missing=False):
278
+ try:
279
+ state_path.unlink()
280
+ except OSError:
281
+ pass
282
+ return 0
283
+
284
+
285
+ def _owns_state(state_path, pid, token, missing=False):
286
+ """True while the state file still names this server. A transient Windows sharing violation counts as owned."""
287
+ try:
288
+ data = json.loads(state_path.read_text(encoding='utf-8'))
289
+ except FileNotFoundError:
290
+ return missing
291
+ except OSError:
292
+ return True
293
+ except ValueError:
294
+ return False
295
+ return isinstance(data, dict) and data.get('pid') == pid and data.get('token') == token
296
+
297
+
298
+ def serve_main(argv, scorer_factory=None):
299
+ if len(argv) != 3:
300
+ return 2
301
+ return serve(json.loads(argv[0]), argv[1], argv[2], scorer_factory=scorer_factory)
302
+
303
+
304
+ def source_revision(direct_url_text):
305
+ """Return the PEP 610 VCS commit recorded at install time, or None when unprovable."""
306
+ try:
307
+ commit = json.loads(direct_url_text)['vcs_info']['commit_id']
308
+ except Exception:
309
+ return None
310
+ return commit if isinstance(commit, str) and re.fullmatch(r'[0-9a-f]{40}', commit) else None
311
+
312
+
313
+ def provenance():
314
+ """Installed runtime/source facts only; never imports torch or loads a checkpoint."""
315
+ import importlib.metadata as metadata
316
+ import platform
317
+
318
+ cua = {'distribution': None, 'version': None, 'sourceRevision': None}
319
+ names = sorted(set(metadata.packages_distributions().get('cua_s1', [])))
320
+ if len(names) == 1:
321
+ dist = metadata.distribution(names[0])
322
+ cua = {'distribution': names[0], 'version': dist.version, 'sourceRevision': source_revision(dist.read_text('direct_url.json'))}
323
+ try:
324
+ torch = metadata.version('torch')
325
+ except metadata.PackageNotFoundError:
326
+ torch = None
327
+ return {'schemaVersion': 1, 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()},
328
+ 'cuaS1': cua, 'torch': torch}
329
+
330
+
331
+ def main():
332
+ try:
333
+ if sys.argv[1:] == ['--provenance']:
334
+ print(json.dumps(provenance(), allow_nan=False))
335
+ return 0
336
+ if sys.argv[1:2] == ['--serve']:
337
+ return serve_main(sys.argv[2:])
338
+ manifest = json.loads(sys.argv[1])
339
+ raw = sys.stdin.buffer.read(2 * 1024 * 1024 + 1)
340
+ if len(raw) > 2 * 1024 * 1024:
341
+ raise ValueError('request-limit')
342
+ request = json.loads(raw)
343
+ result = infer(request, manifest)
344
+ print(json.dumps(result, allow_nan=False))
345
+ except Exception:
346
+ # No raw prompt, selected values, artifact paths, or traceback on the wire.
347
+ print('cua-provider-unavailable', file=sys.stderr)
348
+ return 1
349
+ return 0
350
+
351
+
352
+ if __name__ == '__main__':
353
+ sys.exit(main())
@@ -0,0 +1,98 @@
1
+ 'use strict';
2
+ const { validateRequest } = require('./contract');
3
+ const { TIER_OPTIONS } = require('./router');
4
+ const exact = (obj, keys) => obj && typeof obj === 'object' && !Array.isArray(obj) && Object.keys(obj).length === keys.length && keys.every(k => Object.hasOwn(obj, k));
5
+ const nonempty = v => typeof v === 'string' && v.trim().length > 0;
6
+ const labels = ['tier1', 'tier2', 'tier3', null];
7
+ const finite = v => typeof v === 'number' && Number.isFinite(v);
8
+ const fail = () => { throw new TypeError('invalid-evaluation-input'); };
9
+ function validateCorpus(corpus) {
10
+ if (!exact(corpus, ['schemaVersion', 'cases']) || corpus.schemaVersion !== 1 || !Array.isArray(corpus.cases) || !corpus.cases.length) fail();
11
+ const ids = new Set(); const hashes = new Set(); const families = new Map();
12
+ for (const c of corpus.cases) {
13
+ if (!exact(c, ['id', 'family', 'split', 'language', 'source', 'reviewed', 'request', 'gold'])
14
+ || ![c.id, c.family, c.source].every(nonempty) || !['train', 'validation', 'holdout'].includes(c.split)
15
+ || !['en', 'zh-TW'].includes(c.language) || typeof c.reviewed !== 'boolean' || !labels.includes(c.gold)) fail();
16
+ validateRequest(c.request);
17
+ if (c.request.task !== 'tier' || JSON.stringify(c.request.options) !== JSON.stringify(TIER_OPTIONS)
18
+ || ids.has(c.id) || hashes.has(c.request.requestHash) || (families.has(c.family) && families.get(c.family) !== c.split)) fail();
19
+ ids.add(c.id); hashes.add(c.request.requestHash); families.set(c.family, c.split);
20
+ }
21
+ return true;
22
+ }
23
+ function prediction(decision) {
24
+ if (exact(decision, ['status', 'reason']) && decision.status === 'unavailable' && nonempty(decision.reason)) return null;
25
+ if (!exact(decision, ['status', 'reason', 'selectedId', 'confidence', 'margin', 'model'])
26
+ || !['accepted', 'abstain', 'invalid-output'].includes(decision.status) || !nonempty(decision.reason)
27
+ || !labels.includes(decision.selectedId) || (decision.status === 'accepted') !== (decision.selectedId !== null)) fail();
28
+ for (const key of ['confidence', 'margin']) if (decision[key] !== null && (!finite(decision[key]) || decision[key] < 0 || decision[key] > 1)) fail();
29
+ if (decision.model !== null && (!exact(decision.model, ['id', 'revision', 'domain']) || !Object.values(decision.model).every(nonempty))) fail();
30
+ if (decision.status === 'accepted' && (decision.confidence === null || decision.margin === null || decision.model?.domain !== 'harness-routing-v1')) fail();
31
+ return decision.selectedId;
32
+ }
33
+ // Scored decisions carry the full catalog-order probability vector; unavailable/invalid outputs carry null.
34
+ function checkScores(decision, scores) {
35
+ const scored = ['accepted', 'abstain'].includes(decision.status);
36
+ if (!scored) { if (scores !== null) fail(); return; }
37
+ if (!Array.isArray(scores) || scores.length !== TIER_OPTIONS.length
38
+ || scores.some(p => !finite(p) || p < 0 || p > 1) || Math.abs(scores.reduce((s, p) => s + p, 0) - 1) > 1e-6) fail();
39
+ const ranked = [...scores].sort((a, b) => b - a);
40
+ if (decision.confidence !== ranked[0] || decision.margin !== ranked[0] - ranked[1]) fail();
41
+ }
42
+ function metrics(gold, predicted) {
43
+ const count = gold.length;
44
+ const accepted = predicted.filter(x => x !== null).length;
45
+ const correct = predicted.filter((x, i) => x === gold[i]).length;
46
+ const acceptedCorrect = predicted.filter((x, i) => x !== null && x === gold[i]).length;
47
+ const macroF1 = labels.reduce((sum, label) => {
48
+ const tp = gold.filter((g, i) => g === label && predicted[i] === label).length;
49
+ const fp = gold.filter((g, i) => g !== label && predicted[i] === label).length;
50
+ const fn = gold.filter((g, i) => g === label && predicted[i] !== label).length;
51
+ return sum + (2 * tp + fp + fn ? 2 * tp / (2 * tp + fp + fn) : 0);
52
+ }, 0) / labels.length;
53
+ return { accuracy: correct / count, macroF1, coverage: accepted / count, abstentionRate: 1 - accepted / count,
54
+ acceptedPrecision: accepted ? acceptedCorrect / accepted : null,
55
+ acceptedErrorRate: accepted ? (accepted - acceptedCorrect) / accepted : null };
56
+ }
57
+ function p95(values) { return values.length ? [...values].sort((a, b) => a - b)[Math.ceil(values.length * 0.95) - 1] : null; }
58
+ // sourceEvidence is the provider provenance result; only the pinned cua_s1 revision satisfies the gate.
59
+ function evaluate(corpus, records, sourceEvidence = null) {
60
+ validateCorpus(corpus);
61
+ const cases = corpus.cases.filter(c => c.split === 'holdout');
62
+ if (!cases.length || !Array.isArray(records) || records.length !== cases.length) fail();
63
+ const byId = new Map(); const models = new Set(); const cold = []; const warm = [];
64
+ for (const record of records) {
65
+ if (!exact(record, ['id', 'baseline', 'runs']) || byId.has(record.id) || !cases.some(c => c.id === record.id)
66
+ || !labels.includes(record.baseline) || !Array.isArray(record.runs) || record.runs.length !== 2) fail();
67
+ for (const run of record.runs) {
68
+ if (!exact(run, ['decision', 'scores', 'latencyMs', 'coldStart']) || !finite(run.latencyMs) || run.latencyMs < 0 || typeof run.coldStart !== 'boolean') fail();
69
+ prediction(run.decision);
70
+ checkScores(run.decision, run.scores);
71
+ if (run.decision.model) models.add(JSON.stringify([run.decision.model.id, run.decision.model.revision, run.decision.model.domain]));
72
+ (run.coldStart ? cold : warm).push(run.latencyMs);
73
+ }
74
+ byId.set(record.id, record);
75
+ }
76
+ if (models.size > 1) fail();
77
+ const gold = cases.map(c => c.gold);
78
+ const baseline = metrics(gold, cases.map(c => byId.get(c.id).baseline));
79
+ const model = metrics(gold, cases.map(c => prediction(byId.get(c.id).runs[0].decision)));
80
+ const canonicalRun = ({ decision: d, scores }) => JSON.stringify([d.status, d.reason, d.selectedId ?? null, d.confidence ?? null, d.margin ?? null,
81
+ d.model ? [d.model.id, d.model.revision, d.model.domain] : null, scores]);
82
+ const agreement = records.filter(r => canonicalRun(r.runs[0]) === canonicalRun(r.runs[1])).length / cases.length;
83
+ const latency = { coldP95Ms: p95(cold), warmP95Ms: p95(warm), coldSamples: cold.length, warmSamples: warm.length };
84
+ const gates = {
85
+ reviewedHoldout: cases.length >= 200 && cases.every(c => c.reviewed)
86
+ && ['en', 'zh-TW'].every(lang => cases.filter(c => c.language === lang).length >= 50),
87
+ acceptedPrecision: model.acceptedPrecision !== null && model.acceptedPrecision >= 0.98,
88
+ coverage: model.coverage >= 0.8,
89
+ macroF1: model.macroF1 >= baseline.macroF1,
90
+ repeatability: agreement === 1,
91
+ warmLatency: warm.length >= cases.length && latency.warmP95Ms <= 100,
92
+ sourceProvenance: sourceEvidence?.status === 'recorded' && sourceEvidence.provenance?.cuaS1?.sourceRevisionStatus === 'pinned',
93
+ policyEvidence: false,
94
+ liveHostEvidence: false,
95
+ };
96
+ return { schemaVersion: 1, cases: cases.length, baseline, model, agreement, latency, gates, rolloutReady: Object.values(gates).every(Boolean) };
97
+ }
98
+ module.exports = { validateCorpus, evaluate };
@@ -0,0 +1,161 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+ // Opt-in System One installer: a dedicated venv, pinned cua_s1 source, and a pinned CPU checkpoint.
4
+ // It is never run from hooks, and it never changes the router mode; it prints the environment to set.
5
+ const fs = require('node:fs');
6
+ const os = require('node:os');
7
+ const path = require('node:path');
8
+ const { createHash } = require('node:crypto');
9
+ const { spawnSync } = require('node:child_process');
10
+ const { performance } = require('node:perf_hooks');
11
+ const { validateManifest, readManifest, provenance, score, PINNED_CUA_S1_REVISION } = require('./provider');
12
+ const resident = require('./resident');
13
+ const { createRequest, decide } = require('./contract');
14
+ const { TIER_OPTIONS } = require('./router');
15
+
16
+ const PINS = Object.freeze({
17
+ cuaS1Revision: PINNED_CUA_S1_REVISION,
18
+ cuaS1Spec: `cua-s1 @ git+https://github.com/trycua/cua@${PINNED_CUA_S1_REVISION}#subdirectory=libs/cua-s1/python`,
19
+ torchSpec: 'torch>=2.2,<3',
20
+ torchIndex: 'https://download.pytorch.org/whl/cpu',
21
+ modelId: 'cua-ai/cua-s1-forms',
22
+ modelRevision: 'f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71',
23
+ domain: 'forms-v1',
24
+ // sha256/size from the Hugging Face tree API at modelRevision (LFS oid for weights; the JSON
25
+ // sidecar's git blob id d8f8426a... was re-hashed to confirm these exact CRLF bytes).
26
+ files: Object.freeze({
27
+ weights: Object.freeze({ name: 'cua-s1-forms.safetensors', size: 2828784, sha256: '05954c1caf51c2fb6c13ea4acbfc88a2e7653dea192252bb51dc89e76a356ddc' }),
28
+ config: Object.freeze({ name: 'cua-s1-forms.json', size: 1053, sha256: '62d31e2f9a001a8e9b6f8534c5194d07ebdd3f9d62ef1ac281906622992650ca' }),
29
+ }),
30
+ });
31
+ const USAGE = 'Usage: install.js [--dir <absolute dir>] [--python <3.11-3.13 executable>] [--dry-run]';
32
+
33
+ const defaultDir = () => path.join(os.homedir(), '.agents', 'harness-everything', 'system-one');
34
+ const fileUrl = name => `https://huggingface.co/${PINS.modelId}/resolve/${PINS.modelRevision}/${name}`;
35
+ const checkpointDir = dir => path.join(dir, 'checkpoints', PINS.modelRevision);
36
+ const venvPython = (dir, platform = process.platform) => (platform === 'win32'
37
+ ? path.join(dir, 'venv', 'Scripts', 'python.exe') : path.join(dir, 'venv', 'bin', 'python'));
38
+ function supportedPython(version) {
39
+ const match = typeof version === 'string' && /^3\.(\d+)\.\d+$/.exec(version);
40
+ return Boolean(match) && Number(match[1]) >= 11 && Number(match[1]) <= 13;
41
+ }
42
+ function pythonCandidates(platform = process.platform) {
43
+ return platform === 'win32'
44
+ ? [['py', '-3.13'], ['py', '-3.12'], ['py', '-3.11'], ['python'], ['python3']]
45
+ : [['python3.13'], ['python3.12'], ['python3.11'], ['python3'], ['python']];
46
+ }
47
+ function parseArgs(argv) {
48
+ const opts = { dir: defaultDir(), python: null, dryRun: false };
49
+ const seen = new Set();
50
+ for (let i = 0; i < argv.length; i++) {
51
+ const flag = argv[i];
52
+ if (seen.has(flag)) throw new Error('usage');
53
+ seen.add(flag);
54
+ if (flag === '--dry-run') opts.dryRun = true;
55
+ else if (flag === '--dir' || flag === '--python') {
56
+ const value = argv[++i];
57
+ if (!value || value.startsWith('--')) throw new Error('usage');
58
+ if (flag === '--dir' && !path.isAbsolute(value)) throw new Error('usage');
59
+ opts[flag.slice(2)] = flag === '--dir' ? path.resolve(value) : value;
60
+ } else throw new Error('usage');
61
+ }
62
+ return opts;
63
+ }
64
+ function buildManifest(dir, python) {
65
+ const manifest = { schemaVersion: 1, python, checkpoint: path.join(checkpointDir(dir), PINS.files.weights.name),
66
+ weightsSha256: PINS.files.weights.sha256, configSha256: PINS.files.config.sha256,
67
+ modelId: PINS.modelId, revision: PINS.modelRevision, domain: PINS.domain,
68
+ // One-shot fallback budget (fresh Python + torch import per call); resident keeps the model loaded.
69
+ timeoutMs: 10000, transport: 'resident', idleTimeoutMs: 1800000 };
70
+ validateManifest(manifest);
71
+ return manifest;
72
+ }
73
+ const matches = (buf, spec) => buf.length === spec.size && createHash('sha256').update(buf).digest('hex') === spec.sha256;
74
+ async function httpFetch(url) {
75
+ const res = await fetch(url, { redirect: 'follow' });
76
+ if (!res.ok) throw new Error(`download-failed:${res.status}`);
77
+ return Buffer.from(await res.arrayBuffer());
78
+ }
79
+ async function ensureFile(url, dest, spec, fetcher = httpFetch) {
80
+ if (fs.existsSync(dest) && matches(fs.readFileSync(dest), spec)) return 'verified-existing';
81
+ const bytes = await fetcher(url);
82
+ if (!Buffer.isBuffer(bytes) || !matches(bytes, spec)) throw new Error(`download-verification:${spec.name}`);
83
+ const tmp = `${dest}.${process.pid}.tmp`;
84
+ fs.writeFileSync(tmp, bytes);
85
+ fs.renameSync(tmp, dest);
86
+ return 'downloaded';
87
+ }
88
+ function exec(cmd, args, capture = false) {
89
+ // Child progress goes to stderr (fd 2): stdout carries only the final JSON report.
90
+ const result = spawnSync(cmd, args, { shell: false, windowsHide: true, encoding: 'utf8', stdio: capture ? 'pipe' : ['ignore', 2, 2] });
91
+ if (result.error || result.status !== 0) throw new Error(`command-failed: ${[cmd, ...args].join(' ')}`);
92
+ return capture ? result.stdout.trim() : '';
93
+ }
94
+ const VERSION_PROBE = ['-c', 'import sys; print("%d.%d.%d" % sys.version_info[:3])'];
95
+ function findPython(explicit) {
96
+ for (const [cmd, ...pre] of explicit ? [[explicit]] : pythonCandidates()) {
97
+ const result = spawnSync(cmd, [...pre, ...VERSION_PROBE], { shell: false, windowsHide: true, encoding: 'utf8' });
98
+ const version = result.status === 0 ? result.stdout.trim() : null;
99
+ if (supportedPython(version)) return { cmd, pre, version };
100
+ }
101
+ throw new Error(explicit ? 'python-unsupported: need Python 3.11-3.13' : 'python-not-found: install Python 3.11-3.13 or pass --python');
102
+ }
103
+ function plan(opts) {
104
+ const venvPy = venvPython(opts.dir);
105
+ const manifestPath = path.join(opts.dir, 'manifest.json');
106
+ return { dir: opts.dir, manifest: manifestPath,
107
+ steps: [
108
+ `create venv ${path.join(opts.dir, 'venv')} with ${opts.python || 'first Python 3.11-3.13 found'}`,
109
+ `${venvPy} -m pip install --index-url ${PINS.torchIndex} "${PINS.torchSpec}"`,
110
+ `${venvPy} -m pip install "${PINS.cuaS1Spec}"`,
111
+ ...Object.values(PINS.files).map(f => `download ${fileUrl(f.name)} (sha256 ${f.sha256})`),
112
+ `write ${manifestPath}`,
113
+ 'verify provenance (pinned revision), start the resident server, one warm CPU inference, stop it',
114
+ ],
115
+ env: { HARNESS_SYSTEM_ONE_MODE: 'shadow', HARNESS_SYSTEM_ONE_CONFIG: manifestPath } };
116
+ }
117
+ // Starts the resident server (if needed), scores once warm, then stops a server it started.
118
+ function verify(manifestPath) {
119
+ const source = provenance(manifestPath);
120
+ const manifest = readManifest(manifestPath);
121
+ const wasRunning = resident.status(manifest, manifestPath).running;
122
+ const startMs = performance.now();
123
+ const ready = resident.ensureReady(manifest, manifestPath, 60000);
124
+ const readyMs = Math.round(performance.now() - startMs);
125
+ const request = createRequest('tier', 'fix a typo in README', TIER_OPTIONS);
126
+ const start = performance.now();
127
+ const result = score(request, manifestPath);
128
+ const latencyMs = Math.round(performance.now() - start);
129
+ if (!wasRunning) resident.stop(manifest, manifestPath);
130
+ const decision = result.status === 'scored' ? decide(request, result.response) : result;
131
+ const scores = result.status === 'scored' && decision.status !== 'invalid-output' ? result.response.scores : null;
132
+ const ok = ready && source.status === 'recorded' && source.provenance.cuaS1.sourceRevisionStatus === 'pinned' && scores !== null;
133
+ return { ok, source, residentReady: ready, readyMs, decision, scores, latencyMs };
134
+ }
135
+ async function main(argv) {
136
+ const opts = parseArgs(argv);
137
+ const p = plan(opts);
138
+ if (opts.dryRun) { process.stdout.write(`${JSON.stringify({ dryRun: true, ...p }, null, 2)}\n`); return 0; }
139
+ const base = findPython(opts.python);
140
+ const venvPy = venvPython(opts.dir);
141
+ fs.mkdirSync(checkpointDir(opts.dir), { recursive: true });
142
+ if (!fs.existsSync(venvPy)) exec(base.cmd, [...base.pre, '-m', 'venv', path.join(opts.dir, 'venv')]);
143
+ const venvVersion = exec(venvPy, VERSION_PROBE, true);
144
+ if (!supportedPython(venvVersion)) throw new Error(`python-unsupported: existing venv uses ${venvVersion}`);
145
+ exec(venvPy, ['-m', 'pip', 'install', '--disable-pip-version-check', '--index-url', PINS.torchIndex, PINS.torchSpec]);
146
+ exec(venvPy, ['-m', 'pip', 'install', '--disable-pip-version-check', PINS.cuaS1Spec]);
147
+ for (const spec of Object.values(PINS.files)) {
148
+ console.error(`${spec.name}: ${await ensureFile(fileUrl(spec.name), path.join(checkpointDir(opts.dir), spec.name), spec)}`);
149
+ }
150
+ fs.writeFileSync(p.manifest, `${JSON.stringify(buildManifest(opts.dir, venvPy), null, 2)}\n`);
151
+ const check = verify(p.manifest);
152
+ process.stdout.write(`${JSON.stringify({ dryRun: false, ...p, verification: check }, null, 2)}\n`);
153
+ return check.ok ? 0 : 1;
154
+ }
155
+ if (require.main === module) {
156
+ main(process.argv.slice(2)).then(code => { process.exitCode = code; }, err => {
157
+ if (err.message === 'usage') { console.error(USAGE); process.exitCode = 2; return; }
158
+ console.error(`System One install failed: ${err.message}`); process.exitCode = 1;
159
+ });
160
+ }
161
+ module.exports = { PINS, fileUrl, venvPython, supportedPython, pythonCandidates, parseArgs, buildManifest, ensureFile, exec, plan, verify };