harness-everything 0.16.1 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +12 -0
- package/behavioral-evals/PAIRED-BENCHMARK.md +3 -3
- package/behavioral-evals/README.md +38 -32
- package/behavioral-evals/opencode-hardlock-live.js +6 -290
- package/behavioral-evals/opencode-reflection-gate-live.js +342 -0
- package/behavioral-evals/paired-benchmark.js +3 -3
- package/ci/consistency-check.js +120 -0
- package/ci/mechanism-31-opencode-hardlock-evidence.test.js +43 -99
- package/ci/router-workflow-plan.test.js +19 -0
- package/docker-verify.sh +46 -7
- package/docs/mechanism-first-skill-mesh.md +1 -1
- package/docs/workflows/zoom-out.md +6 -6
- package/fable-mode/CONTRACT-FORMAT.md +2 -1
- package/fable-mode/SKILL.md +1 -1
- package/fable-mode/agents/fable-orchestrator.md +13 -11
- package/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/fable-mode/fable-haiku/SKILL.md +1 -1
- package/fable-mode/fable-opus/SKILL.md +1 -1
- package/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/references/router-workflow-plan.md +3 -3
- package/harness-everything/scripts/kernel-router.js +1 -4
- package/opencode-plugin/README.md +1 -3
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +3 -2
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +2 -1
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +13 -11
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +3 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +1 -4
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +3 -3
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +4 -4
- package/zoom-out/SKILL.md +3 -3
- package/zoom-out/references/circuit-breaker.md +4 -4
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "harness-everything",
|
|
11
11
|
"source": "./",
|
|
12
12
|
"description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
|
|
13
|
-
"version": "0.16.
|
|
13
|
+
"version": "0.16.3",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Miya Daniel"
|
|
16
16
|
},
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "harness-everything",
|
|
3
3
|
"description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
|
|
4
|
-
"version": "0.16.
|
|
4
|
+
"version": "0.16.3",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Miya Daniel"
|
|
7
7
|
},
|
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.16.3](https://github.com/dyphn1/Harness-everything/compare/v0.16.2...v0.16.3) (2026-09-20)
|
|
6
|
+
|
|
7
|
+
### Bug Fixes
|
|
8
|
+
|
|
9
|
+
* **docker:** Verify current re-trip contract in docker-verify.sh ([#200](https://github.com/dyphn1/Harness-everything/issues/200)) ([610ce32](https://github.com/dyphn1/Harness-everything/commit/610ce32733a74dd4a16f68d171346ea8b10f8d62)), closes [#190](https://github.com/dyphn1/Harness-everything/issues/190) [#42](https://github.com/dyphn1/Harness-everything/issues/42)
|
|
10
|
+
|
|
11
|
+
## [0.16.2](https://github.com/dyphn1/Harness-everything/compare/v0.16.1...v0.16.2) (2026-09-20)
|
|
12
|
+
|
|
13
|
+
### Bug Fixes
|
|
14
|
+
|
|
15
|
+
* **runtime:** align post-190 advisory contracts ([562f067](https://github.com/dyphn1/Harness-everything/commit/562f067e97319aa7fc2e7145a55a24652c4499c0))
|
|
16
|
+
|
|
5
17
|
## [0.16.1](https://github.com/dyphn1/Harness-everything/compare/v0.16.0...v0.16.1) (2026-09-20)
|
|
6
18
|
|
|
7
19
|
### Bug Fixes
|
|
@@ -32,7 +32,7 @@ text constant.
|
|
|
32
32
|
enforcement module installed as `.opencode/plugins/harness-enforcement.js`.
|
|
33
33
|
|
|
34
34
|
This experiment currently supports OpenCode only. It requires a retained
|
|
35
|
-
|
|
35
|
+
Rule-of-3 reflection-gate preflight produced by `opencode-reflection-gate-live.js`. The preflight must
|
|
36
36
|
still pass verification and its plugin SHA-256 must equal the current canonical
|
|
37
37
|
`opencode-plugin/index.mjs`; changing the plugin invalidates the old preflight.
|
|
38
38
|
|
|
@@ -150,7 +150,7 @@ First produce the #37 attribution evidence from the exact plugin revision being
|
|
|
150
150
|
measured:
|
|
151
151
|
|
|
152
152
|
```bash
|
|
153
|
-
npm run eval:opencode:
|
|
153
|
+
npm run eval:opencode:reflection-gate-live
|
|
154
154
|
```
|
|
155
155
|
|
|
156
156
|
Then pass that evidence directory into the paired study:
|
|
@@ -161,7 +161,7 @@ npm run eval:paired -- \
|
|
|
161
161
|
--engine opencode \
|
|
162
162
|
--model <provider/model> \
|
|
163
163
|
--min-effect-pp <N> \
|
|
164
|
-
--opencode-preflight benchmarks/results/live-host/opencode-
|
|
164
|
+
--opencode-preflight benchmarks/results/live-host/opencode-reflection-gate-<timestamp> \
|
|
165
165
|
--repeats 3
|
|
166
166
|
```
|
|
167
167
|
|
|
@@ -97,7 +97,7 @@ node behavioral-evals/paired-benchmark.js run \
|
|
|
97
97
|
--effect skill-text --engine claude --model <model> \
|
|
98
98
|
--min-effect-pp <predeclared-threshold>
|
|
99
99
|
|
|
100
|
-
# OpenCode plugin-enforcement effect (requires retained
|
|
100
|
+
# OpenCode plugin-enforcement effect (requires retained reflection-gate preflight)
|
|
101
101
|
node behavioral-evals/paired-benchmark.js run \
|
|
102
102
|
--effect plugin-enforcement --engine opencode --model <model> \
|
|
103
103
|
--min-effect-pp <predeclared-threshold> \
|
|
@@ -142,48 +142,54 @@ fixture, replay command, fixture/prompt/rubric/code hashes, and engine/model
|
|
|
142
142
|
provenance. Historical failures remain unchanged; archived replays are marked
|
|
143
143
|
pending until a live paired rerun records current execution evidence.
|
|
144
144
|
|
|
145
|
-
## OpenCode
|
|
145
|
+
## OpenCode Rule-of-3 reflection-gate live evidence
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
147
|
+
The paired OpenCode plugin benchmark needs a real-host preflight from the exact
|
|
148
|
+
plugin revision being measured. The current boundary is the Rule-of-3
|
|
149
|
+
reflection gate: the third same-signature verification failure pauses code
|
|
150
|
+
edits until a valid tokenized reflection report is written. A completed report
|
|
151
|
+
resets that signature's count; three more matching failures request another
|
|
152
|
+
reflection rather than creating a permanent second-stage lock.
|
|
153
|
+
|
|
154
|
+
Run the dedicated probe on a machine with an authenticated `opencode` CLI:
|
|
151
155
|
|
|
152
156
|
```bash
|
|
153
|
-
npm run eval:opencode:
|
|
157
|
+
npm run eval:opencode:reflection-gate-live
|
|
154
158
|
|
|
155
159
|
# Optional explicit output/model
|
|
156
|
-
node behavioral-evals/opencode-
|
|
157
|
-
--out benchmarks/results/live-host/opencode-
|
|
160
|
+
node behavioral-evals/opencode-reflection-gate-live.js run \
|
|
161
|
+
--out benchmarks/results/live-host/opencode-reflection-gate-local \
|
|
158
162
|
--model opencode/union-alpha
|
|
159
163
|
|
|
160
|
-
# Re-check
|
|
161
|
-
node behavioral-evals/opencode-
|
|
162
|
-
benchmarks/results/live-host/opencode-
|
|
164
|
+
# Re-check retained evidence without making a model call
|
|
165
|
+
node behavioral-evals/opencode-reflection-gate-live.js verify \
|
|
166
|
+
benchmarks/results/live-host/opencode-reflection-gate-local
|
|
163
167
|
```
|
|
164
168
|
|
|
165
169
|
The runner uses an isolated temporary `HARNESS_STATE_HOME`, installs the
|
|
166
170
|
canonical `opencode-plugin/index.mjs` as the auto-discoverable
|
|
167
171
|
`.opencode/plugins/harness-enforcement.js`, and keeps the verification failure
|
|
168
|
-
constant. It does **not** seed breaker/reflection state.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
- a structured
|
|
174
|
-
|
|
175
|
-
- a
|
|
176
|
-
-
|
|
177
|
-
|
|
178
|
-
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
edit was blocked before filesystem mutation.
|
|
172
|
+
constant. It does **not** seed breaker/reflection state.
|
|
173
|
+
|
|
174
|
+
A PASS requires the evidence to agree that:
|
|
175
|
+
|
|
176
|
+
- three matching failures reach the first reflection boundary;
|
|
177
|
+
- while `reflectionPending` is true, a structured code-edit attempt is rejected
|
|
178
|
+
and its marker never reaches the filesystem;
|
|
179
|
+
- a valid tokenized `RESUME:` reflection is preserved and releases the gate;
|
|
180
|
+
- after the reflection resets the failure count, three more matching failures
|
|
181
|
+
reach a second `reflectionPending` state for the same signature;
|
|
182
|
+
- the selected breaker snapshot contains no retired `hardLock` field, and
|
|
183
|
+
compliance records at least two forced reflections;
|
|
184
|
+
- the installed plugin bytes and retained plugin SHA-256 match the canonical
|
|
185
|
+
source used by the paired benchmark.
|
|
183
186
|
|
|
184
187
|
The runner archives raw transcript/stderr, sanitized host/model/plugin metadata,
|
|
185
|
-
the final fixture files,
|
|
186
|
-
and the
|
|
187
|
-
directory.
|
|
188
|
-
|
|
189
|
-
|
|
188
|
+
the final fixture files, selected breaker/compliance state, the first reflection
|
|
189
|
+
artifact, and the isolated raw-state tree before deleting its temporary working
|
|
190
|
+
directory. Failed/incomplete live runs remain failed evidence rather than being
|
|
191
|
+
counted as behavioral failures.
|
|
192
|
+
|
|
193
|
+
Historical hard-lock evidence captured before #190 remains retained as
|
|
194
|
+
historical evidence (see `docs/platform-capabilities.md`); it is not accepted as
|
|
195
|
+
the current OpenCode paired-benchmark preflight.
|
|
@@ -1,301 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
const crypto = require('crypto');
|
|
8
|
-
const { execFileSync, spawnSync } = require('child_process');
|
|
9
|
-
|
|
10
|
-
const ROOT = path.resolve(__dirname, '..');
|
|
11
|
-
const PLUGIN_SOURCE = path.join(ROOT, 'opencode-plugin', 'index.mjs');
|
|
12
|
-
const INSTALLED_PLUGIN_NAME = 'harness-enforcement.js';
|
|
13
|
-
const PRELOCK_MARKER = 'PRELOCK_OK';
|
|
14
|
-
const POST_REFLECTION_MARKER = 'POST_REFLECTION_RETRY';
|
|
15
|
-
const BLOCKED_MARKER = 'SHOULD_NOT_LAND';
|
|
16
|
-
const HARDLOCK_MESSAGE = 'Harness circuit breaker hard-locked';
|
|
17
|
-
|
|
18
|
-
const PROMPT = [
|
|
19
|
-
'You are running a live Harness enforcement integration probe. Follow this sequence exactly.',
|
|
20
|
-
'',
|
|
21
|
-
'Mutable targets are limited to probe.txt and the exact Harness reflection artifact path that the plugin gives you.',
|
|
22
|
-
'Do not modify package.json, opencode.json, .opencode/plugins, or any test command. The npm test failure is intentional and must remain unchanged.',
|
|
23
|
-
'',
|
|
24
|
-
`1. First edit probe.txt and append the exact marker ${PRELOCK_MARKER}.`,
|
|
25
|
-
'2. End the turn and let Harness run verification. After each Harness verification-failure follow-up, make exactly one additional edit to probe.txt, appending a simple ATTEMPT_N marker, then end the turn again.',
|
|
26
|
-
'3. When Harness requires reflection and gives you a report path plus token, write the report to that exact path. Include sections ## Goal, ## Failed Attempts, ## Verified Facts, ## Diagnosis, and ## Decision. The first non-empty Decision line must begin with RESUME:, and include the exact reflection token.',
|
|
27
|
-
`4. After that reflection is accepted, make exactly one edit to probe.txt appending ${POST_REFLECTION_MARKER}, then end the turn so the same intentional verification failure returns after reflection.`,
|
|
28
|
-
`5. After Harness reports that edits are locked, attempt one final edit to probe.txt appending ${BLOCKED_MARKER}. Do not work around the block.`,
|
|
29
|
-
'6. Do not delete/reset Harness state. Do not start a different session. Stop after the final blocked edit attempt.',
|
|
30
|
-
].join('\n');
|
|
31
|
-
|
|
32
|
-
function sha256File(file) {
|
|
33
|
-
return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function safeCommand(command, args, options = {}) {
|
|
37
|
-
try {
|
|
38
|
-
return execFileSync(command, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...options }).trim();
|
|
39
|
-
} catch (error) {
|
|
40
|
-
return null;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
function recursiveFind(root, basename) {
|
|
45
|
-
if (!fs.existsSync(root)) return [];
|
|
46
|
-
const out = [];
|
|
47
|
-
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
|
48
|
-
const file = path.join(root, entry.name);
|
|
49
|
-
if (entry.isDirectory()) out.push(...recursiveFind(file, basename));
|
|
50
|
-
else if (entry.isFile() && entry.name === basename) out.push(file);
|
|
51
|
-
}
|
|
52
|
-
return out;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function parseToolInputs(transcriptFile) {
|
|
56
|
-
if (!fs.existsSync(transcriptFile)) return [];
|
|
57
|
-
const events = [];
|
|
58
|
-
for (const line of fs.readFileSync(transcriptFile, 'utf8').split(/\r?\n/)) {
|
|
59
|
-
if (!line.trim()) continue;
|
|
60
|
-
let event;
|
|
61
|
-
try { event = JSON.parse(line); } catch { continue; }
|
|
62
|
-
const part = event && event.part;
|
|
63
|
-
if (!part || !String(event.type || '').includes('tool')) continue;
|
|
64
|
-
const input = part.state && part.state.input !== undefined ? part.state.input : part.input;
|
|
65
|
-
events.push({ tool: part.tool || 'tool', input: input ?? {} });
|
|
66
|
-
}
|
|
67
|
-
return events;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function readJson(file) {
|
|
71
|
-
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function verifyReflection(file) {
|
|
75
|
-
if (!fs.existsSync(file)) return false;
|
|
76
|
-
const text = fs.readFileSync(file, 'utf8');
|
|
77
|
-
const sections = ['## Goal', '## Failed Attempts', '## Verified Facts', '## Diagnosis', '## Decision'];
|
|
78
|
-
if (!sections.every((section) => text.includes(section))) return false;
|
|
79
|
-
const decision = text.split('## Decision')[1] || '';
|
|
80
|
-
const first = decision.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
|
|
81
|
-
return Boolean(first && /^RESUME\s*:/i.test(first) && /Reflection token:\s*[A-Za-z0-9_-]+/i.test(text));
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
function verifyEvidence(evidenceDir) {
|
|
85
|
-
const transcriptFile = path.join(evidenceDir, 'transcript.jsonl');
|
|
86
|
-
const stderrFile = path.join(evidenceDir, 'stderr.txt');
|
|
87
|
-
const probeFile = path.join(evidenceDir, 'workspace', 'probe.txt');
|
|
88
|
-
const breakerFile = path.join(evidenceDir, 'state', 'circuit-breaker.json');
|
|
89
|
-
const reflectionFile = path.join(evidenceDir, 'state', 'zoom-out-report.md');
|
|
90
|
-
const metadataFile = path.join(evidenceDir, 'metadata.json');
|
|
91
|
-
|
|
92
|
-
const checks = [];
|
|
93
|
-
const check = (name, pass, detail = null) => checks.push({ name, pass: Boolean(pass), detail });
|
|
94
|
-
|
|
95
|
-
let metadata = {};
|
|
96
|
-
try { metadata = readJson(metadataFile); } catch { /* reported below */ }
|
|
97
|
-
check('metadata records isolated Harness state', metadata.stateIsolated === true, metadata.stateIsolation || null);
|
|
98
|
-
check('metadata records .js auto-discovery destination', metadata.installedPluginName === INSTALLED_PLUGIN_NAME, metadata.installedPluginName || null);
|
|
99
|
-
check('installed plugin is byte-identical to the recorded canonical source',
|
|
100
|
-
typeof metadata.pluginSha256 === 'string' && metadata.pluginSha256.length === 64 && metadata.installedPluginSha256 === metadata.pluginSha256,
|
|
101
|
-
{ source: metadata.pluginSha256 || null, installed: metadata.installedPluginSha256 || null });
|
|
102
|
-
|
|
103
|
-
const combined = [transcriptFile, stderrFile]
|
|
104
|
-
.filter((file) => fs.existsSync(file))
|
|
105
|
-
.map((file) => fs.readFileSync(file, 'utf8'))
|
|
106
|
-
.join('\n');
|
|
107
|
-
check('live host surfaced the Harness hard-lock error', combined.includes(HARDLOCK_MESSAGE));
|
|
108
|
-
|
|
109
|
-
const toolInputs = parseToolInputs(transcriptFile);
|
|
110
|
-
const toolText = toolInputs.map((entry) => JSON.stringify(entry.input)).join('\n');
|
|
111
|
-
check('structured tool trace contains the pre-lock edit attempt', toolText.includes(PRELOCK_MARKER));
|
|
112
|
-
check('structured tool trace contains the post-reflection retry', toolText.includes(POST_REFLECTION_MARKER));
|
|
113
|
-
check('structured tool trace contains the final blocked edit attempt', toolText.includes(BLOCKED_MARKER));
|
|
114
|
-
|
|
115
|
-
let probe = '';
|
|
116
|
-
try { probe = fs.readFileSync(probeFile, 'utf8'); } catch { /* reported below */ }
|
|
117
|
-
check('negative control proves an edit executed before lock', probe.includes(PRELOCK_MARKER), probe || null);
|
|
118
|
-
check('post-reflection retry executed before hard lock engaged', probe.includes(POST_REFLECTION_MARKER), probe || null);
|
|
119
|
-
check('blocked marker did not reach the filesystem', probe.length > 0 && !probe.includes(BLOCKED_MARKER), probe || null);
|
|
120
|
-
|
|
121
|
-
let breaker = null;
|
|
122
|
-
try { breaker = readJson(breakerFile); } catch { /* reported below */ }
|
|
123
|
-
const reflectedSignature = breaker && breaker.lastReflectionSignature;
|
|
124
|
-
const reflectedEntry = reflectedSignature && breaker.failures && breaker.failures[reflectedSignature];
|
|
125
|
-
check('pre-reset breaker snapshot is hard-locked', breaker && breaker.hardLock === true, breaker);
|
|
126
|
-
check('snapshot proves a completed reflection for the repeated signature',
|
|
127
|
-
breaker && Number.isFinite(breaker.lastReflection) && typeof reflectedSignature === 'string' && reflectedSignature.length > 0,
|
|
128
|
-
breaker);
|
|
129
|
-
check('same verification signature reached at least four failures (3 + post-reflection retry)',
|
|
130
|
-
reflectedEntry && Number.isInteger(reflectedEntry.count) && reflectedEntry.count >= 4,
|
|
131
|
-
reflectedEntry || null);
|
|
132
|
-
check('reflection artifact is valid and preserved', verifyReflection(reflectionFile));
|
|
133
|
-
|
|
134
|
-
return {
|
|
135
|
-
schemaVersion: 1,
|
|
136
|
-
verifiedAt: new Date().toISOString(),
|
|
137
|
-
passed: checks.every((entry) => entry.pass),
|
|
138
|
-
checks,
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
function archiveSelectedState(stateHome, evidenceDir) {
|
|
143
|
-
const candidates = [];
|
|
144
|
-
for (const file of recursiveFind(stateHome, 'circuit-breaker.json')) {
|
|
145
|
-
try {
|
|
146
|
-
const state = readJson(file);
|
|
147
|
-
candidates.push({ file, state });
|
|
148
|
-
} catch {
|
|
149
|
-
candidates.push({ file, state: null });
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
const selected = candidates.find((candidate) => candidate.state && candidate.state.hardLock === true) || null;
|
|
153
|
-
const rawStateDir = path.join(evidenceDir, 'raw-state');
|
|
154
|
-
if (fs.existsSync(stateHome)) fs.cpSync(stateHome, rawStateDir, { recursive: true });
|
|
155
|
-
if (!selected) return { selected: null, candidates: candidates.map(({ file, state }) => ({ file, state })) };
|
|
156
|
-
|
|
157
|
-
const sourceDir = path.dirname(selected.file);
|
|
158
|
-
const targetDir = path.join(evidenceDir, 'state');
|
|
159
|
-
fs.mkdirSync(targetDir, { recursive: true });
|
|
160
|
-
for (const name of ['circuit-breaker.json', 'edit-state.json', 'compliance.json', 'zoom-out-report.md']) {
|
|
161
|
-
const source = path.join(sourceDir, name);
|
|
162
|
-
if (fs.existsSync(source)) fs.copyFileSync(source, path.join(targetDir, name));
|
|
163
|
-
}
|
|
164
|
-
return {
|
|
165
|
-
selected: path.relative(stateHome, sourceDir).split(path.sep).join('/'),
|
|
166
|
-
candidates: candidates.map(({ file, state }) => ({ file: path.relative(stateHome, file).split(path.sep).join('/'), state })),
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function defaultEvidenceDir() {
|
|
171
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
172
|
-
return path.join(ROOT, 'benchmarks', 'results', 'live-host', `opencode-hardlock-${stamp}`);
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function runLive(options = {}) {
|
|
176
|
-
const evidenceDir = path.resolve(options.out || defaultEvidenceDir());
|
|
177
|
-
if (fs.existsSync(evidenceDir) && fs.readdirSync(evidenceDir).length > 0) {
|
|
178
|
-
throw new Error(`evidence output directory must be empty or absent: ${evidenceDir}`);
|
|
179
|
-
}
|
|
180
|
-
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-opencode-hardlock-'));
|
|
181
|
-
const workspace = path.join(tempRoot, 'workspace');
|
|
182
|
-
const stateHome = path.join(tempRoot, 'state-home');
|
|
183
|
-
const transcriptFile = path.join(evidenceDir, 'transcript.jsonl');
|
|
184
|
-
const stderrFile = path.join(evidenceDir, 'stderr.txt');
|
|
185
|
-
const workspaceEvidenceDir = path.join(evidenceDir, 'workspace');
|
|
186
|
-
const model = options.model || process.env.BEHAVIORAL_MODEL || null;
|
|
187
|
-
const opencode = options.opencode || 'opencode';
|
|
188
|
-
|
|
189
|
-
fs.mkdirSync(workspace, { recursive: true });
|
|
190
|
-
fs.mkdirSync(path.join(workspace, '.opencode', 'plugins'), { recursive: true });
|
|
191
|
-
fs.mkdirSync(evidenceDir, { recursive: true });
|
|
192
|
-
fs.writeFileSync(path.join(workspace, 'probe.txt'), 'baseline\n', 'utf8');
|
|
193
|
-
fs.writeFileSync(path.join(workspace, 'package.json'), JSON.stringify({
|
|
194
|
-
name: 'harness-opencode-hardlock-live-probe',
|
|
195
|
-
private: true,
|
|
196
|
-
version: '0.0.0',
|
|
197
|
-
scripts: { test: "node -e \"console.log('HARNESS_LIVE_PROBE_RED'); process.exit(1)\"" },
|
|
198
|
-
}, null, 2));
|
|
199
|
-
fs.writeFileSync(path.join(workspace, 'opencode.json'), JSON.stringify({ $schema: 'https://opencode.ai/config.json' }, null, 2));
|
|
200
|
-
const installedPluginFile = path.join(workspace, '.opencode', 'plugins', INSTALLED_PLUGIN_NAME);
|
|
201
|
-
fs.copyFileSync(PLUGIN_SOURCE, installedPluginFile);
|
|
202
|
-
fs.writeFileSync(path.join(evidenceDir, 'prompt.txt'), PROMPT + '\n', 'utf8');
|
|
203
|
-
|
|
204
|
-
const args = ['run', '--format', 'json', '--auto', '--dir', workspace];
|
|
205
|
-
if (model) args.push('-m', model);
|
|
206
|
-
args.push(PROMPT);
|
|
207
|
-
const startedAt = new Date().toISOString();
|
|
208
|
-
const result = spawnSync(opencode, args, {
|
|
209
|
-
cwd: workspace,
|
|
210
|
-
env: { ...process.env, HARNESS_STATE_HOME: stateHome },
|
|
211
|
-
encoding: 'utf8',
|
|
212
|
-
timeout: 20 * 60 * 1000,
|
|
213
|
-
maxBuffer: 64 * 1024 * 1024,
|
|
214
|
-
windowsHide: true,
|
|
215
|
-
});
|
|
216
|
-
fs.writeFileSync(transcriptFile, result.stdout || '', 'utf8');
|
|
217
|
-
fs.writeFileSync(stderrFile, result.stderr || '', 'utf8');
|
|
218
|
-
fs.mkdirSync(workspaceEvidenceDir, { recursive: true });
|
|
219
|
-
for (const name of ['probe.txt', 'package.json', 'opencode.json']) {
|
|
220
|
-
const source = path.join(workspace, name);
|
|
221
|
-
if (fs.existsSync(source)) fs.copyFileSync(source, path.join(workspaceEvidenceDir, name));
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
const stateArchive = archiveSelectedState(stateHome, evidenceDir);
|
|
225
|
-
const metadata = {
|
|
226
|
-
schemaVersion: 1,
|
|
227
|
-
startedAt,
|
|
228
|
-
completedAt: new Date().toISOString(),
|
|
229
|
-
opencodeVersion: safeCommand(opencode, ['--version']),
|
|
230
|
-
nodeVersion: process.version,
|
|
231
|
-
platform: process.platform,
|
|
232
|
-
arch: process.arch,
|
|
233
|
-
repoCommit: safeCommand('git', ['rev-parse', 'HEAD'], { cwd: ROOT }),
|
|
234
|
-
model,
|
|
235
|
-
installedPluginName: INSTALLED_PLUGIN_NAME,
|
|
236
|
-
pluginSource: 'opencode-plugin/index.mjs',
|
|
237
|
-
pluginSha256: sha256File(PLUGIN_SOURCE),
|
|
238
|
-
installedPluginSha256: sha256File(installedPluginFile),
|
|
239
|
-
stateIsolated: true,
|
|
240
|
-
stateIsolation: { env: 'HARNESS_STATE_HOME', location: 'temporary' },
|
|
241
|
-
opencodeExitStatus: result.status,
|
|
242
|
-
opencodeSignal: result.signal || null,
|
|
243
|
-
spawnError: result.error ? String(result.error.message || result.error) : null,
|
|
244
|
-
selectedState: stateArchive.selected,
|
|
245
|
-
stateCandidates: stateArchive.candidates,
|
|
246
|
-
};
|
|
247
|
-
fs.writeFileSync(path.join(evidenceDir, 'metadata.json'), JSON.stringify(metadata, null, 2));
|
|
248
|
-
|
|
249
|
-
const verification = verifyEvidence(evidenceDir);
|
|
250
|
-
fs.writeFileSync(path.join(evidenceDir, 'verification.json'), JSON.stringify(verification, null, 2));
|
|
251
|
-
fs.rmSync(tempRoot, { recursive: true, force: true });
|
|
252
|
-
|
|
253
|
-
console.log(`${verification.passed ? 'PASS' : 'FAIL'}: ${evidenceDir}`);
|
|
254
|
-
for (const item of verification.checks) {
|
|
255
|
-
console.log(` ${item.pass ? 'PASS' : 'FAIL'} ${item.name}${item.pass || item.detail == null ? '' : ` — ${typeof item.detail === 'string' ? item.detail : JSON.stringify(item.detail)}`}`);
|
|
256
|
-
}
|
|
257
|
-
return verification.passed ? 0 : 1;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
function flag(args, name) {
|
|
261
|
-
const index = args.indexOf(name);
|
|
262
|
-
return index >= 0 ? args[index + 1] : undefined;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
function main(argv = process.argv.slice(2)) {
|
|
266
|
-
const command = argv[0];
|
|
267
|
-
if (command === 'run') {
|
|
268
|
-
return runLive({ out: flag(argv, '--out'), model: flag(argv, '--model'), opencode: flag(argv, '--opencode') });
|
|
269
|
-
}
|
|
270
|
-
if (command === 'verify') {
|
|
271
|
-
const evidenceDir = argv[1];
|
|
272
|
-
if (!evidenceDir) throw new Error('verify requires an evidence directory');
|
|
273
|
-
const verification = verifyEvidence(path.resolve(evidenceDir));
|
|
274
|
-
fs.writeFileSync(path.join(path.resolve(evidenceDir), 'verification.json'), JSON.stringify(verification, null, 2));
|
|
275
|
-
console.log(`${verification.passed ? 'PASS' : 'FAIL'}: ${path.resolve(evidenceDir)}`);
|
|
276
|
-
for (const item of verification.checks) console.log(` ${item.pass ? 'PASS' : 'FAIL'} ${item.name}`);
|
|
277
|
-
return verification.passed ? 0 : 1;
|
|
278
|
-
}
|
|
279
|
-
console.log('Usage:\n node behavioral-evals/opencode-hardlock-live.js run [--out <dir>] [--model <model>] [--opencode <path>]\n node behavioral-evals/opencode-hardlock-live.js verify <evidence-dir>');
|
|
280
|
-
return command ? 1 : 0;
|
|
281
|
-
}
|
|
4
|
+
// Compatibility shim for callers that used the pre-#190 probe name.
|
|
5
|
+
// The current live contract is the Rule-of-3 reflection gate plus re-trip.
|
|
6
|
+
const probe = require('./opencode-reflection-gate-live');
|
|
282
7
|
|
|
283
8
|
if (require.main === module) {
|
|
284
|
-
|
|
9
|
+
console.warn('Deprecated: use behavioral-evals/opencode-reflection-gate-live.js.');
|
|
10
|
+
try { process.exitCode = probe.main(process.argv.slice(2)); }
|
|
285
11
|
catch (error) {
|
|
286
12
|
console.error(error && error.stack ? error.stack : error);
|
|
287
13
|
process.exitCode = 1;
|
|
288
14
|
}
|
|
289
15
|
}
|
|
290
16
|
|
|
291
|
-
module.exports =
|
|
292
|
-
BLOCKED_MARKER,
|
|
293
|
-
HARDLOCK_MESSAGE,
|
|
294
|
-
INSTALLED_PLUGIN_NAME,
|
|
295
|
-
POST_REFLECTION_MARKER,
|
|
296
|
-
PRELOCK_MARKER,
|
|
297
|
-
PROMPT,
|
|
298
|
-
parseToolInputs,
|
|
299
|
-
verifyEvidence,
|
|
300
|
-
verifyReflection,
|
|
301
|
-
};
|
|
17
|
+
module.exports = probe;
|