harness-everything 0.16.1 → 0.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +6 -0
  4. package/behavioral-evals/PAIRED-BENCHMARK.md +3 -3
  5. package/behavioral-evals/README.md +38 -32
  6. package/behavioral-evals/opencode-hardlock-live.js +6 -290
  7. package/behavioral-evals/opencode-reflection-gate-live.js +342 -0
  8. package/behavioral-evals/paired-benchmark.js +3 -3
  9. package/ci/consistency-check.js +20 -0
  10. package/ci/mechanism-31-opencode-hardlock-evidence.test.js +43 -99
  11. package/ci/router-workflow-plan.test.js +19 -0
  12. package/fable-mode/SKILL.md +1 -1
  13. package/fable-mode/agents/fable-orchestrator.md +7 -6
  14. package/fable-mode/execution-guardrails/SKILL.md +1 -1
  15. package/fable-mode/fable-haiku/SKILL.md +1 -1
  16. package/fable-mode/fable-opus/SKILL.md +1 -1
  17. package/fable-mode/fable-sonnet/SKILL.md +1 -1
  18. package/harness-everything/SKILL.md +1 -1
  19. package/harness-everything/references/router-workflow-plan.md +2 -2
  20. package/harness-everything/scripts/kernel-router.js +1 -4
  21. package/opencode-plugin/README.md +1 -3
  22. package/opencode-plugin/plugin.json +1 -1
  23. package/package.json +3 -2
  24. package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
  25. package/plugins/harness-everything/plugin.json +1 -1
  26. package/plugins/harness-everything/skills/fable-mode/SKILL.md +1 -1
  27. package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +7 -6
  28. package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +1 -1
  29. package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
  30. package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
  31. package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
  32. package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
  33. package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +2 -2
  34. package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +1 -4
  35. package/plugins/harness-everything/skills/zoom-out/SKILL.md +3 -3
  36. package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +4 -4
  37. package/zoom-out/SKILL.md +3 -3
  38. package/zoom-out/references/circuit-breaker.md +4 -4
@@ -0,0 +1,342 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ const fs = require('fs');
5
+ const os = require('os');
6
+ const path = require('path');
7
+ const crypto = require('crypto');
8
+ const { execFileSync, spawnSync } = require('child_process');
9
+
10
+ const ROOT = path.resolve(__dirname, '..');
11
+ const PLUGIN_SOURCE = path.join(ROOT, 'opencode-plugin', 'index.mjs');
12
+ const INSTALLED_PLUGIN_NAME = 'harness-enforcement.js';
13
+ const PRE_REFLECTION_MARKER = 'PRE_REFLECTION_OK';
14
+ const FIRST_CYCLE_MARKERS = ['CYCLE1_ATTEMPT_2', 'CYCLE1_ATTEMPT_3'];
15
+ const BLOCKED_MARKER = 'BLOCKED_WHILE_REFLECTION_PENDING';
16
+ const RETRIP_MARKERS = ['RETRIP_1', 'RETRIP_2', 'RETRIP_3'];
17
+ const REFLECTION_REQUIRED_MESSAGE = 'Harness reflection is required before another code edit.';
18
+ const PROBE_CONTRACT = 'rule-of-3-reflection-retrip-v2';
19
+
20
+ const PROMPT = [
21
+ 'You are running a live Harness Rule-of-3 reflection-gate integration probe. Follow this sequence exactly.',
22
+ '',
23
+ 'Mutable targets are limited to probe.txt and the exact Harness reflection artifact path that the plugin gives you.',
24
+ 'Do not modify package.json, opencode.json, .opencode/plugins, or any test command. The npm test failure is intentional and must remain unchanged.',
25
+ '',
26
+ `1. First edit probe.txt and append the exact marker ${PRE_REFLECTION_MARKER}, then end the turn.`,
27
+ `2. After the first verification-failure follow-up, append ${FIRST_CYCLE_MARKERS[0]} and end the turn. After the second, append ${FIRST_CYCLE_MARKERS[1]} and end the turn. The third matching failure should request reflection.`,
28
+ `3. When reflection is first required, BEFORE writing the report, attempt one edit to probe.txt appending ${BLOCKED_MARKER}. Do not work around the rejection. Then write the required report to the exact path with sections ## Goal, ## Failed Attempts, ## Verified Facts, ## Diagnosis, and ## Decision. The first non-empty Decision line must begin with RESUME:, and include the exact reflection token.`,
29
+ `4. After that reflection is accepted, make three successive edits to probe.txt, one per verification cycle, appending ${RETRIP_MARKERS[0]}, then ${RETRIP_MARKERS[1]}, then ${RETRIP_MARKERS[2]}. End the turn after each edit so the intentional verification failure is observed each time.`,
30
+ '5. When Harness requests reflection for the second three-failure cycle, stop. Do not write a second reflection report, reset Harness state, or start a different session.',
31
+ ].join('\n');
32
+
33
+ function sha256File(file) {
34
+ return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
35
+ }
36
+
37
+ function safeCommand(command, args, options = {}) {
38
+ try {
39
+ return execFileSync(command, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...options }).trim();
40
+ } catch (error) {
41
+ return null;
42
+ }
43
+ }
44
+
45
+ function recursiveFind(root, basename) {
46
+ if (!fs.existsSync(root)) return [];
47
+ const out = [];
48
+ for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
49
+ const file = path.join(root, entry.name);
50
+ if (entry.isDirectory()) out.push(...recursiveFind(file, basename));
51
+ else if (entry.isFile() && entry.name === basename) out.push(file);
52
+ }
53
+ return out;
54
+ }
55
+
56
+ function parseToolInputs(transcriptFile) {
57
+ if (!fs.existsSync(transcriptFile)) return [];
58
+ const events = [];
59
+ for (const line of fs.readFileSync(transcriptFile, 'utf8').split(/\r?\n/)) {
60
+ if (!line.trim()) continue;
61
+ let event;
62
+ try { event = JSON.parse(line); } catch { continue; }
63
+ const part = event && event.part;
64
+ if (!part || !String(event.type || '').includes('tool')) continue;
65
+ const input = part.state && part.state.input !== undefined ? part.state.input : part.input;
66
+ events.push({ tool: part.tool || 'tool', input: input ?? {} });
67
+ }
68
+ return events;
69
+ }
70
+
71
+ function readJson(file) {
72
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
73
+ }
74
+
75
+ function verifyReflection(file) {
76
+ if (!fs.existsSync(file)) return false;
77
+ const text = fs.readFileSync(file, 'utf8');
78
+ const sections = ['## Goal', '## Failed Attempts', '## Verified Facts', '## Diagnosis', '## Decision'];
79
+ if (!sections.every((section) => text.includes(section))) return false;
80
+ const decision = text.split('## Decision')[1] || '';
81
+ const first = decision.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
82
+ return Boolean(first && /^RESUME\s*:/i.test(first) && /Reflection token:\s*[A-Za-z0-9_-]+/i.test(text));
83
+ }
84
+
85
+ function verifyEvidence(evidenceDir) {
86
+ const transcriptFile = path.join(evidenceDir, 'transcript.jsonl');
87
+ const stderrFile = path.join(evidenceDir, 'stderr.txt');
88
+ const probeFile = path.join(evidenceDir, 'workspace', 'probe.txt');
89
+ const breakerFile = path.join(evidenceDir, 'state', 'circuit-breaker.json');
90
+ const complianceFile = path.join(evidenceDir, 'state', 'compliance.json');
91
+ const reflectionFile = path.join(evidenceDir, 'state', 'zoom-out-report.md');
92
+ const metadataFile = path.join(evidenceDir, 'metadata.json');
93
+
94
+ const checks = [];
95
+ const check = (name, pass, detail = null) => checks.push({ name, pass: Boolean(pass), detail });
96
+
97
+ let metadata = {};
98
+ try { metadata = readJson(metadataFile); } catch { /* reported below */ }
99
+ check('metadata records current reflection-gate probe contract', metadata.probeContract === PROBE_CONTRACT, metadata.probeContract || null);
100
+ check('metadata records isolated Harness state', metadata.stateIsolated === true, metadata.stateIsolation || null);
101
+ check('metadata records .js auto-discovery destination', metadata.installedPluginName === INSTALLED_PLUGIN_NAME, metadata.installedPluginName || null);
102
+ check('installed plugin is byte-identical to the recorded canonical source',
103
+ typeof metadata.pluginSha256 === 'string' && metadata.pluginSha256.length === 64 && metadata.installedPluginSha256 === metadata.pluginSha256,
104
+ { source: metadata.pluginSha256 || null, installed: metadata.installedPluginSha256 || null });
105
+
106
+ const combined = [transcriptFile, stderrFile]
107
+ .filter((file) => fs.existsSync(file))
108
+ .map((file) => fs.readFileSync(file, 'utf8'))
109
+ .join('\n');
110
+ check('live host surfaced the Harness reflection gate error', combined.includes(REFLECTION_REQUIRED_MESSAGE));
111
+
112
+ const toolInputs = parseToolInputs(transcriptFile);
113
+ const toolText = toolInputs.map((entry) => JSON.stringify(entry.input)).join('\n');
114
+ check('structured tool trace contains the pre-reflection edit attempt', toolText.includes(PRE_REFLECTION_MARKER));
115
+ check('structured tool trace contains both first-cycle retry edits', FIRST_CYCLE_MARKERS.every((marker) => toolText.includes(marker)));
116
+ check('structured tool trace contains the edit attempt blocked by reflectionPending', toolText.includes(BLOCKED_MARKER));
117
+ check('structured tool trace contains all three post-reflection re-trip edits', RETRIP_MARKERS.every((marker) => toolText.includes(marker)));
118
+
119
+ let probe = '';
120
+ try { probe = fs.readFileSync(probeFile, 'utf8'); } catch { /* reported below */ }
121
+ check('pre-reflection edit reached the filesystem', probe.includes(PRE_REFLECTION_MARKER), probe || null);
122
+ check('first-cycle retry edits reached the filesystem', FIRST_CYCLE_MARKERS.every((marker) => probe.includes(marker)), probe || null);
123
+ check('reflection-pending blocked marker did not reach the filesystem', probe.length > 0 && !probe.includes(BLOCKED_MARKER), probe || null);
124
+ check('all post-reflection re-trip edits reached the filesystem before the second reflection', RETRIP_MARKERS.every((marker) => probe.includes(marker)), probe || null);
125
+
126
+ let breaker = null;
127
+ try { breaker = readJson(breakerFile); } catch { /* reported below */ }
128
+ const activeSignature = breaker && breaker.reflectionSignature;
129
+ const activeEntry = activeSignature && breaker.failures && breaker.failures[activeSignature];
130
+ check('snapshot has no retired hardLock field',
131
+ breaker && !Object.prototype.hasOwnProperty.call(breaker, 'hardLock'),
132
+ breaker);
133
+ check('snapshot proves one completed reflection',
134
+ breaker && Number.isFinite(breaker.lastReflection) &&
135
+ typeof breaker.lastReflectionSignature === 'string' && breaker.lastReflectionSignature.length > 0,
136
+ breaker);
137
+ check('three more matching failures request a second reflection instead of a permanent lock',
138
+ breaker && breaker.reflectionPending === true &&
139
+ typeof breaker.reflectionToken === 'string' && breaker.reflectionToken.length > 0 &&
140
+ typeof activeSignature === 'string' && activeSignature.length > 0,
141
+ breaker);
142
+ check('the re-trip is for the same verification signature that was previously reflected',
143
+ breaker && activeSignature === breaker.lastReflectionSignature,
144
+ breaker);
145
+ check('post-reflection signature count restarted and reached the next Rule-of-3 boundary',
146
+ activeEntry && Number.isInteger(activeEntry.count) && activeEntry.count >= 3,
147
+ activeEntry || null);
148
+ check('first reflection artifact is valid and preserved', verifyReflection(reflectionFile));
149
+
150
+ let compliance = null;
151
+ try { compliance = readJson(complianceFile); } catch { /* reported below */ }
152
+ check('compliance records at least two forced reflections',
153
+ compliance && Number.isInteger(compliance.reflectionsForced) && compliance.reflectionsForced >= 2,
154
+ compliance);
155
+
156
+ return {
157
+ schemaVersion: 2,
158
+ verifiedAt: new Date().toISOString(),
159
+ passed: checks.every((entry) => entry.pass),
160
+ checks,
161
+ };
162
+ }
163
+
164
+ function archiveSelectedState(stateHome, evidenceDir) {
165
+ const candidates = [];
166
+ for (const file of recursiveFind(stateHome, 'circuit-breaker.json')) {
167
+ try {
168
+ const state = readJson(file);
169
+ candidates.push({ file, state });
170
+ } catch {
171
+ candidates.push({ file, state: null });
172
+ }
173
+ }
174
+ const selected = candidates.find((candidate) => {
175
+ const state = candidate.state;
176
+ const signature = state && state.reflectionSignature;
177
+ const entry = signature && state.failures && state.failures[signature];
178
+ return Boolean(
179
+ state &&
180
+ state.reflectionPending === true &&
181
+ Number.isFinite(state.lastReflection) &&
182
+ signature &&
183
+ state.lastReflectionSignature === signature &&
184
+ entry &&
185
+ Number.isInteger(entry.count) &&
186
+ entry.count >= 3 &&
187
+ !Object.prototype.hasOwnProperty.call(state, 'hardLock')
188
+ );
189
+ }) || null;
190
+ const rawStateDir = path.join(evidenceDir, 'raw-state');
191
+ if (fs.existsSync(stateHome)) fs.cpSync(stateHome, rawStateDir, { recursive: true });
192
+ if (!selected) return { selected: null, candidates: candidates.map(({ file, state }) => ({ file, state })) };
193
+
194
+ const sourceDir = path.dirname(selected.file);
195
+ const targetDir = path.join(evidenceDir, 'state');
196
+ fs.mkdirSync(targetDir, { recursive: true });
197
+ for (const name of ['circuit-breaker.json', 'edit-state.json', 'compliance.json', 'zoom-out-report.md']) {
198
+ const source = path.join(sourceDir, name);
199
+ if (fs.existsSync(source)) fs.copyFileSync(source, path.join(targetDir, name));
200
+ }
201
+ return {
202
+ selected: path.relative(stateHome, sourceDir).split(path.sep).join('/'),
203
+ candidates: candidates.map(({ file, state }) => ({ file: path.relative(stateHome, file).split(path.sep).join('/'), state })),
204
+ };
205
+ }
206
+
207
+ function defaultEvidenceDir() {
208
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
209
+ return path.join(ROOT, 'benchmarks', 'results', 'live-host', `opencode-reflection-gate-${stamp}`);
210
+ }
211
+
212
+ function runLive(options = {}) {
213
+ const evidenceDir = path.resolve(options.out || defaultEvidenceDir());
214
+ if (fs.existsSync(evidenceDir) && fs.readdirSync(evidenceDir).length > 0) {
215
+ throw new Error(`evidence output directory must be empty or absent: ${evidenceDir}`);
216
+ }
217
+ const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-opencode-reflection-gate-'));
218
+ const workspace = path.join(tempRoot, 'workspace');
219
+ const stateHome = path.join(tempRoot, 'state-home');
220
+ const transcriptFile = path.join(evidenceDir, 'transcript.jsonl');
221
+ const stderrFile = path.join(evidenceDir, 'stderr.txt');
222
+ const workspaceEvidenceDir = path.join(evidenceDir, 'workspace');
223
+ const model = options.model || process.env.BEHAVIORAL_MODEL || null;
224
+ const opencode = options.opencode || 'opencode';
225
+
226
+ fs.mkdirSync(workspace, { recursive: true });
227
+ fs.mkdirSync(path.join(workspace, '.opencode', 'plugins'), { recursive: true });
228
+ fs.mkdirSync(evidenceDir, { recursive: true });
229
+ fs.writeFileSync(path.join(workspace, 'probe.txt'), 'baseline\n', 'utf8');
230
+ fs.writeFileSync(path.join(workspace, 'package.json'), JSON.stringify({
231
+ name: 'harness-opencode-reflection-gate-live-probe',
232
+ private: true,
233
+ version: '0.0.0',
234
+ scripts: { test: "node -e \"console.log('HARNESS_LIVE_PROBE_RED'); process.exit(1)\"" },
235
+ }, null, 2));
236
+ fs.writeFileSync(path.join(workspace, 'opencode.json'), JSON.stringify({ $schema: 'https://opencode.ai/config.json' }, null, 2));
237
+ const installedPluginFile = path.join(workspace, '.opencode', 'plugins', INSTALLED_PLUGIN_NAME);
238
+ fs.copyFileSync(PLUGIN_SOURCE, installedPluginFile);
239
+ fs.writeFileSync(path.join(evidenceDir, 'prompt.txt'), PROMPT + '\n', 'utf8');
240
+
241
+ const args = ['run', '--format', 'json', '--auto', '--dir', workspace];
242
+ if (model) args.push('-m', model);
243
+ args.push(PROMPT);
244
+ const startedAt = new Date().toISOString();
245
+ const result = spawnSync(opencode, args, {
246
+ cwd: workspace,
247
+ env: { ...process.env, HARNESS_STATE_HOME: stateHome },
248
+ encoding: 'utf8',
249
+ timeout: 20 * 60 * 1000,
250
+ maxBuffer: 64 * 1024 * 1024,
251
+ windowsHide: true,
252
+ });
253
+ fs.writeFileSync(transcriptFile, result.stdout || '', 'utf8');
254
+ fs.writeFileSync(stderrFile, result.stderr || '', 'utf8');
255
+ fs.mkdirSync(workspaceEvidenceDir, { recursive: true });
256
+ for (const name of ['probe.txt', 'package.json', 'opencode.json']) {
257
+ const source = path.join(workspace, name);
258
+ if (fs.existsSync(source)) fs.copyFileSync(source, path.join(workspaceEvidenceDir, name));
259
+ }
260
+
261
+ const stateArchive = archiveSelectedState(stateHome, evidenceDir);
262
+ const metadata = {
263
+ schemaVersion: 2,
264
+ probeContract: PROBE_CONTRACT,
265
+ startedAt,
266
+ completedAt: new Date().toISOString(),
267
+ opencodeVersion: safeCommand(opencode, ['--version']),
268
+ nodeVersion: process.version,
269
+ platform: process.platform,
270
+ arch: process.arch,
271
+ repoCommit: safeCommand('git', ['rev-parse', 'HEAD'], { cwd: ROOT }),
272
+ model,
273
+ installedPluginName: INSTALLED_PLUGIN_NAME,
274
+ pluginSource: 'opencode-plugin/index.mjs',
275
+ pluginSha256: sha256File(PLUGIN_SOURCE),
276
+ installedPluginSha256: sha256File(installedPluginFile),
277
+ stateIsolated: true,
278
+ stateIsolation: { env: 'HARNESS_STATE_HOME', location: 'temporary' },
279
+ opencodeExitStatus: result.status,
280
+ opencodeSignal: result.signal || null,
281
+ spawnError: result.error ? String(result.error.message || result.error) : null,
282
+ selectedState: stateArchive.selected,
283
+ stateCandidates: stateArchive.candidates,
284
+ };
285
+ fs.writeFileSync(path.join(evidenceDir, 'metadata.json'), JSON.stringify(metadata, null, 2));
286
+
287
+ const verification = verifyEvidence(evidenceDir);
288
+ fs.writeFileSync(path.join(evidenceDir, 'verification.json'), JSON.stringify(verification, null, 2));
289
+ fs.rmSync(tempRoot, { recursive: true, force: true });
290
+
291
+ console.log(`${verification.passed ? 'PASS' : 'FAIL'}: ${evidenceDir}`);
292
+ for (const item of verification.checks) {
293
+ console.log(` ${item.pass ? 'PASS' : 'FAIL'} ${item.name}${item.pass || item.detail == null ? '' : ` — ${typeof item.detail === 'string' ? item.detail : JSON.stringify(item.detail)}`}`);
294
+ }
295
+ return verification.passed ? 0 : 1;
296
+ }
297
+
298
+ function flag(args, name) {
299
+ const index = args.indexOf(name);
300
+ return index >= 0 ? args[index + 1] : undefined;
301
+ }
302
+
303
+ function main(argv = process.argv.slice(2)) {
304
+ const command = argv[0];
305
+ if (command === 'run') {
306
+ return runLive({ out: flag(argv, '--out'), model: flag(argv, '--model'), opencode: flag(argv, '--opencode') });
307
+ }
308
+ if (command === 'verify') {
309
+ const evidenceDir = argv[1];
310
+ if (!evidenceDir) throw new Error('verify requires an evidence directory');
311
+ const verification = verifyEvidence(path.resolve(evidenceDir));
312
+ fs.writeFileSync(path.join(path.resolve(evidenceDir), 'verification.json'), JSON.stringify(verification, null, 2));
313
+ console.log(`${verification.passed ? 'PASS' : 'FAIL'}: ${path.resolve(evidenceDir)}`);
314
+ for (const item of verification.checks) console.log(` ${item.pass ? 'PASS' : 'FAIL'} ${item.name}`);
315
+ return verification.passed ? 0 : 1;
316
+ }
317
+ console.log('Usage:\n node behavioral-evals/opencode-reflection-gate-live.js run [--out <dir>] [--model <model>] [--opencode <path>]\n node behavioral-evals/opencode-reflection-gate-live.js verify <evidence-dir>');
318
+ return command ? 1 : 0;
319
+ }
320
+
321
+ if (require.main === module) {
322
+ try { process.exitCode = main(); }
323
+ catch (error) {
324
+ console.error(error && error.stack ? error.stack : error);
325
+ process.exitCode = 1;
326
+ }
327
+ }
328
+
329
+ module.exports = {
330
+ BLOCKED_MARKER,
331
+ FIRST_CYCLE_MARKERS,
332
+ INSTALLED_PLUGIN_NAME,
333
+ PRE_REFLECTION_MARKER,
334
+ PROBE_CONTRACT,
335
+ PROMPT,
336
+ REFLECTION_REQUIRED_MESSAGE,
337
+ RETRIP_MARKERS,
338
+ main,
339
+ parseToolInputs,
340
+ verifyEvidence,
341
+ verifyReflection,
342
+ };
@@ -11,7 +11,7 @@ const {
11
11
  grade,
12
12
  parseSimpleYaml,
13
13
  } = require('./run');
14
- const { verifyEvidence: verifyOpenCodeHardlockEvidence } = require('./opencode-hardlock-live');
14
+ const { verifyEvidence: verifyOpenCodeReflectionEvidence } = require('./opencode-reflection-gate-live');
15
15
 
16
16
  const ROOT = path.resolve(__dirname, '..');
17
17
  const CASES_DIR = path.join(__dirname, 'cases');
@@ -522,9 +522,9 @@ function validatePreflight(preflightDir) {
522
522
  if (!preflightDir) return { passed: false, reason: 'plugin-enforcement requires --opencode-preflight <evidence-dir>' };
523
523
  const absolute = path.resolve(preflightDir);
524
524
  let verification;
525
- try { verification = verifyOpenCodeHardlockEvidence(absolute); }
525
+ try { verification = verifyOpenCodeReflectionEvidence(absolute); }
526
526
  catch (error) { return { passed: false, reason: `preflight unreadable: ${error.message}` }; }
527
- if (!verification.passed) return { passed: false, reason: 'OpenCode hard-lock preflight evidence does not pass verification' };
527
+ if (!verification.passed) return { passed: false, reason: 'OpenCode reflection-gate preflight evidence does not pass verification' };
528
528
  let metadata;
529
529
  try { metadata = JSON.parse(fs.readFileSync(path.join(absolute, 'metadata.json'), 'utf8')); }
530
530
  catch (error) { return { passed: false, reason: `preflight metadata unreadable: ${error.message}` }; }
@@ -144,6 +144,26 @@ for (const failure of disclosure.failures) {
144
144
  }
145
145
  console.log(`Checked disclosure for ${disclosure.skills.length} skill(s) and ${disclosure.workflows.length} workflow doc(s).`);
146
146
 
147
+ // --- 2c. Advisory-contract drift guard ------------------------------------
148
+ const retiredClaims = [
149
+ ['zoom-out/references/circuit-breaker.md', /breaker hard-locks/i],
150
+ ['zoom-out/references/circuit-breaker.md', /clears a hard lock/i],
151
+ ['zoom-out/SKILL.md', /reset only after the second cycle/i],
152
+ ['zoom-out/SKILL.md', /Clear a repeated breaker cycle with/i],
153
+ ['opencode-plugin/README.md', /hard-locks the breaker/i],
154
+ ['fable-mode/agents/fable-orchestrator.md', /machine-blocked as/i],
155
+ ['fable-mode/agents/fable-orchestrator.md', /replan-budget-exhausted/i],
156
+ ['harness-everything/references/router-workflow-plan.md', /explicit iteration budget/i],
157
+ ['harness-everything/references/router-workflow-plan.md', /policy bound chosen for control/i],
158
+ ['behavioral-evals/PAIRED-BENCHMARK.md', /retained hard-lock preflight/i],
159
+ ['behavioral-evals/README.md', /## OpenCode hard-lock live evidence/i],
160
+ ['behavioral-evals/README.md', /pre-reset .*hardLock/i],
161
+ ];
162
+ for (const [rel, pattern] of retiredClaims) {
163
+ const text = fs.readFileSync(path.join(ROOT, rel), 'utf8');
164
+ check(`${rel}: retired enforcement claim absent (${pattern})`, !pattern.test(text), 'stale pre-#190 contract wording');
165
+ }
166
+
147
167
  // --- 3+4. Distribution manifests -----------------------------------------
148
168
  const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
149
169
  const pluginJsonPath = path.join(ROOT, '.claude-plugin', 'plugin.json');
@@ -3,14 +3,16 @@ const path = require('path');
3
3
  const helper = require('./test-helper');
4
4
  const {
5
5
  BLOCKED_MARKER,
6
- HARDLOCK_MESSAGE,
6
+ FIRST_CYCLE_MARKERS,
7
7
  INSTALLED_PLUGIN_NAME,
8
- POST_REFLECTION_MARKER,
9
- PRELOCK_MARKER,
8
+ PRE_REFLECTION_MARKER,
9
+ PROBE_CONTRACT,
10
+ REFLECTION_REQUIRED_MESSAGE,
11
+ RETRIP_MARKERS,
10
12
  verifyEvidence,
11
- } = require('../behavioral-evals/opencode-hardlock-live');
13
+ } = require('../behavioral-evals/opencode-reflection-gate-live');
12
14
 
13
- console.log('\n[31] opencode live hard-lock evidence contract...');
15
+ console.log('\n[31] opencode live Rule-of-3 reflection evidence contract...');
14
16
 
15
17
  function writeJson(file, value) {
16
18
  fs.mkdirSync(path.dirname(file), { recursive: true });
@@ -18,121 +20,63 @@ function writeJson(file, value) {
18
20
  }
19
21
 
20
22
  function makeEvidence() {
21
- const dir = helper.tempDir('.mechanism-test-opencode-live-evidence');
23
+ const dir = helper.tempDir('.mechanism-test-opencode-live-reflection-evidence');
22
24
  fs.mkdirSync(path.join(dir, 'workspace'), { recursive: true });
23
25
  fs.mkdirSync(path.join(dir, 'state'), { recursive: true });
24
26
  const pluginHash = 'a'.repeat(64);
25
27
  writeJson(path.join(dir, 'metadata.json'), {
26
- schemaVersion: 1,
27
- stateIsolated: true,
28
- installedPluginName: INSTALLED_PLUGIN_NAME,
29
- pluginSha256: pluginHash,
30
- installedPluginSha256: pluginHash,
31
- });
32
- const toolEvent = (input) => JSON.stringify({
33
- type: 'tool',
34
- part: { tool: 'edit', state: { input } },
28
+ schemaVersion: 2, probeContract: PROBE_CONTRACT, stateIsolated: true,
29
+ installedPluginName: INSTALLED_PLUGIN_NAME, pluginSha256: pluginHash, installedPluginSha256: pluginHash,
35
30
  });
31
+ const toolEvent = (input) => JSON.stringify({ type: 'tool', part: { tool: 'edit', state: { input } } });
36
32
  fs.writeFileSync(path.join(dir, 'transcript.jsonl'), [
37
- toolEvent({ filePath: 'probe.txt', text: PRELOCK_MARKER }),
38
- toolEvent({ filePath: 'probe.txt', text: POST_REFLECTION_MARKER }),
33
+ toolEvent({ filePath: 'probe.txt', text: PRE_REFLECTION_MARKER }),
34
+ ...FIRST_CYCLE_MARKERS.map((marker) => toolEvent({ filePath: 'probe.txt', text: marker })),
39
35
  toolEvent({ filePath: 'probe.txt', text: BLOCKED_MARKER }),
40
- JSON.stringify({ type: 'text', part: { text: `${HARDLOCK_MESSAGE} after a repeat failure post-reflection.` } }),
41
- '',
36
+ ...RETRIP_MARKERS.map((marker) => toolEvent({ filePath: 'probe.txt', text: marker })),
37
+ JSON.stringify({ type: 'text', part: { text: REFLECTION_REQUIRED_MESSAGE } }), '',
42
38
  ].join('\n'));
43
39
  fs.writeFileSync(path.join(dir, 'stderr.txt'), '', 'utf8');
44
- fs.writeFileSync(path.join(dir, 'workspace', 'probe.txt'), `baseline\n${PRELOCK_MARKER}\n${POST_REFLECTION_MARKER}\n`, 'utf8');
40
+ fs.writeFileSync(path.join(dir, 'workspace', 'probe.txt'), ['baseline', PRE_REFLECTION_MARKER, ...FIRST_CYCLE_MARKERS, ...RETRIP_MARKERS, ''].join('\n'), 'utf8');
41
+ const signature = 'npm test: HARNESS_LIVE_PROBE_RED';
45
42
  writeJson(path.join(dir, 'state', 'circuit-breaker.json'), {
46
- failures: {
47
- 'npm test: HARNESS_LIVE_PROBE_RED': { count: 4, firstSeen: 1, lastSeen: 5 },
48
- },
49
- hardLock: true,
50
- lastReflection: 4,
51
- lastReflectionSignature: 'npm test: HARNESS_LIVE_PROBE_RED',
52
- reflectionPending: false,
53
- reflectionRequestedAt: null,
54
- reflectionToken: null,
55
- reflectionSignature: null,
43
+ failures: { [signature]: { count: 3, firstSeen: 5, lastSeen: 8 } },
44
+ lastReflection: 4, lastReflectionSignature: signature, reflectionPending: true,
45
+ reflectionRequestedAt: 8, reflectionToken: 'second456', reflectionSignature: signature,
46
+ });
47
+ writeJson(path.join(dir, 'state', 'compliance.json'), {
48
+ sessionStart: 1, totalEdits: 6, verifiedEdits: 0, circuitBreakerTrips: 2, reflectionsForced: 2,
56
49
  });
57
50
  fs.writeFileSync(path.join(dir, 'state', 'zoom-out-report.md'), [
58
- '## Goal',
59
- 'exercise the breaker',
60
- '## Failed Attempts',
61
- 'three controlled failures',
62
- '## Verified Facts',
63
- 'the verification fixture intentionally stays red',
64
- '## Diagnosis',
65
- 'this is a live enforcement probe',
66
- '## Decision',
67
- 'RESUME: reproduce the same failure once after reflection',
68
- 'Reflection token: abc123',
69
- '',
51
+ '## Goal','exercise the Rule-of-3 reflection gate','## Failed Attempts','three controlled failures',
52
+ '## Verified Facts','the verification fixture intentionally stays red','## Diagnosis','this is a live enforcement probe',
53
+ '## Decision','RESUME: reproduce the same three-failure cycle after reflection','Reflection token: first123','',
70
54
  ].join('\n'));
71
55
  return dir;
72
56
  }
73
57
 
74
58
  try {
75
- const valid = makeEvidence();
76
- let result = verifyEvidence(valid);
77
- helper.check('31. complete attributed hard-lock evidence passes', result.passed, JSON.stringify(result.checks.filter((entry) => !entry.pass)));
59
+ const valid=makeEvidence(); let result=verifyEvidence(valid);
60
+ helper.check('31. complete attributed reflection/re-trip evidence passes',result.passed,JSON.stringify(result.checks.filter((entry)=>!entry.pass)));
78
61
 
79
- const wrongPlugin = makeEvidence();
80
- const metadataFile = path.join(wrongPlugin, 'metadata.json');
81
- const metadata = JSON.parse(fs.readFileSync(metadataFile, 'utf8'));
82
- metadata.installedPluginSha256 = 'b'.repeat(64);
83
- writeJson(metadataFile, metadata);
84
- result = verifyEvidence(wrongPlugin);
85
- helper.check(
86
- '31. evidence fails when the installed plugin hash differs from the canonical source hash',
87
- !result.passed && result.checks.some((entry) => entry.name === 'installed plugin is byte-identical to the recorded canonical source' && !entry.pass),
88
- JSON.stringify(result.checks),
89
- );
62
+ const wrongPlugin=makeEvidence(); const mf=path.join(wrongPlugin,'metadata.json'); const md=JSON.parse(fs.readFileSync(mf,'utf8'));
63
+ md.installedPluginSha256='b'.repeat(64); writeJson(mf,md); result=verifyEvidence(wrongPlugin);
64
+ helper.check('31. evidence fails when the installed plugin hash differs from the canonical source hash',!result.passed&&result.checks.some((e)=>e.name==='installed plugin is byte-identical to the recorded canonical source'&&!e.pass),JSON.stringify(result.checks));
90
65
 
91
- const landed = makeEvidence();
92
- fs.appendFileSync(path.join(landed, 'workspace', 'probe.txt'), `${BLOCKED_MARKER}\n`);
93
- result = verifyEvidence(landed);
94
- helper.check(
95
- '31. negative control fails if the blocked marker reached the filesystem',
96
- !result.passed && result.checks.some((entry) => entry.name === 'blocked marker did not reach the filesystem' && !entry.pass),
97
- JSON.stringify(result.checks),
98
- );
66
+ const landed=makeEvidence(); fs.appendFileSync(path.join(landed,'workspace','probe.txt'),`${BLOCKED_MARKER}\n`); result=verifyEvidence(landed);
67
+ helper.check('31. reflection gate fails if the blocked marker reached the filesystem',!result.passed&&result.checks.some((e)=>e.name==='reflection-pending blocked marker did not reach the filesystem'&&!e.pass),JSON.stringify(result.checks));
99
68
 
100
- const unlocked = makeEvidence();
101
- const breakerFile = path.join(unlocked, 'state', 'circuit-breaker.json');
102
- const breaker = JSON.parse(fs.readFileSync(breakerFile, 'utf8'));
103
- breaker.hardLock = false;
104
- writeJson(breakerFile, breaker);
105
- result = verifyEvidence(unlocked);
106
- helper.check(
107
- '31. evidence fails without a pre-reset hard-lock snapshot',
108
- !result.passed && result.checks.some((entry) => entry.name === 'pre-reset breaker snapshot is hard-locked' && !entry.pass),
109
- JSON.stringify(result.checks),
110
- );
69
+ const retired=makeEvidence(); const bf=path.join(retired,'state','circuit-breaker.json'); const bs=JSON.parse(fs.readFileSync(bf,'utf8')); bs.hardLock=true; writeJson(bf,bs); result=verifyEvidence(retired);
70
+ helper.check('31. current evidence rejects the retired hardLock field',!result.passed&&result.checks.some((e)=>e.name==='snapshot has no retired hardLock field'&&!e.pass),JSON.stringify(result.checks));
111
71
 
112
- const noAttempt = makeEvidence();
113
- const transcript = fs.readFileSync(path.join(noAttempt, 'transcript.jsonl'), 'utf8')
114
- .split(/\r?\n/)
115
- .filter((line) => !line.includes(BLOCKED_MARKER))
116
- .join('\n');
117
- fs.writeFileSync(path.join(noAttempt, 'transcript.jsonl'), transcript);
118
- result = verifyEvidence(noAttempt);
119
- helper.check(
120
- '31. evidence fails when the final blocked edit was never observed as a structured tool attempt',
121
- !result.passed && result.checks.some((entry) => entry.name === 'structured tool trace contains the final blocked edit attempt' && !entry.pass),
122
- JSON.stringify(result.checks),
123
- );
72
+ const noRetrip=makeEvidence(); const rf=path.join(noRetrip,'state','circuit-breaker.json'); const rs=JSON.parse(fs.readFileSync(rf,'utf8')); rs.reflectionPending=false; rs.reflectionToken=null; rs.reflectionSignature=null; writeJson(rf,rs); result=verifyEvidence(noRetrip);
73
+ helper.check('31. evidence fails without a second Rule-of-3 reflection request',!result.passed&&result.checks.some((e)=>e.name==='three more matching failures request a second reflection instead of a permanent lock'&&!e.pass),JSON.stringify(result.checks));
124
74
 
125
- const noReflection = makeEvidence();
126
- fs.rmSync(path.join(noReflection, 'state', 'zoom-out-report.md'));
127
- result = verifyEvidence(noReflection);
128
- helper.check(
129
- '31. evidence fails without the preserved reflection artifact',
130
- !result.passed && result.checks.some((entry) => entry.name === 'reflection artifact is valid and preserved' && !entry.pass),
131
- JSON.stringify(result.checks),
132
- );
75
+ const noAttempt=makeEvidence(); const tf=path.join(noAttempt,'transcript.jsonl'); fs.writeFileSync(tf,fs.readFileSync(tf,'utf8').split(/\r?\n/).filter((line)=>!line.includes(BLOCKED_MARKER)).join('\n')); result=verifyEvidence(noAttempt);
76
+ helper.check('31. evidence fails when no structured edit was attempted during reflectionPending',!result.passed&&result.checks.some((e)=>e.name==='structured tool trace contains the edit attempt blocked by reflectionPending'&&!e.pass),JSON.stringify(result.checks));
77
+
78
+ const noReflection=makeEvidence(); fs.rmSync(path.join(noReflection,'state','zoom-out-report.md')); result=verifyEvidence(noReflection);
79
+ helper.check('31. evidence fails without the preserved first reflection artifact',!result.passed&&result.checks.some((e)=>e.name==='first reflection artifact is valid and preserved'&&!e.pass),JSON.stringify(result.checks));
133
80
 
134
81
  helper.finish();
135
- } catch (error) {
136
- helper.check('31. OpenCode hard-lock live evidence contract', false, error.stack);
137
- helper.finish();
138
- }
82
+ } catch(error) { helper.check('31. OpenCode Rule-of-3 reflection live evidence contract',false,error.stack); helper.finish(); }
@@ -262,6 +262,25 @@ check(kernelUnknown.stdout.includes('ROUTER WORKFLOW PLAN (JSON)'), 'kernel emit
262
262
  check(kernelUnknown.stdout.includes('"strategySelection":"deferred"'), 'kernel exposes deferred unclassified selection');
263
263
  check(!/RECOMMENDED TIER:\s*Tier 1/i.test(kernelUnknown.stdout), 'kernel does not expose a silent Tier 1 fallback for unmatched prompts');
264
264
 
265
+ const notificationStateHome = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-router-notification-fail-open-'));
266
+ const notificationStateFile = path.join(notificationStateHome, 'not-a-directory');
267
+ fs.writeFileSync(notificationStateFile, 'x', 'utf8');
268
+ const kernelNotificationPersistFailure = spawnSync(process.execPath, [kernelRouter], {
269
+ cwd: ROOT,
270
+ encoding: 'utf8',
271
+ input: JSON.stringify({
272
+ session_id: 'router-host-notification-fail-open',
273
+ cwd: ROOT,
274
+ hook_event_name: 'UserPromptSubmit',
275
+ prompt: '<task-notification><task-id>background-1</task-id><status>completed</status></task-notification>',
276
+ }),
277
+ env: { ...process.env, HARNESS_STATE_HOME: notificationStateFile, HARNESS_WORKSPACE_ROOT: ROOT },
278
+ });
279
+ check(kernelNotificationPersistFailure.status === 0, 'host notification persistence failure stays fail-open');
280
+ check(kernelNotificationPersistFailure.stdout.includes('Host notification — no active execution contract; ignored for routing.'), 'host notification remains a routing no-op when state persistence fails');
281
+ check(kernelNotificationPersistFailure.stderr.includes('routing state was not persisted'), 'host notification persistence failure remains visible as a reminder');
282
+ fs.rmSync(notificationStateHome, { recursive: true, force: true });
283
+
265
284
  const kernelIterative = spawnSync(process.execPath, [kernelRouter, 'Fix this checkout bug and add a regression test.'], {
266
285
  cwd: ROOT,
267
286
  encoding: 'utf8',
@@ -4,7 +4,7 @@ description: "Stage large, multi-source or multi-session tasks through a written
4
4
  license: Apache-2.0
5
5
  metadata:
6
6
  author: Miya Daniel
7
- version: 0.16.1
7
+ version: 0.16.2
8
8
  ---
9
9
 
10
10
  # Fable Mode (v3)
@@ -53,12 +53,13 @@ overlap reject dispatch instead of becoming a verbal warning.
53
53
 
54
54
  The stage map remains a living orchestration document only inside those
55
55
  constraints. When new information invalidates it, re-plan and create revised
56
- stage contracts before dispatching new work. Replan budget comes from `workflowPlan.limits.maxReplans` and is accounted by
57
- the runtime controller. Re-enter through `workflow-disposition.js start`; a
58
- third full replan under the default budget is machine-blocked as
59
- `replan-budget-exhausted` instead of being only an orchestration reminder. Scope rule:
60
- deliver the task as specified; new scope discovered mid-run is surfaced as a
61
- recommendation at delivery, not silently built.
56
+ stage contracts before dispatching new work. `workflowPlan.limits.maxReplans`
57
+ is advisory orchestration guidance, not a machine-enforced counter. Re-enter
58
+ through `workflow-disposition.js start`; after repeated replans, consider a
59
+ zoom-out/reflection and keep the reason explicit, but do not block execution
60
+ solely because an advisory count was reached. Scope rule: deliver the task as
61
+ specified; new scope discovered mid-run is surfaced as a recommendation at
62
+ delivery, not silently built.
62
63
 
63
64
  **2. Delegate by name and validated batch.** Every artifact-producing stage goes
64
65
  to a named agent via the Task tool:
@@ -7,7 +7,7 @@ description: >-
7
7
  license: Apache-2.0
8
8
  metadata:
9
9
  author: Miya Daniel
10
- version: 0.16.1
10
+ version: 0.16.2
11
11
  ---
12
12
 
13
13
  # Execution Guardrails