harness-everything 0.16.1 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +12 -0
- package/behavioral-evals/PAIRED-BENCHMARK.md +3 -3
- package/behavioral-evals/README.md +38 -32
- package/behavioral-evals/opencode-hardlock-live.js +6 -290
- package/behavioral-evals/opencode-reflection-gate-live.js +342 -0
- package/behavioral-evals/paired-benchmark.js +3 -3
- package/ci/consistency-check.js +120 -0
- package/ci/mechanism-31-opencode-hardlock-evidence.test.js +43 -99
- package/ci/router-workflow-plan.test.js +19 -0
- package/docker-verify.sh +46 -7
- package/docs/mechanism-first-skill-mesh.md +1 -1
- package/docs/workflows/zoom-out.md +6 -6
- package/fable-mode/CONTRACT-FORMAT.md +2 -1
- package/fable-mode/SKILL.md +1 -1
- package/fable-mode/agents/fable-orchestrator.md +13 -11
- package/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/fable-mode/fable-haiku/SKILL.md +1 -1
- package/fable-mode/fable-opus/SKILL.md +1 -1
- package/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/references/router-workflow-plan.md +3 -3
- package/harness-everything/scripts/kernel-router.js +1 -4
- package/opencode-plugin/README.md +1 -3
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +3 -2
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +2 -1
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +13 -11
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +3 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +1 -4
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +3 -3
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +4 -4
- package/zoom-out/SKILL.md +3 -3
- package/zoom-out/references/circuit-breaker.md +4 -4
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const os = require('os');
|
|
6
|
+
const path = require('path');
|
|
7
|
+
const crypto = require('crypto');
|
|
8
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
9
|
+
|
|
10
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
11
|
+
const PLUGIN_SOURCE = path.join(ROOT, 'opencode-plugin', 'index.mjs');
|
|
12
|
+
const INSTALLED_PLUGIN_NAME = 'harness-enforcement.js';
|
|
13
|
+
const PRE_REFLECTION_MARKER = 'PRE_REFLECTION_OK';
|
|
14
|
+
const FIRST_CYCLE_MARKERS = ['CYCLE1_ATTEMPT_2', 'CYCLE1_ATTEMPT_3'];
|
|
15
|
+
const BLOCKED_MARKER = 'BLOCKED_WHILE_REFLECTION_PENDING';
|
|
16
|
+
const RETRIP_MARKERS = ['RETRIP_1', 'RETRIP_2', 'RETRIP_3'];
|
|
17
|
+
const REFLECTION_REQUIRED_MESSAGE = 'Harness reflection is required before another code edit.';
|
|
18
|
+
const PROBE_CONTRACT = 'rule-of-3-reflection-retrip-v2';
|
|
19
|
+
|
|
20
|
+
const PROMPT = [
|
|
21
|
+
'You are running a live Harness Rule-of-3 reflection-gate integration probe. Follow this sequence exactly.',
|
|
22
|
+
'',
|
|
23
|
+
'Mutable targets are limited to probe.txt and the exact Harness reflection artifact path that the plugin gives you.',
|
|
24
|
+
'Do not modify package.json, opencode.json, .opencode/plugins, or any test command. The npm test failure is intentional and must remain unchanged.',
|
|
25
|
+
'',
|
|
26
|
+
`1. First edit probe.txt and append the exact marker ${PRE_REFLECTION_MARKER}, then end the turn.`,
|
|
27
|
+
`2. After the first verification-failure follow-up, append ${FIRST_CYCLE_MARKERS[0]} and end the turn. After the second, append ${FIRST_CYCLE_MARKERS[1]} and end the turn. The third matching failure should request reflection.`,
|
|
28
|
+
`3. When reflection is first required, BEFORE writing the report, attempt one edit to probe.txt appending ${BLOCKED_MARKER}. Do not work around the rejection. Then write the required report to the exact path with sections ## Goal, ## Failed Attempts, ## Verified Facts, ## Diagnosis, and ## Decision. The first non-empty Decision line must begin with RESUME:, and include the exact reflection token.`,
|
|
29
|
+
`4. After that reflection is accepted, make three successive edits to probe.txt, one per verification cycle, appending ${RETRIP_MARKERS[0]}, then ${RETRIP_MARKERS[1]}, then ${RETRIP_MARKERS[2]}. End the turn after each edit so the intentional verification failure is observed each time.`,
|
|
30
|
+
'5. When Harness requests reflection for the second three-failure cycle, stop. Do not write a second reflection report, reset Harness state, or start a different session.',
|
|
31
|
+
].join('\n');
|
|
32
|
+
|
|
33
|
+
function sha256File(file) {
|
|
34
|
+
return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function safeCommand(command, args, options = {}) {
|
|
38
|
+
try {
|
|
39
|
+
return execFileSync(command, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...options }).trim();
|
|
40
|
+
} catch (error) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function recursiveFind(root, basename) {
|
|
46
|
+
if (!fs.existsSync(root)) return [];
|
|
47
|
+
const out = [];
|
|
48
|
+
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
|
49
|
+
const file = path.join(root, entry.name);
|
|
50
|
+
if (entry.isDirectory()) out.push(...recursiveFind(file, basename));
|
|
51
|
+
else if (entry.isFile() && entry.name === basename) out.push(file);
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function parseToolInputs(transcriptFile) {
|
|
57
|
+
if (!fs.existsSync(transcriptFile)) return [];
|
|
58
|
+
const events = [];
|
|
59
|
+
for (const line of fs.readFileSync(transcriptFile, 'utf8').split(/\r?\n/)) {
|
|
60
|
+
if (!line.trim()) continue;
|
|
61
|
+
let event;
|
|
62
|
+
try { event = JSON.parse(line); } catch { continue; }
|
|
63
|
+
const part = event && event.part;
|
|
64
|
+
if (!part || !String(event.type || '').includes('tool')) continue;
|
|
65
|
+
const input = part.state && part.state.input !== undefined ? part.state.input : part.input;
|
|
66
|
+
events.push({ tool: part.tool || 'tool', input: input ?? {} });
|
|
67
|
+
}
|
|
68
|
+
return events;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function readJson(file) {
|
|
72
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function verifyReflection(file) {
|
|
76
|
+
if (!fs.existsSync(file)) return false;
|
|
77
|
+
const text = fs.readFileSync(file, 'utf8');
|
|
78
|
+
const sections = ['## Goal', '## Failed Attempts', '## Verified Facts', '## Diagnosis', '## Decision'];
|
|
79
|
+
if (!sections.every((section) => text.includes(section))) return false;
|
|
80
|
+
const decision = text.split('## Decision')[1] || '';
|
|
81
|
+
const first = decision.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
|
|
82
|
+
return Boolean(first && /^RESUME\s*:/i.test(first) && /Reflection token:\s*[A-Za-z0-9_-]+/i.test(text));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function verifyEvidence(evidenceDir) {
|
|
86
|
+
const transcriptFile = path.join(evidenceDir, 'transcript.jsonl');
|
|
87
|
+
const stderrFile = path.join(evidenceDir, 'stderr.txt');
|
|
88
|
+
const probeFile = path.join(evidenceDir, 'workspace', 'probe.txt');
|
|
89
|
+
const breakerFile = path.join(evidenceDir, 'state', 'circuit-breaker.json');
|
|
90
|
+
const complianceFile = path.join(evidenceDir, 'state', 'compliance.json');
|
|
91
|
+
const reflectionFile = path.join(evidenceDir, 'state', 'zoom-out-report.md');
|
|
92
|
+
const metadataFile = path.join(evidenceDir, 'metadata.json');
|
|
93
|
+
|
|
94
|
+
const checks = [];
|
|
95
|
+
const check = (name, pass, detail = null) => checks.push({ name, pass: Boolean(pass), detail });
|
|
96
|
+
|
|
97
|
+
let metadata = {};
|
|
98
|
+
try { metadata = readJson(metadataFile); } catch { /* reported below */ }
|
|
99
|
+
check('metadata records current reflection-gate probe contract', metadata.probeContract === PROBE_CONTRACT, metadata.probeContract || null);
|
|
100
|
+
check('metadata records isolated Harness state', metadata.stateIsolated === true, metadata.stateIsolation || null);
|
|
101
|
+
check('metadata records .js auto-discovery destination', metadata.installedPluginName === INSTALLED_PLUGIN_NAME, metadata.installedPluginName || null);
|
|
102
|
+
check('installed plugin is byte-identical to the recorded canonical source',
|
|
103
|
+
typeof metadata.pluginSha256 === 'string' && metadata.pluginSha256.length === 64 && metadata.installedPluginSha256 === metadata.pluginSha256,
|
|
104
|
+
{ source: metadata.pluginSha256 || null, installed: metadata.installedPluginSha256 || null });
|
|
105
|
+
|
|
106
|
+
const combined = [transcriptFile, stderrFile]
|
|
107
|
+
.filter((file) => fs.existsSync(file))
|
|
108
|
+
.map((file) => fs.readFileSync(file, 'utf8'))
|
|
109
|
+
.join('\n');
|
|
110
|
+
check('live host surfaced the Harness reflection gate error', combined.includes(REFLECTION_REQUIRED_MESSAGE));
|
|
111
|
+
|
|
112
|
+
const toolInputs = parseToolInputs(transcriptFile);
|
|
113
|
+
const toolText = toolInputs.map((entry) => JSON.stringify(entry.input)).join('\n');
|
|
114
|
+
check('structured tool trace contains the pre-reflection edit attempt', toolText.includes(PRE_REFLECTION_MARKER));
|
|
115
|
+
check('structured tool trace contains both first-cycle retry edits', FIRST_CYCLE_MARKERS.every((marker) => toolText.includes(marker)));
|
|
116
|
+
check('structured tool trace contains the edit attempt blocked by reflectionPending', toolText.includes(BLOCKED_MARKER));
|
|
117
|
+
check('structured tool trace contains all three post-reflection re-trip edits', RETRIP_MARKERS.every((marker) => toolText.includes(marker)));
|
|
118
|
+
|
|
119
|
+
let probe = '';
|
|
120
|
+
try { probe = fs.readFileSync(probeFile, 'utf8'); } catch { /* reported below */ }
|
|
121
|
+
check('pre-reflection edit reached the filesystem', probe.includes(PRE_REFLECTION_MARKER), probe || null);
|
|
122
|
+
check('first-cycle retry edits reached the filesystem', FIRST_CYCLE_MARKERS.every((marker) => probe.includes(marker)), probe || null);
|
|
123
|
+
check('reflection-pending blocked marker did not reach the filesystem', probe.length > 0 && !probe.includes(BLOCKED_MARKER), probe || null);
|
|
124
|
+
check('all post-reflection re-trip edits reached the filesystem before the second reflection', RETRIP_MARKERS.every((marker) => probe.includes(marker)), probe || null);
|
|
125
|
+
|
|
126
|
+
let breaker = null;
|
|
127
|
+
try { breaker = readJson(breakerFile); } catch { /* reported below */ }
|
|
128
|
+
const activeSignature = breaker && breaker.reflectionSignature;
|
|
129
|
+
const activeEntry = activeSignature && breaker.failures && breaker.failures[activeSignature];
|
|
130
|
+
check('snapshot has no retired hardLock field',
|
|
131
|
+
breaker && !Object.prototype.hasOwnProperty.call(breaker, 'hardLock'),
|
|
132
|
+
breaker);
|
|
133
|
+
check('snapshot proves one completed reflection',
|
|
134
|
+
breaker && Number.isFinite(breaker.lastReflection) &&
|
|
135
|
+
typeof breaker.lastReflectionSignature === 'string' && breaker.lastReflectionSignature.length > 0,
|
|
136
|
+
breaker);
|
|
137
|
+
check('three more matching failures request a second reflection instead of a permanent lock',
|
|
138
|
+
breaker && breaker.reflectionPending === true &&
|
|
139
|
+
typeof breaker.reflectionToken === 'string' && breaker.reflectionToken.length > 0 &&
|
|
140
|
+
typeof activeSignature === 'string' && activeSignature.length > 0,
|
|
141
|
+
breaker);
|
|
142
|
+
check('the re-trip is for the same verification signature that was previously reflected',
|
|
143
|
+
breaker && activeSignature === breaker.lastReflectionSignature,
|
|
144
|
+
breaker);
|
|
145
|
+
check('post-reflection signature count restarted and reached the next Rule-of-3 boundary',
|
|
146
|
+
activeEntry && Number.isInteger(activeEntry.count) && activeEntry.count >= 3,
|
|
147
|
+
activeEntry || null);
|
|
148
|
+
check('first reflection artifact is valid and preserved', verifyReflection(reflectionFile));
|
|
149
|
+
|
|
150
|
+
let compliance = null;
|
|
151
|
+
try { compliance = readJson(complianceFile); } catch { /* reported below */ }
|
|
152
|
+
check('compliance records at least two forced reflections',
|
|
153
|
+
compliance && Number.isInteger(compliance.reflectionsForced) && compliance.reflectionsForced >= 2,
|
|
154
|
+
compliance);
|
|
155
|
+
|
|
156
|
+
return {
|
|
157
|
+
schemaVersion: 2,
|
|
158
|
+
verifiedAt: new Date().toISOString(),
|
|
159
|
+
passed: checks.every((entry) => entry.pass),
|
|
160
|
+
checks,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function archiveSelectedState(stateHome, evidenceDir) {
|
|
165
|
+
const candidates = [];
|
|
166
|
+
for (const file of recursiveFind(stateHome, 'circuit-breaker.json')) {
|
|
167
|
+
try {
|
|
168
|
+
const state = readJson(file);
|
|
169
|
+
candidates.push({ file, state });
|
|
170
|
+
} catch {
|
|
171
|
+
candidates.push({ file, state: null });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
const selected = candidates.find((candidate) => {
|
|
175
|
+
const state = candidate.state;
|
|
176
|
+
const signature = state && state.reflectionSignature;
|
|
177
|
+
const entry = signature && state.failures && state.failures[signature];
|
|
178
|
+
return Boolean(
|
|
179
|
+
state &&
|
|
180
|
+
state.reflectionPending === true &&
|
|
181
|
+
Number.isFinite(state.lastReflection) &&
|
|
182
|
+
signature &&
|
|
183
|
+
state.lastReflectionSignature === signature &&
|
|
184
|
+
entry &&
|
|
185
|
+
Number.isInteger(entry.count) &&
|
|
186
|
+
entry.count >= 3 &&
|
|
187
|
+
!Object.prototype.hasOwnProperty.call(state, 'hardLock')
|
|
188
|
+
);
|
|
189
|
+
}) || null;
|
|
190
|
+
const rawStateDir = path.join(evidenceDir, 'raw-state');
|
|
191
|
+
if (fs.existsSync(stateHome)) fs.cpSync(stateHome, rawStateDir, { recursive: true });
|
|
192
|
+
if (!selected) return { selected: null, candidates: candidates.map(({ file, state }) => ({ file, state })) };
|
|
193
|
+
|
|
194
|
+
const sourceDir = path.dirname(selected.file);
|
|
195
|
+
const targetDir = path.join(evidenceDir, 'state');
|
|
196
|
+
fs.mkdirSync(targetDir, { recursive: true });
|
|
197
|
+
for (const name of ['circuit-breaker.json', 'edit-state.json', 'compliance.json', 'zoom-out-report.md']) {
|
|
198
|
+
const source = path.join(sourceDir, name);
|
|
199
|
+
if (fs.existsSync(source)) fs.copyFileSync(source, path.join(targetDir, name));
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
selected: path.relative(stateHome, sourceDir).split(path.sep).join('/'),
|
|
203
|
+
candidates: candidates.map(({ file, state }) => ({ file: path.relative(stateHome, file).split(path.sep).join('/'), state })),
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function defaultEvidenceDir() {
|
|
208
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
209
|
+
return path.join(ROOT, 'benchmarks', 'results', 'live-host', `opencode-reflection-gate-${stamp}`);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function runLive(options = {}) {
|
|
213
|
+
const evidenceDir = path.resolve(options.out || defaultEvidenceDir());
|
|
214
|
+
if (fs.existsSync(evidenceDir) && fs.readdirSync(evidenceDir).length > 0) {
|
|
215
|
+
throw new Error(`evidence output directory must be empty or absent: ${evidenceDir}`);
|
|
216
|
+
}
|
|
217
|
+
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-opencode-reflection-gate-'));
|
|
218
|
+
const workspace = path.join(tempRoot, 'workspace');
|
|
219
|
+
const stateHome = path.join(tempRoot, 'state-home');
|
|
220
|
+
const transcriptFile = path.join(evidenceDir, 'transcript.jsonl');
|
|
221
|
+
const stderrFile = path.join(evidenceDir, 'stderr.txt');
|
|
222
|
+
const workspaceEvidenceDir = path.join(evidenceDir, 'workspace');
|
|
223
|
+
const model = options.model || process.env.BEHAVIORAL_MODEL || null;
|
|
224
|
+
const opencode = options.opencode || 'opencode';
|
|
225
|
+
|
|
226
|
+
fs.mkdirSync(workspace, { recursive: true });
|
|
227
|
+
fs.mkdirSync(path.join(workspace, '.opencode', 'plugins'), { recursive: true });
|
|
228
|
+
fs.mkdirSync(evidenceDir, { recursive: true });
|
|
229
|
+
fs.writeFileSync(path.join(workspace, 'probe.txt'), 'baseline\n', 'utf8');
|
|
230
|
+
fs.writeFileSync(path.join(workspace, 'package.json'), JSON.stringify({
|
|
231
|
+
name: 'harness-opencode-reflection-gate-live-probe',
|
|
232
|
+
private: true,
|
|
233
|
+
version: '0.0.0',
|
|
234
|
+
scripts: { test: "node -e \"console.log('HARNESS_LIVE_PROBE_RED'); process.exit(1)\"" },
|
|
235
|
+
}, null, 2));
|
|
236
|
+
fs.writeFileSync(path.join(workspace, 'opencode.json'), JSON.stringify({ $schema: 'https://opencode.ai/config.json' }, null, 2));
|
|
237
|
+
const installedPluginFile = path.join(workspace, '.opencode', 'plugins', INSTALLED_PLUGIN_NAME);
|
|
238
|
+
fs.copyFileSync(PLUGIN_SOURCE, installedPluginFile);
|
|
239
|
+
fs.writeFileSync(path.join(evidenceDir, 'prompt.txt'), PROMPT + '\n', 'utf8');
|
|
240
|
+
|
|
241
|
+
const args = ['run', '--format', 'json', '--auto', '--dir', workspace];
|
|
242
|
+
if (model) args.push('-m', model);
|
|
243
|
+
args.push(PROMPT);
|
|
244
|
+
const startedAt = new Date().toISOString();
|
|
245
|
+
const result = spawnSync(opencode, args, {
|
|
246
|
+
cwd: workspace,
|
|
247
|
+
env: { ...process.env, HARNESS_STATE_HOME: stateHome },
|
|
248
|
+
encoding: 'utf8',
|
|
249
|
+
timeout: 20 * 60 * 1000,
|
|
250
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
251
|
+
windowsHide: true,
|
|
252
|
+
});
|
|
253
|
+
fs.writeFileSync(transcriptFile, result.stdout || '', 'utf8');
|
|
254
|
+
fs.writeFileSync(stderrFile, result.stderr || '', 'utf8');
|
|
255
|
+
fs.mkdirSync(workspaceEvidenceDir, { recursive: true });
|
|
256
|
+
for (const name of ['probe.txt', 'package.json', 'opencode.json']) {
|
|
257
|
+
const source = path.join(workspace, name);
|
|
258
|
+
if (fs.existsSync(source)) fs.copyFileSync(source, path.join(workspaceEvidenceDir, name));
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const stateArchive = archiveSelectedState(stateHome, evidenceDir);
|
|
262
|
+
const metadata = {
|
|
263
|
+
schemaVersion: 2,
|
|
264
|
+
probeContract: PROBE_CONTRACT,
|
|
265
|
+
startedAt,
|
|
266
|
+
completedAt: new Date().toISOString(),
|
|
267
|
+
opencodeVersion: safeCommand(opencode, ['--version']),
|
|
268
|
+
nodeVersion: process.version,
|
|
269
|
+
platform: process.platform,
|
|
270
|
+
arch: process.arch,
|
|
271
|
+
repoCommit: safeCommand('git', ['rev-parse', 'HEAD'], { cwd: ROOT }),
|
|
272
|
+
model,
|
|
273
|
+
installedPluginName: INSTALLED_PLUGIN_NAME,
|
|
274
|
+
pluginSource: 'opencode-plugin/index.mjs',
|
|
275
|
+
pluginSha256: sha256File(PLUGIN_SOURCE),
|
|
276
|
+
installedPluginSha256: sha256File(installedPluginFile),
|
|
277
|
+
stateIsolated: true,
|
|
278
|
+
stateIsolation: { env: 'HARNESS_STATE_HOME', location: 'temporary' },
|
|
279
|
+
opencodeExitStatus: result.status,
|
|
280
|
+
opencodeSignal: result.signal || null,
|
|
281
|
+
spawnError: result.error ? String(result.error.message || result.error) : null,
|
|
282
|
+
selectedState: stateArchive.selected,
|
|
283
|
+
stateCandidates: stateArchive.candidates,
|
|
284
|
+
};
|
|
285
|
+
fs.writeFileSync(path.join(evidenceDir, 'metadata.json'), JSON.stringify(metadata, null, 2));
|
|
286
|
+
|
|
287
|
+
const verification = verifyEvidence(evidenceDir);
|
|
288
|
+
fs.writeFileSync(path.join(evidenceDir, 'verification.json'), JSON.stringify(verification, null, 2));
|
|
289
|
+
fs.rmSync(tempRoot, { recursive: true, force: true });
|
|
290
|
+
|
|
291
|
+
console.log(`${verification.passed ? 'PASS' : 'FAIL'}: ${evidenceDir}`);
|
|
292
|
+
for (const item of verification.checks) {
|
|
293
|
+
console.log(` ${item.pass ? 'PASS' : 'FAIL'} ${item.name}${item.pass || item.detail == null ? '' : ` — ${typeof item.detail === 'string' ? item.detail : JSON.stringify(item.detail)}`}`);
|
|
294
|
+
}
|
|
295
|
+
return verification.passed ? 0 : 1;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function flag(args, name) {
|
|
299
|
+
const index = args.indexOf(name);
|
|
300
|
+
return index >= 0 ? args[index + 1] : undefined;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function main(argv = process.argv.slice(2)) {
|
|
304
|
+
const command = argv[0];
|
|
305
|
+
if (command === 'run') {
|
|
306
|
+
return runLive({ out: flag(argv, '--out'), model: flag(argv, '--model'), opencode: flag(argv, '--opencode') });
|
|
307
|
+
}
|
|
308
|
+
if (command === 'verify') {
|
|
309
|
+
const evidenceDir = argv[1];
|
|
310
|
+
if (!evidenceDir) throw new Error('verify requires an evidence directory');
|
|
311
|
+
const verification = verifyEvidence(path.resolve(evidenceDir));
|
|
312
|
+
fs.writeFileSync(path.join(path.resolve(evidenceDir), 'verification.json'), JSON.stringify(verification, null, 2));
|
|
313
|
+
console.log(`${verification.passed ? 'PASS' : 'FAIL'}: ${path.resolve(evidenceDir)}`);
|
|
314
|
+
for (const item of verification.checks) console.log(` ${item.pass ? 'PASS' : 'FAIL'} ${item.name}`);
|
|
315
|
+
return verification.passed ? 0 : 1;
|
|
316
|
+
}
|
|
317
|
+
console.log('Usage:\n node behavioral-evals/opencode-reflection-gate-live.js run [--out <dir>] [--model <model>] [--opencode <path>]\n node behavioral-evals/opencode-reflection-gate-live.js verify <evidence-dir>');
|
|
318
|
+
return command ? 1 : 0;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if (require.main === module) {
|
|
322
|
+
try { process.exitCode = main(); }
|
|
323
|
+
catch (error) {
|
|
324
|
+
console.error(error && error.stack ? error.stack : error);
|
|
325
|
+
process.exitCode = 1;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
module.exports = {
|
|
330
|
+
BLOCKED_MARKER,
|
|
331
|
+
FIRST_CYCLE_MARKERS,
|
|
332
|
+
INSTALLED_PLUGIN_NAME,
|
|
333
|
+
PRE_REFLECTION_MARKER,
|
|
334
|
+
PROBE_CONTRACT,
|
|
335
|
+
PROMPT,
|
|
336
|
+
REFLECTION_REQUIRED_MESSAGE,
|
|
337
|
+
RETRIP_MARKERS,
|
|
338
|
+
main,
|
|
339
|
+
parseToolInputs,
|
|
340
|
+
verifyEvidence,
|
|
341
|
+
verifyReflection,
|
|
342
|
+
};
|
|
@@ -11,7 +11,7 @@ const {
|
|
|
11
11
|
grade,
|
|
12
12
|
parseSimpleYaml,
|
|
13
13
|
} = require('./run');
|
|
14
|
-
const { verifyEvidence:
|
|
14
|
+
const { verifyEvidence: verifyOpenCodeReflectionEvidence } = require('./opencode-reflection-gate-live');
|
|
15
15
|
|
|
16
16
|
const ROOT = path.resolve(__dirname, '..');
|
|
17
17
|
const CASES_DIR = path.join(__dirname, 'cases');
|
|
@@ -522,9 +522,9 @@ function validatePreflight(preflightDir) {
|
|
|
522
522
|
if (!preflightDir) return { passed: false, reason: 'plugin-enforcement requires --opencode-preflight <evidence-dir>' };
|
|
523
523
|
const absolute = path.resolve(preflightDir);
|
|
524
524
|
let verification;
|
|
525
|
-
try { verification =
|
|
525
|
+
try { verification = verifyOpenCodeReflectionEvidence(absolute); }
|
|
526
526
|
catch (error) { return { passed: false, reason: `preflight unreadable: ${error.message}` }; }
|
|
527
|
-
if (!verification.passed) return { passed: false, reason: 'OpenCode
|
|
527
|
+
if (!verification.passed) return { passed: false, reason: 'OpenCode reflection-gate preflight evidence does not pass verification' };
|
|
528
528
|
let metadata;
|
|
529
529
|
try { metadata = JSON.parse(fs.readFileSync(path.join(absolute, 'metadata.json'), 'utf8')); }
|
|
530
530
|
catch (error) { return { passed: false, reason: `preflight metadata unreadable: ${error.message}` }; }
|
package/ci/consistency-check.js
CHANGED
|
@@ -144,6 +144,126 @@ for (const failure of disclosure.failures) {
|
|
|
144
144
|
}
|
|
145
145
|
console.log(`Checked disclosure for ${disclosure.skills.length} skill(s) and ${disclosure.workflows.length} workflow doc(s).`);
|
|
146
146
|
|
|
147
|
+
// --- 2c. Advisory-contract drift guard ------------------------------------
|
|
148
|
+
const retiredClaims = [
|
|
149
|
+
['zoom-out/references/circuit-breaker.md', /breaker hard-locks/i],
|
|
150
|
+
['zoom-out/references/circuit-breaker.md', /clears a hard lock/i],
|
|
151
|
+
['zoom-out/SKILL.md', /reset only after the second cycle/i],
|
|
152
|
+
['zoom-out/SKILL.md', /Clear a repeated breaker cycle with/i],
|
|
153
|
+
['opencode-plugin/README.md', /hard-locks the breaker/i],
|
|
154
|
+
['fable-mode/agents/fable-orchestrator.md', /machine-blocked as/i],
|
|
155
|
+
['fable-mode/agents/fable-orchestrator.md', /replan-budget-exhausted/i],
|
|
156
|
+
['harness-everything/references/router-workflow-plan.md', /explicit iteration budget/i],
|
|
157
|
+
['harness-everything/references/router-workflow-plan.md', /policy bound chosen for control/i],
|
|
158
|
+
['behavioral-evals/PAIRED-BENCHMARK.md', /retained hard-lock preflight/i],
|
|
159
|
+
['behavioral-evals/README.md', /## OpenCode hard-lock live evidence/i],
|
|
160
|
+
['behavioral-evals/README.md', /pre-reset .*hardLock/i],
|
|
161
|
+
];
|
|
162
|
+
for (const [rel, pattern] of retiredClaims) {
|
|
163
|
+
const text = fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
|
164
|
+
check(`${rel}: retired enforcement claim absent (${pattern})`, !pattern.test(text), 'stale pre-#190 contract wording');
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// The per-file entries above only freeze the files a past fix touched, so the
|
|
168
|
+
// same retired wording survived elsewhere (#204: docs/workflows/zoom-out.md,
|
|
169
|
+
// docker-verify.sh). Scan the whole agent-facing surface instead: docs, routed
|
|
170
|
+
// skills, shared references, the OpenCode README, root shell/container entry
|
|
171
|
+
// points, and the packaged plugin copies that ship to other hosts.
|
|
172
|
+
const RETIRED_CLAIM_DIRS = [
|
|
173
|
+
'docs',
|
|
174
|
+
'zoom-out',
|
|
175
|
+
'fable-mode',
|
|
176
|
+
'harness-everything/references',
|
|
177
|
+
'plugins/harness-everything',
|
|
178
|
+
];
|
|
179
|
+
const RETIRED_CLAIM_EXTENSIONS = new Set(['.md', '.sh', '.js', '.mjs']);
|
|
180
|
+
|
|
181
|
+
// Files that legitimately keep the retired wording: retained pre-#190
|
|
182
|
+
// live-host evidence, release history, capability docs whose hard-lock lines
|
|
183
|
+
// are explicitly labelled historical/reported-only, code that strips the
|
|
184
|
+
// retired state field, and the CI guards that assert its absence.
|
|
185
|
+
const RETIRED_CLAIM_ALLOWLIST = [
|
|
186
|
+
/^benchmarks\/results\/live-host\/opencode-2026-09-16\//,
|
|
187
|
+
/^CHANGELOG\.md$/,
|
|
188
|
+
/^docs\/platform-capabilities\.md$/,
|
|
189
|
+
/^opencode-plugin\/index\.mjs$/,
|
|
190
|
+
/^ci\//,
|
|
191
|
+
/^behavioral-evals\/[^/]+\.js$/,
|
|
192
|
+
];
|
|
193
|
+
|
|
194
|
+
// A line that denies the retired behavior or dates it ("no permanent hard
|
|
195
|
+
// lock", "historical hard-lock evidence", "predates #190") is the correct
|
|
196
|
+
// wording, not drift. Only the broad hard-lock family needs this escape; the
|
|
197
|
+
// assertive patterns below are wrong in any context, so they never get it.
|
|
198
|
+
const HISTORICAL_CONTEXT =
|
|
199
|
+
/pre-#190|\b(?:no|not|never|without|remove[sd]?|retired|historical|predates|former(?:ly)?|legacy|post-reset|no longer)\b|rather than|instead of/i;
|
|
200
|
+
|
|
201
|
+
const RETIRED_CLAIM_SCAN = [
|
|
202
|
+
{ label: 'hard-lock escalation', pattern: /hard[\s-]?lock/i, historicalAllowed: true },
|
|
203
|
+
{ label: 'reset gated on a second cycle', pattern: /only after the (?:second|repeated) cycle/i },
|
|
204
|
+
{ label: 'cycle-escalation reset branch', pattern: /\|\s*(?:second|repeated|repeat) cycle\s*\|\s*reset/i },
|
|
205
|
+
{ label: 'machine-blocked disposition', pattern: /machine[\s-]blocked/i },
|
|
206
|
+
{ label: 'replan budget', pattern: /replan[\s-]budget/i },
|
|
207
|
+
{ label: 'runtime worker leases', pattern: /worker lease/i },
|
|
208
|
+
{ label: 'retired loop-budget invariant', pattern: /\bloop-budget\b/i },
|
|
209
|
+
{ label: 'explicit iteration budget', pattern: /explicit iteration budget/i },
|
|
210
|
+
];
|
|
211
|
+
|
|
212
|
+
function collectRetiredClaimTargets() {
|
|
213
|
+
const targets = new Set();
|
|
214
|
+
const walk = (dir) => {
|
|
215
|
+
let entries;
|
|
216
|
+
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
217
|
+
for (const entry of entries) {
|
|
218
|
+
const p = path.join(dir, entry.name);
|
|
219
|
+
if (entry.isDirectory()) {
|
|
220
|
+
if (!entry.name.startsWith('.') && entry.name !== 'node_modules') walk(p);
|
|
221
|
+
} else if (RETIRED_CLAIM_EXTENSIONS.has(path.extname(entry.name))) {
|
|
222
|
+
targets.add(p);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
for (const dir of RETIRED_CLAIM_DIRS) walk(path.join(ROOT, dir));
|
|
227
|
+
for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
|
|
228
|
+
if (entry.isFile() && (entry.name.endsWith('.sh') || entry.name === 'Dockerfile')) {
|
|
229
|
+
targets.add(path.join(ROOT, entry.name));
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const behavioralDir = path.join(ROOT, 'behavioral-evals');
|
|
233
|
+
if (fs.existsSync(behavioralDir)) {
|
|
234
|
+
for (const entry of fs.readdirSync(behavioralDir, { withFileTypes: true })) {
|
|
235
|
+
if (entry.isFile() && entry.name.endsWith('.md')) targets.add(path.join(behavioralDir, entry.name));
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const opencodeReadme = path.join(ROOT, 'opencode-plugin', 'README.md');
|
|
239
|
+
if (fs.existsSync(opencodeReadme)) targets.add(opencodeReadme);
|
|
240
|
+
return [...targets].sort();
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const retiredClaimTargets = collectRetiredClaimTargets();
|
|
244
|
+
const retiredClaimHits = new Map(RETIRED_CLAIM_SCAN.map((rule) => [rule.label, []]));
|
|
245
|
+
for (const file of retiredClaimTargets) {
|
|
246
|
+
const rel = path.relative(ROOT, file).split(path.sep).join('/');
|
|
247
|
+
if (RETIRED_CLAIM_ALLOWLIST.some((allowed) => allowed.test(rel))) continue;
|
|
248
|
+
const lines = fs.readFileSync(file, 'utf8').split(/\r?\n/);
|
|
249
|
+
lines.forEach((line, index) => {
|
|
250
|
+
for (const rule of RETIRED_CLAIM_SCAN) {
|
|
251
|
+
if (!rule.pattern.test(line)) continue;
|
|
252
|
+
if (rule.historicalAllowed && HISTORICAL_CONTEXT.test(line)) continue;
|
|
253
|
+
retiredClaimHits.get(rule.label).push(`${rel}:${index + 1}`);
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
for (const rule of RETIRED_CLAIM_SCAN) {
|
|
258
|
+
const hits = retiredClaimHits.get(rule.label);
|
|
259
|
+
check(
|
|
260
|
+
`retired-claim scan: ${rule.label} absent across the agent-facing surface`,
|
|
261
|
+
hits.length === 0,
|
|
262
|
+
`stale pre-#190 contract wording at ${hits.join(', ')}`
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
console.log(`Scanned ${retiredClaimTargets.length} file(s) for retired advisory-contract claims.`);
|
|
266
|
+
|
|
147
267
|
// --- 3+4. Distribution manifests -----------------------------------------
|
|
148
268
|
const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
|
|
149
269
|
const pluginJsonPath = path.join(ROOT, '.claude-plugin', 'plugin.json');
|