harness-everything 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +16 -0
- package/ci/fixtures/doc-obligations/reconciliation-prep/cases.json +70 -11
- package/ci/fixtures/doc-obligations/reconciliation-prep/pending-red.json +24 -556
- package/ci/fixtures/doc-obligations/reconciliation-s1/cases.json +6842 -0
- package/ci/fixtures/doc-obligations/reconciliation-s1/fail-session-writes.js +116 -0
- package/ci/lib/doc-reconciliation-prep-fixtures.js +116 -10
- package/ci/mechanism-325-3b-preparation.test.js +8 -8
- package/ci/mechanism-325-3b-s1-review.test.js +831 -0
- package/ci/mechanism-325-3b-s1.test.js +188 -0
- package/ci/prep/doc-reconciliation-red.js +2 -1
- package/docs/doc-workflow-reconciliation.md +217 -26
- package/docs/document-lifecycle.md +4 -4
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/doc-lifecycle.js +1 -1
- package/hooks/scripts/lib/doc-reconciliation-pilot.js +152 -0
- package/hooks/scripts/lib/workflow-obligations.js +44 -0
- package/hooks/scripts/workflow-disposition.js +2 -0
- package/hooks/scripts/workflow-stop-gate.js +24 -1
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-pilot.js +152 -0
- package/plugins/harness-everything/hooks/scripts/lib/workflow-obligations.js +44 -0
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +2 -0
- package/plugins/harness-everything/hooks/scripts/workflow-stop-gate.js +24 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/doc-lifecycle.js +1 -1
- package/sbom.cdx.json +7 -7
|
@@ -0,0 +1,831 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// #325 PR #351 review P2: withholding persistence must apply the credit decision in memory first (RED).
|
|
5
|
+
//
|
|
6
|
+
// Contract under test (docs/doc-workflow-reconciliation.md, "Persistence failure (PR #351 review P2)"):
|
|
7
|
+
// when the S1 pilot withholds credit at a boundary, the withholding decision is applied in memory first
|
|
8
|
+
// (aggregate verify becomes pending, a satisfied workflow becomes running, step:<verify>:pending is
|
|
9
|
+
// reported) and persisting that state is best-effort afterwards. A failed obligations-file write adds the
|
|
10
|
+
// diagnostic withhold-persist-failed and a reminder; it never escapes, never leaves persisted
|
|
11
|
+
// satisfied/PASS counting as credit, and never makes Stop exit non-zero.
|
|
12
|
+
//
|
|
13
|
+
// The seam is hooks/scripts/lib/workflow-obligations.js (withholdPilotVerification): it writes
|
|
14
|
+
// workflow-obligations.json through ./fable-contracts atomicWriteJson before flipping workflow.state. Each
|
|
15
|
+
// leg (source hooks/scripts and packaged plugins/harness-everything/hooks/scripts) is stubbed in require.cache
|
|
16
|
+
// — atomicWriteJson is replaced before that leg's workflow-obligations.js is required, so the stub throws on
|
|
17
|
+
// the obligations-file write only; the pilot report write still succeeds through the same stub. The context is
|
|
18
|
+
// an OFF-seeded satisfied workflow in a temporary git repository whose recorded scope is governed (the pilot
|
|
19
|
+
// baseline is captured through the runtime's own captureBaseline seam), assessed with the switch env exactly 1.
|
|
20
|
+
//
|
|
21
|
+
// This file grants no pilot credit: a FAIL is a real gap (exception escapes, state stays satisfied, the
|
|
22
|
+
// diagnostic is absent); PASS/FAIL/INFRA classification follows the mechanism suite convention
|
|
23
|
+
// (0 pass, 1 failed assertion, 2 infrastructure).
|
|
24
|
+
|
|
25
|
+
const fs = require('fs');
|
|
26
|
+
const os = require('os');
|
|
27
|
+
const path = require('path');
|
|
28
|
+
const { execFileSync } = require('child_process');
|
|
29
|
+
|
|
30
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
31
|
+
const SWITCH = 'HARNESS_DOC_RECONCILIATION_PILOT';
|
|
32
|
+
const OBLIGATION_FILE = 'workflow-obligations.json';
|
|
33
|
+
const PILOT_REPORT_FILE = 'doc-reconciliation-pilot.json';
|
|
34
|
+
const VERIFY_STEP_ID = 'req-verify';
|
|
35
|
+
|
|
36
|
+
const LEGS = [
|
|
37
|
+
{ id: 'source', lib: path.join(ROOT, 'hooks', 'scripts', 'lib') },
|
|
38
|
+
{ id: 'packaged', lib: path.join(ROOT, 'plugins', 'harness-everything', 'hooks', 'scripts', 'lib') },
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
const GOVERNED_CONFIG = '{\n "governed": [\n { "path": "docs/adr/**", "role": "adr" }\n ],\n "domains": ["GRPH"]\n}\n';
|
|
42
|
+
const ADR_BASE = '# GRPH-001 Checkout retry\n\nStatus: accepted\n\nCheckout MUST retry once.\n';
|
|
43
|
+
const ADR_CANDIDATE = '# GRPH-001 Checkout retry\n\nStatus: accepted\n\nCheckout MUST retry twice.\n';
|
|
44
|
+
const REQUIREMENTS = JSON.stringify([
|
|
45
|
+
{ id: 'req-docs', stepType: 'behavior-change', summary: 'Update the ADR text', acceptance: 'normative ADR text is reviewed' },
|
|
46
|
+
{ id: VERIFY_STEP_ID, stepType: 'verification', summary: 'Verify the ADR change', acceptance: 'objective doc checks are recorded' },
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
let passed = 0;
|
|
50
|
+
let failed = 0;
|
|
51
|
+
let infra = 0;
|
|
52
|
+
|
|
53
|
+
function record(status, id, detail) {
|
|
54
|
+
if (status === 'PASS') passed += 1;
|
|
55
|
+
else if (status === 'FAIL') failed += 1;
|
|
56
|
+
else infra += 1;
|
|
57
|
+
console.log(` ${status} ${id}${detail ? ` — ${detail}` : ''}`);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function readJsonOrNull(file) {
|
|
61
|
+
try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch (_) { return null; }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function git(cwd, args) {
|
|
65
|
+
const env = { ...process.env };
|
|
66
|
+
for (const key of Object.keys(env)) if (key.startsWith('GIT_')) delete env[key];
|
|
67
|
+
env.GIT_AUTHOR_DATE = '2026-04-01T12:00:00Z';
|
|
68
|
+
env.GIT_COMMITTER_DATE = '2026-04-01T12:00:00Z';
|
|
69
|
+
return execFileSync('git', ['-C', cwd, ...args], { env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function writeFile(dir, relative, content) {
|
|
73
|
+
const target = path.join(dir, relative);
|
|
74
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
75
|
+
fs.writeFileSync(target, content, 'utf8');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// A temporary git repository with a governed document diff, mirroring the fixture materialisation:
|
|
79
|
+
// the baseline definitions are committed, then the candidate ADR becomes the working tree, so the
|
|
80
|
+
// recorded scope is governed while the workflow itself was seeded satisfied without the pilot.
|
|
81
|
+
function buildGovernedWorkspace(dir) {
|
|
82
|
+
writeFile(dir, '.agents/harness-everything/project-docs.json', GOVERNED_CONFIG);
|
|
83
|
+
writeFile(dir, 'docs/adr/GRPH-001.md', ADR_BASE);
|
|
84
|
+
git(dir, ['init', '-q']);
|
|
85
|
+
git(dir, ['add', '-A']);
|
|
86
|
+
git(dir, ['-c', 'user.name=harness-review', '-c', 'user.email=harness-review@example.com', '-c', 'commit.gpgsign=false', 'commit', '-q', '--no-verify', '-m', 'baseline']);
|
|
87
|
+
writeFile(dir, 'docs/adr/GRPH-001.md', ADR_CANDIDATE);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Installs the stub in require.cache before that leg's workflow-obligations.js is required, then loads
|
|
91
|
+
// the leg fresh. armed=0 keeps writes succeeding (control); armed=1 throws on the obligations-file
|
|
92
|
+
// write only and stashes the payload the withholding decision tried to persist. Returns a restore handle.
|
|
93
|
+
function installLeg(leg) {
|
|
94
|
+
const contractsFile = path.join(leg.lib, 'fable-contracts.js');
|
|
95
|
+
const state = { armed: false, lastFailedWrite: null };
|
|
96
|
+
const cached = new Map();
|
|
97
|
+
for (const key of Object.keys(require.cache)) {
|
|
98
|
+
if (key === leg.lib || key.startsWith(leg.lib + path.sep)) {
|
|
99
|
+
cached.set(key, require.cache[key]);
|
|
100
|
+
delete require.cache[key];
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
const contracts = require(contractsFile);
|
|
104
|
+
const originalWrite = contracts.atomicWriteJson;
|
|
105
|
+
contracts.atomicWriteJson = (target, value) => {
|
|
106
|
+
if (state.armed && path.basename(String(target)) === OBLIGATION_FILE) {
|
|
107
|
+
state.lastFailedWrite = value;
|
|
108
|
+
throw new Error('simulated obligations-file write failure (PR #351 review P2)');
|
|
109
|
+
}
|
|
110
|
+
return originalWrite(target, value);
|
|
111
|
+
};
|
|
112
|
+
const restore = () => {
|
|
113
|
+
contracts.atomicWriteJson = originalWrite;
|
|
114
|
+
for (const key of Object.keys(require.cache)) {
|
|
115
|
+
if (key === leg.lib || key.startsWith(leg.lib + path.sep)) delete require.cache[key];
|
|
116
|
+
}
|
|
117
|
+
for (const [key, module] of cached) require.cache[key] = module;
|
|
118
|
+
};
|
|
119
|
+
try {
|
|
120
|
+
const obligations = require(path.join(leg.lib, 'workflow-obligations.js'));
|
|
121
|
+
return { state, restore, obligations, contracts };
|
|
122
|
+
} catch (error) {
|
|
123
|
+
restore();
|
|
124
|
+
throw error;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Builds an ON context with an OFF-seeded satisfied workflow whose scope is governed, through the leg's own
|
|
129
|
+
// planning API; the pilot baseline is captured through captureBaseline. Must never fault behind the assertions.
|
|
130
|
+
function seedLegScenario(leg, handle, dir) {
|
|
131
|
+
const plan = { strategySelection: 'selected', tier: 'tier2', strategy: 'iterative-single' };
|
|
132
|
+
const sessionDir = path.join(dir, 'session');
|
|
133
|
+
fs.mkdirSync(sessionDir, { recursive: true });
|
|
134
|
+
buildGovernedWorkspace(dir);
|
|
135
|
+
const context = {
|
|
136
|
+
sessionId: 's1-review-persist-' + leg.id,
|
|
137
|
+
sessionDir,
|
|
138
|
+
root: dir,
|
|
139
|
+
workflow: {
|
|
140
|
+
workflowId: 'wf-325b1-review-' + leg.id,
|
|
141
|
+
revision: 1,
|
|
142
|
+
state: 'satisfied',
|
|
143
|
+
strategy: 'iterative-single',
|
|
144
|
+
workflowPlan: plan,
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
handle.obligations.initializePlanningContract(context, plan);
|
|
148
|
+
handle.obligations.recordPlanning(context, REQUIREMENTS, plan, 'iterative-single',
|
|
149
|
+
'two dependent requirements fit a bounded documentation loop');
|
|
150
|
+
handle.obligations.materializeExecutionObligations(context, plan);
|
|
151
|
+
const seeded = handle.obligations.loadObligationSet(context, plan).set;
|
|
152
|
+
if (!seeded) throw new Error(`[${leg.id}] seeding failed: obligation set invalid after planning`);
|
|
153
|
+
for (const step of seeded.requirements) {
|
|
154
|
+
step.status = 'pass';
|
|
155
|
+
step.evidence = 'seeded-off-pass';
|
|
156
|
+
for (const binding of [...step.requiredBindings, ...step.optionalBindings]) binding.status = 'loaded';
|
|
157
|
+
}
|
|
158
|
+
for (const obligation of seeded.obligations) {
|
|
159
|
+
obligation.status = 'pass';
|
|
160
|
+
obligation.evidence = 'seeded-off-pass';
|
|
161
|
+
obligation.reasonCode = null;
|
|
162
|
+
obligation.scope = null;
|
|
163
|
+
}
|
|
164
|
+
seeded.activeStepId = handle.obligations.activeStepSnapshot(seeded)?.id || null;
|
|
165
|
+
handle.contracts.atomicWriteJson(handle.obligations.obligationPath(sessionDir), seeded);
|
|
166
|
+
const pilot = require(path.join(leg.lib, 'doc-reconciliation-pilot.js'));
|
|
167
|
+
pilot.captureBaseline(context);
|
|
168
|
+
if (!context.workflow.docReconciliationBaseline) {
|
|
169
|
+
throw new Error(`[${leg.id}] seeding failed: captureBaseline recorded no baseline`);
|
|
170
|
+
}
|
|
171
|
+
context.workflow.state = 'satisfied';
|
|
172
|
+
return { context, plan };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function withCapturedStderr(fn) {
|
|
176
|
+
const chunks = [];
|
|
177
|
+
const originalWrite = process.stderr.write;
|
|
178
|
+
process.stderr.write = function (chunk, ...rest) {
|
|
179
|
+
chunks.push(typeof chunk === 'string' ? chunk : String(chunk));
|
|
180
|
+
return originalWrite.call(process.stderr, chunk, ...rest);
|
|
181
|
+
};
|
|
182
|
+
try {
|
|
183
|
+
return { value: fn(), stderr: chunks.join('') };
|
|
184
|
+
} finally {
|
|
185
|
+
process.stderr.write = originalWrite;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// One scenario: seed, arm, call, and the four observation shapes. Returns { error, unresolved, stderr } —
|
|
190
|
+
// never throws except for infrastructure (seed failures), which the caller records as INFRA.
|
|
191
|
+
function runLegScenario(leg, mode) {
|
|
192
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-review-'));
|
|
193
|
+
let handle = null;
|
|
194
|
+
try {
|
|
195
|
+
handle = installLeg(leg);
|
|
196
|
+
const seeded = seedLegScenario(leg, handle, dir);
|
|
197
|
+
const { context, plan } = seeded;
|
|
198
|
+
handle.state.armed = mode === 'fail-write';
|
|
199
|
+
let thrown = null;
|
|
200
|
+
let unresolved;
|
|
201
|
+
let stderr = '';
|
|
202
|
+
const call = withCapturedStderr(() => {
|
|
203
|
+
try { return handle.obligations.unresolvedObligations(context); }
|
|
204
|
+
catch (error) { thrown = error; return null; }
|
|
205
|
+
});
|
|
206
|
+
stderr = call.stderr;
|
|
207
|
+
unresolved = call.value;
|
|
208
|
+
handle.state.armed = false;
|
|
209
|
+
// Capture the persisted state before the temporary directory is cleaned up. In the control leg the
|
|
210
|
+
// write succeeded, so disk reflects the in-memory decision; in the fail-write leg the failed write is
|
|
211
|
+
// observed through the stub's stash of the exact in-memory payload it was asked to persist.
|
|
212
|
+
const verifyOnDisk = (readJsonOrNull(path.join(context.sessionDir, OBLIGATION_FILE))?.obligations || [])
|
|
213
|
+
.find(item => item.id === 'verify') || null;
|
|
214
|
+
const reportDiagnostics = readJsonOrNull(path.join(context.sessionDir, PILOT_REPORT_FILE))?.diagnostics || null;
|
|
215
|
+
return {
|
|
216
|
+
panicked: thrown && { message: thrown.message, stack: String(thrown.stack || '').split('\n').slice(0, 4).join(' | ') } || null,
|
|
217
|
+
unresolved, context, plan, stderr, reportDiagnostics, verifyOnDisk, loaded: handle.state.lastFailedWrite,
|
|
218
|
+
};
|
|
219
|
+
} finally {
|
|
220
|
+
if (handle) handle.restore();
|
|
221
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function diagnosePersistence(leg, mode, scenario) {
|
|
226
|
+
const tag = `${leg.id}.${mode}`;
|
|
227
|
+
if (!scenario || !scenario.context) {
|
|
228
|
+
record('INFRA', `${tag}.scenario`, 'seed or load failed; scenario not observable');
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
const verifyOnDisk = scenario.verifyOnDisk || null;
|
|
232
|
+
const verifyInMemory = scenario.loaded
|
|
233
|
+
? (scenario.loaded.obligations?.find(item => item.id === 'verify') || null)
|
|
234
|
+
: verifyOnDisk;
|
|
235
|
+
const diagnosticSeen = String(scenario.stderr).includes('withhold-persist-failed')
|
|
236
|
+
|| (Array.isArray(scenario.reportDiagnostics) && scenario.reportDiagnostics.includes('withhold-persist-failed'));
|
|
237
|
+
record(scenario.panicked === null ? 'PASS' : 'FAIL', `${tag}.no-escape`,
|
|
238
|
+
scenario.panicked ? `exception escaped unresolvedObligations: ${scenario.panicked.message}` : '');
|
|
239
|
+
record(scenario.context.workflow.state === 'running' ? 'PASS' : 'FAIL', `${tag}.state-running`,
|
|
240
|
+
`workflow.state ${JSON.stringify(scenario.context.workflow.state)} where "running" desired`);
|
|
241
|
+
record(Array.isArray(scenario.unresolved) && scenario.unresolved.includes('step:' + VERIFY_STEP_ID + ':pending') ? 'PASS' : 'FAIL',
|
|
242
|
+
`${tag}.step-pending-reported`,
|
|
243
|
+
`unresolved ${JSON.stringify(scenario.unresolved)} lacks step:${VERIFY_STEP_ID}:pending`);
|
|
244
|
+
record(verifyInMemory?.status === 'pending' ? 'PASS' : 'FAIL', `${tag}.verify-pending`,
|
|
245
|
+
`verify obligation ${JSON.stringify(verifyInMemory?.status)} where "pending" desired`);
|
|
246
|
+
if (mode !== 'control') {
|
|
247
|
+
record(diagnosticSeen ? 'PASS' : 'FAIL', `${tag}.persist-failed-diagnostic`,
|
|
248
|
+
'withhold-persist-failed diagnostic absent from stderr and pilot report');
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
console.log('=== #325 PR #351 review P2: withholding persistence applied in memory first (RED) ===');
|
|
253
|
+
|
|
254
|
+
const switchSaved = process.env[SWITCH];
|
|
255
|
+
const harnessEnvSaved = {};
|
|
256
|
+
for (const key of Object.keys(process.env)) {
|
|
257
|
+
if (key.startsWith('HARNESS_') && key !== SWITCH) {
|
|
258
|
+
harnessEnvSaved[key] = process.env[key];
|
|
259
|
+
delete process.env[key];
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
process.env[SWITCH] = '1';
|
|
263
|
+
|
|
264
|
+
try {
|
|
265
|
+
for (const leg of LEGS) {
|
|
266
|
+
for (const mode of ['fail-write', 'control']) {
|
|
267
|
+
let scenario = null;
|
|
268
|
+
try {
|
|
269
|
+
scenario = runLegScenario(leg, mode);
|
|
270
|
+
} catch (error) {
|
|
271
|
+
record('INFRA', `${leg.id}.${mode}.scenario`, error.message.split('\n')[0]);
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
diagnosePersistence(leg, mode, scenario);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
} finally {
|
|
278
|
+
if (switchSaved === undefined) delete process.env[SWITCH];
|
|
279
|
+
else process.env[SWITCH] = switchSaved;
|
|
280
|
+
for (const [key, value] of Object.entries(harnessEnvSaved)) process.env[key] = value;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const persistenceCounts = { passed, failed, infra };
|
|
284
|
+
|
|
285
|
+
console.log(`\nP2 persistence review summary: ${persistenceCounts.passed} PASS, ${persistenceCounts.failed} FAIL, ${persistenceCounts.infra} INFRA across ${LEGS.length} legs (pilotCredit: false)`);
|
|
286
|
+
|
|
287
|
+
// ---- R1f section: "Unwritable session store (PR #351 follow-up)" — integration through the real Stop entry point ----
|
|
288
|
+
//
|
|
289
|
+
// Both obligations and SaveWorkflow persistence are made to fail for one Stop invocation, by preloading
|
|
290
|
+
// fail-session-writes.js onto the real stop-gate child through a real argv element (`node --require <path>`).
|
|
291
|
+
// The preload makes every
|
|
292
|
+
// write/rename/open-for-write/unlink inside that run's session directory throw EACCES (patching fs synchron,
|
|
293
|
+
// fs.promises and the callback wrappers) without chmod, and nothing else. Contract (docs table):
|
|
294
|
+
// Stop exits 0, the Unresolved-workflow-evidence reminder with step:<verify>:pending and a
|
|
295
|
+
// withhold-persist-failed diagnostic are still emitted after the failed saveWorkflow, no uncaught exception
|
|
296
|
+
// text appears, the stored files stay byte-identical (the write really failed), and the stored satisfied/PASS
|
|
297
|
+
// is never confused for credit by an ON disposition attempt. Only a completely unwritable store is under test:
|
|
298
|
+
// the in-memory downgrade is explicitly not counted as durable invalidation.
|
|
299
|
+
//
|
|
300
|
+
// R1g (PR #351 re-review, "Scope of the recovery") addenda in this section:
|
|
301
|
+
// - P2a legacy oracle: with the switch OFF (satisfied workflow, legacy pending token) and for a
|
|
302
|
+
// Fable-strategy workflow ON (unsupported ⇒ legacy path), a failing saveWorkflow() must keep the
|
|
303
|
+
// pre-S1 Stop behaviour: the same exit code and stderr as the pre-PR Stop file run over the same
|
|
304
|
+
// scenario, and no withhold-persist-failed diagnostic.
|
|
305
|
+
// - P2b oracle gap: the exit-2 disposition probe is renamed legacy-closed-guard-exit-2 so it is no
|
|
306
|
+
// longer counted as pilot evidence; a pilot-specific ON-Stop-rerun oracle plus an OFF negative
|
|
307
|
+
// control carry the pilot claim instead.
|
|
308
|
+
|
|
309
|
+
const { spawnSync: r1fSpawnSync } = require('child_process');
|
|
310
|
+
const R1F_PRELOAD = path.join(ROOT, 'ci', 'fixtures', 'doc-obligations', 'reconciliation-s1', 'fail-session-writes.js');
|
|
311
|
+
const R1F_SESSION_ENV = 'HARNESS_S1_REVIEW_SESSION_DIR';
|
|
312
|
+
const R1F_WORKFLOW_FILE = 'workflow-run.json';
|
|
313
|
+
|
|
314
|
+
const R1F_LEGS = [
|
|
315
|
+
{
|
|
316
|
+
id: 'source',
|
|
317
|
+
root: ROOT,
|
|
318
|
+
kernel: path.join(ROOT, 'harness-everything', 'scripts', 'kernel-router.js'),
|
|
319
|
+
controller: path.join(ROOT, 'hooks', 'scripts', 'workflow-disposition.js'),
|
|
320
|
+
stopGate: path.join(ROOT, 'hooks', 'scripts', 'workflow-stop-gate.js'),
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
id: 'packaged',
|
|
324
|
+
root: path.join(ROOT, 'plugins', 'harness-everything'),
|
|
325
|
+
kernel: path.join(ROOT, 'plugins', 'harness-everything', 'skills', 'harness-everything', 'scripts', 'kernel-router.js'),
|
|
326
|
+
controller: path.join(ROOT, 'plugins', 'harness-everything', 'hooks', 'scripts', 'workflow-disposition.js'),
|
|
327
|
+
stopGate: path.join(ROOT, 'plugins', 'harness-everything', 'hooks', 'scripts', 'workflow-stop-gate.js'),
|
|
328
|
+
},
|
|
329
|
+
];
|
|
330
|
+
|
|
331
|
+
function r1fScrub(text, tokens) {
|
|
332
|
+
let out = String(text ?? '');
|
|
333
|
+
for (const token of tokens) out = out.split(token).join('<tmp>');
|
|
334
|
+
return out;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function r1fBuildWorkspace(dir) {
|
|
338
|
+
writeFile(dir, '.agents/harness-everything/project-docs.json', GOVERNED_CONFIG);
|
|
339
|
+
writeFile(dir, 'docs/adr/GRPH-001.md', ADR_BASE);
|
|
340
|
+
git(dir, ['init', '-q']);
|
|
341
|
+
git(dir, ['add', '-A']);
|
|
342
|
+
git(dir, ['-c', 'user.name=harness-review', '-c', 'user.email=harness-review@example.com', '-c', 'commit.gpgsign=false', 'commit', '-q', '--no-verify', '-m', 'baseline']);
|
|
343
|
+
writeFile(dir, 'docs/adr/GRPH-001.md', ADR_CANDIDATE);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Isolated child environment mirroring the replay helper: no inherited HARNESS switches, explicit state
|
|
347
|
+
// home and workspace root; the pilot switch is added only where named. NODE_OPTIONS is cleared so an
|
|
348
|
+
// inherited value from the parent cannot add or strip the preload.
|
|
349
|
+
function r1fChildEnv(dir, stateHome, options) {
|
|
350
|
+
const env = { ...process.env };
|
|
351
|
+
for (const key of Object.keys(env)) if (key.startsWith('HARNESS_')) delete env[key];
|
|
352
|
+
delete env.CLAUDE_PLUGIN_ROOT;
|
|
353
|
+
delete env.PLUGIN_ROOT;
|
|
354
|
+
delete env.NODE_OPTIONS;
|
|
355
|
+
env.HARNESS_STATE_HOME = stateHome;
|
|
356
|
+
env.HARNESS_WORKSPACE_ROOT = dir;
|
|
357
|
+
if (options.pilot !== undefined) env[SWITCH] = String(options.pilot);
|
|
358
|
+
if (options.sessionDirectory) env[R1F_SESSION_ENV] = options.sessionDirectory;
|
|
359
|
+
return env;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// The preload is passed as a real argv element (`node --require <path> <script>`), never through
|
|
363
|
+
// NODE_OPTIONS: option values there are tokenised at spaces, so a checkout or temp directory with a
|
|
364
|
+
// space in its name would demand a truncated module path. argv elements need no quoting.
|
|
365
|
+
function r1fSpawnPreloadArgs(options) {
|
|
366
|
+
if (!options.preload) return [];
|
|
367
|
+
const preloadPath = typeof options.preload === 'string' ? options.preload : R1F_PRELOAD;
|
|
368
|
+
return ['--require', preloadPath];
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function r1fSpawn(leg, script, args, dir, stateHome, options = {}) {
|
|
372
|
+
return r1fSpawnSync(process.execPath, [...r1fSpawnPreloadArgs(options), script, ...args], {
|
|
373
|
+
cwd: dir,
|
|
374
|
+
encoding: 'utf8',
|
|
375
|
+
input: options.input === undefined ? undefined : JSON.stringify(options.input),
|
|
376
|
+
env: r1fChildEnv(dir, stateHome, options),
|
|
377
|
+
timeout: 60000,
|
|
378
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// Seeds a stored satisfied workflow with stored verify pass through the leg's real CLI entry points, with
|
|
383
|
+
// the pilot switch OFF: route → plan → start → both bindings → both step passes → an OFF Stop.
|
|
384
|
+
function r1fSeedSatisfied(leg, dir, stateHome, sessionId) {
|
|
385
|
+
let result = r1fSpawn(leg, leg.kernel, [], dir, stateHome, {
|
|
386
|
+
input: { session_id: sessionId, cwd: dir, prompt: 'Fix this checkout bug and add a regression test.' },
|
|
387
|
+
});
|
|
388
|
+
if (result.status !== 0) return { fault: 'route: ' + (result.stderr || '').split('\n')[0] };
|
|
389
|
+
const args = (command, stepId, extra) => [
|
|
390
|
+
command, '--session-id', sessionId,
|
|
391
|
+
...(stepId ? ['--step-id', stepId] : []), ...extra,
|
|
392
|
+
];
|
|
393
|
+
result = r1fSpawn(leg, leg.controller, args('plan', null, [
|
|
394
|
+
'--requirements-json', REQUIREMENTS, '--strategy', 'iterative-single',
|
|
395
|
+
'--evidence', 'two dependent requirements fit a bounded documentation loop',
|
|
396
|
+
]), dir, stateHome);
|
|
397
|
+
if (result.status !== 0) return { fault: 'plan: ' + (result.stderr || '').split('\n')[0] };
|
|
398
|
+
result = r1fSpawn(leg, leg.controller, args('start', null, []), dir, stateHome);
|
|
399
|
+
if (result.status !== 0) return { fault: 'start: ' + (result.stderr || '').split('\n')[0] };
|
|
400
|
+
for (const [stepId, bindingId] of [['req-docs', 'tdd'], [VERIFY_STEP_ID, 'verification-loop']]) {
|
|
401
|
+
result = r1fSpawn(leg, leg.controller, args('binding', stepId, ['--binding-id', bindingId, '--disposition', 'loaded', '--evidence', 'located']), dir, stateHome);
|
|
402
|
+
if (result.status !== 0) return { fault: `binding ${stepId}: ` + (result.stderr || '').split('\n')[0] };
|
|
403
|
+
result = r1fSpawn(leg, leg.controller, args('step', stepId, ['--disposition', 'pass', '--evidence', 'seeded-off-pass']), dir, stateHome);
|
|
404
|
+
if (result.status !== 0) return { fault: `step ${stepId}: ` + (result.stderr || '').split('\n')[0] };
|
|
405
|
+
}
|
|
406
|
+
result = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, { input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' } });
|
|
407
|
+
if (result.status !== 0) return { fault: 'seed-stop: ' + (result.stderr || '').split('\n')[0] };
|
|
408
|
+
return { fault: null };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// The session directory inside the isolated state home: exactly one workspace key must contain it.
|
|
412
|
+
function r1fFindSessionDir(stateHome, sessionId) {
|
|
413
|
+
const workspacesRoot = path.join(stateHome, 'workspaces');
|
|
414
|
+
const found = [];
|
|
415
|
+
let entries = [];
|
|
416
|
+
try { entries = fs.readdirSync(workspacesRoot, { withFileTypes: true }); } catch (_) {
|
|
417
|
+
return null;
|
|
418
|
+
}
|
|
419
|
+
for (const entry of entries.filter(item => item.isDirectory())) {
|
|
420
|
+
const candidate = path.join(workspacesRoot, entry.name, 'state', 'sessions', sessionId);
|
|
421
|
+
if (fs.existsSync(path.join(candidate, R1F_WORKFLOW_FILE))) found.push(candidate);
|
|
422
|
+
}
|
|
423
|
+
return found.length === 1 ? found[0] : null;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function r1fStoredWorkflow(sessionDir) {
|
|
427
|
+
return readJsonOrNull(path.join(sessionDir, R1F_WORKFLOW_FILE));
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
// One integration scenario: seed OFF, run the real Stop entry point (child) with the switch ON.
|
|
431
|
+
function r1fRunScenario(leg, mode) {
|
|
432
|
+
const tag = `${leg.id}.${mode === 'fail' ? 'unwritable' : 'writable-control'}`;
|
|
433
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1f-'));
|
|
434
|
+
const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1f-state-'));
|
|
435
|
+
const sessionId = 's1-review-r1f-' + leg.id;
|
|
436
|
+
const tokens = [dir, stateHome];
|
|
437
|
+
let spacedPreloadDirForCleanup = null;
|
|
438
|
+
try {
|
|
439
|
+
if (!fs.existsSync(R1F_PRELOAD)) {
|
|
440
|
+
record('INFRA', `${tag}.preload`, 'fail-session-writes.js preload not found at ' + R1F_PRELOAD);
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
r1fBuildWorkspace(dir);
|
|
444
|
+
const seeded = r1fSeedSatisfied(leg, dir, stateHome, sessionId);
|
|
445
|
+
if (seeded.fault) {
|
|
446
|
+
record('INFRA', `${tag}.scenario`, 'seed failed: ' + r1fScrub(seeded.fault, tokens));
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
const sessionDir = r1fFindSessionDir(stateHome, sessionId);
|
|
450
|
+
if (!sessionDir) {
|
|
451
|
+
record('INFRA', `${tag}.scenario`, 'the OFF-seeded session directory was not found (expected exactly one)');
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
const stored = r1fStoredWorkflow(sessionDir);
|
|
455
|
+
const storedObligations = readJsonOrNull(path.join(sessionDir, OBLIGATION_FILE));
|
|
456
|
+
if (!stored || stored.state !== 'satisfied'
|
|
457
|
+
|| storedObligations?.obligations?.find(item => item.id === 'verify')?.status !== 'pass') {
|
|
458
|
+
record('INFRA', `${tag}.scenario`,
|
|
459
|
+
`seed did not leave stored satisfied/verify-pass (workflow ${JSON.stringify(stored?.state)}, verify ${JSON.stringify(storedObligations?.obligations?.find(item => item.id === 'verify')?.status)})`);
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
const before = {
|
|
463
|
+
run: fs.readFileSync(path.join(sessionDir, R1F_WORKFLOW_FILE), 'utf8'),
|
|
464
|
+
obligations: fs.readFileSync(path.join(sessionDir, OBLIGATION_FILE), 'utf8'),
|
|
465
|
+
};
|
|
466
|
+
const stopResult = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, {
|
|
467
|
+
pilot: 1,
|
|
468
|
+
preload: mode === 'fail',
|
|
469
|
+
sessionDirectory: mode === 'fail' ? sessionDir : undefined,
|
|
470
|
+
input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' },
|
|
471
|
+
});
|
|
472
|
+
const stderr = stopResult.stderr || '';
|
|
473
|
+
record(stopResult.status === 0 ? 'PASS' : 'FAIL', `${tag}.stop-exit-0`,
|
|
474
|
+
stopResult.status === 0 ? '' : `exit ${stopResult.status} where 0 desired`);
|
|
475
|
+
if (mode === 'fail') {
|
|
476
|
+
record(stderr.includes('Unresolved workflow evidence') ? 'PASS' : 'FAIL', `${tag}.reminder-unresolved-evidence`,
|
|
477
|
+
stderr.includes('Unresolved workflow evidence') ? '' : 'the Unresolved workflow evidence reminder is missing after the failed saveWorkflow');
|
|
478
|
+
record(stderr.includes('step:' + VERIFY_STEP_ID + ':pending') ? 'PASS' : 'FAIL', `${tag}.reminder-step-pending`,
|
|
479
|
+
stderr.includes('step:' + VERIFY_STEP_ID + ':pending') ? '' : 'step:req-verify:pending absent from the Stop reminder');
|
|
480
|
+
record(stderr.includes('withhold-persist-failed') ? 'PASS' : 'FAIL', `${tag}.persist-failed-diagnostic`,
|
|
481
|
+
stderr.includes('withhold-persist-failed') ? '' : 'withhold-persist-failed diagnostic absent from Stop stderr');
|
|
482
|
+
const exceptionText = stderr.includes('EACCES') || stderr.includes('simulated session store');
|
|
483
|
+
record(exceptionText ? 'FAIL' : 'PASS', `${tag}.no-exception-text`,
|
|
484
|
+
exceptionText ? 'exception text reached stderr: ' + r1fScrub(stderr.trim().split('\n').find(line => line.includes('EACCES') || line.includes('simulated session store')) || '', tokens) : '');
|
|
485
|
+
const afterBytes = {
|
|
486
|
+
run: fs.readFileSync(path.join(sessionDir, R1F_WORKFLOW_FILE), 'utf8'),
|
|
487
|
+
obligations: fs.readFileSync(path.join(sessionDir, OBLIGATION_FILE), 'utf8'),
|
|
488
|
+
};
|
|
489
|
+
record(afterBytes.run === before.run && afterBytes.obligations === before.obligations ? 'PASS' : 'FAIL',
|
|
490
|
+
`${tag}.stored-bytes-identical`,
|
|
491
|
+
afterBytes.run === before.run && afterBytes.obligations === before.obligations
|
|
492
|
+
? '' : 'the unwritable session store changed: the failed write is not proven');
|
|
493
|
+
// P3 preload-path portability: copy the preload into a temp directory whose name contains a space
|
|
494
|
+
// and run one fail-mode Stop child through the same spawn helper. The preload must still load —
|
|
495
|
+
// writes fail (reminder text and the withheld decision are on stderr) and the stored files stay
|
|
496
|
+
// byte-identical — instead of an infrastructure `Cannot find module` failure. The preload path is
|
|
497
|
+
// passed as a real argv element (see r1fSpawnPreloadArgs), which needs no quoting.
|
|
498
|
+
const spacedPreloadDir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness preload space-'));
|
|
499
|
+
spacedPreloadDirForCleanup = spacedPreloadDir;
|
|
500
|
+
const spacedPreload = path.join(spacedPreloadDir, 'fail-session-writes.js');
|
|
501
|
+
fs.copyFileSync(R1F_PRELOAD, spacedPreload);
|
|
502
|
+
const spaceStop = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, {
|
|
503
|
+
pilot: 1,
|
|
504
|
+
preload: spacedPreload,
|
|
505
|
+
sessionDirectory: sessionDir,
|
|
506
|
+
input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' },
|
|
507
|
+
});
|
|
508
|
+
const spaceStderr = spaceStop.stderr || '';
|
|
509
|
+
record(spaceStop.status === 0 ? 'PASS' : 'FAIL', `${tag}.preload-space.stop-exit-0`,
|
|
510
|
+
spaceStop.status === 0 ? '' : `exit ${spaceStop.status} where 0 desired; stderr ${r1fScrub(spaceStderr.trim(), [spacedPreloadDir]).slice(0, 140)}`);
|
|
511
|
+
const spaceLoaded = spaceStderr.includes('Unresolved workflow evidence')
|
|
512
|
+
&& spaceStderr.includes('step:' + VERIFY_STEP_ID + ':pending')
|
|
513
|
+
&& spaceStderr.includes('withhold-persist-failed');
|
|
514
|
+
record(spaceLoaded ? 'PASS' : 'FAIL', `${tag}.preload-space.reminder-and-diagnostic`,
|
|
515
|
+
spaceLoaded ? '' : `preloaded child did not run the withheld path: ${r1fScrub(spaceStderr.trim(), [spacedPreloadDir]).slice(0, 160)}`);
|
|
516
|
+
const spaceBytes = {
|
|
517
|
+
run: fs.readFileSync(path.join(sessionDir, R1F_WORKFLOW_FILE), 'utf8'),
|
|
518
|
+
obligations: fs.readFileSync(path.join(sessionDir, OBLIGATION_FILE), 'utf8'),
|
|
519
|
+
};
|
|
520
|
+
record(spaceBytes.run === before.run && spaceBytes.obligations === before.obligations ? 'PASS' : 'FAIL',
|
|
521
|
+
`${tag}.preload-space.stored-bytes-identical`,
|
|
522
|
+
spaceBytes.run === before.run && spaceBytes.obligations === before.obligations ? '' : 'the spaced-preload child did not fail its session writes: the preload did not load');
|
|
523
|
+
} else {
|
|
524
|
+
record(stderr.includes('Unresolved workflow evidence') && stderr.includes('step:' + VERIFY_STEP_ID + ':pending') ? 'PASS' : 'FAIL',
|
|
525
|
+
`${tag}.reminder-present`,
|
|
526
|
+
stderr.includes('Unresolved workflow evidence') && stderr.includes('step:' + VERIFY_STEP_ID + ':pending') ? '' : 'reminder or pending token missing on the writable control');
|
|
527
|
+
}
|
|
528
|
+
// ON disposition attempt with the switch on, no preload. Over the fail mode, the stored files still
|
|
529
|
+
// say satisfied/verify:pass (the writes failed); over the control, Stop persisted the downgrade.
|
|
530
|
+
const disposition = r1fSpawn(leg, leg.controller, [
|
|
531
|
+
'step', '--session-id', sessionId, '--step-id', VERIFY_STEP_ID, '--disposition', 'pass', '--evidence', 'attempt over stored state',
|
|
532
|
+
], dir, stateHome, { pilot: 1 });
|
|
533
|
+
// P2b oracle-gap fix: this probe proves only the legacy closed-workflow guard (the CLI's OPEN_STATES
|
|
534
|
+
// rule and updateStep's active-step rules over a closed/finished plan); it is NOT pilot evidence and
|
|
535
|
+
// is not counted as such. The pilot-specific oracle below carries that claim instead.
|
|
536
|
+
record(disposition.status === 2 ? 'PASS' : 'FAIL', `${tag}.legacy-closed-guard-exit-2`,
|
|
537
|
+
disposition.status === 2 ? `refused: ${r1fScrub((disposition.stderr || '').trim(), tokens).slice(0, 120)}` : `exit ${disposition.status} where 2 desired`);
|
|
538
|
+
if (mode === 'fail') {
|
|
539
|
+
// P2b pilot-specific oracle: after the failed ON Stop, restore write access and run the real Stop
|
|
540
|
+
// entry point ON again. The ON re-read must re-derive the withheld decision from scratch: reminder
|
|
541
|
+
// with step:<verify>:pending, the stored workflow becomes running, and the runtime pilot report
|
|
542
|
+
// carries pilotCredit false with a non-empty unsatisfied list (never trust stored PASS as credit).
|
|
543
|
+
const rerun = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, {
|
|
544
|
+
pilot: 1,
|
|
545
|
+
input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' },
|
|
546
|
+
});
|
|
547
|
+
const rerunStderr = rerun.stderr || '';
|
|
548
|
+
record(rerun.status === 0 && rerunStderr.includes('Unresolved workflow evidence') && rerunStderr.includes('step:' + VERIFY_STEP_ID + ':pending') ? 'PASS' : 'FAIL',
|
|
549
|
+
`${leg.id}.p2b-on-rerun.reminder-pending`,
|
|
550
|
+
rerun.status === 0 && rerunStderr.includes('Unresolved workflow evidence') && rerunStderr.includes('step:' + VERIFY_STEP_ID + ':pending') ? '' : `exit ${rerun.status}, stderr ${r1fScrub(rerunStderr.trim(), tokens).slice(0, 140)} where reminder step:req-verify:pending desired`);
|
|
551
|
+
const rerunStored = r1fStoredWorkflow(sessionDir);
|
|
552
|
+
record(rerunStored?.state === 'running' ? 'PASS' : 'FAIL', `${leg.id}.p2b-on-rerun.stored-running`,
|
|
553
|
+
rerunStored?.state === 'running' ? '' : `stored workflow.state ${JSON.stringify(rerunStored?.state)} where "running" desired`);
|
|
554
|
+
const rerunReport = readJsonOrNull(path.join(sessionDir, PILOT_REPORT_FILE));
|
|
555
|
+
record(rerunReport && rerunReport.reportOrigin === 'runtime' && rerunReport.pilotCredit === false ? 'PASS' : 'FAIL',
|
|
556
|
+
`${leg.id}.p2b-on-rerun.pilot-credit-false`,
|
|
557
|
+
rerunReport && rerunReport.reportOrigin === 'runtime' && rerunReport.pilotCredit === false ? '' : `pilot report ${JSON.stringify(rerunReport ? { origin: rerunReport.reportOrigin, credit: rerunReport.pilotCredit } : null)} where {origin runtime, credit false} desired`);
|
|
558
|
+
record(rerunReport && Array.isArray(rerunReport.unsatisfied) && rerunReport.unsatisfied.length > 0 ? 'PASS' : 'FAIL',
|
|
559
|
+
`${leg.id}.p2b-on-rerun.unsatisfied-nonempty`,
|
|
560
|
+
rerunReport && Array.isArray(rerunReport.unsatisfied) && rerunReport.unsatisfied.length > 0 ? '' : `unsatisfied ${JSON.stringify(rerunReport?.unsatisfied)} where a non-empty list desired`);
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
// Writable control: the durable downgrade is persisted and claimed (already passing at this head).
|
|
564
|
+
const reread = r1fStoredWorkflow(sessionDir);
|
|
565
|
+
record(reread?.state === 'running' ? 'PASS' : 'FAIL', `${tag}.stored-running`,
|
|
566
|
+
reread?.state === 'running' ? '' : `stored workflow.state ${JSON.stringify(reread?.state)} where "running" desired`);
|
|
567
|
+
} finally {
|
|
568
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
569
|
+
fs.rmSync(stateHome, { recursive: true, force: true });
|
|
570
|
+
if (spacedPreloadDirForCleanup) fs.rmSync(spacedPreloadDirForCleanup, { recursive: true, force: true });
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// Item 4, observable-report form: ON verification-step PASS through the real disposition CLI over a seeded
|
|
575
|
+
// running workflow (its stored verify step still pending), so the pilot assess runs and writes the runtime
|
|
576
|
+
// pilot report; assert exit 2 with pilotCredit false. On this head both item-4 probes already pass.
|
|
577
|
+
function r1fRunOnVerifyRefusal(leg) {
|
|
578
|
+
const tag = `${leg.id}.on-verify-refusal`;
|
|
579
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1f-refusal-'));
|
|
580
|
+
const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1f-refusal-state-'));
|
|
581
|
+
const sessionId = 's1-review-r1f-verify-' + leg.id;
|
|
582
|
+
const tokens = [dir, stateHome];
|
|
583
|
+
try {
|
|
584
|
+
r1fBuildWorkspace(dir);
|
|
585
|
+
let result = r1fSpawn(leg, leg.kernel, [], dir, stateHome, {
|
|
586
|
+
input: { session_id: sessionId, cwd: dir, prompt: 'Fix this checkout bug and add a regression test.' },
|
|
587
|
+
});
|
|
588
|
+
if (result.status !== 0) return record('INFRA', `${tag}.scenario`, 'route: ' + r1fScrub((result.stderr || '').split('\n')[0], tokens));
|
|
589
|
+
result = r1fSpawn(leg, leg.controller, [
|
|
590
|
+
'plan', '--session-id', sessionId,
|
|
591
|
+
'--requirements-json', REQUIREMENTS, '--strategy', 'iterative-single',
|
|
592
|
+
'--evidence', 'two dependent requirements fit a bounded documentation loop',
|
|
593
|
+
], dir, stateHome);
|
|
594
|
+
if (result.status !== 0) return record('INFRA', `${tag}.scenario`, 'plan: ' + r1fScrub((result.stderr || '').split('\n')[0], tokens));
|
|
595
|
+
result = r1fSpawn(leg, leg.controller, ['start', '--session-id', sessionId], dir, stateHome);
|
|
596
|
+
if (result.status !== 0) return record('INFRA', `${tag}.scenario`, 'start: ' + r1fScrub((result.stderr || '').split('\n')[0], tokens));
|
|
597
|
+
result = r1fSpawn(leg, leg.controller, ['binding', '--session-id', sessionId, '--step-id', 'req-docs', '--binding-id', 'tdd', '--disposition', 'loaded', '--evidence', 'located'], dir, stateHome);
|
|
598
|
+
if (result.status !== 0) return record('INFRA', `${tag}.scenario`, 'binding: ' + r1fScrub((result.stderr || '').split('\n')[0], tokens));
|
|
599
|
+
result = r1fSpawn(leg, leg.controller, ['step', '--session-id', sessionId, '--step-id', 'req-docs', '--disposition', 'pass', '--evidence', 'added'], dir, stateHome);
|
|
600
|
+
if (result.status !== 0) return record('INFRA', `${tag}.scenario`, 'step: ' + r1fScrub((result.stderr || '').split('\n')[0], tokens));
|
|
601
|
+
result = r1fSpawn(leg, leg.controller, ['binding', '--session-id', sessionId, '--step-id', VERIFY_STEP_ID, '--binding-id', 'verification-loop', '--disposition', 'loaded', '--evidence', 'located'], dir, stateHome);
|
|
602
|
+
if (result.status !== 0) return record('INFRA', `${tag}.scenario`, 'binding2: ' + r1fScrub((result.stderr || '').split('\n')[0], tokens));
|
|
603
|
+
result = r1fSpawn(leg, leg.controller, ['step', '--session-id', sessionId, '--step-id', VERIFY_STEP_ID, '--disposition', 'pass', '--evidence', 'objective check attempted'], dir, stateHome, { pilot: 1 });
|
|
604
|
+
record(result.status === 2 ? 'PASS' : 'FAIL', `${tag}.exit-2`,
|
|
605
|
+
result.status === 2 ? `refused: ${r1fScrub((result.stderr || '').trim(), tokens).slice(0, 120)}` : `exit ${result.status} where 2 desired`);
|
|
606
|
+
const sessionDir = r1fFindSessionDir(stateHome, sessionId);
|
|
607
|
+
const report = sessionDir ? readJsonOrNull(path.join(sessionDir, PILOT_REPORT_FILE)) : null;
|
|
608
|
+
record(report && report.pilotCredit === false && report.reportOrigin === 'runtime' ? 'PASS' : 'FAIL', `${tag}.pilot-credit-false`,
|
|
609
|
+
report && report.pilotCredit === false && report.reportOrigin === 'runtime' ? '' : `pilot report ${JSON.stringify(report ? { origin: report.reportOrigin, credit: report.pilotCredit } : null)} where {origin runtime, credit false} desired`);
|
|
610
|
+
} finally {
|
|
611
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
612
|
+
fs.rmSync(stateHome, { recursive: true, force: true });
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// ---- R1g: legacy-oracle and negative-control runners (PR #351 re-review) ----
|
|
617
|
+
|
|
618
|
+
const R1G_LEGACY_REFERENCES = {
|
|
619
|
+
// git revision:tree-path strings, read-only `git show` sources — not local file paths.
|
|
620
|
+
source: [ // portability-lint: allow git revision reference, not a local path
|
|
621
|
+
'5f9726c:hooks/scripts/workflow-stop-gate.js', // portability-lint: allow git revision reference, not a local path
|
|
622
|
+
],
|
|
623
|
+
packaged: [ // portability-lint: allow git revision reference, not a local path
|
|
624
|
+
'5f9726c:plugins/harness-everything/hooks/scripts/workflow-stop-gate.js', // portability-lint: allow git revision reference, not a local path
|
|
625
|
+
'5f9726c:hooks/scripts/workflow-stop-gate.js', // portability-lint: allow git revision reference, not a local path
|
|
626
|
+
],
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
// Byte copy of the pre-PR Stop entry point from git history (read-only git access).
|
|
630
|
+
function r1gLegacyStopSource(leg) {
|
|
631
|
+
for (const reference of R1G_LEGACY_REFERENCES[leg.id]) {
|
|
632
|
+
const result = r1fSpawnSync('git', ['show', reference], { cwd: ROOT, encoding: 'utf8', maxBuffer: 2 * 1024 * 1024 });
|
|
633
|
+
if (result.status === 0 && String(result.stdout || '').includes('workflow')) return String(result.stdout);
|
|
634
|
+
}
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// Faithful legacy reference: the pre-PR Stop file written into a scratch copy sitting beside the leg's
|
|
639
|
+
// own supporting tree, so the legacy file's './lib/*' requires resolve to the same modules the current
|
|
640
|
+
// Stop uses. The stop-gate file is the only changed layer; for the OFF and Fable paths the current lib's
|
|
641
|
+
// pilot additions are inert (switch OFF / unsupported strategy), so this composite reproduces pre-S1
|
|
642
|
+
// behaviour. The scratch tree is temporary, outside the repo, and removed again.
|
|
643
|
+
function r1gBuildLegacyScratch(leg, source) {
|
|
644
|
+
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1g-legacy-'));
|
|
645
|
+
fs.cpSync(path.join(leg.root, 'hooks', 'scripts'), path.join(scratch, 'hooks', 'scripts'), { recursive: true });
|
|
646
|
+
fs.cpSync(path.join(leg.root, 'scripts'), path.join(scratch, 'scripts'), { recursive: true });
|
|
647
|
+
fs.writeFileSync(path.join(scratch, 'hooks', 'scripts', 'workflow-stop-gate.js'), source);
|
|
648
|
+
return { scratch, stopGate: path.join(scratch, 'hooks', 'scripts', 'workflow-stop-gate.js') };
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
function r1gNormalise(text, tokens) {
|
|
652
|
+
return r1fScrub(text, tokens).replace(/\.json\.\d+\.\d+\.tmp/g, '.json.tmp');
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// Seeds the P2a fable variant: route → plan(fable-staged), no start (stage-contracted starts are
|
|
656
|
+
// refused); Stop then takes the pending path through 'correlated-run-missing' with the pilot
|
|
657
|
+
// unsupported for the Fable strategy.
|
|
658
|
+
function r1gSeedFablePending(leg, dir, stateHome, sessionId) {
|
|
659
|
+
let result = r1fSpawn(leg, leg.kernel, [], dir, stateHome, {
|
|
660
|
+
input: { session_id: sessionId, cwd: dir, prompt: 'Fix this checkout bug and add a regression test.' },
|
|
661
|
+
});
|
|
662
|
+
if (result.status !== 0) return 'route: ' + (result.stderr || '').split('\n')[0];
|
|
663
|
+
result = r1fSpawn(leg, leg.controller, [
|
|
664
|
+
'plan', '--session-id', sessionId,
|
|
665
|
+
'--requirements-json', REQUIREMENTS, '--strategy', 'fable-staged',
|
|
666
|
+
'--evidence', 'fable legacy-path probe',
|
|
667
|
+
], dir, stateHome);
|
|
668
|
+
if (result.status !== 0) return 'plan: ' + (result.stderr || '').split('\n')[0];
|
|
669
|
+
return null;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// P2a: legacy-oracle comparison through real child Stop processes. variant 'off-pending' seeds a stored
|
|
673
|
+
// satisfied workflow whose handoff records a later edit (a legacy pending token, pilot inert), switch
|
|
674
|
+
// OFF; variant 'fable-on' seeds a pending fable-staged plan, switch ON. Both legs run the current Stop
|
|
675
|
+
// and the pre-PR legacy Stop reference over the same unwritable (preloaded) session directory; the
|
|
676
|
+
// current Stop must keep the legacy exit code and legacy stderr and must not emit withhold-persist-failed.
|
|
677
|
+
function r1gRunLegacyOracle(leg, variant) {
|
|
678
|
+
const tag = `${leg.id}.${variant === 'off-pending' ? 'p2a-off-pending' : 'p2a-fable-on'}`;
|
|
679
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1g-' + (variant === 'off-pending' ? 'off-' : 'fable-')));
|
|
680
|
+
const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1g-' + (variant === 'off-pending' ? 'off-state-' : 'fable-state-')));
|
|
681
|
+
const sessionId = 's1-review-r1g-' + (variant === 'off-pending' ? 'off' : 'fable') + '-' + leg.id;
|
|
682
|
+
let scratch = null;
|
|
683
|
+
try {
|
|
684
|
+
const source = r1gLegacyStopSource(leg);
|
|
685
|
+
if (!source) {
|
|
686
|
+
record('INFRA', `${tag}.legacy-reference`, 'git show could not produce the pre-PR Stop file source');
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
const legacyScratch = r1gBuildLegacyScratch(leg, source);
|
|
690
|
+
scratch = legacyScratch.scratch;
|
|
691
|
+
const tokens = [dir, stateHome, scratch];
|
|
692
|
+
let fault;
|
|
693
|
+
if (variant === 'off-pending') {
|
|
694
|
+
const seeded = r1fSeedSatisfied(leg, dir, stateHome, sessionId);
|
|
695
|
+
fault = seeded.fault;
|
|
696
|
+
if (!fault) {
|
|
697
|
+
const sessionProbe = r1fFindSessionDir(stateHome, sessionId);
|
|
698
|
+
if (sessionProbe) fs.writeFileSync(path.join(sessionProbe, 'handoff-state.json'), JSON.stringify({ lastEditAt: Date.now() + 10000 }));
|
|
699
|
+
else fault = 'the OFF-seeded session directory was not found';
|
|
700
|
+
}
|
|
701
|
+
} else {
|
|
702
|
+
fault = r1gSeedFablePending(leg, dir, stateHome, sessionId);
|
|
703
|
+
}
|
|
704
|
+
if (fault) {
|
|
705
|
+
record('INFRA', `${tag}.scenario`, 'seed failed: ' + r1fScrub(String(fault), tokens));
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
const sessionDir = r1fFindSessionDir(stateHome, sessionId);
|
|
709
|
+
if (!sessionDir) {
|
|
710
|
+
record('INFRA', `${tag}.scenario`, 'the seeded session directory was not found (expected exactly one)');
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
if (!fs.existsSync(path.join(sessionDir, R1F_WORKFLOW_FILE))) {
|
|
714
|
+
record('INFRA', `${tag}.scenario`, 'workflow-run.json missing from the seeded session directory');
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
const stopOptions = {
|
|
718
|
+
preload: true,
|
|
719
|
+
sessionDirectory: sessionDir,
|
|
720
|
+
pilot: variant === 'fable-on' ? 1 : undefined,
|
|
721
|
+
input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' },
|
|
722
|
+
};
|
|
723
|
+
const legacyRun = r1fSpawn(leg, legacyScratch.stopGate, [], dir, stateHome, stopOptions);
|
|
724
|
+
const currentRun = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, stopOptions);
|
|
725
|
+
if (!legacyRun || typeof legacyRun.status !== 'number' || legacyRun.status !== 0) {
|
|
726
|
+
record('INFRA', `${tag}.legacy-reference`,
|
|
727
|
+
'the legacy Stop reference did not run cleanly: ' + JSON.stringify(legacyRun ? legacyRun.status : null) + ' ' + r1fScrub(String(legacyRun?.stderr || '').split('\n')[0], tokens));
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
730
|
+
record(currentRun && currentRun.status === legacyRun.status ? 'PASS' : 'FAIL', `${tag}.exit-equal-legacy`,
|
|
731
|
+
currentRun && typeof currentRun.status === 'number' ? '' : `current exit ${JSON.stringify(currentRun?.status)} where legacy exit ${legacyRun.status} desired`);
|
|
732
|
+
const legacyStderr = r1gNormalise(legacyRun.stderr, tokens);
|
|
733
|
+
const currentStderr = r1gNormalise(currentRun?.stderr, tokens);
|
|
734
|
+
record(legacyStderr === currentStderr ? 'PASS' : 'FAIL', `${tag}.stderr-equal-legacy`,
|
|
735
|
+
legacyStderr === currentStderr ? '' : `stderr differs from the pre-PR legacy reference; legacy ${JSON.stringify(legacyStderr.slice(0, 140))} vs current ${JSON.stringify(currentStderr.slice(0, 140))}`);
|
|
736
|
+
record(currentStderr.includes('withhold-persist-failed') ? 'FAIL' : 'PASS', `${tag}.no-withhold-persist-failed`,
|
|
737
|
+
currentStderr.includes('withhold-persist-failed') ? 'withhold-persist-failed emitted on the non-pilot path (OFF or unsupported Fable)' : '');
|
|
738
|
+
} finally {
|
|
739
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
740
|
+
fs.rmSync(stateHome, { recursive: true, force: true });
|
|
741
|
+
if (scratch) fs.rmSync(scratch, { recursive: true, force: true });
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
// P2b negative control: the ON oracle's exact sequence with the switch OFF throughout. OFF keeps legacy
|
|
746
|
+
// state: both Stop runs exit 0, the stored workflow stays satisfied, no withhold-persist-failed diagnostic
|
|
747
|
+
// is emitted in either run, and no pilot report ever appears — so the ON rerun result provably comes from
|
|
748
|
+
// the pilot, not from the recovery itself.
|
|
749
|
+
function r1gRunOffNegative(leg) {
|
|
750
|
+
const tag = `${leg.id}.p2b-off-negative`;
|
|
751
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1g-neg-'));
|
|
752
|
+
const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-325b1-r1g-neg-state-'));
|
|
753
|
+
const sessionId = 's1-review-r1g-neg-' + leg.id;
|
|
754
|
+
const tokens = [dir, stateHome];
|
|
755
|
+
try {
|
|
756
|
+
const seeded = r1fSeedSatisfied(leg, dir, stateHome, sessionId);
|
|
757
|
+
if (seeded.fault) return record('INFRA', `${tag}.scenario`, 'seed failed: ' + r1fScrub(seeded.fault, tokens));
|
|
758
|
+
const sessionDir = r1fFindSessionDir(stateHome, sessionId);
|
|
759
|
+
if (!sessionDir) return record('INFRA', `${tag}.scenario`, 'the OFF-seeded session directory was not found');
|
|
760
|
+
const first = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, {
|
|
761
|
+
preload: true,
|
|
762
|
+
sessionDirectory: sessionDir,
|
|
763
|
+
input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' },
|
|
764
|
+
});
|
|
765
|
+
const second = r1fSpawn(leg, leg.stopGate, [], dir, stateHome, {
|
|
766
|
+
input: { session_id: sessionId, cwd: dir, hook_event_name: 'Stop' },
|
|
767
|
+
});
|
|
768
|
+
if (first.status !== 0 || second.status !== 0) {
|
|
769
|
+
return record('INFRA', `${tag}.scenario`, `Stop exits ${JSON.stringify(first.status)}, ${JSON.stringify(second.status)} where 0/0 desired`);
|
|
770
|
+
}
|
|
771
|
+
const stored = r1fStoredWorkflow(sessionDir);
|
|
772
|
+
record(stored?.state === 'satisfied' ? 'PASS' : 'FAIL', `${tag}.still-satisfied`,
|
|
773
|
+
stored?.state === 'satisfied' ? '' : `stored workflow.state ${JSON.stringify(stored?.state)} where "satisfied" desired`);
|
|
774
|
+
const combined = (first.stderr || '') + (second.stderr || '');
|
|
775
|
+
record(combined.includes('withhold-persist-failed') ? 'FAIL' : 'PASS', `${tag}.no-withhold-persist-failed`,
|
|
776
|
+
combined.includes('withhold-persist-failed') ? 'withhold-persist-failed emitted although no pilot decision is active' : '');
|
|
777
|
+
record(fs.existsSync(path.join(sessionDir, PILOT_REPORT_FILE)) ? 'FAIL' : 'PASS', `${tag}.no-pilot-report`,
|
|
778
|
+
fs.existsSync(path.join(sessionDir, PILOT_REPORT_FILE)) ? 'doc-reconciliation-pilot.json exists although no pilot decision is active' : '');
|
|
779
|
+
} finally {
|
|
780
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
781
|
+
fs.rmSync(stateHome, { recursive: true, force: true });
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
function runR1fSection() {
|
|
786
|
+
for (const leg of R1F_LEGS) {
|
|
787
|
+
for (const mode of ['fail', 'control']) {
|
|
788
|
+
try {
|
|
789
|
+
r1fRunScenario(leg, mode);
|
|
790
|
+
} catch (error) {
|
|
791
|
+
record('INFRA', `${leg.id}.${mode}.crash`, error.message.split('\n')[0]);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
try {
|
|
795
|
+
r1fRunOnVerifyRefusal(leg);
|
|
796
|
+
} catch (error) {
|
|
797
|
+
record('INFRA', `${leg.id}.on-verify-refusal.crash`, error.message.split('\n')[0]);
|
|
798
|
+
}
|
|
799
|
+
for (const variant of ['off-pending', 'fable-on']) {
|
|
800
|
+
try {
|
|
801
|
+
r1gRunLegacyOracle(leg, variant);
|
|
802
|
+
} catch (error) {
|
|
803
|
+
record('INFRA', `${leg.id}.${variant === 'off-pending' ? 'p2a-off-pending' : 'p2a-fable-on'}.crash`, error.message.split('\n')[0]);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
try {
|
|
807
|
+
r1gRunOffNegative(leg);
|
|
808
|
+
} catch (error) {
|
|
809
|
+
record('INFRA', `${leg.id}.p2b-off-negative.crash`, error.message.split('\n')[0]);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
runR1fSection();
|
|
815
|
+
|
|
816
|
+
const r1fCounts = {
|
|
817
|
+
passed: passed - persistenceCounts.passed,
|
|
818
|
+
failed: failed - persistenceCounts.failed,
|
|
819
|
+
infra: infra - persistenceCounts.infra,
|
|
820
|
+
};
|
|
821
|
+
console.log(`\nR1f/R1g unwritable-session-store summary: ${r1fCounts.passed} PASS, ${r1fCounts.failed} FAIL, ${r1fCounts.infra} INFRA across ${R1F_LEGS.length} legs (pilotCredit: false)`);
|
|
822
|
+
console.log(`\nReview file summary (both sections): ${passed} PASS, ${failed} FAIL, ${infra} INFRA (pilotCredit: false)`);
|
|
823
|
+
if (infra > 0) {
|
|
824
|
+
console.error('S1 review suite: infrastructure faults present; these are not assertion results.');
|
|
825
|
+
process.exitCode = 2;
|
|
826
|
+
} else if (failed > 0) {
|
|
827
|
+
console.error('S1 review suite: assertion FAIL present (expected on the RED base; GREEN must clear every FAIL).');
|
|
828
|
+
process.exitCode = 1;
|
|
829
|
+
} else {
|
|
830
|
+
console.log('S1 review suite: all assertions PASS.');
|
|
831
|
+
}
|