harness-everything 0.16.1 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +12 -0
- package/behavioral-evals/PAIRED-BENCHMARK.md +3 -3
- package/behavioral-evals/README.md +38 -32
- package/behavioral-evals/opencode-hardlock-live.js +6 -290
- package/behavioral-evals/opencode-reflection-gate-live.js +342 -0
- package/behavioral-evals/paired-benchmark.js +3 -3
- package/ci/consistency-check.js +120 -0
- package/ci/mechanism-31-opencode-hardlock-evidence.test.js +43 -99
- package/ci/router-workflow-plan.test.js +19 -0
- package/docker-verify.sh +46 -7
- package/docs/mechanism-first-skill-mesh.md +1 -1
- package/docs/workflows/zoom-out.md +6 -6
- package/fable-mode/CONTRACT-FORMAT.md +2 -1
- package/fable-mode/SKILL.md +1 -1
- package/fable-mode/agents/fable-orchestrator.md +13 -11
- package/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/fable-mode/fable-haiku/SKILL.md +1 -1
- package/fable-mode/fable-opus/SKILL.md +1 -1
- package/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/references/router-workflow-plan.md +3 -3
- package/harness-everything/scripts/kernel-router.js +1 -4
- package/opencode-plugin/README.md +1 -3
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +3 -2
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +2 -1
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +13 -11
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +1 -1
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +3 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +1 -4
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +3 -3
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +4 -4
- package/zoom-out/SKILL.md +3 -3
- package/zoom-out/references/circuit-breaker.md +4 -4
|
@@ -3,14 +3,16 @@ const path = require('path');
|
|
|
3
3
|
const helper = require('./test-helper');
|
|
4
4
|
const {
|
|
5
5
|
BLOCKED_MARKER,
|
|
6
|
-
|
|
6
|
+
FIRST_CYCLE_MARKERS,
|
|
7
7
|
INSTALLED_PLUGIN_NAME,
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
PRE_REFLECTION_MARKER,
|
|
9
|
+
PROBE_CONTRACT,
|
|
10
|
+
REFLECTION_REQUIRED_MESSAGE,
|
|
11
|
+
RETRIP_MARKERS,
|
|
10
12
|
verifyEvidence,
|
|
11
|
-
} = require('../behavioral-evals/opencode-
|
|
13
|
+
} = require('../behavioral-evals/opencode-reflection-gate-live');
|
|
12
14
|
|
|
13
|
-
console.log('\n[31] opencode live
|
|
15
|
+
console.log('\n[31] opencode live Rule-of-3 reflection evidence contract...');
|
|
14
16
|
|
|
15
17
|
function writeJson(file, value) {
|
|
16
18
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
@@ -18,121 +20,63 @@ function writeJson(file, value) {
|
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
function makeEvidence() {
|
|
21
|
-
const dir = helper.tempDir('.mechanism-test-opencode-live-evidence');
|
|
23
|
+
const dir = helper.tempDir('.mechanism-test-opencode-live-reflection-evidence');
|
|
22
24
|
fs.mkdirSync(path.join(dir, 'workspace'), { recursive: true });
|
|
23
25
|
fs.mkdirSync(path.join(dir, 'state'), { recursive: true });
|
|
24
26
|
const pluginHash = 'a'.repeat(64);
|
|
25
27
|
writeJson(path.join(dir, 'metadata.json'), {
|
|
26
|
-
schemaVersion:
|
|
27
|
-
|
|
28
|
-
installedPluginName: INSTALLED_PLUGIN_NAME,
|
|
29
|
-
pluginSha256: pluginHash,
|
|
30
|
-
installedPluginSha256: pluginHash,
|
|
31
|
-
});
|
|
32
|
-
const toolEvent = (input) => JSON.stringify({
|
|
33
|
-
type: 'tool',
|
|
34
|
-
part: { tool: 'edit', state: { input } },
|
|
28
|
+
schemaVersion: 2, probeContract: PROBE_CONTRACT, stateIsolated: true,
|
|
29
|
+
installedPluginName: INSTALLED_PLUGIN_NAME, pluginSha256: pluginHash, installedPluginSha256: pluginHash,
|
|
35
30
|
});
|
|
31
|
+
const toolEvent = (input) => JSON.stringify({ type: 'tool', part: { tool: 'edit', state: { input } } });
|
|
36
32
|
fs.writeFileSync(path.join(dir, 'transcript.jsonl'), [
|
|
37
|
-
toolEvent({ filePath: 'probe.txt', text:
|
|
38
|
-
toolEvent({ filePath: 'probe.txt', text:
|
|
33
|
+
toolEvent({ filePath: 'probe.txt', text: PRE_REFLECTION_MARKER }),
|
|
34
|
+
...FIRST_CYCLE_MARKERS.map((marker) => toolEvent({ filePath: 'probe.txt', text: marker })),
|
|
39
35
|
toolEvent({ filePath: 'probe.txt', text: BLOCKED_MARKER }),
|
|
40
|
-
|
|
41
|
-
'',
|
|
36
|
+
...RETRIP_MARKERS.map((marker) => toolEvent({ filePath: 'probe.txt', text: marker })),
|
|
37
|
+
JSON.stringify({ type: 'text', part: { text: REFLECTION_REQUIRED_MESSAGE } }), '',
|
|
42
38
|
].join('\n'));
|
|
43
39
|
fs.writeFileSync(path.join(dir, 'stderr.txt'), '', 'utf8');
|
|
44
|
-
fs.writeFileSync(path.join(dir, 'workspace', 'probe.txt'),
|
|
40
|
+
fs.writeFileSync(path.join(dir, 'workspace', 'probe.txt'), ['baseline', PRE_REFLECTION_MARKER, ...FIRST_CYCLE_MARKERS, ...RETRIP_MARKERS, ''].join('\n'), 'utf8');
|
|
41
|
+
const signature = 'npm test: HARNESS_LIVE_PROBE_RED';
|
|
45
42
|
writeJson(path.join(dir, 'state', 'circuit-breaker.json'), {
|
|
46
|
-
failures: {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
reflectionPending: false,
|
|
53
|
-
reflectionRequestedAt: null,
|
|
54
|
-
reflectionToken: null,
|
|
55
|
-
reflectionSignature: null,
|
|
43
|
+
failures: { [signature]: { count: 3, firstSeen: 5, lastSeen: 8 } },
|
|
44
|
+
lastReflection: 4, lastReflectionSignature: signature, reflectionPending: true,
|
|
45
|
+
reflectionRequestedAt: 8, reflectionToken: 'second456', reflectionSignature: signature,
|
|
46
|
+
});
|
|
47
|
+
writeJson(path.join(dir, 'state', 'compliance.json'), {
|
|
48
|
+
sessionStart: 1, totalEdits: 6, verifiedEdits: 0, circuitBreakerTrips: 2, reflectionsForced: 2,
|
|
56
49
|
});
|
|
57
50
|
fs.writeFileSync(path.join(dir, 'state', 'zoom-out-report.md'), [
|
|
58
|
-
'## Goal',
|
|
59
|
-
'
|
|
60
|
-
'##
|
|
61
|
-
'three controlled failures',
|
|
62
|
-
'## Verified Facts',
|
|
63
|
-
'the verification fixture intentionally stays red',
|
|
64
|
-
'## Diagnosis',
|
|
65
|
-
'this is a live enforcement probe',
|
|
66
|
-
'## Decision',
|
|
67
|
-
'RESUME: reproduce the same failure once after reflection',
|
|
68
|
-
'Reflection token: abc123',
|
|
69
|
-
'',
|
|
51
|
+
'## Goal','exercise the Rule-of-3 reflection gate','## Failed Attempts','three controlled failures',
|
|
52
|
+
'## Verified Facts','the verification fixture intentionally stays red','## Diagnosis','this is a live enforcement probe',
|
|
53
|
+
'## Decision','RESUME: reproduce the same three-failure cycle after reflection','Reflection token: first123','',
|
|
70
54
|
].join('\n'));
|
|
71
55
|
return dir;
|
|
72
56
|
}
|
|
73
57
|
|
|
74
58
|
try {
|
|
75
|
-
const valid
|
|
76
|
-
|
|
77
|
-
helper.check('31. complete attributed hard-lock evidence passes', result.passed, JSON.stringify(result.checks.filter((entry) => !entry.pass)));
|
|
59
|
+
const valid=makeEvidence(); let result=verifyEvidence(valid);
|
|
60
|
+
helper.check('31. complete attributed reflection/re-trip evidence passes',result.passed,JSON.stringify(result.checks.filter((entry)=>!entry.pass)));
|
|
78
61
|
|
|
79
|
-
const wrongPlugin =
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
metadata.installedPluginSha256 = 'b'.repeat(64);
|
|
83
|
-
writeJson(metadataFile, metadata);
|
|
84
|
-
result = verifyEvidence(wrongPlugin);
|
|
85
|
-
helper.check(
|
|
86
|
-
'31. evidence fails when the installed plugin hash differs from the canonical source hash',
|
|
87
|
-
!result.passed && result.checks.some((entry) => entry.name === 'installed plugin is byte-identical to the recorded canonical source' && !entry.pass),
|
|
88
|
-
JSON.stringify(result.checks),
|
|
89
|
-
);
|
|
62
|
+
const wrongPlugin=makeEvidence(); const mf=path.join(wrongPlugin,'metadata.json'); const md=JSON.parse(fs.readFileSync(mf,'utf8'));
|
|
63
|
+
md.installedPluginSha256='b'.repeat(64); writeJson(mf,md); result=verifyEvidence(wrongPlugin);
|
|
64
|
+
helper.check('31. evidence fails when the installed plugin hash differs from the canonical source hash',!result.passed&&result.checks.some((e)=>e.name==='installed plugin is byte-identical to the recorded canonical source'&&!e.pass),JSON.stringify(result.checks));
|
|
90
65
|
|
|
91
|
-
const landed
|
|
92
|
-
|
|
93
|
-
result = verifyEvidence(landed);
|
|
94
|
-
helper.check(
|
|
95
|
-
'31. negative control fails if the blocked marker reached the filesystem',
|
|
96
|
-
!result.passed && result.checks.some((entry) => entry.name === 'blocked marker did not reach the filesystem' && !entry.pass),
|
|
97
|
-
JSON.stringify(result.checks),
|
|
98
|
-
);
|
|
66
|
+
const landed=makeEvidence(); fs.appendFileSync(path.join(landed,'workspace','probe.txt'),`${BLOCKED_MARKER}\n`); result=verifyEvidence(landed);
|
|
67
|
+
helper.check('31. reflection gate fails if the blocked marker reached the filesystem',!result.passed&&result.checks.some((e)=>e.name==='reflection-pending blocked marker did not reach the filesystem'&&!e.pass),JSON.stringify(result.checks));
|
|
99
68
|
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
const breaker = JSON.parse(fs.readFileSync(breakerFile, 'utf8'));
|
|
103
|
-
breaker.hardLock = false;
|
|
104
|
-
writeJson(breakerFile, breaker);
|
|
105
|
-
result = verifyEvidence(unlocked);
|
|
106
|
-
helper.check(
|
|
107
|
-
'31. evidence fails without a pre-reset hard-lock snapshot',
|
|
108
|
-
!result.passed && result.checks.some((entry) => entry.name === 'pre-reset breaker snapshot is hard-locked' && !entry.pass),
|
|
109
|
-
JSON.stringify(result.checks),
|
|
110
|
-
);
|
|
69
|
+
const retired=makeEvidence(); const bf=path.join(retired,'state','circuit-breaker.json'); const bs=JSON.parse(fs.readFileSync(bf,'utf8')); bs.hardLock=true; writeJson(bf,bs); result=verifyEvidence(retired);
|
|
70
|
+
helper.check('31. current evidence rejects the retired hardLock field',!result.passed&&result.checks.some((e)=>e.name==='snapshot has no retired hardLock field'&&!e.pass),JSON.stringify(result.checks));
|
|
111
71
|
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
.split(/\r?\n/)
|
|
115
|
-
.filter((line) => !line.includes(BLOCKED_MARKER))
|
|
116
|
-
.join('\n');
|
|
117
|
-
fs.writeFileSync(path.join(noAttempt, 'transcript.jsonl'), transcript);
|
|
118
|
-
result = verifyEvidence(noAttempt);
|
|
119
|
-
helper.check(
|
|
120
|
-
'31. evidence fails when the final blocked edit was never observed as a structured tool attempt',
|
|
121
|
-
!result.passed && result.checks.some((entry) => entry.name === 'structured tool trace contains the final blocked edit attempt' && !entry.pass),
|
|
122
|
-
JSON.stringify(result.checks),
|
|
123
|
-
);
|
|
72
|
+
const noRetrip=makeEvidence(); const rf=path.join(noRetrip,'state','circuit-breaker.json'); const rs=JSON.parse(fs.readFileSync(rf,'utf8')); rs.reflectionPending=false; rs.reflectionToken=null; rs.reflectionSignature=null; writeJson(rf,rs); result=verifyEvidence(noRetrip);
|
|
73
|
+
helper.check('31. evidence fails without a second Rule-of-3 reflection request',!result.passed&&result.checks.some((e)=>e.name==='three more matching failures request a second reflection instead of a permanent lock'&&!e.pass),JSON.stringify(result.checks));
|
|
124
74
|
|
|
125
|
-
const
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
!result.passed && result.checks.some((entry) => entry.name === 'reflection artifact is valid and preserved' && !entry.pass),
|
|
131
|
-
JSON.stringify(result.checks),
|
|
132
|
-
);
|
|
75
|
+
const noAttempt=makeEvidence(); const tf=path.join(noAttempt,'transcript.jsonl'); fs.writeFileSync(tf,fs.readFileSync(tf,'utf8').split(/\r?\n/).filter((line)=>!line.includes(BLOCKED_MARKER)).join('\n')); result=verifyEvidence(noAttempt);
|
|
76
|
+
helper.check('31. evidence fails when no structured edit was attempted during reflectionPending',!result.passed&&result.checks.some((e)=>e.name==='structured tool trace contains the edit attempt blocked by reflectionPending'&&!e.pass),JSON.stringify(result.checks));
|
|
77
|
+
|
|
78
|
+
const noReflection=makeEvidence(); fs.rmSync(path.join(noReflection,'state','zoom-out-report.md')); result=verifyEvidence(noReflection);
|
|
79
|
+
helper.check('31. evidence fails without the preserved first reflection artifact',!result.passed&&result.checks.some((e)=>e.name==='first reflection artifact is valid and preserved'&&!e.pass),JSON.stringify(result.checks));
|
|
133
80
|
|
|
134
81
|
helper.finish();
|
|
135
|
-
} catch (error)
|
|
136
|
-
helper.check('31. OpenCode hard-lock live evidence contract', false, error.stack);
|
|
137
|
-
helper.finish();
|
|
138
|
-
}
|
|
82
|
+
} catch(error) { helper.check('31. OpenCode Rule-of-3 reflection live evidence contract',false,error.stack); helper.finish(); }
|
|
@@ -262,6 +262,25 @@ check(kernelUnknown.stdout.includes('ROUTER WORKFLOW PLAN (JSON)'), 'kernel emit
|
|
|
262
262
|
check(kernelUnknown.stdout.includes('"strategySelection":"deferred"'), 'kernel exposes deferred unclassified selection');
|
|
263
263
|
check(!/RECOMMENDED TIER:\s*Tier 1/i.test(kernelUnknown.stdout), 'kernel does not expose a silent Tier 1 fallback for unmatched prompts');
|
|
264
264
|
|
|
265
|
+
const notificationStateHome = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-router-notification-fail-open-'));
|
|
266
|
+
const notificationStateFile = path.join(notificationStateHome, 'not-a-directory');
|
|
267
|
+
fs.writeFileSync(notificationStateFile, 'x', 'utf8');
|
|
268
|
+
const kernelNotificationPersistFailure = spawnSync(process.execPath, [kernelRouter], {
|
|
269
|
+
cwd: ROOT,
|
|
270
|
+
encoding: 'utf8',
|
|
271
|
+
input: JSON.stringify({
|
|
272
|
+
session_id: 'router-host-notification-fail-open',
|
|
273
|
+
cwd: ROOT,
|
|
274
|
+
hook_event_name: 'UserPromptSubmit',
|
|
275
|
+
prompt: '<task-notification><task-id>background-1</task-id><status>completed</status></task-notification>',
|
|
276
|
+
}),
|
|
277
|
+
env: { ...process.env, HARNESS_STATE_HOME: notificationStateFile, HARNESS_WORKSPACE_ROOT: ROOT },
|
|
278
|
+
});
|
|
279
|
+
check(kernelNotificationPersistFailure.status === 0, 'host notification persistence failure stays fail-open');
|
|
280
|
+
check(kernelNotificationPersistFailure.stdout.includes('Host notification — no active execution contract; ignored for routing.'), 'host notification remains a routing no-op when state persistence fails');
|
|
281
|
+
check(kernelNotificationPersistFailure.stderr.includes('routing state was not persisted'), 'host notification persistence failure remains visible as a reminder');
|
|
282
|
+
fs.rmSync(notificationStateHome, { recursive: true, force: true });
|
|
283
|
+
|
|
265
284
|
const kernelIterative = spawnSync(process.execPath, [kernelRouter, 'Fix this checkout bug and add a regression test.'], {
|
|
266
285
|
cwd: ROOT,
|
|
267
286
|
encoding: 'utf8',
|
package/docker-verify.sh
CHANGED
|
@@ -86,24 +86,63 @@ if ! node -e "const s=require('./.claude/harness-everything/state/sessions/defau
|
|
|
86
86
|
fi
|
|
87
87
|
echo "✅ PASS: Valid reflection report released the breaker (self-recovery)."
|
|
88
88
|
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
|
|
89
|
+
# 3a-ter. Re-trip: three more failures on the same signature after a released
|
|
90
|
+
# cycle ask for ANOTHER zoom-out. #190 retired the second-stage escalation, so
|
|
91
|
+
# the contract is exit 2 plus the reflection-required message the hook actually
|
|
92
|
+
# prints - never a permanent lock (ci/mechanism-2a-rule-of-3.test.js:58-64).
|
|
93
|
+
echo -e "\n---> Running 3a-ter: Re-trip requests another zoom-out"
|
|
94
|
+
# Seed through the hooks' own resolver: since #42 runtime state lives in the
|
|
95
|
+
# global, workspace-keyed state root ($HARNESS_STATE_HOME or
|
|
96
|
+
# ~/.agents/harness-everything/...), and a repo-relative seed only reaches the
|
|
97
|
+
# hook while the one-time legacy migration can still move it.
|
|
98
|
+
STATE_DIR=$(node -e "process.stdout.write(require('./hooks/scripts/lib/harness-state').getSessionDir(null, 'default'))")
|
|
99
|
+
RETRIP_AT=$(node -e "process.stdout.write(String(Date.now()))")
|
|
100
|
+
node -e "require('fs').writeFileSync(process.argv[1], JSON.stringify({count:3,lastHash:'verify-test',zoomOutResolved:false,lastFailureAt:Number(process.argv[2]),zoomOutCycles:1}))" "$STATE_DIR/rule-of-3-state.json" "$RETRIP_AT"
|
|
92
101
|
set +e
|
|
93
102
|
node hooks/scripts/rule-of-3.js 2>rule-of-3-err.log
|
|
94
103
|
EXIT_CODE=$?
|
|
95
104
|
set -e
|
|
96
105
|
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
97
|
-
echo "❌ FAIL:
|
|
106
|
+
echo "❌ FAIL: Re-trip did not request another zoom-out (expected 2, got $EXIT_CODE)"
|
|
98
107
|
cat rule-of-3-err.log
|
|
99
108
|
exit 1
|
|
100
109
|
fi
|
|
101
|
-
if ! grep -q "
|
|
102
|
-
echo "❌ FAIL:
|
|
110
|
+
if ! grep -q "RULE OF 3 CIRCUIT BREAKER TRIGGERED" rule-of-3-err.log; then
|
|
111
|
+
echo "❌ FAIL: Re-trip output did not contain the circuit-breaker trigger message"
|
|
103
112
|
cat rule-of-3-err.log
|
|
104
113
|
exit 1
|
|
105
114
|
fi
|
|
106
|
-
|
|
115
|
+
if ! grep -q "ACTION REQUIRED (reflect first)" rule-of-3-err.log; then
|
|
116
|
+
echo "❌ FAIL: Re-trip output did not demand reflection"
|
|
117
|
+
cat rule-of-3-err.log
|
|
118
|
+
exit 1
|
|
119
|
+
fi
|
|
120
|
+
# Negative guard: the retired escalation wording must never come back (#190).
|
|
121
|
+
if grep -qi "hard lock" rule-of-3-err.log; then # this must never match
|
|
122
|
+
echo "❌ FAIL: Re-trip printed the retired hard-lock escalation, which no longer exists"
|
|
123
|
+
cat rule-of-3-err.log
|
|
124
|
+
exit 1
|
|
125
|
+
fi
|
|
126
|
+
echo "✅ PASS: Re-trip asked for another zoom-out, with no retired escalation wording."
|
|
127
|
+
|
|
128
|
+
# A fresh report, written after the re-trip failure, releases the breaker again.
|
|
129
|
+
printf '## Goal\nx\n## Failed Attempts\nx\n## Verified Facts\nx\n## Diagnosis\nx\n## Decision\nRESUME: second untried approach\n' > "$STATE_DIR/zoom-out-report.md"
|
|
130
|
+
set +e
|
|
131
|
+
node hooks/scripts/rule-of-3.js
|
|
132
|
+
EXIT_CODE=$?
|
|
133
|
+
set -e
|
|
134
|
+
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
135
|
+
echo "❌ FAIL: Fresh reflection report did not release the re-tripped breaker (expected 0, got $EXIT_CODE)"
|
|
136
|
+
exit 1
|
|
137
|
+
fi
|
|
138
|
+
if ! node -e "const s=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8')); process.exit(s.count===0 && s.zoomOutResolved===true && s.zoomOutCycles===2 ? 0 : 1)" "$STATE_DIR/rule-of-3-state.json"; then
|
|
139
|
+
echo "❌ FAIL: Breaker state not updated after the second report release"
|
|
140
|
+
cat "$STATE_DIR/rule-of-3-state.json"
|
|
141
|
+
exit 1
|
|
142
|
+
fi
|
|
143
|
+
echo "✅ PASS: A fresh report released the re-tripped breaker (advisory re-trip, not an escalation)."
|
|
144
|
+
|
|
145
|
+
# harness:reset stays an optional manual clear, not a required escalation step.
|
|
107
146
|
npm run harness:reset
|
|
108
147
|
|
|
109
148
|
|
|
@@ -55,7 +55,7 @@ HARNESS_DECISION
|
|
|
55
55
|
tier: 2
|
|
56
56
|
strategy: iterative-single
|
|
57
57
|
workflow_state: active
|
|
58
|
-
required:
|
|
58
|
+
required: scope-lock, objective-verification, loop-awareness
|
|
59
59
|
suggest: tdd, verification-loop
|
|
60
60
|
suggestion_policy: evaluate-applicability
|
|
61
61
|
escape_policy: workflow-uncovered-scope-only
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
Source of truth: `zoom-out/SKILL.md`.
|
|
6
6
|
|
|
7
|
-
Contract summary from SKILL.md — Trigger: three same-signature failures or an explicit loop/rethink request. Output: fact-checked report ending in `RESUME` or `ESCALATE`. State: writes the session `zoom-out-report.md`; reset may clear breaker state. Gate: `hooks/scripts/rule-of-3.js` plus a valid report;
|
|
7
|
+
Contract summary from SKILL.md — Trigger: three same-signature failures or an explicit loop/rethink request. Output: fact-checked report ending in `RESUME` or `ESCALATE`. State: writes the session `zoom-out-report.md`; reset may clear breaker state. Gate: `hooks/scripts/rule-of-3.js` plus a valid report; a later three-failure cycle requests another reflection. USE FOR: "you are stuck in a loop", "the same error keeps failing", "rethink your assumptions". DO NOT USE FOR: routine single-failure debugging, greenfield planning without failures.
|
|
8
8
|
|
|
9
9
|
## 1. Skill Behavior Workflow
|
|
10
10
|
|
|
@@ -16,9 +16,9 @@ graph TD
|
|
|
16
16
|
Fill --> Decide{RESUME or ESCALATE?}
|
|
17
17
|
Decide -->|untried path| Resume[RESUME On Untried Path]
|
|
18
18
|
Decide -->|genuine user decision| Escalate[ESCALATE With Options]
|
|
19
|
-
Resume --> Again{
|
|
20
|
-
Again -->|
|
|
21
|
-
|
|
19
|
+
Resume --> Again{Three More Matching Failures?}
|
|
20
|
+
Again -->|yes| Cycle[Another Zoom-Out Cycle]
|
|
21
|
+
Cycle --> Evolve[Record Insight With self-evolve]
|
|
22
22
|
Again -->|no| Done([Report Complete])
|
|
23
23
|
Escalate --> Done
|
|
24
24
|
```
|
|
@@ -47,7 +47,7 @@ graph TD
|
|
|
47
47
|
Write --> Choice{Untried Path Exists?}
|
|
48
48
|
Choice -->|yes| Resume2[RESUME On Untried Path]
|
|
49
49
|
Choice -->|no| Esc2[ESCALATE With Options]
|
|
50
|
-
Resume2 -->|
|
|
50
|
+
Resume2 -->|three more matching failures| Cycle2[Another Zoom-Out Cycle Then self-evolve]
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
Template: `zoom-out/templates/zoom-out-report.template.md`. Enforcement: `hooks/scripts/rule-of-3.js`. Deep dive: `zoom-out/references/circuit-breaker.md`.
|
|
@@ -58,5 +58,5 @@ Template: `zoom-out/templates/zoom-out-report.template.md`. Enforcement: `hooks/
|
|
|
58
58
|
- [ ] Goal, files, configuration, and logs were rechecked with read-only tools before any resume
|
|
59
59
|
- [ ] `templates/zoom-out-report.template.md` was filled and the session `zoom-out-report.md` ends in `RESUME` or `ESCALATE`
|
|
60
60
|
- [ ] Resume targeted only an untried path; genuine user decisions were escalated with options
|
|
61
|
-
- [ ]
|
|
61
|
+
- [ ] A later three-failure cycle ran another zoom-out; `npm run harness:reset` was used only as an optional manual clear, with the insight recorded via `self-evolve`
|
|
62
62
|
- [ ] Routine single-failure debugging and greenfield planning without failures were not routed here
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Issue #85 Phase 3 makes the stage map a machine-readable execution contract.
|
|
4
4
|
The router still chooses only the execution topology. Fable owns the stage map,
|
|
5
|
-
model selector, worker briefs, stage audit, replan
|
|
5
|
+
model selector, worker briefs, stage audit, advisory replan limits, and cold
|
|
6
|
+
verifier.
|
|
6
7
|
Phase 4 adds an optional bounded ensemble modifier without creating a new base
|
|
7
8
|
topology.
|
|
8
9
|
|
package/fable-mode/SKILL.md
CHANGED
|
@@ -53,12 +53,13 @@ overlap reject dispatch instead of becoming a verbal warning.
|
|
|
53
53
|
|
|
54
54
|
The stage map remains a living orchestration document only inside those
|
|
55
55
|
constraints. When new information invalidates it, re-plan and create revised
|
|
56
|
-
stage contracts before dispatching new work.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
56
|
+
stage contracts before dispatching new work. `workflowPlan.limits.maxReplans`
|
|
57
|
+
is advisory orchestration guidance, not a machine-enforced counter. Re-enter
|
|
58
|
+
through `workflow-disposition.js start`; after repeated replans, consider a
|
|
59
|
+
zoom-out/reflection and keep the reason explicit, but do not block execution
|
|
60
|
+
solely because an advisory count was reached. Scope rule: deliver the task as
|
|
61
|
+
specified; new scope discovered mid-run is surfaced as a recommendation at
|
|
62
|
+
delivery, not silently built.
|
|
62
63
|
|
|
63
64
|
**2. Delegate by name and validated batch.** Every artifact-producing stage goes
|
|
64
65
|
to a named agent via the Task tool:
|
|
@@ -76,11 +77,12 @@ spawn workers.
|
|
|
76
77
|
|
|
77
78
|
For `fable-parallel`, spawn only stages in the same validated execution batch;
|
|
78
79
|
never invent an additional parallel edge. A stage depending on another stage is
|
|
79
|
-
not in the same ready batch.
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
80
|
+
not in the same ready batch. Treat `workflowPlan.limits.maxWorkers` as a
|
|
81
|
+
planning cap you are expected to respect: it is advisory. `workflow-plan-consumer.js`
|
|
82
|
+
returns every dependency-ready stage without chunking to that cap, and nothing
|
|
83
|
+
leases or counts workers at runtime, so exceeding it is your own judgement call,
|
|
84
|
+
not a blocked action. For other Fable strategies, serialize the returned batches
|
|
85
|
+
unless a later validated plan says otherwise.
|
|
84
86
|
|
|
85
87
|
**3. Verify with a check that can fail — external artifacts only.** Each stage
|
|
86
88
|
defines a pass condition an external artifact satisfies: a test that runs, a file
|
|
@@ -54,14 +54,14 @@ If the structured contract is missing or invalid, the kernel emits a degraded/un
|
|
|
54
54
|
| Strategy | Selection rule | Key invariants |
|
|
55
55
|
|---|---|---|
|
|
56
56
|
| `direct-single` | Tier 1 bounded/trivial task | scope lock, verify before claim |
|
|
57
|
-
| `iterative-single` | Tier 2 ordinary diagnosis/TDD/tool loop | objective verification,
|
|
57
|
+
| `iterative-single` | Tier 2 ordinary diagnosis/TDD/tool loop | objective verification, advisory iteration guidance |
|
|
58
58
|
| `fable-staged` | Tier 3 dependent work or parallel independence not proven | stage contracts, cold verifier |
|
|
59
59
|
| `fable-parallel` | Tier 3 with explicit independent workstreams and read-only/disjoint write scope | stage contracts, parallel scope contract, synthesis barrier |
|
|
60
60
|
| `fable-multi-agent-workspace` | durable multi-session task needing reusable specialists | workspace state, handoffs, stage contracts, cold verifier |
|
|
61
61
|
|
|
62
62
|
Tier 3 alone never implies a persistent workspace. Likewise, an `independent` label alone is insufficient for `fable-parallel`: the write set must also be `read-only` or `disjoint`.
|
|
63
63
|
|
|
64
|
-
The Phase 2 `iterative-single`
|
|
64
|
+
The Phase 2 `iterative-single` plan exposes **8 iterations** as advisory planning guidance, not a machine-enforced stop. Continue when evidence justifies it; the separate rule-of-3 repeated-failure reflection gate remains the hard boundary.
|
|
65
65
|
|
|
66
66
|
## Explicit user choice and prohibitions
|
|
67
67
|
|
|
@@ -110,7 +110,7 @@ This does not choose a topology and does not itself enforce the gate. Phase 5 ow
|
|
|
110
110
|
|
|
111
111
|
Examples:
|
|
112
112
|
|
|
113
|
-
- `
|
|
113
|
+
- `loop-awareness` and `objective-verification` are mandatory for `iterative-single`, on top of the base `scope-lock`, `verify-before-claim`, and `replan-after-repeated-failure`;
|
|
114
114
|
- `tdd` and `verification-loop` remain advisory skill suggestions;
|
|
115
115
|
- `stage-contracts` and `cold-verification` are mandatory for Fable topologies;
|
|
116
116
|
- `multi-agent-workspace` is suggested only when the selected topology actually requires durable workspace behavior.
|
|
@@ -120,10 +120,7 @@ function run(raw) {
|
|
|
120
120
|
} catch (error) { failure = error; }
|
|
121
121
|
if (hostNotification && !persisted?.workflow) {
|
|
122
122
|
console.log('\n=> Host notification — no active execution contract; ignored for routing.');
|
|
123
|
-
if (failure)
|
|
124
|
-
console.error('[Workflow Router] ' + failure.message);
|
|
125
|
-
process.exitCode = 2;
|
|
126
|
-
}
|
|
123
|
+
if (failure) console.error('[Workflow Reminder] routing state was not persisted: ' + failure.message);
|
|
127
124
|
return;
|
|
128
125
|
}
|
|
129
126
|
if (result?.sanitized) console.log(result.sanitized);
|
|
@@ -62,10 +62,8 @@ normalized success/failure field to key a signature on for tools in general):
|
|
|
62
62
|
- the follow-up asks for a tokenized `zoom-out-report.md` artifact with the
|
|
63
63
|
required reflection sections; the artifact is accepted through the real
|
|
64
64
|
edit hook and records `lastReflection`
|
|
65
|
-
- the same signature failing again after that reflection was recorded
|
|
66
|
-
hard-locks the breaker
|
|
67
65
|
- after a valid reflection, the signature count resets; another three matching failures trigger another zoom-out
|
|
68
|
-
|
|
66
|
+
- `npm run harness:reset` remains an optional manual state clear; a repeated cycle does not create a permanent second-stage lock
|
|
69
67
|
|
|
70
68
|
Idle handling is bounded: after a failed verification, the plugin records that
|
|
71
69
|
the follow-up is pending and ignores repeated `session.idle` events until a new
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "harness-everything",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.3",
|
|
4
4
|
"description": "Harness OS: An Orchestrated Agent Operating System for AI-driven Development",
|
|
5
5
|
"main": "harness-everything/scripts/bootstrap.js",
|
|
6
6
|
"bin": {
|
|
@@ -40,7 +40,8 @@
|
|
|
40
40
|
"eval:behavioral": "node behavioral-evals/run.js run",
|
|
41
41
|
"eval:behavioral:validate": "node behavioral-evals/run.js validate",
|
|
42
42
|
"eval:behavioral:plugin": "node behavioral-evals/run-with-plugin.js run",
|
|
43
|
-
"eval:opencode:
|
|
43
|
+
"eval:opencode:reflection-gate-live": "node behavioral-evals/opencode-reflection-gate-live.js run",
|
|
44
|
+
"eval:opencode:hardlock-live": "node behavioral-evals/opencode-reflection-gate-live.js run",
|
|
44
45
|
"eval:paired": "node behavioral-evals/paired-benchmark.js run",
|
|
45
46
|
"eval:paired:summarize": "node behavioral-evals/paired-benchmark.js summarize",
|
|
46
47
|
"test:ab": "node ci/ab-test-harness.js validate",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "harness-everything",
|
|
4
|
-
"version": "0.16.
|
|
4
|
+
"version": "0.16.3",
|
|
5
5
|
"description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Miya Daniel",
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Issue #85 Phase 3 makes the stage map a machine-readable execution contract.
|
|
4
4
|
The router still chooses only the execution topology. Fable owns the stage map,
|
|
5
|
-
model selector, worker briefs, stage audit, replan
|
|
5
|
+
model selector, worker briefs, stage audit, advisory replan limits, and cold
|
|
6
|
+
verifier.
|
|
6
7
|
Phase 4 adds an optional bounded ensemble modifier without creating a new base
|
|
7
8
|
topology.
|
|
8
9
|
|
|
@@ -53,12 +53,13 @@ overlap reject dispatch instead of becoming a verbal warning.
|
|
|
53
53
|
|
|
54
54
|
The stage map remains a living orchestration document only inside those
|
|
55
55
|
constraints. When new information invalidates it, re-plan and create revised
|
|
56
|
-
stage contracts before dispatching new work.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
56
|
+
stage contracts before dispatching new work. `workflowPlan.limits.maxReplans`
|
|
57
|
+
is advisory orchestration guidance, not a machine-enforced counter. Re-enter
|
|
58
|
+
through `workflow-disposition.js start`; after repeated replans, consider a
|
|
59
|
+
zoom-out/reflection and keep the reason explicit, but do not block execution
|
|
60
|
+
solely because an advisory count was reached. Scope rule: deliver the task as
|
|
61
|
+
specified; new scope discovered mid-run is surfaced as a recommendation at
|
|
62
|
+
delivery, not silently built.
|
|
62
63
|
|
|
63
64
|
**2. Delegate by name and validated batch.** Every artifact-producing stage goes
|
|
64
65
|
to a named agent via the Task tool:
|
|
@@ -76,11 +77,12 @@ spawn workers.
|
|
|
76
77
|
|
|
77
78
|
For `fable-parallel`, spawn only stages in the same validated execution batch;
|
|
78
79
|
never invent an additional parallel edge. A stage depending on another stage is
|
|
79
|
-
not in the same ready batch.
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
80
|
+
not in the same ready batch. Treat `workflowPlan.limits.maxWorkers` as a
|
|
81
|
+
planning cap you are expected to respect: it is advisory. `workflow-plan-consumer.js`
|
|
82
|
+
returns every dependency-ready stage without chunking to that cap, and nothing
|
|
83
|
+
leases or counts workers at runtime, so exceeding it is your own judgement call,
|
|
84
|
+
not a blocked action. For other Fable strategies, serialize the returned batches
|
|
85
|
+
unless a later validated plan says otherwise.
|
|
84
86
|
|
|
85
87
|
**3. Verify with a check that can fail — external artifacts only.** Each stage
|
|
86
88
|
defines a pass condition an external artifact satisfies: a test that runs, a file
|