harness-everything 0.11.6 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +45 -0
- package/README.md +7 -3
- package/behavioral-evals/README.md +37 -0
- package/behavioral-evals/cases/self-evolve-retrieval-recurrence.yaml +48 -0
- package/behavioral-evals/paired-benchmark.js +211 -5
- package/ci/contract-integrity.test.js +68 -2
- package/ci/mechanism-2a-rule-of-3.test.js +71 -0
- package/ci/mechanism-2ah-workflow-lifecycle.test.js +276 -0
- package/ci/mechanism-2c-state-persist.test.js +45 -2
- package/ci/mechanism-2t-issue-20.test.js +0 -8
- package/ci/mechanism-30-opencode-plugin-loadability.test.js +39 -7
- package/ci/mechanism-32-paired-benchmark.test.js +139 -0
- package/ci/openai-plugin-package.test.js +17 -0
- package/ci/self-evolve-loop.test.js +37 -2
- package/ci/telemetry.test.js +31 -1
- package/docs/architecture.md +17 -0
- package/docs/audit.md +52 -5
- package/docs/release-evidence.md +3 -0
- package/docs/workflow-runtime.md +13 -0
- package/docs/workflows/environment-detection.md +47 -37
- package/docs/workflows/eval-harness.md +45 -38
- package/docs/workflows/fable-discipline.md +45 -33
- package/docs/workflows/git-commit.md +53 -45
- package/docs/workflows/grill-me.md +36 -38
- package/docs/workflows/grill-with-docs.md +36 -46
- package/docs/workflows/harness-everything.md +59 -57
- package/docs/workflows/improve-codebase-architecture.md +36 -44
- package/docs/workflows/install-cognitive-os.md +36 -29
- package/docs/workflows/repo-docs.md +29 -40
- package/docs/workflows/rewrite-commits.md +40 -36
- package/docs/workflows/security-review.md +34 -53
- package/docs/workflows/skill-creator.md +39 -56
- package/docs/workflows/skill-style.md +35 -40
- package/docs/workflows/using-git-worktrees.md +40 -44
- package/docs/workflows/verification-loop.md +43 -48
- package/docs/workflows/verify-before-claim.md +37 -40
- package/docs/workflows/zoom-out.md +39 -51
- package/grill-with-docs/ADR-FORMAT.md +22 -0
- package/grill-with-docs/SKILL.md +1 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/kernel-router.js +38 -10
- package/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/hooks/scripts/lib/telemetry.js +40 -0
- package/hooks/scripts/lib/workflow-isolation.js +30 -1
- package/hooks/scripts/lib/workflow-runtime.js +18 -1
- package/hooks/scripts/rule-of-3-tracker.js +17 -9
- package/hooks/scripts/state-persist.js +19 -7
- package/hooks/scripts/workflow-disposition.js +19 -8
- package/hooks/scripts/workflow-gate.js +21 -4
- package/multi-agent-workspace/SKILL.md +1 -1
- package/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/contract-integrity/scripts/audit.js +536 -0
- package/plugins/harness-everything/hooks/scripts/lib/learning-opportunity.js +22 -0
- package/plugins/harness-everything/hooks/scripts/lib/telemetry.js +40 -0
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +30 -1
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +18 -1
- package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +17 -9
- package/plugins/harness-everything/hooks/scripts/state-persist.js +19 -7
- package/plugins/harness-everything/hooks/scripts/workflow-disposition.js +19 -8
- package/plugins/harness-everything/hooks/scripts/workflow-gate.js +21 -4
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +22 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +38 -10
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +28 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +1 -1
- package/plugins/harness-everything/skills/self-evolve/scripts/lesson-candidate.js +52 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-spec/references/process.md +26 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +14 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +14 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +20 -1
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +3 -2
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +25 -1
- package/plugins/harness-everything/skills/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +3 -0
- package/scripts/sync-openai-plugin.js +12 -0
- package/self-evolve/SKILL.md +1 -1
- package/self-evolve/scripts/lesson-candidate.js +52 -0
- package/telemetry/schemas/event.schema.json +141 -22
- package/telemetry/scripts/report.js +42 -0
- package/to-spec/SKILL.md +1 -1
- package/to-spec/references/process.md +26 -0
- package/to-spec/templates/cli-reference.md +14 -0
- package/to-spec/templates/dev-doc.md +14 -0
- package/to-spec/templates/feature-spec.md +14 -0
- package/to-spec/templates/schema-doc.md +14 -0
- package/to-tickets/SKILL.md +1 -1
- package/to-tickets/references/publishing-and-templates.md +20 -1
- package/verification-loop/SKILL.md +3 -2
- package/verification-loop/references/verification-phases.md +25 -1
- package/verification-loop/scripts/contract-integrity-audit.js +31 -0
- package/verification-loop/templates/verification-report.template.md +3 -0
- package/docs/issue-20-rollup.md +0 -47
- package/docs/reviews/pr-142-issue-131.md +0 -89
|
@@ -68,6 +68,77 @@ helper.check(
|
|
|
68
68
|
`Got exit=${successResult.code}, state=${JSON.stringify(resetState)}`
|
|
69
69
|
);
|
|
70
70
|
|
|
71
|
+
helper.runHook('rule-of-3-tracker.js', claudeFailurePayload);
|
|
72
|
+
const claudeNoExitSuccess = helper.runHook('rule-of-3-tracker.js', {
|
|
73
|
+
hook_event_name: 'PostToolUse',
|
|
74
|
+
tool_name: 'Bash',
|
|
75
|
+
tool_input: { command: 'echo recovered' },
|
|
76
|
+
tool_response: {
|
|
77
|
+
stdout: 'recovered',
|
|
78
|
+
stderr: 'warning: successful command emitted an error-like diagnostic',
|
|
79
|
+
},
|
|
80
|
+
session_id: helper.SESSION_ID,
|
|
81
|
+
});
|
|
82
|
+
const noExitResetState = helper.readState('rule-of-3-state.json');
|
|
83
|
+
helper.check(
|
|
84
|
+
'2a-claude-success-no-exit. PostToolUse without exitCode resets even with stderr diagnostics',
|
|
85
|
+
claudeNoExitSuccess.code === 0 &&
|
|
86
|
+
noExitResetState.count === 0 &&
|
|
87
|
+
noExitResetState.zoomOutResolved === true,
|
|
88
|
+
`Got exit=${claudeNoExitSuccess.code}, state=${JSON.stringify(noExitResetState)}`
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
helper.writeState('rule-of-3-state.json', {
|
|
92
|
+
count: 0, lastHash: null, zoomOutResolved: false, zoomOutCycles: 0, lastFailureAt: 0,
|
|
93
|
+
});
|
|
94
|
+
helper.runHook('rule-of-3-tracker.js', {
|
|
95
|
+
hook_event_name: 'PostToolUseFailure',
|
|
96
|
+
tool_name: 'Bash',
|
|
97
|
+
tool_input: { command: 'diff -rq hooks cache-hooks' },
|
|
98
|
+
error: 'Exit code 1\nworkflow-mutation-denied.js\nFiles hooks/a.js and cache-hooks/a.js differ',
|
|
99
|
+
session_id: helper.SESSION_ID,
|
|
100
|
+
});
|
|
101
|
+
const deniedFilenameState = helper.readState('rule-of-3-state.json');
|
|
102
|
+
helper.check(
|
|
103
|
+
'2a-category-negative. File name containing denied stays unknown/threshold 3',
|
|
104
|
+
deniedFilenameState.category === 'unknown' && deniedFilenameState.threshold === 3,
|
|
105
|
+
`State=${JSON.stringify(deniedFilenameState)}`
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
helper.writeState('rule-of-3-state.json', {
|
|
109
|
+
count: 0, lastHash: null, zoomOutResolved: false, zoomOutCycles: 0, lastFailureAt: 0,
|
|
110
|
+
});
|
|
111
|
+
helper.runHook('rule-of-3-tracker.js', {
|
|
112
|
+
hook_event_name: 'PostToolUseFailure',
|
|
113
|
+
tool_name: 'Bash',
|
|
114
|
+
tool_input: { command: 'cat protected.txt' },
|
|
115
|
+
error: 'EACCES: permission denied, open protected.txt',
|
|
116
|
+
session_id: helper.SESSION_ID,
|
|
117
|
+
});
|
|
118
|
+
const permissionState = helper.readState('rule-of-3-state.json');
|
|
119
|
+
helper.check(
|
|
120
|
+
'2a-category-permission. Real EACCES is permission/threshold 2',
|
|
121
|
+
permissionState.category === 'permission' && permissionState.threshold === 2,
|
|
122
|
+
`State=${JSON.stringify(permissionState)}`
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
helper.writeState('rule-of-3-state.json', {
|
|
126
|
+
count: 0, lastHash: null, zoomOutResolved: false, zoomOutCycles: 0, lastFailureAt: 0,
|
|
127
|
+
});
|
|
128
|
+
helper.runHook('rule-of-3-tracker.js', {
|
|
129
|
+
hook_event_name: 'PostToolUseFailure',
|
|
130
|
+
tool_name: 'Bash',
|
|
131
|
+
tool_input: { command: 'node slow.js' },
|
|
132
|
+
error: 'TimeoutError: operation timed out',
|
|
133
|
+
session_id: helper.SESSION_ID,
|
|
134
|
+
});
|
|
135
|
+
const timeoutState = helper.readState('rule-of-3-state.json');
|
|
136
|
+
helper.check(
|
|
137
|
+
'2a-category-timeout. Real timeout is timeout/threshold 2',
|
|
138
|
+
timeoutState.category === 'timeout' && timeoutState.threshold === 2,
|
|
139
|
+
`State=${JSON.stringify(timeoutState)}`
|
|
140
|
+
);
|
|
141
|
+
|
|
71
142
|
helper.writeState('rule-of-3-state.json', { count: 3, lastHash: 'mech-test', zoomOutResolved: false });
|
|
72
143
|
const tripResult = helper.runHook('rule-of-3.js', { session_id: helper.SESSION_ID });
|
|
73
144
|
helper.check(
|
|
@@ -17,6 +17,7 @@ const env = { ...process.env, HARNESS_STATE_HOME: path.join(temp, 'state'), HARN
|
|
|
17
17
|
process.env.HARNESS_STATE_HOME = env.HARNESS_STATE_HOME;
|
|
18
18
|
const state = require(path.join(runtimeRoot, 'hooks/scripts/lib/harness-state'));
|
|
19
19
|
const workflowRuntime = require(path.join(runtimeRoot, 'hooks/scripts/lib/workflow-runtime'));
|
|
20
|
+
const workflowIsolation = require(path.join(runtimeRoot, 'hooks/scripts/lib/workflow-isolation'));
|
|
20
21
|
const sessionId = 'workflow-lifecycle';
|
|
21
22
|
const payload = { session_id: sessionId, cwd: repo };
|
|
22
23
|
const sessionDir = state.getSessionDir(repo, sessionId);
|
|
@@ -35,6 +36,14 @@ function git(args) {
|
|
|
35
36
|
}
|
|
36
37
|
const route = prompt => node('harness-everything/scripts/kernel-router.js', { ...payload, prompt });
|
|
37
38
|
const gate = (tool, input, cwd = linked) => node('hooks/scripts/workflow-gate.js', { ...payload, cwd, tool_name: tool, tool_input: input });
|
|
39
|
+
const persistDirect = (tool, input, cwd = repo, response = {}) => node('hooks/scripts/state-persist.js', {
|
|
40
|
+
...payload,
|
|
41
|
+
cwd,
|
|
42
|
+
hook_event_name: 'PostToolUse',
|
|
43
|
+
tool_name: tool,
|
|
44
|
+
tool_input: input,
|
|
45
|
+
tool_response: response,
|
|
46
|
+
});
|
|
38
47
|
let toolUseSeq = 0;
|
|
39
48
|
function beginShell(command, cwd = repo) {
|
|
40
49
|
const toolUseId = 'toolu_fixture_' + (++toolUseSeq);
|
|
@@ -77,6 +86,28 @@ const probeFiles = () => {
|
|
|
77
86
|
};
|
|
78
87
|
const stop = extra => node('hooks/scripts/workflow-stop-gate.js', { ...payload, ...extra });
|
|
79
88
|
const control = (...args) => node('hooks/scripts/workflow-disposition.js', null, [args[0], '--session-id', sessionId, ...args.slice(1)]);
|
|
89
|
+
function fixtureSession(fixtureSessionId) {
|
|
90
|
+
const fixturePayload = { session_id: fixtureSessionId, cwd: repo };
|
|
91
|
+
const fixtureSessionDir = state.getSessionDir(repo, fixtureSessionId);
|
|
92
|
+
const fixtureFile = path.join(fixtureSessionDir, 'workflow-run.json');
|
|
93
|
+
return {
|
|
94
|
+
payload: fixturePayload,
|
|
95
|
+
file: fixtureFile,
|
|
96
|
+
sessionDir: fixtureSessionDir,
|
|
97
|
+
route: prompt => node('harness-everything/scripts/kernel-router.js', { ...fixturePayload, prompt }),
|
|
98
|
+
gate: (tool, input, extra = {}) => node('hooks/scripts/workflow-gate.js', { ...fixturePayload, ...extra, tool_name: tool, tool_input: input }),
|
|
99
|
+
persist: (tool, input, response = {}, extra = {}) => node('hooks/scripts/state-persist.js', {
|
|
100
|
+
...fixturePayload,
|
|
101
|
+
...extra,
|
|
102
|
+
hook_event_name: 'PostToolUse',
|
|
103
|
+
tool_name: tool,
|
|
104
|
+
tool_input: input,
|
|
105
|
+
tool_response: response,
|
|
106
|
+
}),
|
|
107
|
+
stop: extra => node('hooks/scripts/workflow-stop-gate.js', { ...fixturePayload, ...extra }),
|
|
108
|
+
control: (...args) => node('hooks/scripts/workflow-disposition.js', null, [args[0], '--session-id', fixtureSessionId, ...args.slice(1)]),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
80
111
|
const stages = [
|
|
81
112
|
{ stageId: 'build', goal: 'fix', agent: 'fable-worker', task: 'fix bounded module', dependsOn: [], writeSet: ['src'], checkCommand: 'node test-build.js', passCondition: 'exit 0' },
|
|
82
113
|
{ stageId: 'optional', goal: 'external scope', agent: 'fable-worker', task: 'external task', dependsOn: [], writeSet: [], checkCommand: 'node external.js', passCondition: 'exit 0' },
|
|
@@ -84,9 +115,95 @@ const stages = [
|
|
|
84
115
|
];
|
|
85
116
|
|
|
86
117
|
try {
|
|
118
|
+
// #162: Tier-2 shell accounting must degrade conservatively when a workspace
|
|
119
|
+
// is not a Git repository. Lack of a Git fingerprint is not an isolation
|
|
120
|
+
// violation for a non-major workflow.
|
|
121
|
+
const nonGit = fixtureSession('workflow-non-git');
|
|
122
|
+
check(nonGit.route('Fix this checkout bug with a regression test').status === 0 &&
|
|
123
|
+
read(nonGit.file).strategy === 'iterative-single',
|
|
124
|
+
'#162 non-git fixture routes to Tier 2 iterative workflow');
|
|
125
|
+
check(nonGit.control('start').status === 0 && read(nonGit.file).state === 'running',
|
|
126
|
+
'#162 non-git Tier 2 workflow starts normally');
|
|
127
|
+
let nonGitIterations = read(nonGit.file).budget?.counters?.iterations || 0;
|
|
128
|
+
for (const [index, command] of ['npm --version', 'node -e "process.exit(0)"', 'cd .', 'git init -q'].entries()) {
|
|
129
|
+
const toolUseId = 'toolu_non_git_pre_' + index;
|
|
130
|
+
const admitted = nonGit.gate('Bash', { command }, { tool_use_id: toolUseId });
|
|
131
|
+
check(admitted.status === 0, '#162 non-git Tier 2 admits untrusted shell: ' + command);
|
|
132
|
+
const afterGate = read(nonGit.file);
|
|
133
|
+
check(afterGate.budget.counters.iterations === ++nonGitIterations &&
|
|
134
|
+
afterGate.lastMutationAt > 0 &&
|
|
135
|
+
afterGate.budget.events.at(-1)?.evidence === 'shell:unobservable-workspace',
|
|
136
|
+
'#162 unobservable non-git shell is conservatively counted once: ' + command);
|
|
137
|
+
check(nonGit.persist('Bash', { command }, { stdout: 'fixture success' }, { tool_use_id: toolUseId }).status === 0,
|
|
138
|
+
'#162 non-git shell PostToolUse stays fail-open without a probe: ' + command);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// A probe may have been created while Git was observable and become
|
|
142
|
+
// unobservable before PostToolUse (for example .git disappears). Tier 2
|
|
143
|
+
// must settle it conservatively instead of blocking the workflow.
|
|
144
|
+
const lostObservationId = 'toolu_non_git_post_fallback';
|
|
145
|
+
const lostObservationPayload = {
|
|
146
|
+
...nonGit.payload,
|
|
147
|
+
tool_name: 'Bash',
|
|
148
|
+
tool_use_id: lostObservationId,
|
|
149
|
+
tool_input: { command: 'node lost-observation.js' },
|
|
150
|
+
};
|
|
151
|
+
const lostObservationContext = workflowRuntime.loadWorkflow(lostObservationPayload);
|
|
152
|
+
const lostObservationKey = workflowIsolation.shellProbeKey(lostObservationPayload, repo);
|
|
153
|
+
workflowRuntime.registerMutationProbe(lostObservationContext, lostObservationKey, 'f'.repeat(64), true);
|
|
154
|
+
const beforeLostObservation = read(nonGit.file);
|
|
155
|
+
check(nonGit.persist('Bash', { command: 'node lost-observation.js' }, { stdout: 'fixture success' },
|
|
156
|
+
{ tool_use_id: lostObservationId }).status === 0,
|
|
157
|
+
'#162 Tier 2 PostToolUse degrades when workspace observation becomes unavailable');
|
|
158
|
+
const afterLostObservation = read(nonGit.file);
|
|
159
|
+
check(afterLostObservation.state === 'running' &&
|
|
160
|
+
afterLostObservation.budget.counters.iterations === beforeLostObservation.budget.counters.iterations + 1 &&
|
|
161
|
+
afterLostObservation.lastMutationAt >= (beforeLostObservation.lastMutationAt || 0),
|
|
162
|
+
'#162 lost post-observation is settled as one conservative mutation without blocking');
|
|
163
|
+
|
|
164
|
+
const nonGitTarget = path.join(repo, 'non-git-edit.txt');
|
|
165
|
+
check(nonGit.gate('Write', { file_path: nonGitTarget }).status === 0,
|
|
166
|
+
'#162 direct edit remains available in non-git Tier 2');
|
|
167
|
+
fs.writeFileSync(nonGitTarget, 'changed\n');
|
|
168
|
+
check(nonGit.persist('Write', { file_path: nonGitTarget }).status === 0,
|
|
169
|
+
'#162 direct non-git edit persists mutation milestone');
|
|
170
|
+
const verifyId = 'toolu_non_git_verify';
|
|
171
|
+
check(nonGit.gate('Bash', { command: 'node run-test.js' }, { tool_use_id: verifyId }).status === 0,
|
|
172
|
+
'#162 verification shell is admitted in non-git Tier 2');
|
|
173
|
+
check(nonGit.persist('Bash', { command: 'node run-test.js' }, { stdout: 'tests passed' },
|
|
174
|
+
{ tool_use_id: verifyId }).status === 0,
|
|
175
|
+
'#162 non-git verification result is persisted');
|
|
176
|
+
check(nonGit.stop().status === 0 && read(nonGit.file).state === 'satisfied',
|
|
177
|
+
'#162 edit then verification can satisfy Stop in non-git Tier 2');
|
|
178
|
+
|
|
179
|
+
const nonGitMajor = fixtureSession('workflow-non-git-major');
|
|
180
|
+
check(nonGitMajor.route('Refactor the entire authentication architecture across all services without fable').status === 0 &&
|
|
181
|
+
read(nonGitMajor.file).tier === 'tier3',
|
|
182
|
+
'#162 non-git major fixture routes to Tier 3');
|
|
183
|
+
check(nonGitMajor.control('start').status === 0 && read(nonGitMajor.file).state === 'running',
|
|
184
|
+
'#162 non-Fable Tier 3 controller can enter running state before mutation');
|
|
185
|
+
const majorBlocked = nonGitMajor.gate('Bash', { command: 'node mutate.js' }, { tool_use_id: 'toolu_non_git_major' });
|
|
186
|
+
check(majorBlocked.status === 2 && /Git worktree isolation/i.test(majorBlocked.stderr),
|
|
187
|
+
'#162 non-git Tier 3 remains fail-closed with an isolation-specific error');
|
|
188
|
+
|
|
189
|
+
fs.unlinkSync(nonGitTarget);
|
|
87
190
|
git(['init']); fs.writeFileSync(path.join(repo, 'README.md'), 'fixture\n'); git(['add', '.']);
|
|
88
191
|
git(['-c', 'user.name=Harness Test', '-c', 'user.email=harness@example.invalid', 'commit', '-m', 'fixture']);
|
|
89
192
|
git(['worktree', 'add', linked, '-b', 'isolated']);
|
|
193
|
+
const taskNotification = '<task-notification><summary>Background command "Run full test suite" failed with exit code 1</summary></task-notification>';
|
|
194
|
+
const unboundNotification = node('harness-everything/scripts/kernel-router.js', { cwd: repo, prompt: taskNotification });
|
|
195
|
+
check(unboundNotification.status === 0 &&
|
|
196
|
+
unboundNotification.stdout.includes('ignored for routing') &&
|
|
197
|
+
!unboundNotification.stdout.includes('WORKFLOW EXECUTION CONTRACT'),
|
|
198
|
+
'host notification without session binding is a routing no-op');
|
|
199
|
+
const noWorkflowNotification = route(taskNotification);
|
|
200
|
+
check(noWorkflowNotification.status === 0 && !fs.existsSync(file),
|
|
201
|
+
'host notification with no active workflow creates no contract');
|
|
202
|
+
check(noWorkflowNotification.stdout.includes('ignored for routing') &&
|
|
203
|
+
!noWorkflowNotification.stdout.includes('iterative-single') &&
|
|
204
|
+
!noWorkflowNotification.stdout.includes('WORKFLOW EXECUTION CONTRACT'),
|
|
205
|
+
'host notification with no active workflow emits no notification-derived routing contract');
|
|
206
|
+
|
|
90
207
|
const routed = route('Refactor the entire repository architecture in dependent stages.');
|
|
91
208
|
check(routed.status === 0, 'prompt hook creates workflow');
|
|
92
209
|
const initial = read(file);
|
|
@@ -149,6 +266,15 @@ try {
|
|
|
149
266
|
check(stop().status === 2, 'verification predating a new mutation is stale');
|
|
150
267
|
write(file, mutationState);
|
|
151
268
|
check(stop().status === 0 && read(file).state === 'satisfied', 'all covered stages and verifier resolve workflow to satisfied');
|
|
269
|
+
const satisfiedWorkflow = read(file);
|
|
270
|
+
const satisfiedNotification = route(taskNotification);
|
|
271
|
+
check(satisfiedNotification.status === 0 &&
|
|
272
|
+
read(file).workflowId === satisfiedWorkflow.workflowId &&
|
|
273
|
+
read(file).state === 'satisfied',
|
|
274
|
+
'satisfied workflow followed by host notification keeps the same satisfied contract');
|
|
275
|
+
check(!satisfiedNotification.stdout.includes('Run full test suite') &&
|
|
276
|
+
satisfiedNotification.stdout.includes('routing unchanged'),
|
|
277
|
+
'satisfied host notification text is not routed as new work');
|
|
152
278
|
check(route('Refactor the entire repository architecture in dependent stages.').status === 0 && read(file).workflowId !== initial.workflowId, 'next completed-task boundary creates a fresh workflow');
|
|
153
279
|
check(stop({ stop_hook_active: true }).status === 0 && read(file).state === 'blocked', 'Stop retry reports blocked instead of faking completion');
|
|
154
280
|
check(gate('Write', { file_path: path.join(linked, 'src.js') }).status === 2, 'blocked state still prohibits mutation');
|
|
@@ -194,11 +320,161 @@ try {
|
|
|
194
320
|
promotedTier3.mutationIsolation?.required === true && !promotedTier3.pendingPlan, 'pending Tier 3 isolation becomes active atomically on start');
|
|
195
321
|
check(workflowRuntime.isMajorWorkflow(promotedTier3), 'major-workflow classification flips only after explicit activation');
|
|
196
322
|
check(gate('Write', { file_path: path.join(repo, 'premature-primary.js') }, repo).status === 2, 'activated Tier 3 plan still blocks primary-tree mutation');
|
|
323
|
+
|
|
324
|
+
// #163: a queued Tier-3 plan can intentionally remain single-agent when
|
|
325
|
+
// Fable is prohibited. Explicit start must still promote the whole plan,
|
|
326
|
+
// including isolation and budget metadata, before any mutation is admitted.
|
|
327
|
+
fs.unlinkSync(file);
|
|
328
|
+
check(route('Fix this checkout bug with a regression test').status === 0 && read(file).strategy === 'iterative-single',
|
|
329
|
+
'#163 fixture starts from a Tier 2 iterative workflow');
|
|
330
|
+
check(control('start').status === 0 && read(file).state === 'running',
|
|
331
|
+
'#163 fixture activates Tier 2 before a stronger route arrives');
|
|
332
|
+
check(route('Refactor the entire authentication architecture across all services and migrate the database schema without fable').status === 0,
|
|
333
|
+
'#163 stronger Tier 3 route without Fable is retained for explicit replan');
|
|
334
|
+
const queuedNonFableTier3 = read(file);
|
|
335
|
+
check(queuedNonFableTier3.state === 'blocked' &&
|
|
336
|
+
queuedNonFableTier3.pendingPlan?.tier === 'tier3' &&
|
|
337
|
+
queuedNonFableTier3.pendingPlan?.strategy === 'iterative-single' &&
|
|
338
|
+
queuedNonFableTier3.pendingPlan?.mutationIsolation?.required === true,
|
|
339
|
+
'#163 queued Tier 3 plan remains non-Fable but still requires isolation');
|
|
340
|
+
write(file, {
|
|
341
|
+
...queuedNonFableTier3,
|
|
342
|
+
budget: {
|
|
343
|
+
...queuedNonFableTier3.budget,
|
|
344
|
+
limits: { ...queuedNonFableTier3.budget.limits, maxIterations: 1 },
|
|
345
|
+
},
|
|
346
|
+
});
|
|
347
|
+
check(control('start').status === 0, '#163 explicit start activates queued non-Fable Tier 3 plan');
|
|
348
|
+
const promotedNonFableTier3 = read(file);
|
|
349
|
+
check(promotedNonFableTier3.workflowPlan?.tier === 'tier3' &&
|
|
350
|
+
promotedNonFableTier3.tier === 'tier3' &&
|
|
351
|
+
promotedNonFableTier3.strategy === 'iterative-single' &&
|
|
352
|
+
promotedNonFableTier3.mutationIsolation?.required === true &&
|
|
353
|
+
!promotedNonFableTier3.pendingPlan,
|
|
354
|
+
'#163 non-Fable Tier 3 plan replaces all authoritative active-plan fields');
|
|
355
|
+
check(promotedNonFableTier3.budget?.limits?.maxIterations === promotedNonFableTier3.workflowPlan?.limits?.maxIterations &&
|
|
356
|
+
promotedNonFableTier3.budget?.limits?.maxIterations !== 1,
|
|
357
|
+
'#163 plan activation re-derives budget limits from the promoted plan');
|
|
358
|
+
check(workflowRuntime.isMajorWorkflow(promotedNonFableTier3),
|
|
359
|
+
'#163 non-Fable Tier 3 promotion enables major-workflow isolation');
|
|
360
|
+
check(gate('Write', { file_path: path.join(repo, 'non-fable-primary.js') }, repo).status === 2,
|
|
361
|
+
'#163 promoted non-Fable Tier 3 blocks primary-tree mutation');
|
|
362
|
+
check(gate('Write', { file_path: path.join(linked, 'non-fable-linked.js') }, linked).status === 0,
|
|
363
|
+
'#163 promoted non-Fable Tier 3 admits linked-worktree mutation');
|
|
364
|
+
|
|
197
365
|
fs.unlinkSync(file);
|
|
198
366
|
check(route('Fix this checkout bug with a regression test').status === 0 && read(file).strategy === 'iterative-single', 'fresh bounded fix returns to Tier 2 iterative lifecycle');
|
|
367
|
+
let tier2 = read(file);
|
|
368
|
+
const tier2WorkflowId = tier2.workflowId;
|
|
369
|
+
write(file, { ...tier2, state: 'deferred' });
|
|
370
|
+
const deferredNotification = route(taskNotification);
|
|
371
|
+
check(deferredNotification.status === 0 &&
|
|
372
|
+
read(file).workflowId === tier2WorkflowId &&
|
|
373
|
+
read(file).state === 'deferred',
|
|
374
|
+
'deferred workflow followed by host notification keeps the same deferred contract');
|
|
375
|
+
check(!deferredNotification.stdout.includes('Run full test suite') &&
|
|
376
|
+
deferredNotification.stdout.includes('routing unchanged'),
|
|
377
|
+
'deferred host notification text is not routed as new work');
|
|
378
|
+
write(file, tier2);
|
|
379
|
+
|
|
199
380
|
const iterationLimit = read(file).workflowPlan.limits.maxIterations;
|
|
200
381
|
check(Number.isInteger(iterationLimit) && iterationLimit > 0, 'iterative route exposes a numeric iteration budget');
|
|
201
382
|
|
|
383
|
+
// #165: direct Edit/Write/apply_patch accounting follows workspace effects.
|
|
384
|
+
// Bookkeeping outside the repository must not consume Tier-2 iterations or
|
|
385
|
+
// advance workflow/Stop-gate mutation milestones.
|
|
386
|
+
const handoffFile = path.join(sessionDir, 'handoff-state.json');
|
|
387
|
+
const outsideTarget = path.join(temp, 'scratch-note.txt');
|
|
388
|
+
const beforeOutside = read(file);
|
|
389
|
+
const beforeOutsideIterations = beforeOutside.budget?.counters?.iterations || 0;
|
|
390
|
+
const beforeOutsideMutationAt = beforeOutside.lastMutationAt || 0;
|
|
391
|
+
const beforeOutsideEditAt = fs.existsSync(handoffFile) ? (read(handoffFile).lastEditAt || 0) : 0;
|
|
392
|
+
for (let i = 0; i < 3; i++) {
|
|
393
|
+
check(gate('Write', { file_path: outsideTarget }, repo).status === 0,
|
|
394
|
+
'out-of-workspace Write is admitted without Tier-2 mutation accounting');
|
|
395
|
+
fs.writeFileSync(outsideTarget, 'scratch ' + i + '\n');
|
|
396
|
+
const outsideResponse = i === 0
|
|
397
|
+
? { stderr: 'warning: successful bookkeeping write emitted diagnostic text' }
|
|
398
|
+
: {};
|
|
399
|
+
check(persistDirect('Write', { file_path: outsideTarget }, repo, outsideResponse).status === 0,
|
|
400
|
+
'out-of-workspace Write post-tool state persists without mutation milestone');
|
|
401
|
+
}
|
|
402
|
+
const afterOutside = read(file);
|
|
403
|
+
const afterOutsideHandoff = read(handoffFile);
|
|
404
|
+
check((afterOutside.budget?.counters?.iterations || 0) === beforeOutsideIterations &&
|
|
405
|
+
(afterOutside.lastMutationAt || 0) === beforeOutsideMutationAt &&
|
|
406
|
+
(afterOutsideHandoff.lastEditAt || 0) === beforeOutsideEditAt &&
|
|
407
|
+
afterOutsideHandoff.status !== 'failed',
|
|
408
|
+
'out-of-workspace Writes leave iterations/milestones unchanged and PostToolUse stderr is not a failure');
|
|
409
|
+
check(stop().status === 0, 'out-of-workspace-only bookkeeping does not force verification before Stop');
|
|
410
|
+
|
|
411
|
+
check(route('Fix this checkout bug with a regression test').status === 0 && read(file).strategy === 'iterative-single',
|
|
412
|
+
'real follow-up user prompt after notification/accounting checks routes normally');
|
|
413
|
+
let observedDirectIterations = read(file).budget?.counters?.iterations || 0;
|
|
414
|
+
const internalTarget = path.join(repo, 'direct-inside.txt');
|
|
415
|
+
const beforeInternalMutationAt = read(file).lastMutationAt || 0;
|
|
416
|
+
check(gate('Write', { file_path: internalTarget }, repo).status === 0, 'in-workspace Write is admitted');
|
|
417
|
+
fs.writeFileSync(internalTarget, 'inside\n');
|
|
418
|
+
check(persistDirect('Write', { file_path: internalTarget }, repo).status === 0, 'in-workspace Write post-tool state persists');
|
|
419
|
+
check(read(file).budget.counters.iterations === ++observedDirectIterations &&
|
|
420
|
+
read(file).lastMutationAt >= beforeInternalMutationAt &&
|
|
421
|
+
read(handoffFile).lastEditAt > beforeOutsideEditAt,
|
|
422
|
+
'in-workspace Write consumes exactly one iteration and advances mutation milestones');
|
|
423
|
+
|
|
424
|
+
const mixedPatch = [
|
|
425
|
+
'*** Begin Patch',
|
|
426
|
+
`*** Update File: ${outsideTarget}`,
|
|
427
|
+
'@@',
|
|
428
|
+
'-scratch',
|
|
429
|
+
'+scratch updated',
|
|
430
|
+
`*** Update File: ${path.join(repo, 'README.md')}`,
|
|
431
|
+
'@@',
|
|
432
|
+
'-fixture',
|
|
433
|
+
'+fixture updated',
|
|
434
|
+
'*** End Patch',
|
|
435
|
+
].join('\n');
|
|
436
|
+
check(gate('apply_patch', { patch: mixedPatch }, repo).status === 0, 'mixed apply_patch is admitted');
|
|
437
|
+
check(persistDirect('apply_patch', { patch: mixedPatch }, repo).status === 0, 'mixed apply_patch post-tool state persists');
|
|
438
|
+
check(read(file).budget.counters.iterations === ++observedDirectIterations,
|
|
439
|
+
'mixed apply_patch counts exactly once when any target is inside the workspace');
|
|
440
|
+
|
|
441
|
+
const pureOutsidePatch = [
|
|
442
|
+
'*** Begin Patch',
|
|
443
|
+
`*** Update File: ${outsideTarget}`,
|
|
444
|
+
'@@',
|
|
445
|
+
'-scratch updated',
|
|
446
|
+
'+scratch external only',
|
|
447
|
+
'*** End Patch',
|
|
448
|
+
].join('\n');
|
|
449
|
+
const beforePureOutside = read(file);
|
|
450
|
+
const beforePureOutsideEditAt = read(handoffFile).lastEditAt || 0;
|
|
451
|
+
check(gate('apply_patch', { patch: pureOutsidePatch }, repo).status === 0, 'pure external apply_patch is admitted');
|
|
452
|
+
check(persistDirect('apply_patch', { patch: pureOutsidePatch }, repo).status === 0, 'pure external apply_patch post-tool state persists');
|
|
453
|
+
check(read(file).budget.counters.iterations === observedDirectIterations &&
|
|
454
|
+
read(file).lastMutationAt === beforePureOutside.lastMutationAt &&
|
|
455
|
+
(read(handoffFile).lastEditAt || 0) === beforePureOutsideEditAt,
|
|
456
|
+
'pure external apply_patch consumes no iteration and advances no mutation milestone');
|
|
457
|
+
|
|
458
|
+
const linkedTarget = path.join(linked, 'direct-linked.txt');
|
|
459
|
+
check(gate('Write', { file_path: linkedTarget }, linked).status === 0,
|
|
460
|
+
'Write inside a linked worktree is treated as workspace mutation');
|
|
461
|
+
fs.writeFileSync(linkedTarget, 'linked\n');
|
|
462
|
+
check(persistDirect('Write', { file_path: linkedTarget }, linked).status === 0,
|
|
463
|
+
'linked-worktree Write post-tool state persists');
|
|
464
|
+
check(read(file).budget.counters.iterations === ++observedDirectIterations,
|
|
465
|
+
'linked-worktree Write consumes exactly one iteration');
|
|
466
|
+
|
|
467
|
+
// Keep the #165 regression fixture from consuming the budget used by the
|
|
468
|
+
// pre-existing lifecycle/exhaustion assertions below.
|
|
469
|
+
fs.unlinkSync(internalTarget);
|
|
470
|
+
fs.unlinkSync(linkedTarget);
|
|
471
|
+
fs.unlinkSync(file);
|
|
472
|
+
if (fs.existsSync(handoffFile)) fs.unlinkSync(handoffFile);
|
|
473
|
+
check(route('Fix this checkout bug with a regression test').status === 0 &&
|
|
474
|
+
read(file).strategy === 'iterative-single' &&
|
|
475
|
+
(read(file).budget?.counters?.iterations || 0) === 0,
|
|
476
|
+
'#165 regression fixture resets into a fresh Tier-2 lifecycle baseline');
|
|
477
|
+
|
|
202
478
|
// #155/#157 regression lock: read-only shell composition, including quoted
|
|
203
479
|
// and escaped separator characters inside arguments, must never look like an
|
|
204
480
|
// iterative mutation or advance the mutation clock.
|
|
@@ -16,14 +16,57 @@ helper.check(
|
|
|
16
16
|
|
|
17
17
|
helper.runHook('state-persist.js', {
|
|
18
18
|
session_id: helper.SESSION_ID,
|
|
19
|
+
hook_event_name: 'PostToolUse',
|
|
19
20
|
tool_name: 'Bash',
|
|
20
|
-
tool_response: {
|
|
21
|
+
tool_response: {
|
|
22
|
+
stdout: 'ok',
|
|
23
|
+
stderr: 'warning: successful command emitted diagnostic text',
|
|
24
|
+
},
|
|
21
25
|
});
|
|
22
26
|
const resolvedHandoff = helper.readState('handoff-state.json');
|
|
23
27
|
helper.check(
|
|
24
|
-
'2c.
|
|
28
|
+
'2c. Claude-shaped PostToolUse without exitCode clears failed state even with stderr diagnostics',
|
|
25
29
|
resolvedHandoff.status === 'idle' && !!resolvedHandoff.lastResolved,
|
|
26
30
|
`Got ${JSON.stringify(resolvedHandoff)}`
|
|
27
31
|
);
|
|
28
32
|
|
|
33
|
+
helper.runHook('state-persist.js', {
|
|
34
|
+
session_id: helper.SESSION_ID,
|
|
35
|
+
hook_event_name: 'PostToolUse',
|
|
36
|
+
tool_name: 'Bash',
|
|
37
|
+
tool_response: { stdout: '', stderr: 'numeric failure', exitCode: 2 },
|
|
38
|
+
});
|
|
39
|
+
const numericFailure = helper.readState('handoff-state.json');
|
|
40
|
+
helper.check(
|
|
41
|
+
'2c. Numeric non-zero exit still records failure on hosts that report exit codes',
|
|
42
|
+
numericFailure.status === 'failed' && numericFailure.exitCode === 2,
|
|
43
|
+
`Got ${JSON.stringify(numericFailure)}`
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
helper.runHook('state-persist.js', {
|
|
47
|
+
session_id: helper.SESSION_ID,
|
|
48
|
+
hook_event_name: 'PostToolUse',
|
|
49
|
+
tool_name: 'Bash',
|
|
50
|
+
tool_response: { stdout: 'ok', exitCode: 0 },
|
|
51
|
+
});
|
|
52
|
+
const numericResolved = helper.readState('handoff-state.json');
|
|
53
|
+
helper.check(
|
|
54
|
+
'2c. Numeric zero exit still clears failure state',
|
|
55
|
+
numericResolved.status === 'idle' && !!numericResolved.lastResolved,
|
|
56
|
+
`Got ${JSON.stringify(numericResolved)}`
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
helper.runHook('state-persist.js', {
|
|
60
|
+
session_id: helper.SESSION_ID,
|
|
61
|
+
hook_event_name: 'PostToolUseFailure',
|
|
62
|
+
tool_name: 'Bash',
|
|
63
|
+
error: 'host-declared failure without numeric exit code',
|
|
64
|
+
});
|
|
65
|
+
const eventFailure = helper.readState('handoff-state.json');
|
|
66
|
+
helper.check(
|
|
67
|
+
'2c. PostToolUseFailure remains authoritative without numeric exitCode',
|
|
68
|
+
eventFailure.status === 'failed',
|
|
69
|
+
`Got ${JSON.stringify(eventFailure)}`
|
|
70
|
+
);
|
|
71
|
+
|
|
29
72
|
helper.finish();
|
|
@@ -64,12 +64,4 @@ helper.check(
|
|
|
64
64
|
`exit ${negativeControls.status}\n${negativeControls.stdout}\n${negativeControls.stderr}`
|
|
65
65
|
);
|
|
66
66
|
|
|
67
|
-
const rollup = fs.readFileSync(path.join(helper.root, 'docs', 'issue-20-rollup.md'), 'utf8');
|
|
68
|
-
const deferredWork = ['baseline-debugging', 'baseline-performance', 'baseline-security-review', 'baseline-simple-bugfix', 'verify-before-done', 'verification-loop', 'zoom-out', 'security-review'];
|
|
69
|
-
helper.check(
|
|
70
|
-
'2t. issue rollup records unresolved acceptance work accurately',
|
|
71
|
-
/Issue #20 remains open/.test(rollup) && deferredWork.every(term => rollup.includes(term)),
|
|
72
|
-
'rollup must name the open issue and each deferred baseline/skill item'
|
|
73
|
-
);
|
|
74
|
-
|
|
75
67
|
helper.finish();
|
|
@@ -40,13 +40,29 @@ function verifyInstall(text) {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
function isWorktreeRoot(dir) {
|
|
44
|
+
try {
|
|
45
|
+
return fs.existsSync(path.join(dir, '.git'));
|
|
46
|
+
} catch (_) {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
43
51
|
function walk(directory) {
|
|
44
52
|
return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
|
|
45
|
-
if (['node_modules', '.git'].includes(entry.name) || entry.isSymbolicLink()) return [];
|
|
53
|
+
if (['node_modules', '.git', '.worktrees'].includes(entry.name) || entry.isSymbolicLink()) return [];
|
|
46
54
|
const file = path.join(directory, entry.name);
|
|
47
55
|
const relative = path.relative(root, file).split(path.sep).join('/');
|
|
48
56
|
if (relative === 'benchmarks/results' || relative === 'CHANGELOG.md') return [];
|
|
49
|
-
|
|
57
|
+
if (relative === '.worktrees' || relative.startsWith('.worktrees/')) return [];
|
|
58
|
+
if (entry.isDirectory()) {
|
|
59
|
+
// Skip linked worktrees / nested checkouts: a `.git` file marks a
|
|
60
|
+
// linked worktree root, a `.git` dir marks a nested repo. Either way
|
|
61
|
+
// the content belongs to another checkout, not this one (#170).
|
|
62
|
+
if (isWorktreeRoot(file)) return [];
|
|
63
|
+
return walk(file);
|
|
64
|
+
}
|
|
65
|
+
return [file];
|
|
50
66
|
});
|
|
51
67
|
}
|
|
52
68
|
|
|
@@ -68,12 +84,28 @@ try {
|
|
|
68
84
|
}
|
|
69
85
|
assert.deepStrictEqual(invalidTargets('opencode-plugin/index.mjs .opencode/plugins/example.js'), []);
|
|
70
86
|
|
|
71
|
-
const
|
|
72
|
-
|
|
87
|
+
const scanViolations = (readText = file => fs.readFileSync(file, 'utf8')) => {
|
|
88
|
+
const violations = [];
|
|
89
|
+
for (const file of walk(root).filter(file => /\.(?:md|js|mjs|json|jsonc|yml|yaml|txt)$/.test(file))) {
|
|
90
|
+
const text = readText(file);
|
|
91
|
+
for (const target of invalidTargets(text)) violations.push(`${path.relative(root, file)}: ${target}`);
|
|
92
|
+
}
|
|
93
|
+
return violations;
|
|
94
|
+
};
|
|
95
|
+
assert.deepStrictEqual(scanViolations(), [], 'Unsupported OpenCode install destinations; only .js/.ts are auto-discovered');
|
|
96
|
+
|
|
97
|
+
const scannedFiles = walk(root);
|
|
98
|
+
assert.ok(scannedFiles.includes(readmeFile), 'Tracked OpenCode README must remain in the repository scan');
|
|
99
|
+
const plantedTarget = '.opencode/plugins/' + 'negative-control.mjs';
|
|
100
|
+
const planted = scanViolations(file => {
|
|
73
101
|
const text = fs.readFileSync(file, 'utf8');
|
|
74
|
-
|
|
75
|
-
}
|
|
76
|
-
assert.
|
|
102
|
+
return file === readmeFile ? text + '\n' + plantedTarget + '\n' : text;
|
|
103
|
+
});
|
|
104
|
+
assert.ok(
|
|
105
|
+
planted.some(item => item === `${path.relative(root, readmeFile)}: ${plantedTarget}`),
|
|
106
|
+
'Scanner negative control must fail when a tracked file contains an unsupported .mjs install target'
|
|
107
|
+
);
|
|
108
|
+
|
|
77
109
|
const readme = fs.readFileSync(readmeFile, 'utf8');
|
|
78
110
|
for (const destination of ['.opencode/plugins/harness-enforcement.js', '~/.config/opencode/plugins/harness-enforcement.js']) {
|
|
79
111
|
assert.ok(readme.includes(`cp opencode-plugin/index.mjs ${destination}`));
|