harness-everything 0.11.5 → 0.11.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +32 -0
- package/README.md +7 -3
- package/ci/mechanism-2a-rule-of-3.test.js +71 -0
- package/ci/mechanism-2ah-workflow-lifecycle.test.js +314 -8
- package/ci/mechanism-2c-state-persist.test.js +45 -2
- package/ci/mechanism-2t-issue-20.test.js +0 -8
- package/ci/mechanism-30-opencode-plugin-loadability.test.js +39 -7
- package/docs/architecture.md +17 -0
- package/docs/audit.md +52 -5
- package/docs/release-evidence.md +3 -0
- package/docs/workflow-runtime.md +13 -0
- package/docs/workflows/environment-detection.md +47 -37
- package/docs/workflows/eval-harness.md +45 -38
- package/docs/workflows/fable-discipline.md +45 -33
- package/docs/workflows/git-commit.md +53 -45
- package/docs/workflows/grill-me.md +36 -38
- package/docs/workflows/grill-with-docs.md +36 -46
- package/docs/workflows/harness-everything.md +59 -57
- package/docs/workflows/improve-codebase-architecture.md +36 -44
- package/docs/workflows/install-cognitive-os.md +36 -29
- package/docs/workflows/repo-docs.md +29 -40
- package/docs/workflows/rewrite-commits.md +40 -36
- package/docs/workflows/security-review.md +34 -53
- package/docs/workflows/skill-creator.md +39 -56
- package/docs/workflows/skill-style.md +35 -40
- package/docs/workflows/using-git-worktrees.md +40 -44
- package/docs/workflows/verification-loop.md +43 -48
- package/docs/workflows/verify-before-claim.md +37 -40
- package/docs/workflows/zoom-out.md +39 -51
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/kernel-router.js +38 -10
- package/hooks/hooks.json +25 -1
- package/hooks/scripts/lib/workflow-isolation.js +111 -6
- package/hooks/scripts/lib/workflow-runtime.js +138 -2
- package/hooks/scripts/rule-of-3-tracker.js +17 -9
- package/hooks/scripts/state-persist.js +52 -9
- package/hooks/scripts/workflow-gate.js +22 -7
- package/hooks/scripts/workflow-mutation-denied.js +17 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/hooks.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +111 -6
- package/plugins/harness-everything/hooks/scripts/lib/workflow-runtime.js +138 -2
- package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +17 -9
- package/plugins/harness-everything/hooks/scripts/state-persist.js +52 -9
- package/plugins/harness-everything/hooks/scripts/workflow-gate.js +22 -7
- package/plugins/harness-everything/hooks/scripts/workflow-mutation-denied.js +17 -0
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +38 -10
- package/docs/issue-20-rollup.md +0 -47
- package/docs/reviews/pr-142-issue-131.md +0 -89
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const { getWorkspaceRoot, getSessionDir } = require('./lib/harness-state');
|
|
5
|
+
const { loadWorkflow, saveWorkflow, peekMutationProbe, settleMutationProbe } = require('./lib/workflow-runtime');
|
|
6
|
+
const { cwdOf, shellProbeKey, workspaceFingerprint, directMutationInWorkspace } = require('./lib/workflow-isolation');
|
|
5
7
|
const { observeTool } = require('./lib/telemetry');
|
|
6
8
|
|
|
7
9
|
// Commands that count as "verification ran" for the Stop gate
|
|
@@ -28,7 +30,31 @@ process.stdin.on('end', () => {
|
|
|
28
30
|
}
|
|
29
31
|
});
|
|
30
32
|
|
|
33
|
+
function observeWorkspaceMutation(payload, root) {
|
|
34
|
+
const toolName = payload && (payload.tool_name || payload.tool);
|
|
35
|
+
if (toolName !== 'Bash' && toolName !== 'PowerShell' && toolName !== 'exec_command') return false;
|
|
36
|
+
const context = loadWorkflow(payload);
|
|
37
|
+
if (!context.workflow || context.workflow.state === 'deferred') return false;
|
|
38
|
+
const cwd = cwdOf(payload, root);
|
|
39
|
+
const probeKey = shellProbeKey(payload, cwd);
|
|
40
|
+
if (!peekMutationProbe(context, probeKey)) return false;
|
|
41
|
+
let after;
|
|
42
|
+
try {
|
|
43
|
+
after = workspaceFingerprint(cwd);
|
|
44
|
+
} catch (error) {
|
|
45
|
+
context.workflow.state = 'blocked';
|
|
46
|
+
context.workflow.blockReason = 'mutation-observation-failed';
|
|
47
|
+
saveWorkflow(context);
|
|
48
|
+
const critical = new Error('cannot verify whether shell execution changed workspace content');
|
|
49
|
+
critical.code = 'HARNESS_MUTATION_OBSERVATION';
|
|
50
|
+
throw critical;
|
|
51
|
+
}
|
|
52
|
+
const result = settleMutationProbe(context, probeKey, after, toolName);
|
|
53
|
+
return Boolean(result && result.changed);
|
|
54
|
+
}
|
|
55
|
+
|
|
31
56
|
function processState(payload) {
|
|
57
|
+
let exitCodeForHook = 0;
|
|
32
58
|
try {
|
|
33
59
|
const sessionId = payload && (payload.session_id || payload.sessionId);
|
|
34
60
|
const root = getWorkspaceRoot(payload);
|
|
@@ -48,12 +74,17 @@ function processState(payload) {
|
|
|
48
74
|
// it's actually a number; fall back to a stderr signal otherwise.
|
|
49
75
|
const toolResponse = payload.tool_response || {};
|
|
50
76
|
const stdout = toolResponse.stdout ?? payload.stdout ?? '';
|
|
51
|
-
const stderr = toolResponse.stderr ?? payload.stderr ?? '';
|
|
77
|
+
const stderr = toolResponse.stderr ?? payload.stderr ?? payload.error ?? '';
|
|
52
78
|
const rawExitCode = toolResponse.exitCode ?? toolResponse.exit_code ?? payload.exitCode;
|
|
53
79
|
const exitCode = typeof rawExitCode === 'number' ? rawExitCode : undefined;
|
|
54
80
|
const stderrSignal = typeof stderr === 'string' && stderr.trim().length > 0;
|
|
55
|
-
const
|
|
81
|
+
const hookEvent = payload.hook_event_name || payload.hookEventName || '';
|
|
82
|
+
const isSuccessEvent = hookEvent === 'PostToolUse';
|
|
83
|
+
const isFailed = hookEvent === 'PostToolUseFailure' ||
|
|
84
|
+
(exitCode !== undefined && exitCode !== 0) ||
|
|
85
|
+
(!isSuccessEvent && exitCode === undefined && stderrSignal);
|
|
56
86
|
const toolName = payload.tool_name || payload.tool || 'command';
|
|
87
|
+
const observedMutation = observeWorkspaceMutation(payload, root);
|
|
57
88
|
|
|
58
89
|
if (isFailed) {
|
|
59
90
|
// Truncate output to avoid state bloat
|
|
@@ -67,8 +98,13 @@ function processState(payload) {
|
|
|
67
98
|
state.tool = toolName;
|
|
68
99
|
state.exitCode = exitCode;
|
|
69
100
|
state.errorSummary = output.trim();
|
|
70
|
-
} else if (
|
|
71
|
-
// Clear failed state or mark as idle/resolved
|
|
101
|
+
} else if (!isFailed && state.status === 'failed') {
|
|
102
|
+
// Clear failed state or mark as idle/resolved.
|
|
103
|
+
// NOTE (#166, follows #153): Claude Code Bash PostToolUse payloads
|
|
104
|
+
// carry no numeric exit code, so `exitCode === 0` never holds there.
|
|
105
|
+
// A PostToolUse event with no numeric non-zero exit status counts as
|
|
106
|
+
// success even when stderr contains diagnostics. Hosts that omit the
|
|
107
|
+
// lifecycle event name retain the conservative stderr fallback.
|
|
72
108
|
state.lastResolved = { tool: state.tool, timestamp: state.timestamp };
|
|
73
109
|
state.status = 'idle';
|
|
74
110
|
state.timestamp = new Date().toISOString();
|
|
@@ -77,10 +113,13 @@ function processState(payload) {
|
|
|
77
113
|
}
|
|
78
114
|
|
|
79
115
|
// Milestones for the Stop gate: when did the last mutation happen, and
|
|
80
|
-
// has any verification-ish command succeeded since.
|
|
81
|
-
|
|
116
|
+
// has any verification-ish command succeeded since. Direct-tool edits
|
|
117
|
+
// outside the workspace advance neither milestone (#165).
|
|
118
|
+
const isDirectTool = toolName === 'Edit' || toolName === 'Write' || toolName === 'apply_patch';
|
|
119
|
+
if ((isDirectTool && directMutationInWorkspace(payload, cwdOf(payload, root), root)) || observedMutation) {
|
|
82
120
|
state.lastEditAt = Date.now();
|
|
83
|
-
}
|
|
121
|
+
}
|
|
122
|
+
if ((toolName === 'Bash' || toolName === 'PowerShell') && !isFailed) {
|
|
84
123
|
const command = (payload.tool_input && payload.tool_input.command) || '';
|
|
85
124
|
if (VERIFY_COMMAND_RE.test(command)) {
|
|
86
125
|
state.lastVerifyAt = Date.now();
|
|
@@ -100,7 +139,11 @@ function processState(payload) {
|
|
|
100
139
|
}
|
|
101
140
|
}
|
|
102
141
|
} catch (err) {
|
|
103
|
-
|
|
142
|
+
if (err && (err.code === 'HARNESS_MUTATION_OBSERVATION' || err.code === 'HARNESS_WORKFLOW_BUDGET')) {
|
|
143
|
+
console.error('[State Persist] ' + err.message);
|
|
144
|
+
exitCodeForHook = 2;
|
|
145
|
+
}
|
|
146
|
+
// Non-critical persistence/telemetry failures remain fail-open.
|
|
104
147
|
}
|
|
105
|
-
process.exit(
|
|
148
|
+
process.exit(exitCodeForHook);
|
|
106
149
|
}
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const fs = require('fs');
|
|
6
|
-
const { loadWorkflow, saveWorkflow, isMajorWorkflow, matchingRun, recordBudgetEvent, readHookInput, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
7
|
-
const { key, classifyShell, cwdOf, commandOf, mutationPaths, linkedWorktree, assertTargets, assertShellScope } = require('./lib/workflow-isolation');
|
|
6
|
+
const { loadWorkflow, saveWorkflow, isMajorWorkflow, matchingRun, registerMutationProbe, recordBudgetEvent, readHookInput, WORKFLOW_CONTROLLER_COMMANDS } = require('./lib/workflow-runtime');
|
|
7
|
+
const { key, classifyShell, workspaceFingerprint, shellProbeKey, cwdOf, commandOf, mutationPaths, directMutationInWorkspace, linkedWorktree, assertTargets, assertShellScope } = require('./lib/workflow-isolation');
|
|
8
8
|
|
|
9
9
|
const DIRECT = new Set(['Edit', 'Write', 'apply_patch']);
|
|
10
10
|
const SHELL = new Set(['Bash', 'PowerShell', 'exec_command']);
|
|
@@ -27,8 +27,8 @@ function decide(payload) {
|
|
|
27
27
|
const cwd = cwdOf(payload, root);
|
|
28
28
|
const command = commandOf(payload);
|
|
29
29
|
const commandClass = SHELL.has(tool) ? classifyShell(command) : 'direct-mutation';
|
|
30
|
-
if (commandClass === 'read-only' || commandClass === 'worktree-setup') return;
|
|
31
30
|
if (SHELL.has(tool) && isController(command, cwd)) return;
|
|
31
|
+
if (commandClass === 'read-only' || commandClass === 'worktree-setup') return;
|
|
32
32
|
// Only the stage specification is allowed before run entry. Other runtime
|
|
33
33
|
// state, including workflow-run.json, cannot use this bootstrap exception.
|
|
34
34
|
const targets = DIRECT.has(tool) ? mutationPaths(payload) : [];
|
|
@@ -47,11 +47,26 @@ function decide(payload) {
|
|
|
47
47
|
if (String(workflow.strategy || '').startsWith('fable-') && !matchingRun(context)) {
|
|
48
48
|
throw new Error('selected Fable workflow has no correlated run. Write workflow-stages.json at the displayed session path, then use workflow-disposition.js start.');
|
|
49
49
|
}
|
|
50
|
-
if (
|
|
51
|
-
|
|
50
|
+
if (DIRECT.has(tool)) {
|
|
51
|
+
// Bookkeeping outside the workspace (scratchpad, memory files, temp
|
|
52
|
+
// notes) is not a workspace mutation: it consumes no Tier-2 iteration
|
|
53
|
+
// and advances no mutation milestone (#165).
|
|
54
|
+
if (!directMutationInWorkspace(payload, cwd, root)) return;
|
|
55
|
+
if (workflow.strategy === 'iterative-single') {
|
|
56
|
+
recordBudgetEvent(context, 'iteration', { evidence: `${tool}:direct-mutation` });
|
|
57
|
+
}
|
|
58
|
+
workflow.lastMutationAt = Date.now();
|
|
59
|
+
saveWorkflow(context);
|
|
60
|
+
return;
|
|
52
61
|
}
|
|
53
|
-
|
|
54
|
-
|
|
62
|
+
|
|
63
|
+
// Shell safety and shell mutation accounting are intentionally separate.
|
|
64
|
+
// `untrusted` means "not proven read-only", not "a mutation occurred".
|
|
65
|
+
// Capture an opaque pre-execution workspace fingerprint; the post-tool
|
|
66
|
+
// observer consumes an iteration only when workspace content actually changed.
|
|
67
|
+
const fingerprint = workspaceFingerprint(cwd);
|
|
68
|
+
const probeKey = shellProbeKey(payload, cwd);
|
|
69
|
+
registerMutationProbe(context, probeKey, fingerprint, workflow.strategy === 'iterative-single');
|
|
55
70
|
}
|
|
56
71
|
|
|
57
72
|
readHookInput(decide);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const { loadWorkflow, discardMutationProbe, readHookInput } = require('./lib/workflow-runtime');
|
|
5
|
+
const { cwdOf, shellProbeKey } = require('./lib/workflow-isolation');
|
|
6
|
+
|
|
7
|
+
function discardDenied(payload) {
|
|
8
|
+
if (!payload) return;
|
|
9
|
+
const tool = payload.tool_name || payload.tool;
|
|
10
|
+
if (tool !== 'Bash' && tool !== 'PowerShell' && tool !== 'exec_command') return;
|
|
11
|
+
const context = loadWorkflow(payload);
|
|
12
|
+
if (!context.workflow || context.workflow.state === 'deferred') return;
|
|
13
|
+
const cwd = cwdOf(payload, context.root);
|
|
14
|
+
discardMutationProbe(context, shellProbeKey(payload, cwd));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
readHookInput(discardDenied);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
|
|
3
3
|
"name": "harness-everything",
|
|
4
|
-
"version": "0.11.
|
|
4
|
+
"version": "0.11.7",
|
|
5
5
|
"description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Miya Daniel",
|
|
@@ -33,12 +33,26 @@ function issueMemoryCapability(workflow) {
|
|
|
33
33
|
return token;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
// Host-generated background-task notifications arrive as UserPromptSubmit
|
|
37
|
+
// hook payloads but are not user input (#168). Routing them like prompts
|
|
38
|
+
// replaces the workflow contract mid-turn (e.g. deferred -> a new Tier-2).
|
|
39
|
+
// A prompt that is entirely a notification envelope must never create or
|
|
40
|
+
// replace a workflow; the current contract is kept unchanged.
|
|
41
|
+
function isHostNotificationPrompt(prompt) {
|
|
42
|
+
const text = String(prompt || '').trim();
|
|
43
|
+
if (!text) return false;
|
|
44
|
+
return /^<task-notification[\s>][\s\S]*<\/task-notification>\s*$/i.test(text);
|
|
45
|
+
}
|
|
46
|
+
|
|
36
47
|
function persistWorkflow(plan, payload, prompt) {
|
|
37
48
|
if (!payload || !(payload.session_id || payload.sessionId)) return null;
|
|
38
49
|
const { runtime, hooksRoot } = loadRuntime();
|
|
39
50
|
const context = runtime.loadWorkflow(payload);
|
|
40
51
|
const { writeCurrentSession } = require(path.join(hooksRoot, 'lib/harness-state'));
|
|
41
52
|
writeCurrentSession(context.root, context.sessionId);
|
|
53
|
+
if (isHostNotificationPrompt(prompt)) {
|
|
54
|
+
return { ...context, hooksRoot, retained: true, hostNotification: true };
|
|
55
|
+
}
|
|
42
56
|
if (context.workflow && runtime.OPEN_STATES.has(context.workflow.state)) {
|
|
43
57
|
if (!context.workflow.workflowPlan) throw new Error('legacy unresolved workflow needs explicit migration; refusing to replace its obligations');
|
|
44
58
|
// Steering/status prompts cannot drop an unresolved execution contract.
|
|
@@ -92,17 +106,30 @@ function run(raw) {
|
|
|
92
106
|
try { payload = JSON.parse(raw); } catch (_) { console.error('[Workflow Router] Invalid hook payload; lifecycle persistence unavailable.'); }
|
|
93
107
|
}
|
|
94
108
|
const prompt = promptArg || (typeof payload?.prompt === 'string' ? payload.prompt : '');
|
|
95
|
-
const
|
|
96
|
-
|
|
109
|
+
const hostNotification = isHostNotificationPrompt(prompt);
|
|
110
|
+
// Host-generated notifications are lifecycle events, not user-authored task
|
|
111
|
+
// input. Never feed their text into the router: when there is an active
|
|
112
|
+
// contract we retain it below; when there is none, the event is a no-op.
|
|
113
|
+
const result = hostNotification ? null : core.route(promptArg, raw || null);
|
|
114
|
+
let plan = result?.contract?.workflowPlan || null;
|
|
97
115
|
let persisted = null;
|
|
98
116
|
let failure = null;
|
|
99
117
|
try {
|
|
100
118
|
persisted = persistWorkflow(plan, payload, prompt);
|
|
101
|
-
if (persisted?.retained) plan = persisted.workflow.workflowPlan;
|
|
119
|
+
if (persisted?.retained && persisted.workflow) plan = persisted.workflow.workflowPlan;
|
|
102
120
|
} catch (error) { failure = error; }
|
|
103
|
-
if (
|
|
104
|
-
|
|
105
|
-
|
|
121
|
+
if (hostNotification && !persisted?.workflow) {
|
|
122
|
+
console.log('\n=> Host notification — no active execution contract; ignored for routing.');
|
|
123
|
+
if (failure) {
|
|
124
|
+
console.error('[Workflow Router] ' + failure.message);
|
|
125
|
+
process.exitCode = 2;
|
|
126
|
+
}
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (result?.sanitized) console.log(result.sanitized);
|
|
130
|
+
if (result?.stderr) process.stderr.write(result.stderr);
|
|
131
|
+
if (persisted?.hostNotification) console.log('\n=> Host notification — routing unchanged; keeping the active execution contract.');
|
|
132
|
+
else if (persisted?.retained) console.log('\n=> Retaining unresolved workflow from earlier prompts; the checkpoint below is the active execution contract.');
|
|
106
133
|
core.printWorkflowPlan(plan);
|
|
107
134
|
core.printRoutingCheckpoint(plan);
|
|
108
135
|
core.printKernelContract(plan);
|
|
@@ -116,7 +143,7 @@ function run(raw) {
|
|
|
116
143
|
console.log(' - Tier 3 / Fable source and artifact mutations require Git worktree isolation; unavailable isolation means BLOCKED.');
|
|
117
144
|
console.log(' - Escape is limited to workflow-uncovered-scope or host-capability-unavailable, with stage, scope and evidence; other obligations remain mandatory.');
|
|
118
145
|
}
|
|
119
|
-
if (persisted) {
|
|
146
|
+
if (persisted?.workflow) {
|
|
120
147
|
const controller = path.join(persisted.hooksRoot, 'workflow-disposition.js');
|
|
121
148
|
console.log(' - Workflow id: ' + persisted.workflow.workflowId);
|
|
122
149
|
console.log(' - Memory write disposition: ' + (persisted.workflow.workflowPlan.memory?.write || 'none'));
|
|
@@ -130,12 +157,13 @@ function run(raw) {
|
|
|
130
157
|
if (failure) {
|
|
131
158
|
console.error('[Workflow Router] ' + failure.message);
|
|
132
159
|
process.exitCode = 2;
|
|
133
|
-
} else if (result.status !== 0) process.exitCode = result.status || 1;
|
|
160
|
+
} else if (result && result.status !== 0) process.exitCode = result.status || 1;
|
|
134
161
|
}
|
|
135
162
|
|
|
163
|
+
if (typeof module !== 'undefined') module.exports = { isHostNotificationPrompt };
|
|
164
|
+
|
|
136
165
|
if (process.argv.length > 2 || process.stdin.isTTY) run('');
|
|
137
|
-
else {
|
|
138
|
-
let input = '';
|
|
166
|
+
else { let input = '';
|
|
139
167
|
let finished = false;
|
|
140
168
|
const finish = () => { if (finished) return; finished = true; clearTimeout(timeout); run(input); };
|
|
141
169
|
const timeout = setTimeout(finish, 500);
|
package/docs/issue-20-rollup.md
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
# Issue #20 umbrella rollup
|
|
2
|
-
|
|
3
|
-
This rollup turns issue #20's historical checklist into bounded acceptance decisions. The review snapshot is `main` at `ba7853b`, checked on 2026-09-07. The umbrella owns coordination and release evidence; implementation details remain with the linked issue.
|
|
4
|
-
|
|
5
|
-
## Closure status
|
|
6
|
-
|
|
7
|
-
Issue #20 remains open after this PR. This branch adds deterministic parser,
|
|
8
|
-
collision-gate, plugin-runner, and negative-control coverage, but it does not
|
|
9
|
-
provide live opencode session evidence, a paid behavioral rerun, the proposed
|
|
10
|
-
100-case study, or the three skill-specific improvements below. Those gaps are
|
|
11
|
-
recorded explicitly rather than treated as closed by the deterministic gates.
|
|
12
|
-
|
|
13
|
-
## Workstream decisions
|
|
14
|
-
|
|
15
|
-
| Workstream | Evidence in the review snapshot | Acceptance state | Remaining claim boundary |
|
|
16
|
-
| --- | --- | --- | --- |
|
|
17
|
-
| #37 opencode enforcement | The real plugin API is implemented and mechanism coverage exists in `ci/mechanism-2n-opencode-plugin.test.js`; the integrated reflection lifecycle is `d790e7f` on the integration branch. | Integrated, pending final integration gates | No live opencode loading or session smoke evidence was available; do not claim live enforcement. |
|
|
18
|
-
| #52 behavioral transcript parsing | Claude stream fixtures and parser coverage are integrated in `77efb06`; deterministic parser tests are available. | Integrated, pending final integration gates | No new paid behavioral rerun is part of this umbrella. Parse failures and missing tool visibility remain evidence states, not skill failures. |
|
|
19
|
-
| Systematic gate hardening | The local gates now execute real negative controls; YAML frontmatter parsing fails closed in `ci/description-collision.js`, `ci/routing-check.js` skips clearly when waza is absent, and `ci/mechanism-2t-issue-20.test.js` covers CRLF behavioral parsing and plugin validation. | Deterministic hardening integrated | Live platform loading and model behavior remain unverified. |
|
|
20
|
-
| #42 workspace identity and migration | Workspace-keyed state and migration coverage are integrated in `9dd2e3d`; the issue review identified session and migration acceptance as the relevant boundary. | Integrated, pending final integration gates | Do not claim every host/session migration path is live verified until the integration gate and targeted fixtures pass. |
|
|
21
|
-
| #43 multi-agent workspace layout | The reviewed plan makes its root contract depend on #42. No implementation commit was present in the issue #20 review snapshot. | Deferred to its owner | Do not duplicate its layout or runtime work in this umbrella. |
|
|
22
|
-
| #44 progressive disclosure | Documentation and reference checks are integrated in `b33f598`; `ci/disclosure-check.js` and its mechanism coverage are present. | Integrated, pending final integration gates | This records repository documentation evidence; it does not establish behavioral-model effectiveness. |
|
|
23
|
-
| #56 behavioral rubric triage | The review found grader/source-boundary issues, including `command_exit_0` proving the grader command rather than the agent action. | Deferred to its owner | Do not reuse the historical pass-rate claims until the rubric and trace evidence are corrected. |
|
|
24
|
-
| Extended 100-case study | The original issue proposed 100 cases, but the review did not establish a decision threshold or clean attribution between skill text, plugin enforcement, and grader behavior. | Explicitly deferred | No uncontrolled 100-case run is authorized by this rollup. Define paired design, failure taxonomy, and decision rule first. |
|
|
25
|
-
| Five historical baseline failures | The committed 2026-08-27 records show one failed expectation in each of `baseline-debugging` (edge-case consideration), `baseline-performance` (binary-search suggestion), `baseline-security-review` (MD5 identification), `baseline-simple-bugfix` (function understanding), and `verify-before-done` (verification actually run). | Explicitly deferred | These are historical observations, not a new causal investigation or rerun; do not claim they were fixed by this PR. |
|
|
26
|
-
| Skill-specific improvements | No targeted changes were made to `verification-loop`, `zoom-out`, or `security-review` in this branch. | Explicitly deferred | Strengthening verification enforcement, reflection-report detection, and vulnerability detection requires separate design and behavioral evidence. |
|
|
27
|
-
| Release | The tag-triggered full gate and provenance publish workflow already exists. Historical `0.3.6` evidence is recorded in [docs/release-evidence.md](release-evidence.md). | Documentation complete; next release unproven | A prepared `0.3.7-beta` version is not evidence of a tag, npm publication, or GitHub release. |
|
|
28
|
-
|
|
29
|
-
## Focused acceptance checks
|
|
30
|
-
|
|
31
|
-
The integration owner should run these checks after incorporating the issue branches:
|
|
32
|
-
|
|
33
|
-
```text
|
|
34
|
-
npm test
|
|
35
|
-
npm run test:mechanism
|
|
36
|
-
npm run test:consistency
|
|
37
|
-
npm run test:references
|
|
38
|
-
npm run test:release
|
|
39
|
-
npm run test:collision
|
|
40
|
-
npm run test:routing:skills
|
|
41
|
-
```
|
|
42
|
-
|
|
43
|
-
The checks establish deterministic repository and routing health. They do not substitute for live opencode loading, a paid behavioral run, or npm/GitHub release evidence. Those claims require the artifacts named in [RELEASING.md](../RELEASING.md) and [docs/release-evidence.md](release-evidence.md).
|
|
44
|
-
|
|
45
|
-
## Historical evidence boundary
|
|
46
|
-
|
|
47
|
-
The local review artifact `.git/issue-review-20260907/release-evidence.md` records a successful `0.3.6` workflow (`33466676632`) and npm observation. It explicitly says that this is not proof of a `0.3.7-beta` release. This rollup preserves that distinction so an old successful release cannot be mistaken for current publication evidence.
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
# PR #142 / issue #131 completion review
|
|
2
|
-
|
|
3
|
-
This review covers the implementation in the commit containing this report,
|
|
4
|
-
starting from PR head `8b99baec44745e171f078fe9eb3dac6fd812b5c8`. It evaluates
|
|
5
|
-
Harness engineering and the latest #131 decision, including mandatory mutation
|
|
6
|
-
isolation. It is not a measured agent-performance benchmark.
|
|
7
|
-
|
|
8
|
-
## Findings resolved
|
|
9
|
-
|
|
10
|
-
| Original gap | Delivered behavior | Regression evidence |
|
|
11
|
-
| --- | --- | --- |
|
|
12
|
-
| Escape changed the whole workflow state and skipped isolation | Stage-scoped escape; isolation survives legacy escape and blocked states | `mechanism-2af-worktree-isolation.test.js`, `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
13
|
-
| Only cwd was checked | Registered linked checkout of the bound repo; direct/patch targets, move destinations, traversal, and links checked | `mechanism-2af-worktree-isolation.test.js` |
|
|
14
|
-
| Windows runner used 8.3 paths while Git reported long names | Native physical-path resolution equates both spellings without relaxing isolation | Windows path regression and worktree suites |
|
|
15
|
-
| Read-only shell prefixes accepted writes | Reject redirects, substitutions, chained commands and dangerous options; require isolated cwd for mutation/unknown commands | `mechanism-2af-worktree-isolation.test.js` |
|
|
16
|
-
| Next prompt erased unresolved obligations | Stable workflow identity across steering prompts; stronger route requires replan | `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
17
|
-
| Same strategy/timestamp loosely matched any Fable run | Exact workflow, session and run correlation | `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
18
|
-
| Entry gate blocked its own bootstrap path | One bounded stage-specification file and trusted start controller; state/source writes remain gated | `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
19
|
-
| Stop trusted directory contents and pass labels | Expected manifest stages, correlated exit-zero evidence, worker attribution and fresh verifier required | `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
20
|
-
| Stop retry silently passed unresolved work | Retry records blocked; no satisfied state without required evidence | `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
21
|
-
| Failures could not recover cleanly | Observed fail-to-pass checks; explicit bounded replan; retained prior run evidence | `mechanism-2ah-workflow-lifecycle.test.js` |
|
|
22
|
-
| Runtime policy scraped human-readable output | Kernel imports the structured core result; complete invariants persist with the plan | routing tests and lifecycle tests |
|
|
23
|
-
| Current documents contradicted the PR | README, platform matrix, routing, architecture, Fable/worktree references and mirrors aligned | consistency/capability/reference gates |
|
|
24
|
-
| Existing CI selected the first matching hook group | Action-gate regression finds its actual adapter among shared tool matchers | `ci/action-gate.test.js` |
|
|
25
|
-
|
|
26
|
-
The first added isolation regression run failed 16 assertions against the PR
|
|
27
|
-
baseline, then passed after repair. The baseline's four failing GitHub CI jobs
|
|
28
|
-
also reported the action-gate matcher defect. The initial local checkout lacked
|
|
29
|
-
dependencies, producing a separate CRLF-fixture setup failure; `npm ci` resolved
|
|
30
|
-
that environment prerequisite.
|
|
31
|
-
|
|
32
|
-
## Verification
|
|
33
|
-
|
|
34
|
-
Local environment: Windows, PowerShell, Node.js 24.15.0, npm 11.12.1. Work was
|
|
35
|
-
performed in an isolated linked worktree; the primary checkout remained clean.
|
|
36
|
-
|
|
37
|
-
All 14 required gate categories passed: `npm test`, mutations, mechanism,
|
|
38
|
-
consistency, repository contract, capability docs, references, release,
|
|
39
|
-
collision, positive skill routing, routing invariants, OpenAI package/runtime,
|
|
40
|
-
public submission, and platform compatibility. Failed checks were corrected and
|
|
41
|
-
the affected gates rerun. The final `npm test` includes **44 mechanism suites**.
|
|
42
|
-
|
|
43
|
-
The new lifecycle suite exercises **50 assertions against source and the same
|
|
44
|
-
50 against the packaged OpenAI layout**, using actual child processes, Git
|
|
45
|
-
repositories, worktrees, controller commands and hook payloads. Host payloads
|
|
46
|
-
are fixtures: these are deterministic mechanism tests, not live agent sessions.
|
|
47
|
-
|
|
48
|
-
Waza passed 30 canonical/nested skill readiness checks, 26 routing-spec checks,
|
|
49
|
-
and token validation (57 successful commands). Positive routing covered 34
|
|
50
|
-
cases across 26 direct skills and classified four nested skills.
|
|
51
|
-
|
|
52
|
-
Build/types/lint have no separate project scripts; repository syntax and
|
|
53
|
-
consistency gates apply. No coverage percentage is claimed. The security scanner
|
|
54
|
-
reported 12 pre-existing documentation/test-pattern matches, including its own
|
|
55
|
-
patterns and an explicitly fake token fixture; changed production code added no
|
|
56
|
-
scanner findings. `npm ci` reported zero dependency vulnerabilities. Diff review
|
|
57
|
-
and `git diff --check` passed. See the retained [gate summary](../../benchmarks/results/reviews/pr142-verification.json).
|
|
58
|
-
|
|
59
|
-
Remote CI is a separate evidence layer. Its result must be read at the final PR
|
|
60
|
-
head; this local record does not assert a future GitHub Actions outcome.
|
|
61
|
-
|
|
62
|
-
## Reassessment
|
|
63
|
-
|
|
64
|
-
Scores are review judgments about engineering readiness, each out of 100.
|
|
65
|
-
Each row has four equally weighted criteria (25 points each), with the observed
|
|
66
|
-
deductions shown. They are not success probabilities or benchmark results.
|
|
67
|
-
|
|
68
|
-
| Dimension | Criteria scores | Total | Remaining deductions |
|
|
69
|
-
| --- | --- | ---: | --- |
|
|
70
|
-
| Contract coherence | entry 22, transitions 23, scoped exceptions 24, docs 23 | 92 | Skill applicability and iterative budgets still partly rely on executor compliance |
|
|
71
|
-
| Mutation safety | worktree identity 23, targets 23, shell handling 17, state trust 12 | 75 | Arbitrary script effects need a host sandbox; runtime state is agent-writable |
|
|
72
|
-
| Deterministic verification | gate breadth 25, negative cases 24, package parity 24, evidence checks 18 | 91 | Host-shaped fixtures cannot establish real host behavior or semantic check quality |
|
|
73
|
-
| Operation and maintenance | entry/recovery 22, modularity 21, distribution 23, diagnostics/migration 18 | 84 | Manual stage setup, missing-metadata blocks and legacy-state repair add operator work |
|
|
74
|
-
|
|
75
|
-
Equal-weight engineering score: **85.5/100** (approximately **86/100**).
|
|
76
|
-
The system now has substantially stronger tested lifecycle and isolation
|
|
77
|
-
contracts. It is suitable for further host validation with explicit boundaries.
|
|
78
|
-
|
|
79
|
-
**Live-host reliability of these new gates remains unverified. Behavioral gain
|
|
80
|
-
from this PR remains unmeasured.** Neither receives an invented effectiveness
|
|
81
|
-
score. Older retained OpenCode evidence and older behavioral cases cannot prove
|
|
82
|
-
this workflow change. This review does not claim superiority over other skill
|
|
83
|
-
collections, reduced token cost, or improved task success rates.
|
|
84
|
-
|
|
85
|
-
The next evidence needed is a retained Claude/Codex session showing routing,
|
|
86
|
-
entry, mutation denial/allow, numeric stage results, worker correlation and Stop
|
|
87
|
-
resolution; then paired same-model/same-task behavioral runs. OpenCode needs its
|
|
88
|
-
own adapter before parity can be claimed. See [runtime limits](../workflow-runtime.md)
|
|
89
|
-
and [platform capabilities](../platform-capabilities.md).
|