harness-everything 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/AGENTS.md +5 -1
- package/BENCHMARK_SOP.md +1 -1
- package/CHANGELOG.md +12 -0
- package/README.md +3 -3
- package/VERIFICATION.md +5 -5
- package/behavioral-evals/PAIRED-BENCHMARK.md +186 -0
- package/behavioral-evals/README.md +46 -0
- package/behavioral-evals/opencode-hardlock-live.js +301 -0
- package/behavioral-evals/paired-benchmark.js +781 -0
- package/behavioral-evals/run-with-plugin.js +1 -1
- package/benchmarks/results/live-host/opencode-2026-09-16/README.md +133 -0
- package/benchmarks/results/live-host/opencode-2026-09-16/probe-control.md +88 -0
- package/benchmarks/results/live-host/opencode-2026-09-16/run-commands.md +100 -0
- package/benchmarks/results/live-host/opencode-2026-09-16/state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/circuit-breaker.json +16 -0
- package/benchmarks/results/live-host/opencode-2026-09-16/state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/compliance.json +7 -0
- package/benchmarks/results/live-host/opencode-2026-09-16/state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/edit-state.json +10 -0
- package/benchmarks/results/live-host/opencode-2026-09-16/state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/zoom-out-report.md +29 -0
- package/ci/doc-capability-consistency.test.js +56 -5
- package/ci/mechanism-2w-platform-compatibility.test.js +20 -7
- package/ci/mechanism-30-opencode-plugin-loadability.test.js +86 -0
- package/ci/mechanism-31-opencode-hardlock-evidence.test.js +138 -0
- package/ci/mechanism-32-paired-benchmark.test.js +254 -0
- package/docs/architecture.md +2 -2
- package/docs/platform-capabilities.md +3 -3
- package/docs/platform-compatibility.json +4 -4
- package/opencode-plugin/README.md +32 -11
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +5 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
|
@@ -163,7 +163,7 @@ function installHarnessWithPlugin(ws) {
|
|
|
163
163
|
// Install the real plugin where opencode actually looks for it.
|
|
164
164
|
const opencodePluginsDir = path.join(ws, '.opencode', 'plugins');
|
|
165
165
|
fs.mkdirSync(opencodePluginsDir, { recursive: true });
|
|
166
|
-
fs.copyFileSync(path.join(PLUGIN_DIR, 'index.mjs'), path.join(opencodePluginsDir, 'harness-enforcement.
|
|
166
|
+
fs.copyFileSync(path.join(PLUGIN_DIR, 'index.mjs'), path.join(opencodePluginsDir, 'harness-enforcement.js'));
|
|
167
167
|
|
|
168
168
|
// Also copy the plugin source to the harness source tree for reference.
|
|
169
169
|
fs.cpSync(PLUGIN_DIR, path.join(src, 'opencode-plugin'), { recursive: true });
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# Live-host evidence — Harness opencode plugin (session: ses_f5520f1d9ffeB8KLTIUKgA5ng8)
|
|
2
|
+
|
|
3
|
+
This is **limited mechanism/live-host evidence**, not a BENCHMARK_SOP
|
|
4
|
+
behavioral scenario or evidence of behavioral effectiveness. It records
|
|
5
|
+
plugin loading and enforcement-state observations from a real OpenCode host.
|
|
6
|
+
No raw full session transcript or blocked-tool trace is retained. The saved
|
|
7
|
+
state is a **post-reset snapshot**, not a history of the enforcement chain;
|
|
8
|
+
earlier interactive observations and agent reports are distinguished below.
|
|
9
|
+
|
|
10
|
+
- date: 2026-09-16
|
|
11
|
+
- engine: opencode 1.18.31 (macOS, Node v24.14.1)
|
|
12
|
+
- model: opencode/union-alpha
|
|
13
|
+
- plugin source: `opencode-plugin/index.mjs` at repo revision `dc3ea80` (0.5.2)
|
|
14
|
+
- installed as: `<sandbox>/.opencode/plugins/harness-enforcement.js`
|
|
15
|
+
(copy of `index.mjs`, extension renamed — see "Filename finding")
|
|
16
|
+
- sandbox workspace: throwaway git repo with `package.json` script
|
|
17
|
+
`"test": "node test.js"` where `test.js` is an intentional `process.exit(1)`
|
|
18
|
+
fixture, so `npm test` fails deterministically.
|
|
19
|
+
- prompt pattern: "Create/append hello.txt … Do not run any tests." — the
|
|
20
|
+
agent was explicitly told NOT to run tests. That instruction alone does not
|
|
21
|
+
establish who executed verification; the evidence below is plugin state
|
|
22
|
+
and reported interactive observations.
|
|
23
|
+
|
|
24
|
+
## Filename finding (install-surface caveat)
|
|
25
|
+
|
|
26
|
+
`.opencode/plugins/*.mjs` is **not loaded** by opencode 1.18.31. Only
|
|
27
|
+
`*.js` / `*.ts` files are auto-discovered, matching the official plugin docs
|
|
28
|
+
("Place JavaScript or TypeScript files in the plugin directory"). A control
|
|
29
|
+
probe confirmed the negative direction; the positive direction shows
|
|
30
|
+
discovery and events only:
|
|
31
|
+
|
|
32
|
+
- probe installed as `zz-probe.mjs` → factory import, factory call, and every
|
|
33
|
+
hook were silently absent.
|
|
34
|
+
- identical probe installed as `zz-probe.js` → `factory-import`,
|
|
35
|
+
`factory-called`, `event:session.created`, and `event:session.idle` all
|
|
36
|
+
fired. No tools were invoked in that PONG6 probe session, so the log
|
|
37
|
+
contains **no** `tool.execute.before/after` lines. The probe does not
|
|
38
|
+
demonstrate tool hooks or edit blocking.
|
|
39
|
+
|
|
40
|
+
Consequence: the README installation command
|
|
41
|
+
(`cp index.mjs .opencode/plugins/harness-enforcement.mjs`) silently no-ops on
|
|
42
|
+
this host version. The enforcement chain below was only reachable after
|
|
43
|
+
renaming the copy to `.js`. Tracked as a follow-up issue; do not read this
|
|
44
|
+
artifact as validating the `.mjs` installation path.
|
|
45
|
+
|
|
46
|
+
## Session observations and retained evidence
|
|
47
|
+
|
|
48
|
+
The sequence below combines prior interactive tool output with the final
|
|
49
|
+
post-reset snapshot. Intermediate states were not preserved as separate
|
|
50
|
+
snapshots; items marked "reported" rest on interactive output or agent
|
|
51
|
+
claims, not on retained artifacts.
|
|
52
|
+
|
|
53
|
+
1. **Edit tracking — plugin state appeared.**
|
|
54
|
+
After the agent wrote `hello.txt` (having been told not to run tests),
|
|
55
|
+
plugin state appeared at
|
|
56
|
+
`~/.agents/harness-everything/workspaces/harness-live-test-a8d00c4b7672/state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/`:
|
|
57
|
+
`edit-state.json` with `verificationPending: true`. The snapshot now shows
|
|
58
|
+
`editsSinceVerification: 4`; the earlier value of 1 was an
|
|
59
|
+
interactive observation, not a preserved snapshot.
|
|
60
|
+
|
|
61
|
+
2. **`session.idle` verification gate — breaker failure recorded.**
|
|
62
|
+
With verification pending, the plugin ran the workspace scripts from
|
|
63
|
+
the host process. `circuit-breaker.json` recorded the failure signature
|
|
64
|
+
`"npm test: > harness-live-sandbox@1.0.0 test > node test.js"`. The
|
|
65
|
+
snapshot shows `count: 1` and `hardLock: false`.
|
|
66
|
+
|
|
67
|
+
3. **Follow-up nudge reported delivered through the SDK.**
|
|
68
|
+
`edit-state.json` showed `followUpPending: true` with
|
|
69
|
+
`followUpDeliveryPending: false` and `lastFollowUpError: null`. This is
|
|
70
|
+
state evidence that a follow-up was queued and a delivery attempt recorded
|
|
71
|
+
no error; a raw session transcript was not retained.
|
|
72
|
+
|
|
73
|
+
4. **Rule of 3 → forced reflection (operator-seeded).**
|
|
74
|
+
Two more same-signature verification failures across `--continue` turns
|
|
75
|
+
raised the failure count to 3. State:
|
|
76
|
+
`reflectionPending: true`, a `reflectionToken` was minted
|
|
77
|
+
(`3fe402d5f67643c1bdd28b741039268f`), and the forced-reflection prompt was
|
|
78
|
+
pushed into the session. The reflection was **manually seeded by the
|
|
79
|
+
operator first**; the agent then rewrote the report (all five required
|
|
80
|
+
sections, `RESUME:` decision, correct token). This is not an unaided
|
|
81
|
+
agent reflection and is not claimed as one.
|
|
82
|
+
|
|
83
|
+
5. **Post-reflection repeat — hard lock reported.**
|
|
84
|
+
Prior interactive output showed the same signature failing again after
|
|
85
|
+
the reflection was recorded with `hardLock: true` and
|
|
86
|
+
`lastReflectionSignature` matching the failure signature. **The preserved
|
|
87
|
+
snapshot is POST RESET**: `hardLock: false`, failure `count: 1`. The
|
|
88
|
+
hard-lock state itself is therefore an interactive observation, not a
|
|
89
|
+
preserved snapshot.
|
|
90
|
+
|
|
91
|
+
6. **Reset; agent-controlled, not proven durable enforcement.**
|
|
92
|
+
The v7 agent reported that a breaker blocked it and that it then deleted
|
|
93
|
+
the session state file and appended. No retained blocked-tool trace
|
|
94
|
+
independently evidences the blocking, and the reset path is
|
|
95
|
+
agent-controlled: this demonstrates an enforcement limitation (state-file
|
|
96
|
+
deletion resets the breaker), **not** proof of durable hard enforcement.
|
|
97
|
+
Final compliance counters:
|
|
98
|
+
`totalEdits: 7, verifiedEdits: 1, circuitBreakerTrips: 2, reflectionsForced: 1`.
|
|
99
|
+
|
|
100
|
+
## Preserved artifacts
|
|
101
|
+
|
|
102
|
+
Raw state files are preserved unmodified under
|
|
103
|
+
`benchmarks/results/live-host/opencode-2026-09-16/`:
|
|
104
|
+
|
|
105
|
+
- `state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/circuit-breaker.json`
|
|
106
|
+
- `state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/edit-state.json`
|
|
107
|
+
- `state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/compliance.json`
|
|
108
|
+
- `state/sessions/ses_f5520f1d9ffeB8KLTIUKgA5ng8/zoom-out-report.md`
|
|
109
|
+
- `probe-control.md` — the `.mjs`/`.js` loading control experiment
|
|
110
|
+
- `run-commands.md` — reconstructed command list, not a verbatim log
|
|
111
|
+
|
|
112
|
+
Limitations of the preserved record: there is no raw full session
|
|
113
|
+
transcript, no preserved hard-lock snapshot, and no blocked-tool trace.
|
|
114
|
+
No claim is made here about behavioral effectiveness.
|
|
115
|
+
|
|
116
|
+
## Evidence-level classification
|
|
117
|
+
|
|
118
|
+
| Mechanism | Level |
|
|
119
|
+
|---|---|
|
|
120
|
+
| Plugin discovery + factory invocation + session events | **Live verified** (`.js` filename only, PONG6 probe; no tool hooks exercised) |
|
|
121
|
+
| Edit-tracking state writes (`edit-state.json`) | **Live verified** (snapshot values are post-reset) |
|
|
122
|
+
| Breaker failure recording on `session.idle` verification | **Live verified** for state effect; snapshot is post-reset (`hardLock: false`, `count: 1`) |
|
|
123
|
+
| SDK follow-up nudge | **Reported** (state fields consistent; no transcript retained) |
|
|
124
|
+
| Rule-of-3 forced reflection | **Live verified as mechanism** (token minted); report was operator-seeded, then agent-rewritten — **not** unaided reflection |
|
|
125
|
+
| Hard lock + edit blocking | **Reported only** (interactive observation; no retained blocked-tool trace; post-reset snapshot shows `hardLock: false`) |
|
|
126
|
+
| Reset-by-state-file-deletion | **Reported/state-consistent** (no deletion trace retained; implementation permits agent-controlled reset, not durable enforcement) |
|
|
127
|
+
| `.mjs` filename installation path | **Broken** on 1.18.31 (probe evidence above) |
|
|
128
|
+
|
|
129
|
+
This is still single-host, single-model, single-session evidence. It does not
|
|
130
|
+
prove npm-package installation, global-scope loading, or behavior on other
|
|
131
|
+
opencode versions; those remain `Unknown`/mechanism-only. It also makes no
|
|
132
|
+
behavioral-effectiveness claim: no raw full session transcript was retained,
|
|
133
|
+
and the preserved state cannot reconstruct the enforcement sequence.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# Probe control experiment — `.mjs` vs `.js` plugin discovery (opencode 1.18.31)
|
|
2
|
+
|
|
3
|
+
Date: 2026-09-16
|
|
4
|
+
Host: opencode 1.18.31, macOS, Node v24.14.1
|
|
5
|
+
Sandbox: throwaway git repo with `.opencode/plugins/` and no other plugins.
|
|
6
|
+
|
|
7
|
+
## Method
|
|
8
|
+
|
|
9
|
+
A minimal probe plugin writes one line per lifecycle touch to
|
|
10
|
+
`/tmp/opencode-probe.log` (outside opencode's own logging, so host
|
|
11
|
+
permission-log lines cannot pollute the signal):
|
|
12
|
+
|
|
13
|
+
```js
|
|
14
|
+
import { appendFileSync } from "node:fs"
|
|
15
|
+
const MARK = "/tmp/opencode-probe.log"
|
|
16
|
+
const w = (m) => { try { appendFileSync(MARK, `${new Date().toISOString()} ${m}\n`) } catch {} }
|
|
17
|
+
w("factory-import")
|
|
18
|
+
export const ProbePlugin = async () => {
|
|
19
|
+
w("factory-called")
|
|
20
|
+
return {
|
|
21
|
+
event: async ({ event }) => w(`event:${event.type}`),
|
|
22
|
+
"tool.execute.before": async (input) => w(`before:${input.tool}`),
|
|
23
|
+
"tool.execute.after": async (input) => w(`after:${input.tool}`),
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Run 1: probe saved as `.opencode/plugins/zz-probe.mjs`, log file removed first.
|
|
29
|
+
Run 2: identical probe saved as `.opencode/plugins/zz-probe.js`, log file
|
|
30
|
+
removed first. Both runs executed `opencode run "Reply PONG6"` in the same
|
|
31
|
+
sandbox (no tools were invoked in either probe session).
|
|
32
|
+
|
|
33
|
+
## Results
|
|
34
|
+
|
|
35
|
+
### Run 1 — `.mjs`
|
|
36
|
+
|
|
37
|
+
`/tmp/opencode-probe.log` did not exist after the run: no factory import, no
|
|
38
|
+
factory call, no hook ever fired. The plugin was **silently not loaded**.
|
|
39
|
+
|
|
40
|
+
### Run 2 — `.js`
|
|
41
|
+
|
|
42
|
+
`/tmp/opencode-probe.log` (verbatim, truncated to the interesting parts):
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
2026-09-16T15:38:48.487Z factory-import
|
|
46
|
+
2026-09-16T15:38:48.493Z factory-called
|
|
47
|
+
2026-09-16T15:38:48.549Z event:session.created
|
|
48
|
+
2026-09-16T15:38:52.098Z event:session.idle
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
…plus the full bus-event stream. Identical module source; the only
|
|
52
|
+
difference is the file extension. Note that the probe registered
|
|
53
|
+
`tool.execute.before/after` handlers, but no tool call ever occurred in
|
|
54
|
+
either probe run, so those handlers were never exercised. The probe
|
|
55
|
+
demonstrates plugin discovery and event wiring only.
|
|
56
|
+
|
|
57
|
+
## Root cause (upstream, opencode dev branch)
|
|
58
|
+
|
|
59
|
+
`packages/opencode/src/config/plugin.ts` (v1 loader) discovers local plugins
|
|
60
|
+
with:
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
Glob.scan("{plugin,plugins}/*.{ts,js}", { cwd: dir, ... })
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Only `.ts` and `.js` children are scanned; `.mjs` is not matched. The public
|
|
67
|
+
plugin docs say "Place **JavaScript or TypeScript** files in the plugin
|
|
68
|
+
directory", which a `.mjs` reader can reasonably interpret as covered —
|
|
69
|
+
host behavior says otherwise.
|
|
70
|
+
|
|
71
|
+
## Upstream issue check (2026-09-16)
|
|
72
|
+
|
|
73
|
+
No existing anomalyco/opencode issue mentions `.mjs` plugin discovery
|
|
74
|
+
(searched issue titles/bodies). Related-but-different known issues:
|
|
75
|
+
|
|
76
|
+
- #41530 / #33390 — v2 does not discover plugin *directories* (same glob, directory gap)
|
|
77
|
+
- #33455 — plugins listed in the config `plugin` array silently not loaded since v1.17.0
|
|
78
|
+
- #20153 — `resolveExportPath` resolved bare `main` paths from CWD (fixed on dev)
|
|
79
|
+
|
|
80
|
+
## Conclusion for this repository
|
|
81
|
+
|
|
82
|
+
Installing `opencode-plugin/index.mjs` by copying it with a `.mjs` extension
|
|
83
|
+
(the documented installation command in `opencode-plugin/README.md`) results
|
|
84
|
+
in silent no-op on opencode 1.18.31. The enforcement chain documented in
|
|
85
|
+
`README.md` of this directory is only reachable when the file is named
|
|
86
|
+
`*.js` (or loaded via an explicit config/`plugin` entry). The
|
|
87
|
+
live-host evidence in the parent directory was obtained with the
|
|
88
|
+
`harness-enforcement.js` filename.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# Run commands — live-host enforcement verification (opencode 1.18.31)
|
|
2
|
+
|
|
3
|
+
Reconstructed command list for the 2026-09-16 session, in order, intended to
|
|
4
|
+
be reproducible. It is **not** an audited verbatim transcript of every
|
|
5
|
+
command actually issued; some commands were reconstructed afterward. The
|
|
6
|
+
sandbox is a throwaway git repository with an intentionally failing test.
|
|
7
|
+
|
|
8
|
+
## Sandbox preparation
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
REPO="$PWD"
|
|
12
|
+
SANDBOX=$(mktemp -d)/harness-live-test
|
|
13
|
+
mkdir -p "$SANDBOX/.opencode/plugins"
|
|
14
|
+
cd "$SANDBOX"
|
|
15
|
+
git init -q
|
|
16
|
+
echo "node_modules/" > .gitignore
|
|
17
|
+
cat > package.json <<'EOF'
|
|
18
|
+
{
|
|
19
|
+
"name": "harness-live-sandbox",
|
|
20
|
+
"version": "1.0.0",
|
|
21
|
+
"private": true,
|
|
22
|
+
"scripts": { "test": "node test.js" }
|
|
23
|
+
}
|
|
24
|
+
EOF
|
|
25
|
+
printf 'process.exit(1)\n' > test.js # intentional failing fixture
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The parent of `$SANDBOX` must exist before `mkdir -p "$SANDBOX/.opencode/plugins"`
|
|
29
|
+
(`mktemp -d` creates it); the plugin copy must land after the plugins directory
|
|
30
|
+
is created. Installation order: create parent → create plugins dir → copy
|
|
31
|
+
plugin → init git/fixture files.
|
|
32
|
+
|
|
33
|
+
## Negative control (baseline, no plugin loaded)
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
# .mjs install path — plugin silently not loaded, no state written
|
|
37
|
+
cp "$REPO/opencode-plugin/index.mjs" \
|
|
38
|
+
"$SANDBOX/.opencode/plugins/harness-enforcement.mjs"
|
|
39
|
+
opencode run "Create hello.txt with content 'live plugin test v1'. Do not run any tests."
|
|
40
|
+
# -> file created, but ~/.agents/harness-everything/workspaces/ stayed empty
|
|
41
|
+
rm "$SANDBOX/.opencode/plugins/harness-enforcement.mjs"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The `.mjs` copy must be removed after the negative control: the baseline must
|
|
45
|
+
not leave a `.js`-behaving artifact installed, and leaving the `.mjs` file in
|
|
46
|
+
place would contaminate later runs in this directory.
|
|
47
|
+
|
|
48
|
+
## Probe control
|
|
49
|
+
|
|
50
|
+
See `probe-control.md` (`.mjs` not loaded; identical `.js` probe fires
|
|
51
|
+
factory and session events; no tool hooks were exercised — PONG6 involved no
|
|
52
|
+
tool calls).
|
|
53
|
+
|
|
54
|
+
## Enforcement chain (plugin installed as `.js`)
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
cp "$REPO/opencode-plugin/index.mjs" "$SANDBOX/.opencode/plugins/harness-enforcement.js"
|
|
58
|
+
|
|
59
|
+
# 1. edit tracking + idle verification + breaker failure #1 + SDK nudge
|
|
60
|
+
opencode run "Create hello.txt with content 'live plugin test v2'. Do not run any tests."
|
|
61
|
+
|
|
62
|
+
# 2. more edits, same failing signature (count -> 3 => force_reflection)
|
|
63
|
+
printf 'process.exit(1)\n' > test.js # restore failing fixture (agent fixed it once)
|
|
64
|
+
opencode run --continue "Append 'v4' to hello.txt. Do not run tests and do not modify test.js."
|
|
65
|
+
opencode run --continue "Append 'v5' to hello.txt. Do not run tests and do not modify test.js."
|
|
66
|
+
|
|
67
|
+
# 3. Operator seeded the reflection artifact; the agent subsequently rewrote
|
|
68
|
+
# it with a RESUME decision. This reconstruction omits that manual step;
|
|
69
|
+
# it is not an unattended deterministic reproduction of reflection.
|
|
70
|
+
|
|
71
|
+
# 4. same signature fails again after reflection -> hardLock: true
|
|
72
|
+
opencode run --continue "Append 'v6' to hello.txt. Do not run tests and do not modify test.js."
|
|
73
|
+
|
|
74
|
+
# 5. Agent reported resetting the breaker and appending; no retained
|
|
75
|
+
# blocked-tool or deletion trace independently confirms those steps.
|
|
76
|
+
opencode run --continue "Append 'v7' to hello.txt. Do not run tests and do not modify test.js."
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## State observation commands
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
KEY=<workspace-slug-and-hash, e.g. harness-live-test-a8d00c4b7672>
|
|
83
|
+
S=~/.agents/harness-everything/workspaces/$KEY/state/sessions/<session-id>
|
|
84
|
+
cat "$S/edit-state.json" "$S/circuit-breaker.json" "$S/compliance.json"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Final counters (`compliance.json`):
|
|
88
|
+
`totalEdits: 7, verifiedEdits: 1, circuitBreakerTrips: 2, reflectionsForced: 1`.
|
|
89
|
+
|
|
90
|
+
## Environment notes
|
|
91
|
+
|
|
92
|
+
- Model: `opencode/union-alpha` (the only configured provider/model on the
|
|
93
|
+
test host; `--model anthropic/claude-sonnet-4-5` fails with a server error
|
|
94
|
+
on this host and is unrelated to the plugin).
|
|
95
|
+
- Plugin state root: `~/.agents/harness-everything/` (no
|
|
96
|
+
`HARNESS_STATE_HOME` override).
|
|
97
|
+
- One deviation from the strict test script: in run 2 the live agent also
|
|
98
|
+
"fixed" `test.js` (turning the failing fixture into a passing one), which
|
|
99
|
+
the plugin correctly credited as a verification pass
|
|
100
|
+
(`verifiedEdits: 1`). The fixture was restored before run 3.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"failures": {
|
|
3
|
+
"npm test: > harness-live-sandbox@1.0.0 test > node test.js": {
|
|
4
|
+
"count": 1,
|
|
5
|
+
"firstSeen": 1789573913728,
|
|
6
|
+
"lastSeen": 1789573913728
|
|
7
|
+
}
|
|
8
|
+
},
|
|
9
|
+
"hardLock": false,
|
|
10
|
+
"lastReflection": null,
|
|
11
|
+
"lastReflectionSignature": null,
|
|
12
|
+
"reflectionPending": false,
|
|
13
|
+
"reflectionRequestedAt": null,
|
|
14
|
+
"reflectionToken": null,
|
|
15
|
+
"reflectionSignature": null
|
|
16
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Zoom-Out Report
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Follow harness instructions: create and append version lines to hello.txt while keeping `npm test` passing. Latest instruction: append 'v6' to hello.txt without running tests or modifying test.js.
|
|
6
|
+
|
|
7
|
+
## Failed Attempts
|
|
8
|
+
|
|
9
|
+
- Attempt 1: Created hello.txt with 'live plugin test v2'; `npm test` failed because test.js was a `process.exit(1)` stub.
|
|
10
|
+
- Attempt 2: Rewrote test.js to assert hello.txt equals `live plugin test v2\nv3\n`, appended 'v3'. Verification still failed: output showed only the npm banner with no test output or error text.
|
|
11
|
+
- Attempt 3: Appended 'v4' and then 'v5' without touching test.js. `npm test` kept failing with the same bare output.
|
|
12
|
+
|
|
13
|
+
## Verified Facts
|
|
14
|
+
|
|
15
|
+
- test.js currently asserts `readFileSync("hello.txt", "utf8") === "live plugin test v2\nv3\n"` and logs "hello.txt verified" on success.
|
|
16
|
+
- hello.txt now contains `live plugin test v2\nv3\nv4\nv5\n` (confirmed by reading it).
|
|
17
|
+
- package.json defines `test` as `node test.js`, so `npm test` runs test.js directly.
|
|
18
|
+
- The harness reports only the npm run banner (`> harness-live-sandbox@1.0.0 test` / `> node test.js`) with no stdout, stderr, or exit-code detail.
|
|
19
|
+
- Instructed not to run tests and not to modify test.js; the failure signature is unchanged across all three attempts.
|
|
20
|
+
|
|
21
|
+
## Diagnosis
|
|
22
|
+
|
|
23
|
+
The recurring failure is almost certainly a stale-expectation mismatch: test.js (fixed earlier) expects exactly `live plugin test v2\nv3\n`, but hello.txt legitimately grew to v4 and v5 per later instructions. Each append re-triggers verification against an outdated assertion, so the identical failure repeats. Alternatively the harness output truncation hides a real error (e.g., crash before assert), but the assertion-mismatch theory best fits the unchanged signature.
|
|
24
|
+
|
|
25
|
+
## Decision
|
|
26
|
+
|
|
27
|
+
RESUME: Append 'v6' to hello.txt as instructed, then request permission to update test.js's expected content to match the full file (`...v4\nv5\nv6\n`) so verification can pass; if modifying test.js remains forbidden, ESCALATE to the harness operator to clarify which file is authoritative.
|
|
28
|
+
|
|
29
|
+
Reflection token: 3fe402d5f67643c1bdd28b741039268f
|
|
@@ -72,7 +72,6 @@ for (const rel of CURRENT_SURFACES) {
|
|
|
72
72
|
requireText('docs/platform-capabilities.md', [
|
|
73
73
|
{ name: 'local Codex plugin boundary', re: /Codex \/ local OpenAI plugin/i },
|
|
74
74
|
{ name: 'public Skills-only boundary', re: /Public OpenAI Skills-only plugin/i },
|
|
75
|
-
{ name: 'OpenCode live-unverified boundary', re: /live plugin loading remains unverified/i },
|
|
76
75
|
{ name: 'general Codex installer boundary', re: /general `--codex` installer|Legacy\/general installer target/i },
|
|
77
76
|
{ name: 'Codex project skill target', re: /Codex[^\n]*`\.agents\/skills\/`/i },
|
|
78
77
|
{ name: 'Continue global skill target', re: /Continue\.dev[^\n]*`~\/\.continue\/skills\/`/i },
|
|
@@ -92,7 +91,6 @@ requireText('docs/platform-compatibility.json', [
|
|
|
92
91
|
requireText('README.md', [
|
|
93
92
|
{ name: 'local Codex plugin surface', re: /Codex \/ local OpenAI plugin|local OpenAI plugin/i },
|
|
94
93
|
{ name: 'public Skills-only distinction', re: /Public OpenAI[^\n]*Skills-only|public OpenAI \*\*Skills-only\*\*/i },
|
|
95
|
-
{ name: 'OpenCode live-unverified qualifier', re: /OpenCode[^\n]*(?:live plugin loading|live loading) remains unverified/i },
|
|
96
94
|
{ name: 'canonical matrix link', re: /docs\/platform-capabilities\.md/ },
|
|
97
95
|
{ name: 'Codex repo Agent Skills path', re: /Codex[^\n]*`?\.agents\/skills\/?`?/i },
|
|
98
96
|
{ name: 'Continue native global skills path', re: /`~\/\.continue\/skills\/`/i },
|
|
@@ -109,12 +107,10 @@ requireText('AGENTS.md', [
|
|
|
109
107
|
requireText('BENCHMARK_SOP.md', [
|
|
110
108
|
{ name: 'behavior vs mechanism separation', re: /behavior[^\n]{0,120}mechanism|mechanism\/live-host evidence/i },
|
|
111
109
|
{ name: 'Codex multi-surface distinction', re: /Codex \/ local OpenAI plugin|general Codex installer path/i },
|
|
112
|
-
{ name: 'OpenCode live-unverified qualifier', re: /OpenCode[^\n]*(?:live plugin loading|live loading) remains unverified/i },
|
|
113
110
|
]);
|
|
114
111
|
|
|
115
112
|
requireText('docs/architecture.md', [
|
|
116
113
|
{ name: 'Codex local plugin host adapter', re: /Codex[^\n]*local OpenAI plugin/i },
|
|
117
|
-
{ name: 'OpenCode live loading remains unverified', re: /live (plugin )?loading remains unverified/i },
|
|
118
114
|
]);
|
|
119
115
|
|
|
120
116
|
requireText('VERIFICATION.md', [
|
|
@@ -158,9 +154,64 @@ requireText('submission/openai/README.md', [
|
|
|
158
154
|
{ name: 'submission excludes local hooks', re: /does not contain the local `\.codex-plugin` lifecycle hooks/i },
|
|
159
155
|
]);
|
|
160
156
|
|
|
157
|
+
const OPENCODE_EVIDENCE = 'benchmarks/results/live-host/opencode-2026-09-16/README.md';
|
|
158
|
+
const OPENCODE_SURFACES = [
|
|
159
|
+
'README.md',
|
|
160
|
+
'AGENTS.md',
|
|
161
|
+
'BENCHMARK_SOP.md',
|
|
162
|
+
'VERIFICATION.md',
|
|
163
|
+
'docs/architecture.md',
|
|
164
|
+
'docs/platform-capabilities.md',
|
|
165
|
+
'docs/platform-compatibility.json',
|
|
166
|
+
'opencode-plugin/README.md',
|
|
167
|
+
];
|
|
168
|
+
|
|
169
|
+
read(OPENCODE_EVIDENCE);
|
|
170
|
+
for (const rel of OPENCODE_SURFACES) {
|
|
171
|
+
requireText(rel, [
|
|
172
|
+
{ name: 'OpenCode retained artifact path', re: /benchmarks\/results\/live-host\/opencode-2026-09-16\/README\.md/ },
|
|
173
|
+
{ name: 'OpenCode host/version/OS scope', re: /OpenCode 1\.18\.31[^\n]{0,40}macOS/i },
|
|
174
|
+
{ name: 'OpenCode project .js scope', re: /project[- ]scope[^\n]{0,100}\.js/i },
|
|
175
|
+
{ name: 'OpenCode loading and state evidence', re: /loading[^\n]{0,120}edit\/verification state/i },
|
|
176
|
+
{ name: 'OpenCode post-reset snapshot boundary', re: /post-reset/i },
|
|
177
|
+
{ name: 'OpenCode operator-seeded reflection boundary', re: /operator-seeded/i },
|
|
178
|
+
{ name: 'OpenCode hard-lock observation boundary', re: /hard[- ]lock[^\n]{0,100}interactive observation/i },
|
|
179
|
+
{ name: 'OpenCode missing blocked-tool trace', re: /no (?:retained )?blocked-tool trace/i },
|
|
180
|
+
{ name: 'OpenCode unverified remainder', re: /Global scope, npm-package installation, and other (?:OpenCode|host) versions remain unverified/i },
|
|
181
|
+
]);
|
|
182
|
+
const text = read(rel);
|
|
183
|
+
for (const re of [/fired the full (?:enforcement )?chain/i, /Hard, live-verified/i, /live (?:plugin )?loading remains unverified until/i]) {
|
|
184
|
+
if (re.test(text)) fail(`${rel}: contains unsupported OpenCode claim ${re}`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const matrix = JSON.parse(read('docs/platform-compatibility.json'));
|
|
189
|
+
const opencode = matrix.platforms.find(platform => platform.id === 'opencode');
|
|
190
|
+
for (const dimension of ['hooksLifecycle', 'liveHostVerification']) {
|
|
191
|
+
if (opencode?.capabilities[dimension]?.status !== 'Partial') {
|
|
192
|
+
fail(`OpenCode ${dimension}: must remain Partial`);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (opencode?.capabilities.liveHostVerification.evidence !== OPENCODE_EVIDENCE) {
|
|
196
|
+
fail('OpenCode liveHostVerification: missing exact retained artifact path');
|
|
197
|
+
}
|
|
198
|
+
for (const platform of matrix.platforms.filter(platform => platform.id !== 'opencode')) {
|
|
199
|
+
if (platform.capabilities.liveHostVerification.status !== 'Unknown') {
|
|
200
|
+
fail(`${platform.id}: liveHostVerification must remain Unknown without its own evidence`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const capabilitySummaryCount = read('docs/platform-capabilities.md').match(/^## Current capability summary$/gm)?.length;
|
|
205
|
+
if (capabilitySummaryCount !== 1) fail('docs/platform-capabilities.md: expected one current capability summary');
|
|
206
|
+
|
|
161
207
|
requireText('opencode-plugin/README.md', [
|
|
162
|
-
{ name: '
|
|
208
|
+
{ name: 'project install directory creation', re: /mkdir -p \.opencode\/plugins\s+cp opencode-plugin\/index\.mjs \.opencode\/plugins\/harness-enforcement\.js/ },
|
|
209
|
+
{ name: 'global install directory creation', re: /mkdir -p ~\/\.config\/opencode\/plugins\s+cp opencode-plugin\/index\.mjs ~\/\.config\/opencode\/plugins\/harness-enforcement\.js/ },
|
|
210
|
+
{ name: 'restart after installation', re: /restart OpenCode/i },
|
|
163
211
|
]);
|
|
212
|
+
if (/package names, not local file paths/i.test(read('opencode-plugin/README.md'))) {
|
|
213
|
+
fail('opencode-plugin/README.md: incorrect local plugin path exclusion');
|
|
214
|
+
}
|
|
164
215
|
|
|
165
216
|
requireText('docs/audit.md', [
|
|
166
217
|
{ name: 'historical snapshot boundary', re: /Historical snapshot boundary/i },
|
|
@@ -70,11 +70,24 @@ for (const platform of matrix.platforms) {
|
|
|
70
70
|
const capability = platform.capabilities[dimension];
|
|
71
71
|
assert.ok(capability && STATUSES.has(capability.status), `${platform.id}/${dimension}: invalid status`);
|
|
72
72
|
}
|
|
73
|
-
|
|
74
|
-
platform.capabilities.liveHostVerification
|
|
75
|
-
'
|
|
76
|
-
|
|
77
|
-
|
|
73
|
+
if (platform.id === 'opencode') {
|
|
74
|
+
const capability = platform.capabilities.liveHostVerification;
|
|
75
|
+
assert.strictEqual(capability.status, 'Partial');
|
|
76
|
+
assert.strictEqual(capability.evidence, 'benchmarks/results/live-host/opencode-2026-09-16/README.md');
|
|
77
|
+
const artifactRoot = path.dirname(path.join(ROOT, capability.evidence));
|
|
78
|
+
for (const file of ['README.md', 'probe-control.md', 'run-commands.md']) {
|
|
79
|
+
assert.ok(fs.statSync(path.join(artifactRoot, file)).size > 0, `Missing OpenCode evidence: ${file}`);
|
|
80
|
+
}
|
|
81
|
+
const session = path.join(artifactRoot, 'state', 'sessions', 'ses_f5520f1d9ffeB8KLTIUKgA5ng8');
|
|
82
|
+
const breaker = JSON.parse(fs.readFileSync(path.join(session, 'circuit-breaker.json'), 'utf8'));
|
|
83
|
+
const compliance = JSON.parse(fs.readFileSync(path.join(session, 'compliance.json'), 'utf8'));
|
|
84
|
+
assert.strictEqual(breaker.hardLock, false, 'Retained snapshot is post-reset, not hard-lock proof');
|
|
85
|
+
assert.ok(compliance.totalEdits > 0);
|
|
86
|
+
assert.ok(fs.statSync(path.join(session, 'edit-state.json')).size > 0);
|
|
87
|
+
assert.ok(fs.statSync(path.join(session, 'zoom-out-report.md')).size > 0);
|
|
88
|
+
} else {
|
|
89
|
+
assert.strictEqual(platform.capabilities.liveHostVerification.status, 'Unknown', `${platform.id}: no preserved live evidence`);
|
|
90
|
+
}
|
|
78
91
|
}
|
|
79
92
|
|
|
80
93
|
const installerTargetSpecs = {
|
|
@@ -142,7 +155,7 @@ try {
|
|
|
142
155
|
const capabilityDocs = fs.readFileSync(path.join(ROOT, 'docs', 'platform-capabilities.md'), 'utf8');
|
|
143
156
|
assert.match(capabilityDocs, /platform-compatibility\.json/);
|
|
144
157
|
assert.match(capabilityDocs, /Continue\.dev[\s\S]*`Unknown` for standalone `SKILL\.md` discovery/);
|
|
145
|
-
assert.match(capabilityDocs, /live
|
|
146
|
-
assert.match(capabilityDocs, /no fresh host-session artifacts/i);
|
|
158
|
+
assert.match(capabilityDocs, /live-host evidence|Partial` live-host loading/i);
|
|
159
|
+
assert.match(capabilityDocs, /No fresh artifacts exist yet|no fresh host-session artifacts/i);
|
|
147
160
|
|
|
148
161
|
console.log('Platform compatibility verified: official-source matrix, installer targets, package manifests, and public bundle boundaries.');
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
const assert = require('assert');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const vm = require('vm');
|
|
6
|
+
const helper = require('./test-helper');
|
|
7
|
+
|
|
8
|
+
const root = helper.root;
|
|
9
|
+
const sourceFile = path.join(root, 'opencode-plugin', 'index.mjs');
|
|
10
|
+
const runnerFile = path.join(root, 'behavioral-evals', 'run-with-plugin.js');
|
|
11
|
+
const readmeFile = path.join(root, 'opencode-plugin', 'README.md');
|
|
12
|
+
|
|
13
|
+
function pluginInstallTargets(text) {
|
|
14
|
+
return [...text.matchAll(/(?:\.opencode|(?:~[\\/])?\.config[\\/]opencode)[\\/]plugins?[\\/][\w.@/-]+/g)]
|
|
15
|
+
.map(match => match[0].replace(/[\\/]+$/, ''));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function invalidTargets(text) {
|
|
19
|
+
return pluginInstallTargets(text).filter(target => !/\.(?:js|ts)$/.test(target));
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function installStatement(text) {
|
|
23
|
+
const statements = text.match(/fs\.copyFileSync\(path\.join\(PLUGIN_DIR,[^;]+;/g) || [];
|
|
24
|
+
assert.strictEqual(statements.length, 1, 'Expected one canonical plugin installation');
|
|
25
|
+
return statements[0];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function verifyInstall(text) {
|
|
29
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-loadability-'));
|
|
30
|
+
try {
|
|
31
|
+
vm.runInNewContext(installStatement(text), {
|
|
32
|
+
fs, path,
|
|
33
|
+
PLUGIN_DIR: path.dirname(sourceFile),
|
|
34
|
+
opencodePluginsDir: directory,
|
|
35
|
+
}, { timeout: 1000 });
|
|
36
|
+
assert.deepStrictEqual(fs.readdirSync(directory), ['harness-enforcement.js']);
|
|
37
|
+
assert.ok(fs.readFileSync(sourceFile).equals(fs.readFileSync(path.join(directory, 'harness-enforcement.js'))));
|
|
38
|
+
} finally {
|
|
39
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function walk(directory) {
|
|
44
|
+
return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
|
|
45
|
+
if (['node_modules', '.git'].includes(entry.name) || entry.isSymbolicLink()) return [];
|
|
46
|
+
const file = path.join(directory, entry.name);
|
|
47
|
+
const relative = path.relative(root, file).split(path.sep).join('/');
|
|
48
|
+
if (relative === 'benchmarks/results' || relative === 'CHANGELOG.md') return [];
|
|
49
|
+
return entry.isDirectory() ? walk(file) : [file];
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
const runner = fs.readFileSync(runnerFile, 'utf8');
|
|
55
|
+
verifyInstall(runner);
|
|
56
|
+
assert.throws(() => verifyInstall(runner.replace('harness-enforcement.js', 'harness-enforcement.mjs')));
|
|
57
|
+
assert.throws(() => verifyInstall(runner.replace("'index.mjs'", "'plugin.json'")));
|
|
58
|
+
|
|
59
|
+
for (const extension of ['mjs', 'cjs', 'mts', 'cts', 'jsx', 'tsx', 'json']) {
|
|
60
|
+
for (const base of ['.opencode/plugins/', '~/.config/opencode/plugins/']) {
|
|
61
|
+
assert.strictEqual(invalidTargets(`${base}example.${extension}`).length, 1);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
for (const base of ['.opencode/plugins/', '~/.config/opencode/plugins/']) {
|
|
65
|
+
assert.strictEqual(invalidTargets(`${base}example`).length, 1);
|
|
66
|
+
assert.deepStrictEqual(invalidTargets(`${base}example.js`), []);
|
|
67
|
+
assert.deepStrictEqual(invalidTargets(`${base}example.ts`), []);
|
|
68
|
+
}
|
|
69
|
+
assert.deepStrictEqual(invalidTargets('opencode-plugin/index.mjs .opencode/plugins/example.js'), []);
|
|
70
|
+
|
|
71
|
+
const violations = [];
|
|
72
|
+
for (const file of walk(root).filter(file => /\.(?:md|js|mjs|json|jsonc|yml|yaml|txt)$/.test(file))) {
|
|
73
|
+
const text = fs.readFileSync(file, 'utf8');
|
|
74
|
+
for (const target of invalidTargets(text)) violations.push(`${path.relative(root, file)}: ${target}`);
|
|
75
|
+
}
|
|
76
|
+
assert.deepStrictEqual(violations, [], 'Unsupported OpenCode install destinations; only .js/.ts are auto-discovered');
|
|
77
|
+
const readme = fs.readFileSync(readmeFile, 'utf8');
|
|
78
|
+
for (const destination of ['.opencode/plugins/harness-enforcement.js', '~/.config/opencode/plugins/harness-enforcement.js']) {
|
|
79
|
+
assert.ok(readme.includes(`cp opencode-plugin/index.mjs ${destination}`));
|
|
80
|
+
}
|
|
81
|
+
helper.check('30. real install statement produces a byte-identical .js/.ts-discoverable file; non-allowlisted filenames and content mutations fail', true);
|
|
82
|
+
helper.finish();
|
|
83
|
+
} catch (error) {
|
|
84
|
+
helper.check('30. OpenCode installation contract', false, error.stack);
|
|
85
|
+
helper.finish();
|
|
86
|
+
}
|