harness-everything 0.47.1 → 0.47.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.github/workflows/ci.yml +14 -0
- package/CHANGELOG.md +18 -0
- package/benchmarks/issue290-pilot/README.md +199 -0
- package/benchmarks/issue290-pilot/analyze.py +299 -0
- package/benchmarks/issue290-pilot/analyze.test.py +146 -0
- package/benchmarks/issue290-pilot/check.py +19 -0
- package/benchmarks/issue290-pilot/evidence/archive.json +7 -0
- package/benchmarks/issue290-pilot/evidence/group-summary.json +684 -0
- package/benchmarks/issue290-pilot/evidence/inventory.json +3728 -0
- package/benchmarks/issue290-pilot/evidence/ledger.json +4644 -0
- package/benchmarks/issue290-pilot/evidence/replay/pairs.json +482 -0
- package/benchmarks/issue290-pilot/evidence/replay/provenance.json +24 -0
- package/benchmarks/issue290-pilot/evidence/replay/tables.md +37 -0
- package/benchmarks/issue290-pilot/evidence.tar.gz +0 -0
- package/benchmarks/issue290-pilot/export.py +110 -0
- package/benchmarks/issue290-pilot/export.test.py +65 -0
- package/benchmarks/issue290-pilot/fable-supplement.py +55 -0
- package/benchmarks/issue290-pilot/fixture.test.py +28 -0
- package/benchmarks/issue290-pilot/fixtures/polyglot/instructions.md +19 -0
- package/benchmarks/issue290-pilot/fixtures/polyglot/package.json +29 -0
- package/benchmarks/issue290-pilot/fixtures/polyglot/simple-linked-list.js +44 -0
- package/benchmarks/issue290-pilot/fixtures/polyglot/simple-linked-list.spec.js +131 -0
- package/benchmarks/issue290-pilot/fixtures/quixbugs/LICENSE +7 -0
- package/benchmarks/issue290-pilot/fixtures/quixbugs/cases.json +7 -0
- package/benchmarks/issue290-pilot/fixtures/quixbugs/find_in_sorted.py +31 -0
- package/benchmarks/issue290-pilot/fixtures/quixbugs/test_find_in_sorted.py +15 -0
- package/benchmarks/issue290-pilot/frozen-documents.json +41 -0
- package/benchmarks/issue290-pilot/grader.cjs +43 -0
- package/benchmarks/issue290-pilot/grader.test.cjs +10 -0
- package/benchmarks/issue290-pilot/hook.test.cjs +27 -0
- package/benchmarks/issue290-pilot/host-hook.cjs +35 -0
- package/benchmarks/issue290-pilot/metrics.test.py +131 -0
- package/benchmarks/issue290-pilot/preflight/amendment.md +32 -0
- package/benchmarks/issue290-pilot/preflight/environment.json +15 -0
- package/benchmarks/issue290-pilot/preflight/fable-amendment.md +20 -0
- package/benchmarks/issue290-pilot/preflight/fable-frozen-files.json +4 -0
- package/benchmarks/issue290-pilot/preflight/fable-infrastructure-amendment.md +27 -0
- package/benchmarks/issue290-pilot/preflight/fable-retry-frozen-files.json +4 -0
- package/benchmarks/issue290-pilot/preflight/fable-supplement-v1.py +47 -0
- package/benchmarks/issue290-pilot/preflight/frozen-files.json +18 -0
- package/benchmarks/issue290-pilot/preflight/graders.json +67 -0
- package/benchmarks/issue290-pilot/pricing-assumptions.json +29 -0
- package/benchmarks/issue290-pilot/protocol.json +32 -0
- package/benchmarks/issue290-pilot/read_metrics.py +166 -0
- package/benchmarks/issue290-pilot/replay.py +167 -0
- package/benchmarks/issue290-pilot/reporting.py +82 -0
- package/benchmarks/issue290-pilot/run.py +262 -0
- package/benchmarks/issue290-pilot/sources.json +10 -0
- package/benchmarks/issue290-pilot/verification/analyzer-tests.log +5 -0
- package/benchmarks/issue290-pilot/verification/final-gates.json +58 -0
- package/benchmarks/issue290-pilot/verification/fixture-tests.log +5 -0
- package/benchmarks/issue290-pilot/verification/gates.json +58 -0
- package/benchmarks/issue290-pilot/verification/npm-audit.json +84 -0
- package/benchmarks/issue290-pilot/verification/offline-correction.json +59 -0
- package/benchmarks/issue290-pilot/verification/post-report/helper-tests.log +22 -0
- package/benchmarks/issue290-pilot/verification/post-report/test-consistency.log +626 -0
- package/benchmarks/issue290-pilot/verification/post-report/test-references.log +6 -0
- package/benchmarks/issue290-pilot/verification/post-report/waza-tokens.log +710 -0
- package/benchmarks/issue290-pilot/verification/test-collision.log +28 -0
- package/benchmarks/issue290-pilot/verification/test-consistency.log +28 -0
- package/benchmarks/issue290-pilot/verification/test-docs-capabilities.log +131 -0
- package/benchmarks/issue290-pilot/verification/test-mechanism.log +1829 -0
- package/benchmarks/issue290-pilot/verification/test-mutations.log +16 -0
- package/benchmarks/issue290-pilot/verification/test-platform-compatibility.log +9 -0
- package/benchmarks/issue290-pilot/verification/test-plugin-openai.log +6 -0
- package/benchmarks/issue290-pilot/verification/test-plugin-submission.log +5 -0
- package/benchmarks/issue290-pilot/verification/test-references.log +6 -0
- package/benchmarks/issue290-pilot/verification/test-release.log +59 -0
- package/benchmarks/issue290-pilot/verification/test-repo-contract.log +5 -0
- package/benchmarks/issue290-pilot/verification/test-routing-invariants.log +785 -0
- package/benchmarks/issue290-pilot/verification/test-routing-skills.log +28 -0
- package/benchmarks/issue290-pilot/verification/test.log +3373 -0
- package/benchmarks/issue290-pilot/verification/verification-report.md +76 -0
- package/benchmarks/issue290-pilot/verification/waza-tokens.log +710 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/gates.json +26 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/test-collision.log +8 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/test-consistency.log +626 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/test-mechanism.log +1693 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/test-release.log +38 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/test-routing-skills.log +44 -0
- package/benchmarks/issue290-pilot/verification/with-dependencies/test.log +3253 -0
- package/ci/lib/opencode-verification-loader.js +24 -0
- package/ci/mechanism-2p-opencode-v2-entrypoint.test.js +8 -1
- package/ci/mechanism-325-f3-derived-views.test.js +42 -2
- package/ci/mechanism-338-opencode-async.test.js +229 -0
- package/ci/mechanism-341-opencode-lane.test.js +14 -3
- package/ci/mechanism-36-verify-timeout.test.js +7 -168
- package/docs/platform-capabilities.md +8 -0
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/doc-check.js +5 -4
- package/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
- package/opencode-plugin/README.md +28 -30
- package/opencode-plugin/index.mjs +249 -112
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/doc-check.js +5 -4
- package/sbom.cdx.json +7 -7
|
@@ -247,11 +247,12 @@ function globToRegExp(pattern) {
|
|
|
247
247
|
return new RegExp(`^${body}$`);
|
|
248
248
|
}
|
|
249
249
|
|
|
250
|
-
//
|
|
250
|
+
// Keep 64-bit device/file IDs exact (Windows IDs can exceed Number's safe
|
|
251
|
+
// integer range), while still rejecting symlink and hard-link aliases.
|
|
251
252
|
function sameFile(a, b) {
|
|
252
|
-
const x = fs.statSync(a);
|
|
253
|
-
const y = fs.statSync(b);
|
|
254
|
-
return x.ino >
|
|
253
|
+
const x = fs.statSync(a, { bigint: true });
|
|
254
|
+
const y = fs.statSync(b, { bigint: true });
|
|
255
|
+
return x.ino > 0n && x.ino === y.ino && x.dev === y.dev;
|
|
255
256
|
}
|
|
256
257
|
|
|
257
258
|
// Open without blocking (O_NONBLOCK keeps a FIFO from waiting for a writer), then require a regular file before reading.
|
|
@@ -101,7 +101,7 @@ function isUncPathPrefix(token, command, index) {
|
|
|
101
101
|
return token === '' && command[index] === '\\' && command[index + 1] === '\\';
|
|
102
102
|
}
|
|
103
103
|
// Canonical SHA-256 of harness-everything/scripts/doc-check.js (CRLF normalised to LF). Re-pin deliberately when that script changes.
|
|
104
|
-
const SHIPPED_DOC_CHECK_DIGEST = '
|
|
104
|
+
const SHIPPED_DOC_CHECK_DIGEST = '0e93f252b2350b145c34dcb5354aa3b4246f3c59e948a62c3a912305b355a475';
|
|
105
105
|
// This install's own doc-check runtime, relative to hooks/scripts/lib: the source checkout or the packaged plugin.
|
|
106
106
|
const DOC_CHECK_INSTALL_LAYOUTS = [
|
|
107
107
|
['..', '..', '..', 'harness-everything', 'scripts', 'doc-check.js'],
|
|
@@ -52,39 +52,37 @@ and on failure calls `client.session.prompt()` to push a synthetic follow-up
|
|
|
52
52
|
message into the session - forcing the agent to keep working instead of
|
|
53
53
|
actually stopping.
|
|
54
54
|
|
|
55
|
-
The
|
|
56
|
-
|
|
57
|
-
`HARNESS_VERIFY_TIMEOUT_MS
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
`execution_error` (
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
follow-up in issue #338.
|
|
55
|
+
The async runner streams stdout/stderr and gives the **whole ordered sequence**
|
|
56
|
+
(`test`, `lint`, `build`) one wall-clock budget: 10 minutes by default, overridden
|
|
57
|
+
by `HARNESS_VERIFY_TIMEOUT_MS`. It reports progress every 15 seconds in the
|
|
58
|
+
plugin log. No later script starts after the budget expires. Each run is
|
|
59
|
+
classified as `pass` (exit 0), `test_failure` (a real nonzero exit), `timeout`
|
|
60
|
+
(the sequence deadline expired), or `execution_error` (spawn failure, external
|
|
61
|
+
signal, or cancellation). Only `test_failure` advances the Rule-of-3 counter.
|
|
62
|
+
|
|
63
|
+
| Resource | Bound / behavior |
|
|
64
|
+
|---|---|
|
|
65
|
+
| Retained output | First and last 2,000 characters per stream; the final diagnostic shares a 2,000-character budget across streams, plus labels/elision markers |
|
|
66
|
+
| Failure identity | Incremental whitespace-normalized digest of both complete streams; first recognized test marker and suite context tracked separately |
|
|
67
|
+
| Marker parsing | First 4,096 characters of each line; an overlong marker/context selects the complete-stream digest; ordinary long banners preserve later complete markers |
|
|
68
|
+
| Timeout/cancel | Terminate the shell/npm/script process group, then force-kill after 150 ms on POSIX; Windows uses `taskkill /T /F` |
|
|
69
|
+
| Concurrent lifecycle | Duplicate idle events share the active run; a code edit or session reset aborts it; an idle arriving during cancellation waits for cleanup before verifying the newer edits; the V2 event stream continues consuming cancellation events; disposal awaits active child cleanup; stale results cannot clear newer edits or recreate deleted state |
|
|
70
|
+
|
|
71
|
+
The event loop remains available while scripts run. The output-retention limit
|
|
72
|
+
never becomes a check verdict. Mechanism tests cover large output, shared
|
|
73
|
+
budget, child cleanup and concurrent hooks; these are deterministic mechanism
|
|
74
|
+
evidence, not a new live-host verification claim. Processes deliberately
|
|
75
|
+
escaping the process group are outside this cleanup boundary.
|
|
77
76
|
|
|
78
77
|
### 3. Rule-of-3 zoom-out boundary (`tool.execute.before` + the verification observer above)
|
|
79
78
|
|
|
80
79
|
Enforces Rule of 3 on repeated verification failures (same failing command +
|
|
81
80
|
normalized failure evidence: the first test-level failing marker - TAP
|
|
82
81
|
`not ok`, node:test/jest symbols, Jest test blocks, assertion lines - over the
|
|
83
|
-
|
|
82
|
+
streamed stdout **and** stderr, with a preceding suite header such as
|
|
84
83
|
Jest's `FAIL <file>` carried as context so different tests in one file stay
|
|
85
84
|
distinct; without a test-level marker the signature falls back to a digest of
|
|
86
|
-
|
|
87
|
-
only to the user-visible diagnostic excerpt, never to signature analysis; not
|
|
85
|
+
those complete streams, never to a suite header alone; the diagnostic retention bound does not limit the stream digest; not
|
|
88
86
|
arbitrary tool failures, since
|
|
89
87
|
`tool.execute.after` has no normalized success/failure field to key a
|
|
90
88
|
signature on for tools in general):
|
|
@@ -146,17 +144,17 @@ installation surfaces and were not verified in this run.
|
|
|
146
144
|
|
|
147
145
|
## Manual verification CLI
|
|
148
146
|
|
|
149
|
-
**File:** `hooks/verify.js` -
|
|
150
|
-
`index.mjs`'s verification gate, kept as a standalone command:
|
|
147
|
+
**File:** `hooks/verify.js` - a standalone "run available npm scripts" command:
|
|
151
148
|
|
|
152
149
|
```bash
|
|
153
150
|
node opencode-plugin/hooks/verify.js
|
|
154
151
|
```
|
|
155
152
|
|
|
156
153
|
Run it from the workspace under test, never from the Harness repo root - from
|
|
157
|
-
there it would re-enter `npm test` from inside `npm test`.
|
|
158
|
-
|
|
159
|
-
|
|
154
|
+
there it would re-enter `npm test` from inside `npm test`. The manual CLI retains its synchronous per-command budget and 16 MiB capture
|
|
155
|
+
limit; the async sequence budget and streaming behavior above apply to the
|
|
156
|
+
plugin only. Keeping its implementation separate preserves the single-file
|
|
157
|
+
plugin installation contract.
|
|
160
158
|
|
|
161
159
|
`plugin.json` records this standalone helper for repository parity checks; it
|
|
162
160
|
does not configure opencode. The runtime module is `index.mjs`.
|