harness-everything 0.41.3 → 0.42.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +18 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/README.md +68 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/green-final.log +162 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/green.log +158 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/live-installed-runtime-rejection.json +31 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/luna-action-family-cross-product.js +143 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/luna-action-family-cross-product.log +7 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/luna-codex-action-gate-prehook-smoke.js +78 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/luna-codex-action-gate-prehook-smoke.log +8 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/luna-review.md +151 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/original-rejection.json +31 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/red.log +158 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/review-descriptor-green.log +192 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/review-descriptor-red.log +192 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/review-family-green.log +206 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/review-family-red.log +206 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/review-hints-green.log +174 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/review-hints-red.log +174 -0
- package/benchmarks/results/codex-pr-metadata-2026-10-26/verification.md +31 -0
- package/benchmarks/results/doc-evidence-e1/README.md +68 -0
- package/benchmarks/results/doc-evidence-e1/evidence.json +360 -0
- package/benchmarks/results/doc-evidence-e1/pilot-report.json +175 -0
- package/benchmarks/results/h1-provider-independence-qualification/README.md +95 -0
- package/benchmarks/results/live-host/opencode-d1-d2-2026-10-25/README.md +30 -0
- package/benchmarks/results/live-host/opencode-d1-d2-2026-10-25/evidence.json +128 -0
- package/ci/action-gate.test.js +86 -0
- package/ci/mechanism-2aa-codex-action-gate-policy.test.js +47 -0
- package/ci/mechanism-325-3b-h1-qualification.test.js +206 -0
- package/ci/mechanism-341-opencode-lane.test.js +465 -0
- package/ci/mechanism-task-git-authorization.test.js +12 -0
- package/docs/codex-action-gate.md +27 -1
- package/docs/doc-workflow-reconciliation.md +4 -3
- package/docs/document-lifecycle.md +3 -3
- package/docs/platform-capabilities.md +8 -2
- package/hooks/scripts/action-gate.js +35 -8
- package/hooks/scripts/lib/doc-reconciliation-pilot.js +40 -10
- package/hooks/scripts/lib/doc-reconciliation-provider.js +159 -11
- package/hooks/scripts/workflow-stop-gate.js +4 -1
- package/opencode-plugin/index.mjs +77 -5
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/action-gate.js +35 -8
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-pilot.js +40 -10
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-provider.js +159 -11
- package/plugins/harness-everything/hooks/scripts/workflow-stop-gate.js +4 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/sbom.cdx.json +7 -7
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# OpenCode D1/D2 live-host observations — 2026-10-25
|
|
2
|
+
|
|
3
|
+
**Retained negative observations only.** These attempts do not demonstrate a completed task, successful enforcement, or behavioral effectiveness.
|
|
4
|
+
|
|
5
|
+
## Run record
|
|
6
|
+
|
|
7
|
+
| Run | Product / fixture baseline | Requested model | Captured result |
|
|
8
|
+
|---|---|---|---|
|
|
9
|
+
| D1 | `37c97046`; fixture `f486e57d` | `opencode/beam-free#default` | Sanitized export records `failed` with direct error type `provider.free-limit`. Shell pipeline exit `1`; OpenCode process exit **UNKNOWN**; sanitized export exit `0`. |
|
|
10
|
+
| D2 | `37c97046`; fixture `39aa0898` | `nvidia/deepseek-ai/deepseek-v4.1-flash` | Process exit `1`, signal `null`, timeout `false`, spawn failure `null`. One filtered stdout event had `type=error` and no session ID; stderr was `0` bytes. Session existence, provider-request status, and cause are **UNKNOWN**. |
|
|
11
|
+
|
|
12
|
+
The user requested switching from the exhausted free quota to NVIDIA; one D2 invocation was attempted. NVIDIA's catalog listing and configured-auth-presence boolean came only from non-session preflight. The model variant was not shown. D2's filtered error omitted its discriminant, so its cause is unknown; this is not evidence of an NVIDIA outage.
|
|
13
|
+
|
|
14
|
+
## Setup and retained evidence
|
|
15
|
+
|
|
16
|
+
Both commands used `opencode run --standalone --format json` and process-local `XDG_CONFIG_HOME`, `OPENCODE_CONFIG_DIR`, `HARNESS_STATE_HOME`, `HARNESS_DOC_RECONCILIATION_PILOT=1`, and `HARNESS_D1_TRACE_FILE`. D1 used the beam-free model above; D2 requested the NVIDIA model above. No user-global credentials or configuration were changed.
|
|
17
|
+
|
|
18
|
+
| Observation | D1 | D2 |
|
|
19
|
+
|---|---|---|
|
|
20
|
+
| Fixture | Commit clean before/after; ADR hash unchanged | Commit clean before/after; ADR hash unchanged |
|
|
21
|
+
| Natural completion | Not observed | Not observed |
|
|
22
|
+
| Observer | Load marker and trace absent | Trace absent; observer load **not observed** |
|
|
23
|
+
| Refusal/report | No pilot report | Report capture **UNKNOWN**; no session ID was available for lookup |
|
|
24
|
+
| Global Harness absence | **UNCHECKED** | **UNCHECKED** |
|
|
25
|
+
|
|
26
|
+
The sanitized D1 export is not copied into this record; its SHA-256 is retained in [`evidence.json`](evidence.json). D2 raw stdout/stderr were not retained; only allowlisted metadata and hashes remain. `LOCAL_ONLY` source paths in `evidence.json` identify supporting packets and artifacts that are not shipped in this PR. The D2 runner and its exit/timeout self-check are also local-only; their self-check is deterministic mechanism evidence, not live-host evidence. This PR does not make the full runs publicly reproducible.
|
|
27
|
+
|
|
28
|
+
## Interpretation
|
|
29
|
+
|
|
30
|
+
Neither negative attempt qualifies as an E1 cycle or supports E1 GO, benefit, qualification, or promotion. D1 stopped on the free-provider limit. D2 returned an error event without a session ID; no cause or provider request can be established from retained metadata. This PR preserves negative observations only.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"clientDate": "2026-10-25",
|
|
4
|
+
"classification": "retained-negative-observations-only",
|
|
5
|
+
"publicReproduction": "LOCAL_ONLY supporting packets, fixtures, and runner are not shipped; this PR does not reproduce full runs",
|
|
6
|
+
"productBaseline": "37c97046cb1e620c4a3585b4da5a2593501a48be",
|
|
7
|
+
"runs": [
|
|
8
|
+
{
|
|
9
|
+
"id": "D1",
|
|
10
|
+
"engine": "OpenCode v2.1.4",
|
|
11
|
+
"requestedModel": "opencode/beam-free#default",
|
|
12
|
+
"fixtureCommit": "f486e57d1ba17d92e585b0b42dd4b9c71b2fda4a",
|
|
13
|
+
"sessionId": "ses_e8de7be3cffeBTghRaz6CN1T01",
|
|
14
|
+
"sessionOutcome": "failed",
|
|
15
|
+
"providerFailureType": "provider.free-limit",
|
|
16
|
+
"directSanitizedExportErrorTypes": [
|
|
17
|
+
"provider.free-limit",
|
|
18
|
+
"provider.free-limit"
|
|
19
|
+
],
|
|
20
|
+
"process": {
|
|
21
|
+
"shellPipelineExit": 1,
|
|
22
|
+
"opencodeProcessExit": null,
|
|
23
|
+
"opencodeProcessExitStatus": "UNKNOWN: pipeline status captured without PIPESTATUS"
|
|
24
|
+
},
|
|
25
|
+
"sanitizedExport": {
|
|
26
|
+
"exitCode": 0,
|
|
27
|
+
"messageCount": 4,
|
|
28
|
+
"sha256": "91f80a5e8b2c94015a5a5602022aa45436cff2a52295615f7eb022aed91eee2b",
|
|
29
|
+
"source": "LOCAL_ONLY:.worktrees/packets/resume-2026-10-25/D1-current/fixture/trace/native-export.json"
|
|
30
|
+
},
|
|
31
|
+
"fixture": {
|
|
32
|
+
"gitCleanBeforeAndAfter": true,
|
|
33
|
+
"configSha256": "422eb8c6481aaadbf7910d4b415e76b2e60723bd0da4c532a94fad1879c38bcd",
|
|
34
|
+
"adrBeforeSha256": "65c903eedc1be2ca6022638fd9cab1f70c08c278e9e03be5c73ad0b2ecab4d18",
|
|
35
|
+
"adrAfterSha256": "65c903eedc1be2ca6022638fd9cab1f70c08c278e9e03be5c73ad0b2ecab4d18",
|
|
36
|
+
"taskPromptSha256": "524e58eeb6bd64e33986777b19a7fca8ec0829154fc2eb4b0c4a26b8e13b7b5e"
|
|
37
|
+
},
|
|
38
|
+
"observations": {
|
|
39
|
+
"naturalCompletionObserved": false,
|
|
40
|
+
"observerLoadMarkerObserved": false,
|
|
41
|
+
"observerTracePresent": false,
|
|
42
|
+
"pilotReportPresent": false,
|
|
43
|
+
"innerConsumerCause": "UNCHECKED",
|
|
44
|
+
"globalHarnessAbsence": "UNCHECKED"
|
|
45
|
+
},
|
|
46
|
+
"sourceArtifactSha256": {
|
|
47
|
+
"beforeState": "91018ac09c0e58537b53ffbef6adb1d2ecd5cc5b079cccc60484493aa592d474",
|
|
48
|
+
"afterState": "d2f729d1f5eb29b6a17b552fb06a31833ecbe4ef3eda5096e895ebfab212518c",
|
|
49
|
+
"filteredRunLog": "b1f1b48237c94733a6753440617e52c85ee754d1287a78b94d2513e5fdfa07a5",
|
|
50
|
+
"observerImplementation": "33a3a3c8da4f5dc495f7462952b11ee8ad46c4fa9ba0882152acd0eebf0ed56f",
|
|
51
|
+
"taskLocalPluginEntrypoint": "ad4eef698164e644e52eeb0c6f36bef56ca75a5c66d7cd71345d0091108f8fbe"
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "D2",
|
|
56
|
+
"engine": "OpenCode CLI invocation; session identity UNKNOWN",
|
|
57
|
+
"requestedModel": "nvidia/deepseek-ai/deepseek-v4.1-flash",
|
|
58
|
+
"productBaseline": "37c97046cb1e620c4a3585b4da5a2593501a48be",
|
|
59
|
+
"fixtureCommit": "39aa08987e0da2fc2ff3ea3556b84bd6d37f4e79",
|
|
60
|
+
"nvidiaPreflight": {
|
|
61
|
+
"catalogModel": "nvidia/deepseek-ai/deepseek-v4.1-flash",
|
|
62
|
+
"configuredAuthPresence": true,
|
|
63
|
+
"variant": "UNKNOWN",
|
|
64
|
+
"evidenceLayer": "non-session preflight only",
|
|
65
|
+
"providerRequestMadeDuringPreflight": false
|
|
66
|
+
},
|
|
67
|
+
"process": {
|
|
68
|
+
"exitCode": 1,
|
|
69
|
+
"signal": null,
|
|
70
|
+
"timedOut": false,
|
|
71
|
+
"spawnFailureReason": null,
|
|
72
|
+
"stdoutJsonlRecords": 1,
|
|
73
|
+
"filteredEventType": "error",
|
|
74
|
+
"sessionId": null,
|
|
75
|
+
"sessionExistence": "UNKNOWN",
|
|
76
|
+
"stderrBytes": 0,
|
|
77
|
+
"rawOutputRetained": false
|
|
78
|
+
},
|
|
79
|
+
"errorInterpretation": {
|
|
80
|
+
"errorDiscriminantRetained": false,
|
|
81
|
+
"captureLimit": "allowlist omitted the error discriminant",
|
|
82
|
+
"cause": "UNKNOWN",
|
|
83
|
+
"providerRequestStatus": "UNKNOWN"
|
|
84
|
+
},
|
|
85
|
+
"fixture": {
|
|
86
|
+
"gitCleanBeforeAndAfter": true,
|
|
87
|
+
"configSha256": "422eb8c6481aaadbf7910d4b415e76b2e60723bd0da4c532a94fad1879c38bcd",
|
|
88
|
+
"adrBeforeSha256": "65c903eedc1be2ca6022638fd9cab1f70c08c278e9e03be5c73ad0b2ecab4d18",
|
|
89
|
+
"adrAfterSha256": "65c903eedc1be2ca6022638fd9cab1f70c08c278e9e03be5c73ad0b2ecab4d18",
|
|
90
|
+
"taskPromptSha256": "524e58eeb6bd64e33986777b19a7fca8ec0829154fc2eb4b0c4a26b8e13b7b5e",
|
|
91
|
+
"adrChanged": false
|
|
92
|
+
},
|
|
93
|
+
"observations": {
|
|
94
|
+
"naturalCompletionObserved": false,
|
|
95
|
+
"observerTracePresent": false,
|
|
96
|
+
"observerLoad": "NOT_OBSERVED",
|
|
97
|
+
"sanitizedExportAttempted": false,
|
|
98
|
+
"reportCapture": "UNKNOWN: no session ID was available for lookup",
|
|
99
|
+
"globalHarnessAbsence": "UNCHECKED",
|
|
100
|
+
"isolation": "UNCHECKED"
|
|
101
|
+
},
|
|
102
|
+
"sourceArtifactSha256": {
|
|
103
|
+
"exitMetadata": "61f8904e70fa9266f52437d536a43d9f8c34b338fa1c0f55827cf1527b7f7582",
|
|
104
|
+
"filteredRunLog": "b2261fe453f0645fb8d7606979e0c4a8b7c08095a6fc1fa3dbb9a670fbf78806",
|
|
105
|
+
"observerImplementation": "33a3a3c8da4f5dc495f7462952b11ee8ad46c4fa9ba0882152acd0eebf0ed56f",
|
|
106
|
+
"taskLocalPluginEntrypoint": "ad4eef698164e644e52eeb0c6f36bef56ca75a5c66d7cd71345d0091108f8fbe"
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
],
|
|
110
|
+
"localRunnerSelfCheck": {
|
|
111
|
+
"result": "PASS",
|
|
112
|
+
"checks": [
|
|
113
|
+
"child exit code 23",
|
|
114
|
+
"timeout and signal capture",
|
|
115
|
+
"spawn failure maps to null exit plus reason"
|
|
116
|
+
],
|
|
117
|
+
"evidenceLayer": "deterministic mechanism only; not live-host evidence",
|
|
118
|
+
"runnerSource": "LOCAL_ONLY:.worktrees/packets/resume-2026-10-25/D2-nvidia-current/scripts/d2-runner.mjs"
|
|
119
|
+
},
|
|
120
|
+
"qualification": {
|
|
121
|
+
"e1Cycle": false,
|
|
122
|
+
"e1Go": false,
|
|
123
|
+
"behaviorBenefit": false,
|
|
124
|
+
"qualifies": false,
|
|
125
|
+
"nvidiaOutageClaim": false,
|
|
126
|
+
"prClaim": "retained negative observations only"
|
|
127
|
+
}
|
|
128
|
+
}
|
package/ci/action-gate.test.js
CHANGED
|
@@ -353,6 +353,92 @@ try {
|
|
|
353
353
|
|
|
354
354
|
check(fs.readFileSync(rulesPath, 'utf8') === fs.readFileSync(pluginRules, 'utf8'), 'canonical and plugin action-gate rule tables are byte-identical');
|
|
355
355
|
check(fs.existsSync(pluginScript), 'OpenAI plugin packages action-gate runtime');
|
|
356
|
+
|
|
357
|
+
// Codex PR metadata scope: exact operation AND actual argument allowlist.
|
|
358
|
+
const metadataInput = { repository_full_name: 'acme/repo', pr_number: 376, body: 'metadata fixture' };
|
|
359
|
+
const metadataPayload = (input = metadataInput, tool = 'mcp__codex_apps__github__update_pull_request', extra = {}) => ({
|
|
360
|
+
...payload('', { tool }), tool_input: input, ...extra,
|
|
361
|
+
});
|
|
362
|
+
for (const tool of ['update_pull_request', 'mcp__github__update_pull_request', 'mcp__codex_apps__github__update_pull_request', 'mcp__codex_apps__github_update_pull_request']) {
|
|
363
|
+
for (const fields of [{ body: '' }, { title: 'new title' }, { title: 'new title', body: 'new body' }]) {
|
|
364
|
+
const candidate = metadataPayload({ repository_full_name: 'acme/repo', pr_number: 376, ...fields }, tool);
|
|
365
|
+
const result = actionGate.evaluatePreToolUse(candidate, { host: 'codex', sessionDir, ruleTable: table });
|
|
366
|
+
check(result.kind === 'allow' && result.stdout === null && result.exitCode === 0, `${tool} ${Object.keys(fields)} defers without a permission decision`);
|
|
367
|
+
const descriptor = actionGate.normalizeActionDescriptor(candidate);
|
|
368
|
+
check(descriptor.action === 'mutate' && descriptor.classification === 'gated', 'metadata remains a non-safe mutation');
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
for (const selector of [{ owner: 'acme', repo: 'repo', pullNumber: 376 }, { owner: 'acme', repo: 'repo', pull_number: 376 }]) {
|
|
372
|
+
check(actionGate.evaluatePreToolUse(metadataPayload({ ...selector, body: 'provider-neutral' }), { host: 'codex', sessionDir, ruleTable: table }).kind === 'allow', 'provider-neutral selectors support metadata updates');
|
|
373
|
+
}
|
|
374
|
+
const ownUnknown = JSON.parse('{"repository_full_name":"acme/repo","pr_number":376,"body":"fixture","__proto__":{"state":"closed"}}');
|
|
375
|
+
check(actionGate.evaluatePreToolUse(metadataPayload(ownUnknown), { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', 'own __proto__ field cannot escape the argument allowlist');
|
|
376
|
+
const explicitDelete = metadataPayload(metadataInput, undefined, { action_descriptor: { action: 'delete' } });
|
|
377
|
+
check(actionGate.evaluatePreToolUse(explicitDelete, { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', 'explicit higher-risk action cannot inherit metadata deferral');
|
|
378
|
+
for (const extra of [
|
|
379
|
+
{ state: 'closed' }, { state: 'open' }, { state: null }, { base_branch: 'other' },
|
|
380
|
+
{ maintainer_can_modify: false }, { maintainer_can_modify: null }, { unknown: true },
|
|
381
|
+
{ title: 'mixed', state: 'closed' }, { draft: true }, { base: 'other' },
|
|
382
|
+
{ action_descriptor: { action: 'delete' } },
|
|
383
|
+
]) {
|
|
384
|
+
const result = actionGate.evaluatePreToolUse(metadataPayload({ ...metadataInput, ...extra }), { host: 'codex', sessionDir, ruleTable: table });
|
|
385
|
+
check(result.kind === 'block' && result.exitCode === 2, `metadata with ${JSON.stringify(extra)} remains gated`);
|
|
386
|
+
}
|
|
387
|
+
for (const input of [null, [], 'body', {}, { repository_full_name: 'acme/repo', pr_number: 376 },
|
|
388
|
+
{ ...metadataInput, body: null }, { ...metadataInput, title: 2 }, { ...metadataInput, body: {} }]) {
|
|
389
|
+
check(actionGate.evaluatePreToolUse(metadataPayload(input), { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', 'missing or malformed metadata remains gated');
|
|
390
|
+
}
|
|
391
|
+
for (const tool of ['mcp__demo__update_pull_requests', 'mcp__demo__foo_update_pull_request', 'mcp__demo__update_pull_request_and_delete', 'mcp__codex_apps__github_update_pull_request_extra', 'mcp__demo__merge_pull_request', 'mcp__demo__close_pull_request']) {
|
|
392
|
+
check(actionGate.evaluatePreToolUse(metadataPayload(metadataInput, tool), { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', `${tool} receives no metadata exception`);
|
|
393
|
+
}
|
|
394
|
+
for (const host of ['claude', 'opencode', 'unknown']) {
|
|
395
|
+
const result = actionGate.evaluatePreToolUse(metadataPayload(), { host, sessionDir, ruleTable: table });
|
|
396
|
+
check(result.rule && result.rule.id === 'structured-mutate', `${host} retains existing mutation rule`);
|
|
397
|
+
}
|
|
398
|
+
for (const hint of [{ actionGate: { required: true } }, { workflowPlan: { actionGate: { required: true } } }, { harness: { workflowPlan: { actionGate: { required: true } } } }, { routerContract: { workflowPlan: { actionGate: { required: true } } } }]) {
|
|
399
|
+
check(actionGate.evaluatePreToolUse(metadataPayload(metadataInput, undefined, hint), { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', 'explicit required metadata gate cannot be bypassed');
|
|
400
|
+
}
|
|
401
|
+
for (const action of ['read', 'unknown']) {
|
|
402
|
+
for (const descriptor of [{ action_descriptor: { action } }, { actionDescriptor: { action } }, { harness: { actionDescriptor: { action } } }]) {
|
|
403
|
+
const candidate = metadataPayload({ ...metadataInput, state: 'closed' }, undefined, {
|
|
404
|
+
...descriptor, workflowPlan: { actionGate: { required: true } },
|
|
405
|
+
});
|
|
406
|
+
const result = actionGate.evaluatePreToolUse(candidate, { host: 'codex', sessionDir, ruleTable: table });
|
|
407
|
+
check(result.kind === 'block' && result.exitCode === 2, `known Codex PR mutation cannot be downgraded to ${action}`);
|
|
408
|
+
check(result.record && result.record.actionDescriptor.action === 'mutate', 'downgrade refusal remains an audited mutation');
|
|
409
|
+
}
|
|
410
|
+
const inputDescriptor = metadataPayload({ ...metadataInput, state: 'closed', action_descriptor: { action } }, undefined, { actionGate: { required: true } });
|
|
411
|
+
check(actionGate.evaluatePreToolUse(inputDescriptor, { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', `input ${action} descriptor cannot hide a required PR mutation gate`);
|
|
412
|
+
}
|
|
413
|
+
const misleadingProvider = metadataPayload({ ...metadataInput, state: 'closed' }, 'mcp__get_provider__update_pull_request');
|
|
414
|
+
check(actionGate.evaluatePreToolUse(misleadingProvider, { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', 'provider namespace read token cannot downgrade exact PR mutation');
|
|
415
|
+
for (const host of ['claude', 'opencode', 'unknown']) {
|
|
416
|
+
const describedRead = metadataPayload(metadataInput, undefined, { action_descriptor: { action: 'read' } });
|
|
417
|
+
check(actionGate.evaluatePreToolUse(describedRead, { host, sessionDir, ruleTable: table }).kind === 'allow', `${host} explicit descriptor behavior remains unchanged`);
|
|
418
|
+
}
|
|
419
|
+
for (const action of ['read', 'unknown', 'mutate', 'delete', 'publish', 'deploy', 'send']) {
|
|
420
|
+
const candidate = metadataPayload({ ...metadataInput, state: 'closed', action_descriptor: { toolFamily: 'shell', action } }, undefined, { actionGate: { required: true } });
|
|
421
|
+
const result = actionGate.evaluatePreToolUse(candidate, { host: 'codex', sessionDir, ruleTable: table });
|
|
422
|
+
check(result.kind === 'block' && result.exitCode === 2, `shell family cannot disguise exact Codex PR ${action}`);
|
|
423
|
+
check(result.record && result.record.actionDescriptor.toolFamily === 'mcp', 'known MCP identity is preserved in audit');
|
|
424
|
+
}
|
|
425
|
+
const gateEnvelopes = gate => [
|
|
426
|
+
{ actionGate: gate }, { workflowPlan: { actionGate: gate } },
|
|
427
|
+
{ routerContract: { workflowPlan: { actionGate: gate } } },
|
|
428
|
+
{ harness: { workflowPlan: { actionGate: gate } } },
|
|
429
|
+
];
|
|
430
|
+
for (let requiredAt = 0; requiredAt < 4; requiredAt++) {
|
|
431
|
+
const optionalHints = Object.assign({}, ...gateEnvelopes({ required: false }));
|
|
432
|
+
const hints = { ...optionalHints, ...gateEnvelopes({ required: true })[requiredAt] };
|
|
433
|
+
const candidate = metadataPayload(metadataInput, undefined, hints);
|
|
434
|
+
const result = actionGate.evaluatePreToolUse(candidate, { host: 'codex', sessionDir, ruleTable: table });
|
|
435
|
+
check(result.kind === 'block' && result.exitCode === 2, `required hint envelope ${requiredAt} wins over optional envelopes`);
|
|
436
|
+
check(result.record && result.record.routerActionGate.required === true, 'audit retains required hint instead of a conflicting optional hint');
|
|
437
|
+
const creation = { ...candidate, tool_name: 'mcp__github__create_pull_request' };
|
|
438
|
+
check(actionGate.evaluatePreToolUse(creation, { host: 'codex', sessionDir, ruleTable: table }).kind === 'block', 'PR creation still honors the documented explicit required gate');
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
|
|
356
442
|
} catch (err) {
|
|
357
443
|
console.error(err.stack || err.message);
|
|
358
444
|
failed++;
|
|
@@ -127,6 +127,53 @@ try {
|
|
|
127
127
|
assert.strictEqual(attributedBlock.exitCode, 2);
|
|
128
128
|
assert.ok(attributedBlock.attribution.file && fs.existsSync(attributedBlock.attribution.file), 'destructive hook entry must be attributable before the block result');
|
|
129
129
|
|
|
130
|
+
// Execute both canonical and packaged pre-entrypoints with real MCP argument shapes.
|
|
131
|
+
for (const gate of [actionGate, require(path.join(pluginRoot, 'hooks', 'scripts', 'action-gate.js'))]) {
|
|
132
|
+
for (const outcome of ['success', 'failure']) {
|
|
133
|
+
const candidate = {
|
|
134
|
+
...payload('', { toolUseId: `metadata-${outcome}`, turnId: 'metadata-turn' }),
|
|
135
|
+
tool_name: 'mcp__codex_apps__github__update_pull_request',
|
|
136
|
+
tool_input: { repository_full_name: 'acme/repo', pr_number: 376, body: 'private fixture body' },
|
|
137
|
+
};
|
|
138
|
+
const pre = gate.evaluatePreToolUse(candidate, { host: 'codex', sessionDir, ruleTable: table });
|
|
139
|
+
assert.strictEqual(pre.kind, 'allow');
|
|
140
|
+
assert.strictEqual(pre.stdout, null);
|
|
141
|
+
assert.strictEqual(pre.record.disposition, 'deferred-to-host');
|
|
142
|
+
assert.strictEqual(pre.record.host, 'codex');
|
|
143
|
+
assert.strictEqual(pre.record.toolUseId, candidate.tool_use_id);
|
|
144
|
+
assert.strictEqual(pre.record.matchedRule, 'routine-pr-metadata-update');
|
|
145
|
+
const hash = gate.hashExact(gate.exactPayloadText(candidate));
|
|
146
|
+
assert.strictEqual(pre.record.commandHash, hash);
|
|
147
|
+
assert.strictEqual(pre.record.actionDescriptor.payloadHash, hash);
|
|
148
|
+
assert.ok(!JSON.stringify(pre.record).includes('private fixture body'));
|
|
149
|
+
const post = gate.finalizeExecution(candidate, outcome, { sessionDir });
|
|
150
|
+
assert.strictEqual(post.record.disposition, 'executed');
|
|
151
|
+
assert.strictEqual(post.record.executionOutcome, outcome);
|
|
152
|
+
assert.strictEqual(post.record.commandHash, hash);
|
|
153
|
+
|
|
154
|
+
gate.evaluatePreToolUse(candidate, { host: 'codex', sessionDir, ruleTable: table });
|
|
155
|
+
const altered = { ...candidate, tool_input: { ...candidate.tool_input, state: 'closed' } };
|
|
156
|
+
assert.strictEqual(gate.finalizeExecution(altered, 'success', { sessionDir }).record.disposition, 'integrity-violation');
|
|
157
|
+
const denied = gate.evaluatePreToolUse(altered, { host: 'codex', sessionDir, ruleTable: table });
|
|
158
|
+
assert.strictEqual(denied.kind, 'block');
|
|
159
|
+
assert.strictEqual(denied.exitCode, 2);
|
|
160
|
+
assert.strictEqual(denied.stdout, null);
|
|
161
|
+
assert.strictEqual(denied.record.disposition, 'rejected');
|
|
162
|
+
assert.strictEqual(denied.record.rejectionReason, 'host-ask-decision-unverified-or-unsupported');
|
|
163
|
+
assert.notStrictEqual(denied.record.commandHash, hash);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const metadataEntry = preAdapter.processPayload({
|
|
167
|
+
...payload('', { toolUseId: 'metadata-entry' }),
|
|
168
|
+
tool_name: 'mcp__codex_apps__github_update_pull_request',
|
|
169
|
+
tool_input: { repository_full_name: 'acme/repo', pr_number: 376, title: 'fixture' },
|
|
170
|
+
}, { host: 'codex', sessionDir, ruleTable: table, attributionDir, runNonce: 'metadata-fixture' });
|
|
171
|
+
assert.strictEqual(metadataEntry.kind, 'allow');
|
|
172
|
+
assert.strictEqual(metadataEntry.attribution.record.payloadHash, metadataEntry.record.commandHash);
|
|
173
|
+
assert.strictEqual(metadataEntry.attribution.record.toolUseId, metadataEntry.record.toolUseId);
|
|
174
|
+
assert.strictEqual(fs.readFileSync(path.join(ROOT, 'hooks/scripts/action-gate.js'), 'utf8'),
|
|
175
|
+
fs.readFileSync(path.join(pluginRoot, 'hooks/scripts/action-gate.js'), 'utf8'));
|
|
176
|
+
|
|
130
177
|
const blockedParent = path.join(tempRoot, 'not-a-directory');
|
|
131
178
|
fs.writeFileSync(blockedParent, 'fixture', 'utf8');
|
|
132
179
|
const failedAttribution = preAdapter.processPayload(payload('git push --force origin main', {
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// #325 H1 deterministic qualification: provider independence/preparation.
|
|
5
|
+
// H1 only: advisory suggestions are never passed to the F4 consumer.
|
|
6
|
+
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
|
|
10
|
+
const fx = require(path.join(__dirname, 'lib', 'doc-reconciliation-prep-fixtures'));
|
|
11
|
+
const { stage } = require(path.join(__dirname, 'lib', 'doc-reconciliation-staged-tree'));
|
|
12
|
+
const F4_CASES = path.join(__dirname, 'fixtures', 'doc-obligations', 'reconciliation-s2b', 'f4', 'cases.json');
|
|
13
|
+
|
|
14
|
+
const REPO_ROOT = path.resolve(__dirname, '..');
|
|
15
|
+
const SOURCE_OBLIGATIONS = require(path.join(REPO_ROOT, 'harness-everything', 'scripts', 'doc-obligations.js'));
|
|
16
|
+
const PACKAGED_OBLIGATIONS = require(path.join(REPO_ROOT, 'plugins', 'harness-everything', 'skills', 'harness-everything', 'scripts', 'doc-obligations.js'));
|
|
17
|
+
const SOURCE_PROVIDER = require(path.join(REPO_ROOT, 'harness-everything', 'scripts', 'doc-providers.js'));
|
|
18
|
+
const PACKAGED_PROVIDER = require(path.join(REPO_ROOT, 'plugins', 'harness-everything', 'skills', 'harness-everything', 'scripts', 'doc-providers.js'));
|
|
19
|
+
|
|
20
|
+
const LEGS = ['source', 'packaged'];
|
|
21
|
+
const CASE = id => fx.readCaseCorpus(F4_CASES).cases.find(item => item.id === id);
|
|
22
|
+
|
|
23
|
+
const EXPECTED_MUSTS = [
|
|
24
|
+
{ id: 'doc-change:doc-lint', capability: 'doc-lint', strength: 'MUST', applicability: 'applicable', scope: { paths: ['docs/a.md'] }, acceptance: 'Run document lifecycle lint for every changed governed document.' },
|
|
25
|
+
{ id: 'doc-change:link-check', capability: 'link-check', strength: 'MUST', applicability: 'applicable', scope: { paths: ['docs/a.md'] }, acceptance: 'Check in-repository links for every changed governed document.' }
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
const PROVIDER_A_INPUT = {
|
|
29
|
+
activeStep: { id: 'verify-docs', stepType: 'verification' },
|
|
30
|
+
neededCapabilities: ['doc-lint'],
|
|
31
|
+
availableProviderMetadata: [
|
|
32
|
+
{ providerId: 'fixture-lint-a', capabilities: ['doc-lint'], stepTypes: ['verification'], requiresHostCapabilities: ['execution-events'], locator: { kind: 'tool', id: 'fixture-lint-a' } }
|
|
33
|
+
],
|
|
34
|
+
hostCapabilities: ['execution-events']
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const PROVIDER_B_INPUT = {
|
|
38
|
+
activeStep: { id: 'verify-docs', stepType: 'verification' },
|
|
39
|
+
neededCapabilities: ['doc-lint'],
|
|
40
|
+
availableProviderMetadata: [
|
|
41
|
+
{ providerId: 'fixture-lint-b', capabilities: ['doc-lint'], stepTypes: ['verification'], requiresHostCapabilities: ['execution-events'], locator: { kind: 'tool', id: 'fixture-lint-b' } }
|
|
42
|
+
],
|
|
43
|
+
hostCapabilities: ['execution-events']
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const PROVIDER_OMITTED_INPUT = {
|
|
47
|
+
activeStep: { id: 'verify-docs', stepType: 'verification' },
|
|
48
|
+
neededCapabilities: ['doc-lint'],
|
|
49
|
+
availableProviderMetadata: [],
|
|
50
|
+
hostCapabilities: []
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const PROVIDER_A_SUGGESTION = [{ capability: 'doc-lint', providerId: 'fixture-lint-a', availability: 'available', reason: 'active-step capability and host requirements match' }];
|
|
54
|
+
const PROVIDER_B_SUGGESTION = [{ capability: 'doc-lint', providerId: 'fixture-lint-b', availability: 'available', reason: 'active-step capability and host requirements match' }];
|
|
55
|
+
const PROVIDER_OMITTED_UNAVAILABLE = [{ capability: 'doc-lint', reason: 'no available provider metadata for the active-step capability' }];
|
|
56
|
+
|
|
57
|
+
let passCount = 0;
|
|
58
|
+
let failCount = 0;
|
|
59
|
+
let infraCount = 0;
|
|
60
|
+
|
|
61
|
+
function deepEqual(left, right) {
|
|
62
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function record(status, id, detail) {
|
|
66
|
+
const suffix = detail ? ` - ${detail}` : '';
|
|
67
|
+
if (status === 'PASS') passCount += 1;
|
|
68
|
+
else if (status === 'FAIL') failCount += 1;
|
|
69
|
+
else infraCount += 1;
|
|
70
|
+
console.log(` ${status.padEnd(5)} ${id}${suffix}`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function cloneZero(caseDef) {
|
|
74
|
+
return {
|
|
75
|
+
...caseDef,
|
|
76
|
+
id: `${caseDef.id}-zero-evidence`,
|
|
77
|
+
actions: caseDef.actions.filter(action => action.id !== 'lint-run' && action.id !== 'post-lint')
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function checkProvider(leg, providerCase) {
|
|
82
|
+
const module = leg === 'source' ? SOURCE_PROVIDER : PACKAGED_PROVIDER;
|
|
83
|
+
let input;
|
|
84
|
+
let expectedSuggestions;
|
|
85
|
+
let expectedUnavailable;
|
|
86
|
+
if (providerCase === 'A') {
|
|
87
|
+
input = PROVIDER_A_INPUT;
|
|
88
|
+
expectedSuggestions = PROVIDER_A_SUGGESTION;
|
|
89
|
+
expectedUnavailable = [];
|
|
90
|
+
} else if (providerCase === 'B') {
|
|
91
|
+
input = PROVIDER_B_INPUT;
|
|
92
|
+
expectedSuggestions = PROVIDER_B_SUGGESTION;
|
|
93
|
+
expectedUnavailable = [];
|
|
94
|
+
} else {
|
|
95
|
+
input = PROVIDER_OMITTED_INPUT;
|
|
96
|
+
expectedSuggestions = [];
|
|
97
|
+
expectedUnavailable = PROVIDER_OMITTED_UNAVAILABLE;
|
|
98
|
+
}
|
|
99
|
+
const result = module.suggestDocProviders(input);
|
|
100
|
+
const ok = deepEqual(result.suggestions, expectedSuggestions) && deepEqual(result.unavailable, expectedUnavailable);
|
|
101
|
+
record(ok ? 'PASS' : 'FAIL', `H1.${leg}.${providerCase}.provider-suggestion`, ok ? '' : `suggestions=${JSON.stringify(result.suggestions)}, unavailable=${JSON.stringify(result.unavailable)}`);
|
|
102
|
+
return ok;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function checkDerivedMusts(leg, label) {
|
|
106
|
+
const module = leg === 'source' ? SOURCE_OBLIGATIONS : PACKAGED_OBLIGATIONS;
|
|
107
|
+
const caseDef = CASE('F4-POS-body-edit');
|
|
108
|
+
const config = JSON.parse(caseDef.input.baseFiles['.agents/harness-everything/project-docs.json']);
|
|
109
|
+
const base = { files: caseDef.input.baseFiles };
|
|
110
|
+
const candidate = { files: { ...caseDef.input.baseFiles, ...caseDef.input.candidateFiles } };
|
|
111
|
+
const { facts } = module.deriveDocFacts({ base, candidate, config, inventory: { complete: true } });
|
|
112
|
+
const obligations = module.deriveDocObligations(facts);
|
|
113
|
+
const fullMusts = obligations.filter(item => item.strength === 'MUST' && item.applicability === 'applicable');
|
|
114
|
+
|
|
115
|
+
let ok = deepEqual(fullMusts, EXPECTED_MUSTS);
|
|
116
|
+
for (const item of fullMusts) {
|
|
117
|
+
if (!item.id || !item.capability || !item.strength || !item.applicability || !item.scope || !Array.isArray(item.scope.paths) || item.scope.paths.some(pathValue => !pathValue) || !item.acceptance) ok = false;
|
|
118
|
+
}
|
|
119
|
+
record(ok ? 'PASS' : 'FAIL', `${label}.derived-full-musts`, ok ? '' : JSON.stringify(fullMusts));
|
|
120
|
+
return ok;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function runCell(leg, providerCase, evidenceMode) {
|
|
124
|
+
const label = `H1.${leg}.${providerCase}.${evidenceMode}`;
|
|
125
|
+
const staged = stage(leg);
|
|
126
|
+
try {
|
|
127
|
+
checkProvider(leg, providerCase);
|
|
128
|
+
checkDerivedMusts(leg, label);
|
|
129
|
+
|
|
130
|
+
staged.writeExport(staged.validExport());
|
|
131
|
+
|
|
132
|
+
const baseCase = CASE('F4-POS-body-edit');
|
|
133
|
+
const caseDef = evidenceMode === 'valid' ? baseCase : cloneZero(baseCase);
|
|
134
|
+
|
|
135
|
+
const root = fx.replayCase(caseDef, staged.dist, { pilot: '1' });
|
|
136
|
+
const faults = fx.infrastructureFaults(caseDef, root, 'on');
|
|
137
|
+
if (faults.length) {
|
|
138
|
+
record('INFRA', label, faults.map(fault => `${fault.actionId}: ${fault.message}`).join('; '));
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const pilot = root.pilot || {};
|
|
143
|
+
const checks = [];
|
|
144
|
+
|
|
145
|
+
checks.push(deepEqual(root.pilotReportOrigin, 'runtime'));
|
|
146
|
+
checks.push(deepEqual(pilot.reportOrigin, 'runtime'));
|
|
147
|
+
checks.push(deepEqual(pilot.expectedMustIds, ['doc-change:doc-lint', 'doc-change:link-check']));
|
|
148
|
+
const verifyAction = root.actions && root.actions['verify-pass'];
|
|
149
|
+
const verifyRequirement = verifyAction?.state?.obligations?.requirements?.find(item => item.id === 'req-verify');
|
|
150
|
+
checks.push(verifyRequirement?.requiredBindings?.some(binding => binding.id === 'verification-loop' && binding.status === 'loaded') === true);
|
|
151
|
+
|
|
152
|
+
if (evidenceMode === 'valid') {
|
|
153
|
+
checks.push(pilot.pilotCredit === true);
|
|
154
|
+
checks.push(pilot.pilotStatus === 'credited');
|
|
155
|
+
checks.push(deepEqual(pilot.unsatisfied, []));
|
|
156
|
+
checks.push(deepEqual(pilot.creditedCapabilities, ['doc-lint', 'link-check']));
|
|
157
|
+
checks.push(Array.isArray(pilot.acceptedRefs) && pilot.acceptedRefs.some(ref => ref.endsWith('#doc-lint')) && pilot.acceptedRefs.some(ref => ref.endsWith('#link-check')));
|
|
158
|
+
checks.push(verifyAction && verifyAction.exitCode === 0);
|
|
159
|
+
const stopAction = root.actions && root.actions.stop;
|
|
160
|
+
checks.push(stopAction && stopAction.exitCode === 0);
|
|
161
|
+
checks.push(stopAction && stopAction.state && stopAction.state.workflow && stopAction.state.workflow.state === 'satisfied');
|
|
162
|
+
} else {
|
|
163
|
+
checks.push(pilot.pilotCredit === false);
|
|
164
|
+
checks.push(pilot.pilotStatus === 'unresolved');
|
|
165
|
+
checks.push(Array.isArray(pilot.unsatisfied) && pilot.unsatisfied.length >= 2);
|
|
166
|
+
checks.push(pilot.unsatisfied.some(item => item.obligationId === 'doc-change:doc-lint'));
|
|
167
|
+
checks.push(pilot.unsatisfied.some(item => item.obligationId === 'doc-change:link-check'));
|
|
168
|
+
checks.push(deepEqual(pilot.creditedCapabilities, []));
|
|
169
|
+
checks.push(deepEqual(pilot.acceptedRefs, []));
|
|
170
|
+
checks.push(verifyAction && verifyAction.exitCode === 2);
|
|
171
|
+
const stopAction = root.actions && root.actions.stop;
|
|
172
|
+
checks.push(stopAction && stopAction.exitCode === 0);
|
|
173
|
+
checks.push(stopAction && stopAction.state && stopAction.state.workflow && stopAction.state.workflow.state === 'running');
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
record(checks.every(Boolean) ? 'PASS' : 'FAIL', label, checks.every(Boolean) ? '' : `checks=${JSON.stringify(checks)}, pilot=${JSON.stringify(pilot).slice(0, 500)}`);
|
|
177
|
+
} catch (error) {
|
|
178
|
+
record('FAIL', label, error.message);
|
|
179
|
+
} finally {
|
|
180
|
+
staged.cleanup();
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function main() {
|
|
185
|
+
console.log('\n== H1 qualification: provider side input and F4 replay matrix ==');
|
|
186
|
+
for (const leg of LEGS) {
|
|
187
|
+
for (const providerCase of ['A', 'B', 'omitted']) {
|
|
188
|
+
for (const evidenceMode of ['valid', 'zero']) {
|
|
189
|
+
runCell(leg, providerCase, evidenceMode);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
console.log('\n== H1 qualification evidence ==');
|
|
195
|
+
console.log(`passing checks: ${passCount}`);
|
|
196
|
+
console.log(`unexpected failures: ${failCount}`);
|
|
197
|
+
console.log(`infrastructure faults: ${infraCount}`);
|
|
198
|
+
if (failCount === 0 && infraCount === 0) {
|
|
199
|
+
console.log('\nSTATUS: GREEN (all assertions passed)');
|
|
200
|
+
return 0;
|
|
201
|
+
}
|
|
202
|
+
console.log('\nSTATUS: NOT-GREEN');
|
|
203
|
+
return 1;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
process.exitCode = main();
|