harness-everything 0.47.2 → 0.47.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.github/workflows/ci.yml +2 -0
- package/CHANGELOG.md +16 -0
- package/behavioral-evals/README.md +10 -0
- package/behavioral-evals/case-validator.js +7 -0
- package/behavioral-evals/cases/baseline-profile-lookup-json-only.yaml +23 -0
- package/behavioral-evals/run.js +85 -6
- package/benchmarks/issue290-pilot/metrics.test.py +60 -2
- package/ci/lib/opencode-verification-loader.js +24 -0
- package/ci/mechanism-2p-opencode-v2-entrypoint.test.js +8 -1
- package/ci/mechanism-325-f3-derived-views.test.js +42 -2
- package/ci/mechanism-338-opencode-async.test.js +229 -0
- package/ci/mechanism-341-opencode-lane.test.js +14 -3
- package/ci/mechanism-36-verify-timeout.test.js +7 -168
- package/ci/test-behavioral-case-validation.js +148 -3
- package/docs/platform-capabilities.md +8 -0
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/doc-check.js +5 -4
- package/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
- package/opencode-plugin/README.md +28 -30
- package/opencode-plugin/index.mjs +249 -112
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/doc-check.js +5 -4
- package/sbom.cdx.json +7 -7
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "harness-everything",
|
|
11
11
|
"source": "./",
|
|
12
12
|
"description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
|
|
13
|
-
"version": "0.47.
|
|
13
|
+
"version": "0.47.4",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Miya Daniel"
|
|
16
16
|
},
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "harness-everything",
|
|
3
3
|
"description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
|
|
4
|
-
"version": "0.47.
|
|
4
|
+
"version": "0.47.4",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Miya Daniel"
|
|
7
7
|
},
|
package/.github/workflows/ci.yml
CHANGED
|
@@ -118,6 +118,8 @@ jobs:
|
|
|
118
118
|
node-version: 24
|
|
119
119
|
- name: Install dependencies for regression suite
|
|
120
120
|
run: npm ci
|
|
121
|
+
- name: Verify OpenCode async process fixtures before the full suite
|
|
122
|
+
run: node ci/mechanism-338-opencode-async.test.js
|
|
121
123
|
- name: Run hermetic self-regression suite
|
|
122
124
|
run: npm test
|
|
123
125
|
- name: Run focused mutation checks
|
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.47.4](https://github.com/dyphn1/Harness-everything/compare/v0.47.3...v0.47.4) (2026-11-03)
|
|
6
|
+
|
|
7
|
+
### Bug Fixes
|
|
8
|
+
|
|
9
|
+
* **behavioral-evals:** grade the whole final assistant message for json-only ([ebbe33c](https://github.com/dyphn1/Harness-everything/commit/ebbe33c7ac8beb3d2b104f81666f06a0ebfcd77d)), closes [#308](https://github.com/dyphn1/Harness-everything/issues/308)
|
|
10
|
+
* **behavioral-evals:** recombine final text parts exactly for json-only grading ([de8341c](https://github.com/dyphn1/Harness-everything/commit/de8341c0621c6aacf319467a9e3c9b1420f51f37)), closes [#308](https://github.com/dyphn1/Harness-everything/issues/308)
|
|
11
|
+
|
|
12
|
+
## [0.47.3](https://github.com/dyphn1/Harness-everything/compare/v0.47.2...v0.47.3) (2026-10-31)
|
|
13
|
+
|
|
14
|
+
### Bug Fixes
|
|
15
|
+
|
|
16
|
+
* **docs:** preserve exact file identities in derived comparisons ([8bc3ddb](https://github.com/dyphn1/Harness-everything/commit/8bc3ddbac38eaffa7495e70922dc1240f9aa63eb))
|
|
17
|
+
* **docs:** repin shipped verifier after exact file identity fix ([c24d0f1](https://github.com/dyphn1/Harness-everything/commit/c24d0f100ce392edd02275aa06b58f141ade6cdd))
|
|
18
|
+
* **opencode:** invalidate idle waiters on session reset ([41b2663](https://github.com/dyphn1/Harness-everything/commit/41b2663f66edbe77188c3ca9fa3b0add6f7ab76d))
|
|
19
|
+
* **opencode:** stream verification asynchronously within a shared budget ([1470247](https://github.com/dyphn1/Harness-everything/commit/1470247249548ec6ccf2acaf97463824e0268be4)), closes [#338](https://github.com/dyphn1/Harness-everything/issues/338)
|
|
20
|
+
|
|
5
21
|
## [0.47.2](https://github.com/dyphn1/Harness-everything/compare/v0.47.1...v0.47.2) (2026-10-31)
|
|
6
22
|
|
|
7
23
|
### Bug Fixes
|
|
@@ -171,6 +171,16 @@ additional arguments) or `tool_denied` when denial is the behavior under test.
|
|
|
171
171
|
Legacy one-object Claude JSON and incomplete streams are graded inconclusive
|
|
172
172
|
for execution evidence.
|
|
173
173
|
|
|
174
|
+
Use `final_output_json_only` when the final message must be only a JSON object
|
|
175
|
+
(raw, or inside one fenced block) with no prose outside it. Hidden
|
|
176
|
+
`<!-- harness-label -->` lines are ignored. Optional `required_keys` lists keys
|
|
177
|
+
that must be present. Extracting the selection fields from prose does not pass.
|
|
178
|
+
The final message is every assistant text part after the last tool call, joined
|
|
179
|
+
by exact concatenation (so JSON streamed in chunks reassembles exactly), with
|
|
180
|
+
newline-separated blocks as an alternative; the case passes if either reading is
|
|
181
|
+
a single JSON output. Claude's repeated `result` is used only when no parts exist.
|
|
182
|
+
Prose streamed in a separate part still fails under both readings.
|
|
183
|
+
|
|
174
184
|
Grading is best-effort mechanical (trace keyword/state assertions), not a
|
|
175
185
|
substitute for reading the transcript — every result JSON records the full
|
|
176
186
|
trace path so humans can audit what the grader concluded.
|
|
@@ -25,6 +25,7 @@ const EXPECTATION_TYPES = new Set([
|
|
|
25
25
|
'tool_completed',
|
|
26
26
|
'tool_denied',
|
|
27
27
|
'execution_evidence',
|
|
28
|
+
'final_output_json_only',
|
|
28
29
|
]);
|
|
29
30
|
|
|
30
31
|
const PRESSURE_CATEGORIES = new Set([
|
|
@@ -118,6 +119,12 @@ function validateExpectation(expectation, index) {
|
|
|
118
119
|
errors.push(`expectations[${index}] ${type} requires a string value`);
|
|
119
120
|
}
|
|
120
121
|
if (type === 'command_exit_0' && (typeof expectation.command !== 'string' || !expectation.command.trim())) errors.push(`expectations[${index}] command_exit_0 requires a non-empty command`);
|
|
122
|
+
if (type === 'final_output_json_only' && expectation.required_keys !== undefined) {
|
|
123
|
+
const keys = expectation.required_keys;
|
|
124
|
+
if (!Array.isArray(keys) || keys.some(key => typeof key !== 'string' || !key.trim())) {
|
|
125
|
+
errors.push(`expectations[${index}] required_keys must be an array of non-empty strings`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
121
128
|
if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence'].includes(type)) {
|
|
122
129
|
if (expectation.value === undefined && expectation.tool === undefined && expectation.name === undefined && expectation.command === undefined) {
|
|
123
130
|
errors.push(`expectations[${index}] ${type} requires a tool/value target`);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Baseline case: Profile lookup returns only the JSON selection record
|
|
2
|
+
id: baseline-profile-lookup-json-only
|
|
3
|
+
name: Profile lookup answers with a JSON-only selection record
|
|
4
|
+
discipline: fable-mode
|
|
5
|
+
loaded_skills:
|
|
6
|
+
- fable-mode
|
|
7
|
+
fixture:
|
|
8
|
+
files:
|
|
9
|
+
- path: README.md
|
|
10
|
+
content: |
|
|
11
|
+
Profile lookup fixture. No orchestration or delegation is needed.
|
|
12
|
+
prompt: "Explain what 'fable on sonnect' means; return only a small JSON selection record with requestedProfile, effectiveProfile, profileAlias, assignedRole, runtimeModel, runtimeEffort. No orchestration or delegation."
|
|
13
|
+
max_turns: 4
|
|
14
|
+
expectations:
|
|
15
|
+
- type: final_output_json_only
|
|
16
|
+
required_keys:
|
|
17
|
+
- requestedProfile
|
|
18
|
+
- effectiveProfile
|
|
19
|
+
- profileAlias
|
|
20
|
+
- assignedRole
|
|
21
|
+
- runtimeModel
|
|
22
|
+
- runtimeEffort
|
|
23
|
+
description: final message is exactly one JSON selection record (raw or fenced) with no prose outside it, and carries all six selection fields; selection-field extraction alone is not completion
|
package/behavioral-evals/run.js
CHANGED
|
@@ -28,7 +28,7 @@ const {
|
|
|
28
28
|
parseTranscriptFile,
|
|
29
29
|
} = require('./transcript-parser');
|
|
30
30
|
const { commandMatches, normalizeRepoPath } = require('../scripts/lib/execution-contract');
|
|
31
|
-
const { resolveFixtureSetup, validateFixture } = require('./case-validator');
|
|
31
|
+
const { resolveFixtureSetup, validateFixture, validateExpectation } = require('./case-validator');
|
|
32
32
|
|
|
33
33
|
const ROOT = path.resolve(__dirname, '..');
|
|
34
34
|
const CASES_DIR = path.join(__dirname, 'cases');
|
|
@@ -150,6 +150,7 @@ function validate(cases) {
|
|
|
150
150
|
const EXPECT_TYPES = new Set([
|
|
151
151
|
'trace_contains', 'trace_not_contains', 'file_contains', 'file_not_exists', 'command_exit_0',
|
|
152
152
|
'tool_call', 'tool_attempted', 'tool_completed', 'tool_executed', 'tool_denied', 'execution_evidence',
|
|
153
|
+
'final_output_json_only',
|
|
153
154
|
]);
|
|
154
155
|
for (const c of cases) {
|
|
155
156
|
const problems = [];
|
|
@@ -165,7 +166,9 @@ function validate(cases) {
|
|
|
165
166
|
if (!resolveTreatmentSkills(c).length) problems.push('treatment must load at least one named Harness skill');
|
|
166
167
|
for (const e of c.expectations || []) {
|
|
167
168
|
if (!EXPECT_TYPES.has(e.type)) problems.push(`unknown expectation type "${e.type}"`);
|
|
168
|
-
if (e.
|
|
169
|
+
if (e.type === 'final_output_json_only') {
|
|
170
|
+
problems.push(...validateExpectation(e, c.expectations.indexOf(e)));
|
|
171
|
+
} else if (e.value === undefined && e.command === undefined && e.tool === undefined && e.name === undefined && e.count === undefined) {
|
|
169
172
|
problems.push(`expectation ${e.type} needs value, tool, name, count, or command`);
|
|
170
173
|
}
|
|
171
174
|
}
|
|
@@ -492,8 +495,10 @@ function parseTranscriptEvents(transcriptPath) {
|
|
|
492
495
|
const events = [];
|
|
493
496
|
const pendingById = new Map();
|
|
494
497
|
let sequence = 0;
|
|
495
|
-
|
|
496
|
-
|
|
498
|
+
// `source` separates assistant content parts from Claude's terminal `result`
|
|
499
|
+
// field, which repeats the final message and must not be joined with it.
|
|
500
|
+
const addText = (text, source = 'content') => {
|
|
501
|
+
if (typeof text === 'string' && text) events.push({ kind: 'text', text, source, sequence: sequence++ });
|
|
497
502
|
};
|
|
498
503
|
const addTool = (event, part) => {
|
|
499
504
|
const type = eventType(event);
|
|
@@ -533,7 +538,7 @@ function parseTranscriptEvents(transcriptPath) {
|
|
|
533
538
|
}
|
|
534
539
|
if (value.message && value.message.content) visit(value.message.content, value.type !== 'user');
|
|
535
540
|
if (value.content && value.content !== value.message) visit(value.content, includeText);
|
|
536
|
-
if (!value.part && typeof value.result === 'string' && includeText) addText(value.result);
|
|
541
|
+
if (!value.part && typeof value.result === 'string' && includeText) addText(value.result, 'result');
|
|
537
542
|
};
|
|
538
543
|
let structured = false;
|
|
539
544
|
for (const line of raw.split('\n')) {
|
|
@@ -548,7 +553,7 @@ function parseTranscriptEvents(transcriptPath) {
|
|
|
548
553
|
if (!structured) {
|
|
549
554
|
try {
|
|
550
555
|
const value = JSON.parse(raw);
|
|
551
|
-
if (value && typeof value.result === 'string') addText(value.result);
|
|
556
|
+
if (value && typeof value.result === 'string') addText(value.result, 'result');
|
|
552
557
|
else visit(value);
|
|
553
558
|
} catch { /* raw fallback handled by renderAgentTrace */ }
|
|
554
559
|
}
|
|
@@ -644,6 +649,60 @@ function isExecutionLikeTraceValue(value) {
|
|
|
644
649
|
|| /(?:^|[/\\])[\w.-]+\.(?:c?js|mjs|py|sh|ps1|bat|cmd)\b/i.test(text);
|
|
645
650
|
}
|
|
646
651
|
|
|
652
|
+
// Harness turn-label lines are hidden bookkeeping (AGENTS.md rule 14) and are
|
|
653
|
+
// ignored when judging whether the final message is JSON-only.
|
|
654
|
+
const HARNESS_LABEL_LINE_RE = /^[ \t]*<!--\s*harness-label\b.*?-->[ \t\r]*$/gm;
|
|
655
|
+
|
|
656
|
+
// Pure check for `final_output_json_only`: the final assistant text must be
|
|
657
|
+
// exactly one JSON object, raw or inside a single fenced block, with no prose.
|
|
658
|
+
function checkJsonOnlyOutput(text, requiredKeys = []) {
|
|
659
|
+
const fail = reason => ({ pass: false, reason });
|
|
660
|
+
if (typeof text !== 'string') return fail('final assistant text is missing');
|
|
661
|
+
const body = text.replace(HARNESS_LABEL_LINE_RE, '').trim();
|
|
662
|
+
if (!body) return fail('final assistant text is empty');
|
|
663
|
+
let json;
|
|
664
|
+
if (body.startsWith('```')) {
|
|
665
|
+
const fence = /^```(?:json)?[ \t]*\r?\n([\s\S]*)\r?\n```$/i.exec(body);
|
|
666
|
+
if (!fence) return fail('final output has text outside the single JSON code fence');
|
|
667
|
+
if (fence[1].includes('```')) return fail('final output contains more than one code fence');
|
|
668
|
+
json = fence[1];
|
|
669
|
+
} else if (body.startsWith('{') || body.startsWith('[')) {
|
|
670
|
+
json = body;
|
|
671
|
+
} else {
|
|
672
|
+
try { JSON.parse(body); }
|
|
673
|
+
catch { return fail('final output has prose before the JSON object'); }
|
|
674
|
+
return fail('final JSON value must be an object');
|
|
675
|
+
}
|
|
676
|
+
let value;
|
|
677
|
+
try { value = JSON.parse(json); }
|
|
678
|
+
catch (error) { return fail(`final output is not a single valid JSON value (${error.message}); prose after the JSON also fails`); }
|
|
679
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) return fail('final JSON value must be an object');
|
|
680
|
+
const missing = requiredKeys.filter(key => !Object.prototype.hasOwnProperty.call(value, key));
|
|
681
|
+
if (missing.length) return fail(`missing required JSON keys: ${missing.join(', ')}`);
|
|
682
|
+
return { pass: true, reason: null };
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
// The final assistant message is every assistant text part after the last tool
|
|
686
|
+
// event. A final turn may stream prose and JSON as separate parts, so all parts
|
|
687
|
+
// count (#308). Parts can be transport chunks of one message (exact
|
|
688
|
+
// concatenation) or separate complete blocks (newline-separated). Candidates
|
|
689
|
+
// are returned exact-concatenation first. Claude's terminal `result` repeats
|
|
690
|
+
// that message, so it is used only when the transcript has no content parts.
|
|
691
|
+
function finalAssistantTextCandidates(events) {
|
|
692
|
+
let lastToolIndex = -1;
|
|
693
|
+
events.forEach((event, index) => { if (event.kind === 'tool') lastToolIndex = index; });
|
|
694
|
+
const texts = events.slice(lastToolIndex + 1).filter(event => event.kind === 'text');
|
|
695
|
+
const parts = texts.filter(event => event.source !== 'result');
|
|
696
|
+
if (!parts.length) return [texts.length ? texts[texts.length - 1].text : ''];
|
|
697
|
+
const candidates = [parts.map(event => event.text).join(''), parts.map(event => event.text).join('\n')];
|
|
698
|
+
return [...new Set(candidates)];
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// Exact text of the final message (first candidate).
|
|
702
|
+
function finalAssistantText(events) {
|
|
703
|
+
return finalAssistantTextCandidates(events)[0];
|
|
704
|
+
}
|
|
705
|
+
|
|
647
706
|
function grade(c, ws, transcriptPath, engine = 'auto') {
|
|
648
707
|
const parsed = parseTranscriptFile(transcriptPath, engine);
|
|
649
708
|
const ordered = parseTranscriptEvents(transcriptPath);
|
|
@@ -675,6 +734,23 @@ function grade(c, ws, transcriptPath, engine = 'auto') {
|
|
|
675
734
|
pass = e.type === 'trace_contains' ? trace.includes(e.value) : !trace.includes(e.value);
|
|
676
735
|
status = pass ? 'pass' : 'fail';
|
|
677
736
|
}
|
|
737
|
+
} else if (e.type === 'final_output_json_only') {
|
|
738
|
+
if (parsed.parseStatus !== 'parsed') {
|
|
739
|
+
status = 'inconclusive';
|
|
740
|
+
reason = parsed.parseError || 'transcript parser did not produce a complete structured trace';
|
|
741
|
+
} else {
|
|
742
|
+
// Neither join can hide prose: prose stays outside the JSON either way.
|
|
743
|
+
// Exact concatenation covers transport chunks of one JSON message;
|
|
744
|
+
// newline join covers separate complete blocks such as a fenced JSON
|
|
745
|
+
// block followed by a separate `<!-- harness-label ... -->` part.
|
|
746
|
+
// PASS if any candidate passes; otherwise report the exact-text reason.
|
|
747
|
+
const verdicts = finalAssistantTextCandidates(ordered.events)
|
|
748
|
+
.map(candidate => checkJsonOnlyOutput(candidate, e.required_keys || []));
|
|
749
|
+
const verdict = verdicts.find(v => v.pass) || verdicts[0];
|
|
750
|
+
pass = verdict.pass;
|
|
751
|
+
status = pass ? 'pass' : 'fail';
|
|
752
|
+
reason = verdict.reason;
|
|
753
|
+
}
|
|
678
754
|
} else if (e.type === 'file_contains') {
|
|
679
755
|
pass = fs.readFileSync(path.join(ws, e.path), 'utf8').includes(e.value);
|
|
680
756
|
status = pass ? 'pass' : 'fail';
|
|
@@ -998,6 +1074,9 @@ module.exports = {
|
|
|
998
1074
|
extractAgentEvents,
|
|
999
1075
|
extractAgentTrace,
|
|
1000
1076
|
extractSessionMetadata,
|
|
1077
|
+
checkJsonOnlyOutput,
|
|
1078
|
+
finalAssistantText,
|
|
1079
|
+
finalAssistantTextCandidates,
|
|
1001
1080
|
grade,
|
|
1002
1081
|
parseTranscriptEvents,
|
|
1003
1082
|
pairVerdict,
|
|
@@ -55,9 +55,9 @@ class ReadMetricTest(unittest.TestCase):
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
class AccountingTest(unittest.TestCase):
|
|
58
|
-
def episode(self, root, engine='claude', workload='profile-lookup', events=None):
|
|
58
|
+
def episode(self, root, engine='claude', workload='profile-lookup', events=None, usage=None):
|
|
59
59
|
row = {'model': {'engine': engine, 'model': 'fixture'}, 'workload': workload,
|
|
60
|
-
'grader': {'pass': False}, 'usage':
|
|
60
|
+
'grader': {'pass': False}, 'usage': usage, 'router_attributed': True}
|
|
61
61
|
(root/'analysis.json').write_text(json.dumps(row))
|
|
62
62
|
(root/'manifest.json').write_text('{}')
|
|
63
63
|
(root/'transcript.jsonl').write_text('\n'.join(json.dumps(e) for e in (events or [])))
|
|
@@ -81,6 +81,64 @@ class AccountingTest(unittest.TestCase):
|
|
|
81
81
|
self.assertIsNone(row['api_equivalent_cost_usd'])
|
|
82
82
|
self.assertIn('unknown', row['child_usage_observability'])
|
|
83
83
|
|
|
84
|
+
def codex_episode(self, root, spawn):
|
|
85
|
+
usage = {'input_tokens': 3000, 'cached_input_tokens': 700, 'output_tokens': 130,
|
|
86
|
+
'cache_write_input_tokens': 0}
|
|
87
|
+
self.episode(root, engine='codex', usage=usage,
|
|
88
|
+
events=[{'type': 'thread.started', 'thread_id': 'T1'}])
|
|
89
|
+
rollout = root/'.codex'/'sessions'/'2026'/'10'/'14'/'rollout-T1.jsonl'
|
|
90
|
+
rollout.parent.mkdir(parents=True)
|
|
91
|
+
lines = []
|
|
92
|
+
if spawn:
|
|
93
|
+
lines.append({'type': 'response_item', 'payload': {'type': 'function_call', 'name': 'spawn_agent',
|
|
94
|
+
'call_id': 'c1', 'arguments': json.dumps({'task_name': 'cold_verifier',
|
|
95
|
+
'fork_turns': 'none', 'reasoning_effort': 'high'})}})
|
|
96
|
+
lines.append({'type': 'response_item', 'payload': {'type': 'function_call_output',
|
|
97
|
+
'call_id': 'c1', 'output': '{"agent_id": "w1"}'}})
|
|
98
|
+
counters = [
|
|
99
|
+
({'input_tokens': 1000, 'cached_input_tokens': 200, 'output_tokens': 50, 'cache_write_input_tokens': 0},
|
|
100
|
+
{'input_tokens': 1000, 'cached_input_tokens': 200, 'output_tokens': 50, 'cache_write_input_tokens': 0}),
|
|
101
|
+
({'input_tokens': 3000, 'cached_input_tokens': 700, 'output_tokens': 130, 'cache_write_input_tokens': 0},
|
|
102
|
+
{'input_tokens': 2000, 'cached_input_tokens': 500, 'output_tokens': 80, 'cache_write_input_tokens': 0}),
|
|
103
|
+
]
|
|
104
|
+
for total, last in counters:
|
|
105
|
+
lines.append({'type': 'event_msg', 'payload': {'type': 'token_count',
|
|
106
|
+
'info': {'total_token_usage': total, 'last_token_usage': last}}})
|
|
107
|
+
rollout.write_text('\n'.join(json.dumps(x) for x in lines))
|
|
108
|
+
|
|
109
|
+
def test_codex_spawn_agent_child_cost_is_unknown_not_zero(self):
|
|
110
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
111
|
+
root = Path(directory)
|
|
112
|
+
self.codex_episode(root, spawn=True)
|
|
113
|
+
with patch.object(Path, 'home', return_value=root):
|
|
114
|
+
row = analyze.inspect_episode(root, documents={})
|
|
115
|
+
self.assertIsNone(row['api_equivalent_cost_usd'])
|
|
116
|
+
self.assertIsNone(row['api_equivalent_input_cost_usd'])
|
|
117
|
+
self.assertIsNone(row['reconstructed_total_cost_usd'])
|
|
118
|
+
self.assertIn('unknown', row['child_usage_observability'])
|
|
119
|
+
self.assertEqual(len(row['private_collaboration_dispatch']), 1)
|
|
120
|
+
dispatch = row['private_collaboration_dispatch'][0]
|
|
121
|
+
self.assertEqual(dispatch['tool'], 'spawn_agent')
|
|
122
|
+
self.assertEqual(dispatch['arguments']['task_name'], 'cold_verifier')
|
|
123
|
+
self.assertEqual(len(dispatch['outputs']), 1)
|
|
124
|
+
# Parent cost is measured and retained; worker cost is never zero-filled.
|
|
125
|
+
self.assertIsInstance(row['parent_api_equivalent_input_cost_usd'], float)
|
|
126
|
+
self.assertAlmostEqual(row['parent_api_equivalent_input_cost_usd'], 0.000237)
|
|
127
|
+
self.assertIsInstance(row['parent_api_equivalent_cost_usd'], float)
|
|
128
|
+
self.assertAlmostEqual(row['parent_api_equivalent_cost_usd'], 0.000302)
|
|
129
|
+
|
|
130
|
+
def test_codex_without_spawn_keeps_measured_cost_and_no_unknown_child(self):
|
|
131
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
132
|
+
root = Path(directory)
|
|
133
|
+
self.codex_episode(root, spawn=False)
|
|
134
|
+
with patch.object(Path, 'home', return_value=root):
|
|
135
|
+
row = analyze.inspect_episode(root, documents={})
|
|
136
|
+
self.assertEqual(row['private_collaboration_dispatch'], [])
|
|
137
|
+
self.assertEqual(row['child_usage_observability'], 'no attributable native spawn in inspected parent rollout')
|
|
138
|
+
self.assertAlmostEqual(row['api_equivalent_input_cost_usd'], 0.000237)
|
|
139
|
+
self.assertAlmostEqual(row['api_equivalent_cost_usd'], 0.000302)
|
|
140
|
+
self.assertAlmostEqual(row['reconstructed_total_cost_usd'], 0.000302)
|
|
141
|
+
|
|
84
142
|
def test_polyglot_grader_timeout_is_retained_failure(self):
|
|
85
143
|
with tempfile.TemporaryDirectory() as directory:
|
|
86
144
|
root = Path(directory)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const vm = require('vm');
|
|
4
|
+
const acorn = require('acorn');
|
|
5
|
+
const childProcess = require('child_process');
|
|
6
|
+
|
|
7
|
+
// Load the private runner without adding exports to the portable plugin's
|
|
8
|
+
// host-discovery surface. Parsing preserves async declarations and boundaries.
|
|
9
|
+
module.exports = function loadRunner(root, overrides = {}) {
|
|
10
|
+
const source = fs.readFileSync(path.join(root, 'opencode-plugin/index.mjs'), 'utf8');
|
|
11
|
+
const names = new Set(['FAILURE_MARKER_PATTERNS', 'CONTEXT_MARKER_PATTERNS',
|
|
12
|
+
'VERIFY_TIMEOUT_ENV', 'DEFAULT_VERIFY_TIMEOUT_MS', 'VERIFY_OUTPUT_LIMIT',
|
|
13
|
+
'VERIFY_LINE_LIMIT', 'VERIFY_KILL_GRACE_MS', 'normalizeFailureLine',
|
|
14
|
+
'firstFailureMarker', 'buildFailureSignature', 'getAvailableScripts',
|
|
15
|
+
'verifyTimeoutMs', 'createVerificationCapture', 'terminateVerificationChild',
|
|
16
|
+
'runVerificationCommand', 'runVerification']);
|
|
17
|
+
const ast = acorn.parse(source, { ecmaVersion: 'latest', sourceType: 'module' });
|
|
18
|
+
const declarations = ast.body.filter(node => names.has(node.id?.name || node.declarations?.[0]?.id?.name));
|
|
19
|
+
const context = { process, Buffer, setTimeout, clearTimeout, setInterval, clearInterval,
|
|
20
|
+
createHash: require('crypto').createHash, spawn: childProcess.spawn,
|
|
21
|
+
existsSync: fs.existsSync, readFileSync: fs.readFileSync, join: path.join, ...overrides };
|
|
22
|
+
return vm.runInNewContext(declarations.map(node => source.slice(node.start, node.end)).join('\n')
|
|
23
|
+
+ '\n;({ runVerification, runVerificationCommand, createVerificationCapture, buildFailureSignature, verifyTimeoutMs })', context);
|
|
24
|
+
};
|
|
@@ -77,6 +77,13 @@ helper.check(
|
|
|
77
77
|
|
|
78
78
|
const push = (event) => eventQueue.push(event);
|
|
79
79
|
const settle = () => new Promise((resolve) => setTimeout(resolve, 50));
|
|
80
|
+
const waitFor = async (predicate) => {
|
|
81
|
+
const deadline = Date.now() + 5000;
|
|
82
|
+
while (!predicate()) {
|
|
83
|
+
if (Date.now() > deadline) throw new Error('V2 async verification did not settle');
|
|
84
|
+
await new Promise(resolve => setTimeout(resolve, 20));
|
|
85
|
+
}
|
|
86
|
+
};
|
|
80
87
|
|
|
81
88
|
const mod = await import(pathToFileURL(path.join(helper.root, 'opencode-plugin', 'index.mjs')).href);
|
|
82
89
|
const definition = mod.default;
|
|
@@ -113,7 +120,7 @@ helper.check(
|
|
|
113
120
|
for (let i = 1; i <= 3; i++) {
|
|
114
121
|
await after({ tool: 'edit', sessionID: 's1', id: `c${i}`, input: {} });
|
|
115
122
|
push({ type: 'session.idle', sessionID: 's1' });
|
|
116
|
-
await
|
|
123
|
+
await waitFor(() => prompts.length === i);
|
|
117
124
|
}
|
|
118
125
|
|
|
119
126
|
helper.check(
|
|
@@ -228,7 +228,7 @@ function caseD4(leg) {
|
|
|
228
228
|
fs.writeFileSync(candidate, differing);
|
|
229
229
|
const before = sha256(path.join(ws, 'docs', 'index.md'));
|
|
230
230
|
const result = runCompare(leg.dir, { workspace: ws, config: CONFIG, view: 'docs/index.md', candidate, status: 'ok' });
|
|
231
|
-
same(result.status, 1,
|
|
231
|
+
same(result.status, 1, `exit; stdout=${truncate(result.stdout)}; stderr=${truncate(result.stderr)}`);
|
|
232
232
|
if (!JSON.stringify(jsonOut(result)).includes('stale')) throw new Error(`result state is not stale: ${truncate(result.stdout)}`);
|
|
233
233
|
same(sha256(path.join(ws, 'docs', 'index.md')), before, 'target preserved');
|
|
234
234
|
}
|
|
@@ -364,6 +364,46 @@ function caseD13(leg) {
|
|
|
364
364
|
}
|
|
365
365
|
assertPerLeg('case-d13:crlf-candidate-not-normalized', leg => caseD13(leg));
|
|
366
366
|
|
|
367
|
+
// Windows file IDs can exceed Number's exact integer range. Exercise the real
|
|
368
|
+
// CLI with deterministic stat identities so runner allocation cannot hide it.
|
|
369
|
+
assertPerLeg('case-d13b:64-bit-file-identities-remain-distinct', leg => {
|
|
370
|
+
const { ws } = viewWorkspace();
|
|
371
|
+
writeTarget(ws);
|
|
372
|
+
const dir = candidateDir();
|
|
373
|
+
const candidate = path.join(dir, 'candidate.md');
|
|
374
|
+
const target = path.join(ws, 'docs', 'index.md');
|
|
375
|
+
fs.writeFileSync(candidate, 'different bytes\n');
|
|
376
|
+
const large = 2n ** 60n;
|
|
377
|
+
same(Number(large), Number(large + 1n), 'fixture must collide as Number');
|
|
378
|
+
for (const scenario of ['distinct-inode', 'distinct-device', 'alias']) {
|
|
379
|
+
const preload = path.join(dir, `${scenario}.cjs`);
|
|
380
|
+
fs.writeFileSync(preload, `
|
|
381
|
+
const fs = require('fs');
|
|
382
|
+
const path = require('path');
|
|
383
|
+
const original = fs.statSync;
|
|
384
|
+
fs.statSync = (file, options) => {
|
|
385
|
+
const result = original(file, options);
|
|
386
|
+
const name = path.resolve(String(file));
|
|
387
|
+
if (name === ${JSON.stringify(candidate)} || name === ${JSON.stringify(target)}) {
|
|
388
|
+
const candidate = name === ${JSON.stringify(candidate)};
|
|
389
|
+
const convert = value => options && options.bigint ? value : Number(value);
|
|
390
|
+
result.ino = convert(${large}n + (candidate && ${JSON.stringify(scenario)} === 'distinct-inode' ? 1n : 0n));
|
|
391
|
+
result.dev = convert(${large}n + (candidate && ${JSON.stringify(scenario)} === 'distinct-device' ? 1n : 0n));
|
|
392
|
+
}
|
|
393
|
+
return result;
|
|
394
|
+
};
|
|
395
|
+
`);
|
|
396
|
+
const result = runCompare(leg.dir, {
|
|
397
|
+
workspace: ws, config: CONFIG, view: 'docs/index.md', candidate, status: 'ok',
|
|
398
|
+
extra: { env: { ...process.env, NODE_OPTIONS: `--require "${preload.replace(/\\/g, '/')}"` } }
|
|
399
|
+
});
|
|
400
|
+
const alias = scenario === 'alias';
|
|
401
|
+
same(result.status, alias ? 2 : 1, `${scenario} exit`);
|
|
402
|
+
same(jsonOut(result).result, alias ? 'comparison-unavailable' : 'stale', `${scenario} verdict`);
|
|
403
|
+
if (alias) same(jsonOut(result).reason, 'candidate aliases the target', 'alias protection retained');
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
|
|
367
407
|
// D14: every compare-derived lane above (plus D1) printed exactly one JSON object on stdout.
|
|
368
408
|
{
|
|
369
409
|
const id = 'case-d14:stdout-single-json-object';
|
|
@@ -487,7 +527,7 @@ function mp3Apply() {
|
|
|
487
527
|
const dir = scriptCopy('f3 derived mp3 ');
|
|
488
528
|
// The guard's only decision line — elsewhere in the mutant the script stays intact, so the
|
|
489
529
|
// aliased run proceeds to a real byte comparison (exit 0, equal) instead of crashing.
|
|
490
|
-
applyMutation(dir, 'return x.ino >
|
|
530
|
+
applyMutation(dir, 'return x.ino > 0n && x.ino === y.ino && x.dev === y.dev;', 'return false;', 'mp3 sameFile identity guard');
|
|
491
531
|
return dir;
|
|
492
532
|
}
|
|
493
533
|
|