harness-everything 0.47.2 → 0.47.4

package/ci/test-behavioral-case-validation.js

@@ -5,8 +5,8 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const { parseSimpleYaml } = require('../behavioral-evals/run');
-const { grade, parseTranscriptEvents } = require('../behavioral-evals/run');
-const { validateCase, validateDirectory, negativeControls } = require('../behavioral-evals/case-validator');
+const { grade, parseTranscriptEvents, checkJsonOnlyOutput, finalAssistantText } = require('../behavioral-evals/run');
+const { validateCase, validateDirectory, validateExpectation, negativeControls } = require('../behavioral-evals/case-validator');
 const { canonicalRubric, extractTrace } = require('./ab-test-harness');
 
 const casesDir = path.join(__dirname, '..', 'behavioral-evals', 'cases');
@@ -85,4 +85,149 @@ try {
   fs.rmSync(transcriptDir, { recursive: true, force: true });
 }
 
-console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence).`);
+// final_output_json_only: pure checker, validator schema, shipped #308 case, and
+// end-to-end grading over the final assistant message.
+const SELECTION_KEYS = ['requestedProfile', 'effectiveProfile', 'profileAlias', 'assignedRole', 'runtimeModel', 'runtimeEffort'];
+const SELECTION = { requestedProfile: 'fable', effectiveProfile: 'sonnet', profileAlias: 'fable-on-sonnet', assignedRole: 'worker', runtimeModel: 'claude-haiku-5-5', runtimeEffort: 'low' };
+const LABEL = '<!-- harness-label {"judgement":"done","routing":"direct"} -->';
+const RAW = JSON.stringify(SELECTION);
+const FENCED = '```json\n' + JSON.stringify(SELECTION, null, 2) + '\n```';
+const PASS_CASES = [
+  ['raw JSON object', RAW],
+  ['fenced json block', FENCED],
+  ['untagged fence', '```\n' + RAW + '\n```'],
+  ['raw JSON followed by harness-label line', RAW + '\n' + LABEL],
+  ['fenced JSON followed by harness-label line', FENCED + '\n' + LABEL],
+];
+for (const [name, text] of PASS_CASES) {
+  const verdict = checkJsonOnlyOutput(text, SELECTION_KEYS);
+  assert.strictEqual(verdict.pass, true, `${name} should pass: ${verdict.reason}`);
+}
+const FAIL_CASES = [
+  // The #308 pilot: correct fields, explanatory prose outside the JSON.
+  ['prose before JSON (#308)', `"Fable on sonnect" means the fable profile runs on sonnet.\n\n${FENCED}`, /prose before/],
+  ['prose after fence', `${FENCED}\nThat is the selection record.`, /outside the single JSON code fence/],
+  ['prose after raw JSON', `${RAW}\nThat is the selection record.`, /not a single valid JSON value/],
+  ['two fenced blocks', `${FENCED}\n${FENCED}`, /more than one code fence/],
+  ['missing required key', JSON.stringify({ ...SELECTION, runtimeEffort: undefined }), /missing required JSON keys: runtimeEffort/],
+  ['non-object JSON (array)', JSON.stringify([SELECTION]), /must be an object/],
+  ['non-object JSON (scalar)', '"fable"', /must be an object/],
+  ['empty final text', '', /empty/],
+  ['label-only final text', LABEL, /empty/],
+  ['missing final text', undefined, /missing/],
+];
+for (const [name, text, expected] of FAIL_CASES) {
+  const verdict = checkJsonOnlyOutput(text, SELECTION_KEYS);
+  assert.strictEqual(verdict.pass, false, `${name} should fail`);
+  assert.match(verdict.reason, expected, `${name} should give a clear reason`);
+}
+assert.strictEqual(checkJsonOnlyOutput(RAW).pass, true, 'required_keys defaults to no key requirement');
+
+for (const requiredKeys of ['requestedProfile', [''], ['  '], [1], ['ok', null]]) {
+  const errors = validateExpectation({ type: 'final_output_json_only', required_keys: requiredKeys }, 0);
+  assert.ok(errors.some(error => /required_keys must be an array of non-empty strings/.test(error)), `required_keys ${JSON.stringify(requiredKeys)} should be rejected`);
+}
+assert.deepStrictEqual(validateExpectation({ type: 'final_output_json_only' }, 0), [], 'final_output_json_only without required_keys is valid');
+assert.deepStrictEqual(validateExpectation({ type: 'final_output_json_only', required_keys: SELECTION_KEYS }, 0), [], 'final_output_json_only with string keys is valid');
+
+const profileCasePath = path.join(casesDir, 'baseline-profile-lookup-json-only.yaml');
+const profileCase = parseSimpleYaml(fs.readFileSync(profileCasePath, 'utf8'));
+assert.deepStrictEqual(validateCase(profileCase, { fileName: 'baseline-profile-lookup-json-only.yaml' }), [], 'profile-lookup JSON-only case should validate');
+assert.deepStrictEqual(profileCase.expectations.map(expectation => expectation.type), ['final_output_json_only'], 'profile-lookup case should assert JSON-only output');
+assert.deepStrictEqual(profileCase.expectations[0].required_keys, SELECTION_KEYS, 'profile-lookup case should require the six selection keys');
+assert.match(profileCase.prompt, /return only a small JSON selection record with requestedProfile, effectiveProfile, profileAlias, assignedRole, runtimeModel, runtimeEffort\. No orchestration or delegation\./, 'profile-lookup prompt should match the #308 wording');
+
+const jsonOnlyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-json-only-'));
+try {
+  const writeTranscript = (name, finalText) => {
+    const transcriptPath = path.join(jsonOnlyDir, name);
+    const events = [
+      { type: 'tool_use', part: { type: 'tool', tool: 'Read', callID: 'read-1', state: { status: 'completed', input: { file_path: 'README.md' } } } },
+      { type: 'text', part: { type: 'text', text: 'Checking the profile table first.' } },
+      { type: 'tool_use', part: { type: 'tool', tool: 'Read', callID: 'read-2', state: { status: 'completed', input: { file_path: 'profiles.md' } } } },
+      { type: 'text', part: { type: 'text', text: finalText } },
+    ];
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return transcriptPath;
+  };
+  const jsonOnlyExpectation = { expectations: [{ type: 'final_output_json_only', required_keys: SELECTION_KEYS }] };
+  let transcriptCount = 0;
+  const gradeFinal = finalText => grade(jsonOnlyExpectation, jsonOnlyDir, writeTranscript(`final-${transcriptCount++}.jsonl`, finalText));
+  assert.strictEqual(gradeFinal(RAW).passed, true, 'raw JSON final message should pass end to end');
+  assert.strictEqual(gradeFinal(`${FENCED}\n${LABEL}`).passed, true, 'fenced JSON with harness-label should pass end to end');
+  const prose = gradeFinal(`Fable on sonnect means a profile.\n${FENCED}`);
+  assert.strictEqual(prose.passed, false, 'prose before JSON must fail end to end');
+  assert.strictEqual(prose.results[0].status, 'fail');
+  assert.match(prose.results[0].reason, /prose before/);
+  // Prose from an earlier turn (before the last tool call) is not the final message.
+  const earlierProse = writeTranscript('earlier-prose.jsonl', RAW);
+  assert.strictEqual(grade(jsonOnlyExpectation, jsonOnlyDir, earlierProse).passed, true, 'only the final assistant message is graded');
+
+  // Multi-part final messages: the final turn may stream several text parts.
+  const writeParts = (name, finalParts) => {
+    const transcriptPath = path.join(jsonOnlyDir, name);
+    const events = [
+      { type: 'tool_use', part: { type: 'tool', tool: 'Read', callID: 'read-1', state: { status: 'completed', input: { file_path: 'profiles.md' } } } },
+      ...finalParts.map(text => ({ type: 'text', part: { type: 'text', text } })),
+    ];
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return transcriptPath;
+  };
+  const proseThenJson = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts('prose-part-json-part.jsonl', ['"Fable on sonnect" means the fable profile on sonnet.', FENCED]));
+  assert.strictEqual(proseThenJson.passed, false, 'prose part + JSON part must fail (#308 false pass)');
+  assert.match(proseThenJson.results[0].reason, /prose before/);
+  const jsonThenProse = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts('json-part-prose-part.jsonl', [FENCED, 'That is the selection record.']));
+  assert.strictEqual(jsonThenProse.passed, false, 'JSON part + prose part must fail');
+  const splitBody = JSON.stringify(SELECTION, null, 2);
+  const cut = splitBody.indexOf('"assignedRole"');
+  const splitJson = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts('split-json.jsonl', ['```json\n' + splitBody.slice(0, cut), splitBody.slice(cut) + '\n```']));
+  assert.strictEqual(splitJson.passed, true, `JSON split across text parts should pass: ${splitJson.results[0].reason}`);
+
+  // Transport chunks cut inside a JSON string value, a key, and the fence opener.
+  // Exact concatenation must rebuild the message; a '\n' join would break it.
+  const valueCut = RAW.indexOf('fable') + 2;
+  const keyCut = RAW.indexOf('requestedProfile') + 5;
+  const splitValueParts = [RAW.slice(0, valueCut), RAW.slice(valueCut)];
+  const splitValuePath = writeParts('split-inside-value.jsonl', splitValueParts);
+  assert.strictEqual(finalAssistantText(parseTranscriptEvents(splitValuePath).events), RAW, 'split-inside-value parts must concatenate to the exact raw JSON');
+  assert.throws(() => JSON.parse(RAW.slice(0, valueCut) + '\n' + RAW.slice(valueCut)), 'a newline join inside a JSON string must not parse (the #308 reviewer case)');
+  const POSITIVE_SPLITS = [
+    ['split inside a JSON string value', splitValueParts],
+    ['split inside a JSON key', [RAW.slice(0, keyCut), RAW.slice(keyCut)]],
+    ['split inside the fence opener', ['```js', 'on\n' + JSON.stringify(SELECTION, null, 2) + '\n```']],
+    ['fenced JSON block followed by a harness-label part', [FENCED, LABEL]],
+  ];
+  POSITIVE_SPLITS.forEach(([name, parts], index) => {
+    const result = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts(`positive-split-${index}.jsonl`, parts));
+    assert.strictEqual(result.passed, true, `${name} should pass end to end: ${result.results[0].reason}`);
+  });
+  const NEGATIVE_SPLITS = [
+    ['prose chunk glued to JSON start', ['Here:', RAW], /prose before/],
+    ['prose after a split JSON', [RAW.slice(0, 10), RAW.slice(10), ' done'], /not a single valid JSON value/],
+  ];
+  NEGATIVE_SPLITS.forEach(([name, parts, expected], index) => {
+    const result = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts(`negative-split-${index}.jsonl`, parts));
+    assert.strictEqual(result.passed, false, `${name} must fail end to end`);
+    assert.match(result.results[0].reason, expected, `${name} should give a clear reason`);
+  });
+
+  // Claude stream-json: assistant content parts plus a terminal `result` that
+  // repeats them. The repeat must not be double-counted, and must not mask prose.
+  const writeClaude = (name, contentTexts, resultText) => {
+    const transcriptPath = path.join(jsonOnlyDir, name);
+    const lines = [
+      { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'tu-1', name: 'Read', input: { file_path: 'profiles.md' } }] } },
+      { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'tu-1', content: 'ok' }] } },
+      { type: 'assistant', message: { content: contentTexts.map(text => ({ type: 'text', text })) } },
+      { type: 'result', subtype: 'success', result: resultText },
+    ];
+    fs.writeFileSync(transcriptPath, lines.map(line => JSON.stringify(line)).join('\n'));
+    return transcriptPath;
+  };
+  assert.strictEqual(grade(jsonOnlyExpectation, jsonOnlyDir, writeClaude('claude-result-repeat.jsonl', [FENCED], FENCED)).passed, true, 'Claude result repeat must not double the JSON');
+  assert.strictEqual(grade(jsonOnlyExpectation, jsonOnlyDir, writeClaude('claude-prose-parts.jsonl', ['Here is the record.', FENCED], FENCED)).passed, false, 'Claude prose content part must fail even when result holds only JSON');
+} finally {
+  fs.rmSync(jsonOnlyDir, { recursive: true, force: true });
+}
+
+console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence + json-only final output).`);

package/docs/platform-capabilities.md

@@ -183,6 +183,14 @@ evidence supports project-scope `.js` loading and edit/verification state on
 OpenCode 1.18.31 (macOS): [live-host evidence](../benchmarks/results/live-host/opencode-2026-09-16/README.md). The
 retained snapshot predates #190 and remains historical evidence only. Current OpenCode removes the permanent post-reflection hard lock; only the third-failure reflection boundary pauses edits. Reflection was operator-seeded, then agent-rewritten, so do not promote this into a behavioral-effectiveness claim. `.mjs` auto-discovery is broken on this host version (issue #127). npm-package installation and behavioral V2-hook fire remain unverified.
 
+The plugin verification runner uses asynchronous child processes with a shared
+sequence budget and bounded streamed diagnostics. Deterministic mechanism
+coverage checks event-loop responsiveness, large-output verdicts, timeout/process
+cleanup, and concurrent edit/session reset handling. The standalone manual CLI
+retains its synchronous per-command runner. This change supplies no additional
+live-host evidence; the historical installation and behavior boundaries above
+remain unchanged.
+
 ## Official sources
 
 The matrix records the official documentation reviewed for each platform. Important source families include [OpenAI plugin packaging](https://developers.openai.com/plugins/build/plugins), [OpenAI public submission](https://developers.openai.com/plugins/deploy/submission), [Codex skills](https://learn.chatgpt.com/docs/build-skills), [Codex hooks](https://learn.chatgpt.com/docs/hooks), [Claude skills](https://code.claude.com/docs/en/skills), [OpenCode skills](https://opencode.ai/docs/skills), [GitHub Copilot Agent Skills](https://docs.github.com/en/copilot/how-tos/copilot-on-github/customize-copilot/customize-cloud-agent/add-skills), [Cursor skills](https://cursor.com/docs/skills), [Continue rules](https://docs.continue.dev/customize/rules), and [Hermes skills](https://hermes-agent.nousresearch.com/docs/user-guide/features/skills).

package/harness-everything/SKILL.md

@@ -4,7 +4,7 @@ description: "Route software/project work through Harness: classify tier/topolog
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.47.1
+  version: 0.47.3
 ---
 
 # Harness Everything

package/harness-everything/scripts/doc-check.js

@@ -247,11 +247,12 @@ function globToRegExp(pattern) {
   return new RegExp(`^${body}$`);
 }
 
-// Same file when both paths stat to one device and inode, so symlinks and hard links are caught.
+// Keep 64-bit device/file IDs exact (Windows IDs can exceed Number's safe
+// integer range), while still rejecting symlink and hard-link aliases.
 function sameFile(a, b) {
-  const x = fs.statSync(a);
-  const y = fs.statSync(b);
-  return x.ino > 0 && x.ino === y.ino && x.dev === y.dev;
+  const x = fs.statSync(a, { bigint: true });
+  const y = fs.statSync(b, { bigint: true });
+  return x.ino > 0n && x.ino === y.ino && x.dev === y.dev;
 }
 
 // Open without blocking (O_NONBLOCK keeps a FIFO from waiting for a writer), then require a regular file before reading.

package/hooks/scripts/lib/doc-reconciliation-provider.js

@@ -101,7 +101,7 @@ function isUncPathPrefix(token, command, index) {
   return token === '' && command[index] === '\\' && command[index + 1] === '\\';
 }
 // Canonical SHA-256 of harness-everything/scripts/doc-check.js (CRLF normalised to LF). Re-pin deliberately when that script changes.
-const SHIPPED_DOC_CHECK_DIGEST = '3ca386666d4dae9a98ffa27adf2f326bf171a783af90515375f0b328ca35f194';
+const SHIPPED_DOC_CHECK_DIGEST = '0e93f252b2350b145c34dcb5354aa3b4246f3c59e948a62c3a912305b355a475';
 // This install's own doc-check runtime, relative to hooks/scripts/lib: the source checkout or the packaged plugin.
 const DOC_CHECK_INSTALL_LAYOUTS = [
   ['..', '..', '..', 'harness-everything', 'scripts', 'doc-check.js'],

package/opencode-plugin/README.md

@@ -52,39 +52,37 @@ and on failure calls `client.session.prompt()` to push a synthetic follow-up
 message into the session - forcing the agent to keep working instead of
 actually stopping.
 
-The verification run has a budget, not a fixed 60s cap: a healthy self-regression
-suite takes around two minutes, so the default is 10 minutes and
-`HARNESS_VERIFY_TIMEOUT_MS` overrides it. Every run is classified as `pass`
-(exit 0), `test_failure` (a real non-zero exit), `timeout` (the budget actually
-expired: `ETIMEDOUT`, or a kill with no exit status once the budget elapsed) or
-`execution_error` (a spawn-level failure such as `ENOBUFS`/`ENOENT`, or an
-external signal such as `SIGTERM`/`SIGINT` before the budget - a signal alone
-is not a budget expiry).
-Only `test_failure` confirms a failing check and advances the Rule-of-3
-counter; `timeout` and `execution_error` are reported as uncompleted
-verifications and never impersonate repeated failures. A timeout is reported as
-`timed out after <n>s (set HARNESS_VERIFY_TIMEOUT_MS to raise it); last
-output: ...`, a spawn-level failure as `could not run to completion (execution
-error: ENOBUFS); output: ...` (an external signal names the signal).
-
-Known operational limits of the synchronous runner: `execSync` blocks the
-plugin's event loop for the duration of each script (worst case one budget per
-script), and captured output is buffered up to 16 MiB. A run whose output
-exceeds that buffer is classified as an `execution_error`, never as a failed
-check, so buffer pressure cannot flip the circuit breaker. Moving verification
-to an async `spawn` with streaming, bounded diagnostics is tracked as a
-follow-up in issue #338.
+The async runner streams stdout/stderr and gives the **whole ordered sequence**
+(`test`, `lint`, `build`) one wall-clock budget: 10 minutes by default, overridden
+by `HARNESS_VERIFY_TIMEOUT_MS`. It reports progress every 15 seconds in the
+plugin log. No later script starts after the budget expires. Each run is
+classified as `pass` (exit 0), `test_failure` (a real nonzero exit), `timeout`
+(the sequence deadline expired), or `execution_error` (spawn failure, external
+signal, or cancellation). Only `test_failure` advances the Rule-of-3 counter.
+
+| Resource | Bound / behavior |
+|---|---|
+| Retained output | First and last 2,000 characters per stream; the final diagnostic shares a 2,000-character budget across streams, plus labels/elision markers |
+| Failure identity | Incremental whitespace-normalized digest of both complete streams; first recognized test marker and suite context tracked separately |
+| Marker parsing | First 4,096 characters of each line; an overlong marker/context selects the complete-stream digest; ordinary long banners preserve later complete markers |
+| Timeout/cancel | Terminate the shell/npm/script process group, then force-kill after 150 ms on POSIX; Windows uses `taskkill /T /F` |
+| Concurrent lifecycle | Duplicate idle events share the active run; a code edit or session reset aborts it; an idle arriving during cancellation waits for cleanup before verifying the newer edits; the V2 event stream continues consuming cancellation events; disposal awaits active child cleanup; stale results cannot clear newer edits or recreate deleted state |
+
+The event loop remains available while scripts run. The output-retention limit
+never becomes a check verdict. Mechanism tests cover large output, shared
+budget, child cleanup and concurrent hooks; these are deterministic mechanism
+evidence, not a new live-host verification claim. Processes deliberately
+escaping the process group are outside this cleanup boundary.
 
 ### 3. Rule-of-3 zoom-out boundary (`tool.execute.before` + the verification observer above)
 
 Enforces Rule of 3 on repeated verification failures (same failing command +
 normalized failure evidence: the first test-level failing marker - TAP
 `not ok`, node:test/jest symbols, Jest test blocks, assertion lines - over the
-full captured stdout **and** stderr, with a preceding suite header such as
+streamed stdout **and** stderr, with a preceding suite header such as
 Jest's `FAIL <file>` carried as context so different tests in one file stay
 distinct; without a test-level marker the signature falls back to a digest of
-that full evidence, never to a suite header alone; the 2 KiB bound applies
-only to the user-visible diagnostic excerpt, never to signature analysis; not
+those complete streams, never to a suite header alone; the diagnostic retention bound does not limit the stream digest; not
 arbitrary tool failures, since
 `tool.execute.after` has no normalized success/failure field to key a
 signature on for tools in general):
@@ -146,17 +144,17 @@ installation surfaces and were not verified in this run.
 
 ## Manual verification CLI
 
-**File:** `hooks/verify.js` - the same "run available npm scripts" logic as
-`index.mjs`'s verification gate, kept as a standalone command:
+**File:** `hooks/verify.js` - a standalone "run available npm scripts" command:
 
 ```bash
 node opencode-plugin/hooks/verify.js
 ```
 
 Run it from the workspace under test, never from the Harness repo root - from
-there it would re-enter `npm test` from inside `npm test`. It is intentionally
-a separate copy of the logic, not a shared import, so `index.mjs` stays a
-single portable file for the installation step above.
+there it would re-enter `npm test` from inside `npm test`. The manual CLI retains its synchronous per-command budget and 16 MiB capture
+limit; the async sequence budget and streaming behavior above apply to the
+plugin only. Keeping its implementation separate preserves the single-file
+plugin installation contract.
 
 `plugin.json` records this standalone helper for repository parity checks; it
 does not configure opencode. The runtime module is `index.mjs`.