harness-everything 0.47.4 → 0.47.5

package/ci/test-behavioral-case-validation.js

@@ -230,4 +230,218 @@ try {
   fs.rmSync(jsonOnlyDir, { recursive: true, force: true });
 }
 
-console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence + json-only final output).`);
+// Skill-core completion: passing project tests is artifact evidence only; the
+// TDD case also requires an executed quality gate command.
+const tddCase = parseSimpleYaml(fs.readFileSync(path.join(casesDir, 'baseline-tdd-workflow.yaml'), 'utf8'));
+const qualityGate = tddCase.expectations.find(expectation => expectation.type === 'tdd_quality_gate');
+assert.ok(qualityGate, 'baseline TDD case must require a passed TDD quality gate');
+// The gate is graded by independently re-evaluating retained evidence, so the
+// workspace holds a real passing evidence file and a failing variant.
+const PASS_EVIDENCE = fs.readFileSync(path.join(__dirname, '..', 'tdd', 'fixtures', 'pass-evidence.json'), 'utf8');
+const FAIL_EVIDENCE = (() => {
+  const evidence = JSON.parse(PASS_EVIDENCE);
+  evidence.requirements = evidence.requirements.map(requirement => ({ ...requirement, sourceConformance: 'FAIL' }));
+  return JSON.stringify(evidence);
+})();
+const gateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-tdd-quality-gate-'));
+try {
+  fs.writeFileSync(path.join(gateDir, 'evidence.json'), PASS_EVIDENCE);
+  fs.writeFileSync(path.join(gateDir, 'failing-evidence.json'), FAIL_EVIDENCE);
+  fs.writeFileSync(path.join(gateDir, 'broken-evidence.json'), '{ not json');
+  // A string command defaults to the gate's own success line when it is a
+  // quality-gate invocation, so existing positives stay meaningful. An object
+  // command states the observed output and exit code explicitly.
+  const QUALITY_GATE_COMMAND_RE = /quality-gate\.js|tdd:quality|test:tdd-quality/;
+  const gradeCommands = (name, commands) => {
+    const transcriptPath = path.join(gateDir, `${name}.jsonl`);
+    const events = commands.map((entry, index) => {
+      const { command, output, exit } = typeof entry === 'string'
+        ? { command: entry, output: QUALITY_GATE_COMMAND_RE.test(entry) ? 'TDD Quality Gate: PASS' : '', exit: undefined }
+        : entry;
+      const state = { status: 'completed', input: { command } };
+      if (output !== undefined) state.output = output;
+      if (exit !== undefined) state.metadata = { exit };
+      return { type: 'tool_use', part: { type: 'tool', tool: 'Bash', callID: `${name}-${index}`, state } };
+    });
+    events.push({ type: 'text', part: { type: 'text', text: 'done' } });
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return grade({ expectations: [qualityGate] }, gateDir, transcriptPath);
+  };
+  const testsOnly = gradeCommands('tests-only', ['npm test']);
+  assert.strictEqual(testsOnly.passed, false, 'passing project tests without the quality gate must fail the TDD case');
+  assert.strictEqual(testsOnly.results[0].status, 'fail', 'missing quality gate must be a definitive fail, not inconclusive evidence');
+  assert.strictEqual(gradeCommands('npm-gate', ['npm test', 'npm run tdd:quality -- evidence.json']).passed, true, 'npm run tdd:quality with evidence must satisfy the gate');
+  assert.strictEqual(gradeCommands('node-gate', ['npm test', 'node tdd/scripts/quality-gate.js evidence.json']).passed, true, 'node quality-gate.js with evidence must satisfy the gate');
+  assert.strictEqual(gradeCommands('installed-skill-gate', ['npm test', 'node .claude/skills/tdd/scripts/quality-gate.js evidence.json']).passed, true, 'quality gate under the installed skill dir must satisfy the gate');
+  assert.strictEqual(gradeCommands('absolute-gate', ['npm test', 'node /tmp/ws/.harness/tdd/scripts/quality-gate.js evidence.json']).passed, true, 'absolute quality-gate path must satisfy the gate');
+  for (const [name, commands] of [
+    ['echo-mention', ['echo npm run tdd:quality']],
+    ['compound-gate', ['npm run tdd:quality -- evidence.json && rm -rf build']],
+    ['similar-name', ['npm run tdd:quality-notes']],
+  ]) {
+    assert.strictEqual(gradeCommands(name, commands).passed, false, `${name} must not count as an executed quality gate`);
+  }
+  // A completed tool call is not completion evidence: the gate outcome is
+  // re-evaluated from the retained evidence, independent of transcript claims.
+  const gateFailures = [
+    ['nonzero-exit', ['npm test', 'npm run tdd:quality -- failing-evidence.json'], /re-evaluation of failing-evidence\.json is FAIL/],
+    ['missing-evidence', ['npm test', 'npm run tdd:quality -- absent.json'], /not retained/],
+    ['invalid-evidence', ['npm test', 'node tdd/scripts/quality-gate.js broken-evidence.json'], /not valid JSON/],
+    ['no-evidence-arg', ['npm test', 'npm run tdd:quality'], /without an evidence file/],
+    ['self-test-alias', ['npm test', 'npm run test:tdd-quality'], /repository self-test/],
+    ['escaping-evidence', ['npm test', 'npm run tdd:quality -- ../evidence.json'], /outside the workspace/],
+    ['missing-output-report', ['npm test', 'npm run tdd:quality -- evidence.json --output reports/quality.json'], /quality report reports\/quality\.json was not retained/],
+    ['last-run-counts', ['npm run tdd:quality -- evidence.json', 'npm run tdd:quality -- failing-evidence.json'], /FAIL/],
+  ];
+  for (const [name, commands, reason] of gateFailures) {
+    const graded = gradeCommands(name, commands);
+    assert.strictEqual(graded.passed, false, `${name} must not satisfy the quality gate`);
+    assert.strictEqual(graded.results[0].status, 'fail', `${name} must be a definitive fail`);
+    assert.match(graded.results[0].reason, reason, `${name} should explain why`);
+  }
+  // OpenCode reports state.status "completed" even when the gate exits 1.
+  const opencodeExit1 = path.join(gateDir, 'opencode-exit1.jsonl');
+  fs.writeFileSync(opencodeExit1, [
+    { type: 'tool_use', part: { type: 'tool', tool: 'bash', callID: 'gate', state: { status: 'completed', input: { command: 'npm run tdd:quality -- failing-evidence.json' }, output: 'TDD Quality Gate: FAIL\nexit code 1' } } },
+    { type: 'text', part: { type: 'text', text: 'Quality gate done.' } },
+  ].map(event => JSON.stringify(event)).join('\n'));
+  assert.strictEqual(grade({ expectations: [qualityGate] }, gateDir, opencodeExit1).passed, false, 'completed OpenCode event with exit 1 must fail');
+  // Fixing evidence and re-running is allowed: the last completed run counts.
+  assert.strictEqual(gradeCommands('rerun-after-fix', ['npm run tdd:quality -- failing-evidence.json', 'npm run tdd:quality -- evidence.json']).passed, true, 'a passing re-run after a failing run should satisfy the gate');
+  // Observed command outcome: the transcript must show the gate's own success
+  // line and no error or non-zero exit before the evidence is trusted.
+  const { evaluateEvidence } = require('../tdd/scripts/quality-gate.js');
+  const observedGate = 'npm run tdd:quality -- evidence.json';
+  const observedFailures = [
+    ['failed-command-pass-evidence', { command: observedGate, output: 'npm ERR! Missing script: "tdd:quality"', exit: 1 }, /quality gate command failed/],
+    ['nonzero-exit-no-error-text', { command: observedGate, output: 'TDD Quality Gate: PASS', exit: 1 }, /exited with code 1/],
+    ['gate-printed-fail-evidence-now-pass', { command: observedGate, output: 'TDD Quality Gate: FAIL' }, /printed TDD Quality Gate: FAIL/],
+  ];
+  for (const [name, command, reason] of observedFailures) {
+    const graded = gradeCommands(name, [command]);
+    assert.strictEqual(graded.passed, false, `${name} must not satisfy the quality gate`);
+    assert.strictEqual(graded.results[0].status, 'fail', `${name} must be a definitive fail`);
+    assert.match(graded.results[0].reason, reason, `${name} should explain why`);
+  }
+  const unobservable = gradeCommands('unobservable-outcome', [{ command: observedGate, output: '' }]);
+  assert.strictEqual(unobservable.results[0].status, 'inconclusive', 'a gate run with no observable output or exit code must be inconclusive');
+  assert.strictEqual(unobservable.passed, false, 'an inconclusive gate run must not pass');
+  assert.match(unobservable.results[0].reason, /not observable in the transcript/, 'inconclusive reason should say the outcome is not observable');
+  // Definitive workspace failures win over an unobservable outcome.
+  const missingNoOutput = gradeCommands('missing-evidence-no-output', [{ command: 'npm run tdd:quality -- nope.json', output: '' }]);
+  assert.strictEqual(missingNoOutput.results[0].status, 'fail', 'missing evidence must be a definitive fail even with no observable output');
+  assert.match(missingNoOutput.results[0].reason, /not retained/, 'missing evidence should be named');
+  // Only the error signal is line-anchored: prose mentioning npm errors passes,
+  // while a real npm error line still fails.
+  assert.strictEqual(gradeCommands('prose-npm-error', [{ command: observedGate, output: 'TDD Quality Gate: PASS\n- note: avoid npm error handling', exit: 0 }]).passed, true, 'prose mentioning npm error must not fail a passing gate');
+  const npmErrorLine = gradeCommands('npm-error-line', [{ command: observedGate, output: 'npm error Missing script: "tdd:quality"' }]);
+  assert.strictEqual(npmErrorLine.results[0].status, 'fail', 'an npm error line must be a definitive fail');
+  assert.match(npmErrorLine.results[0].reason, /npm error/, 'npm error line should be named');
+  // npm ERR!/error lines after a PASS line must fail even without exit metadata;
+  // the boundary after the token must match whitespace or end of line.
+  const npmTrailer = (name, output) => gradeCommands(name, [{ command: observedGate, output }]);
+  for (const [name, output] of [
+    ['npm-err-bang-eacces', 'TDD Quality Gate: PASS\nnpm ERR! code EACCES'],
+    ['npm-err-bang-eresolve', 'TDD Quality Gate: PASS\nnpm ERR! code ERESOLVE'],
+    ['npm-error-elifecycle', 'TDD Quality Gate: PASS\nnpm error code ELIFECYCLE'],
+    ['npm-err-bang-eol', 'TDD Quality Gate: PASS\nnpm ERR!'],
+  ]) {
+    const graded = npmTrailer(name, output);
+    assert.strictEqual(graded.results[0].status, 'fail', `${name} must be a definitive fail`);
+    assert.strictEqual(graded.passed, false, `${name} must not satisfy the quality gate`);
+    assert.match(graded.results[0].reason, /npm error/i, `${name} should name the npm error`);
+  }
+  assert.strictEqual(npmTrailer('npm-errors-word', 'TDD Quality Gate: PASS\nnpm ERRORS are documented').passed, true, 'a word that only starts with ERR must not be an npm error');
+  const { GATE_COMMAND_ERROR_RE } = require('../behavioral-evals/run');
+  assert.strictEqual(GATE_COMMAND_ERROR_RE.test('npm ERR! code EACCES'), true, 'npm ERR! line must match the error regex');
+  assert.strictEqual(GATE_COMMAND_ERROR_RE.test('npm ERRORS'), false, 'npm ERRORS must not match the error regex');
+
+  // Claude stream-json: is_error on the tool_result is the host's failure
+  // signal; the call is recorded as failed, so it never counts as executed.
+  const claudeGate = (name, toolResult) => {
+    const transcriptPath = path.join(gateDir, `${name}.jsonl`);
+    fs.writeFileSync(transcriptPath, [
+      { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'bash-1', name: 'Bash', input: { command: observedGate } }] } },
+      { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'bash-1', ...toolResult }] } },
+      { type: 'result', subtype: 'success', result: 'done' },
+    ].map(event => JSON.stringify(event)).join('\n'));
+    return grade({ expectations: [qualityGate] }, gateDir, transcriptPath);
+  };
+  const claudeError = claudeGate('claude-error', { is_error: true, content: 'npm ERR! Missing script' });
+  assert.strictEqual(claudeError.passed, false, 'Claude tool_result with is_error must not satisfy the gate');
+  assert.strictEqual(claudeError.results[0].status, 'fail', 'Claude tool_result error must be a definitive fail');
+  assert.match(claudeError.results[0].reason, /last quality gate run \(bash-1\) failed/, 'Claude errored gate call should be the failing last run');
+  assert.strictEqual(claudeGate('claude-success', { is_error: false, content: [{ type: 'text', text: 'TDD Quality Gate: PASS' }] }).passed, true, 'Claude tool_result carrying the gate success line should pass');
+  // Runs are graded in transcript order: the last matching attempt decides, so an
+  // earlier PASS never hides a later failed or unresolved run.
+  const claudeRuns = (name, runs) => {
+    const transcriptPath = path.join(gateDir, `${name}.jsonl`);
+    const events = [];
+    runs.forEach(({ command, result }, index) => {
+      const id = `${name}-bash-${index}`;
+      events.push({ type: 'assistant', message: { content: [{ type: 'tool_use', id, name: 'Bash', input: { command } }] } });
+      if (result) events.push({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, ...result }] } });
+    });
+    events.push({ type: 'assistant', message: { content: [{ type: 'text', text: 'done' }] } });
+    events.push({ type: 'result', subtype: 'success', result: 'done' });
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return grade({ expectations: [qualityGate] }, gateDir, transcriptPath);
+  };
+  const gatePass = { is_error: false, content: [{ type: 'text', text: 'TDD Quality Gate: PASS' }] };
+  const gateFail = { is_error: true, content: 'TDD Quality Gate: FAIL' };
+  const lastRunError = claudeRuns('claude-pass-then-error', [
+    { command: observedGate, result: gatePass },
+    { command: observedGate, result: gateFail },
+  ]);
+  assert.strictEqual(lastRunError.results[0].status, 'fail', 'a failed last run must be a definitive fail even after an earlier PASS');
+  assert.strictEqual(lastRunError.passed, false, 'an earlier PASS must not satisfy the gate after a failed last run');
+  assert.match(lastRunError.results[0].reason, /last quality gate run/, 'the failed last run should be named');
+  const lastRunUnresolved = claudeRuns('claude-pass-then-unresolved', [
+    { command: observedGate, result: gatePass },
+    { command: observedGate },
+  ]);
+  assert.strictEqual(lastRunUnresolved.results[0].status, 'inconclusive', 'an unresolved last run must be inconclusive, not a pass');
+  assert.strictEqual(lastRunUnresolved.passed, false, 'an unresolved last run must not pass the gate');
+  assert.match(lastRunUnresolved.results[0].reason, /no observable result/, 'the unresolved last run should be named');
+  const otherEvidenceError = claudeRuns('claude-pass-then-other-evidence', [
+    { command: observedGate, result: gatePass },
+    { command: 'node tdd/scripts/quality-gate.js failing-evidence.json', result: gateFail },
+  ]);
+  assert.strictEqual(otherEvidenceError.results[0].status, 'fail', 'a failed run against other evidence must fail the gate');
+  assert.match(otherEvidenceError.results[0].reason, /last quality gate run/, 'the failed last run against other evidence should be named');
+  assert.strictEqual(claudeRuns('claude-fail-then-retry', [
+    { command: observedGate, result: gateFail },
+    { command: observedGate, result: gatePass },
+  ]).passed, true, 'a failed run followed by a successful retry must pass');
+  // Retained --output report: a full report must be present and must equal the
+  // fresh re-evaluation over every top-level key.
+  const realReport = evaluateEvidence(JSON.parse(PASS_EVIDENCE));
+  assert.strictEqual(realReport.result, 'PASS', 'the pass fixture must re-evaluate to PASS');
+  fs.mkdirSync(path.join(gateDir, 'reports'), { recursive: true });
+  const reportPath = path.join(gateDir, 'reports', 'quality.json');
+  fs.writeFileSync(reportPath, JSON.stringify({ result: 'PASS' }));
+  const markerOnly = gradeCommands('marker-output-report', ['npm run tdd:quality -- evidence.json --output reports/quality.json']);
+  assert.strictEqual(markerOnly.passed, false, 'a {"result":"PASS"} marker must not satisfy a retained report');
+  assert.match(markerOnly.results[0].reason, /missing required keys: .*requirements/, 'a marker-only retained report should name the missing keys');
+  fs.writeFileSync(reportPath, JSON.stringify(realReport, null, 2));
+  assert.strictEqual(gradeCommands('retained-output-report', ['npm run tdd:quality -- evidence.json --output reports/quality.json']).passed, true, 'retained --output report equal to the re-evaluation should pass');
+  fs.writeFileSync(reportPath, JSON.stringify({ ...realReport, score: realReport.score - 1 }, null, 2));
+  const staleScore = gradeCommands('stale-output-report', ['npm run tdd:quality -- evidence.json --output reports/quality.json']);
+  assert.strictEqual(staleScore.passed, false, 'a retained report with a stale score must fail');
+  assert.match(staleScore.results[0].reason, /differs from re-evaluation at "score"/, 'a stale retained report should name the differing key');
+  // Without --output the gate prints its report to stdout; that printed report
+  // must equal the fresh re-evaluation of the evidence the command names.
+  const printedText = report => `TDD Quality Gate: PASS\n${JSON.stringify(report, null, 2)}`;
+  assert.strictEqual(gradeCommands('stdout-report-equal', [{ command: observedGate, output: printedText(realReport) }]).passed, true, 'a printed report equal to the re-evaluation should pass');
+  const otherEvidence = JSON.parse(PASS_EVIDENCE);
+  otherEvidence.requirements[0].requirementId = 'REQ-OTHER';
+  fs.writeFileSync(path.join(gateDir, 'other-evidence.json'), JSON.stringify(otherEvidence));
+  const otherReport = evaluateEvidence(otherEvidence);
+  const stdoutMismatch = gradeCommands('stdout-report-other', [{ command: observedGate, output: printedText(otherReport) }]);
+  assert.strictEqual(stdoutMismatch.passed, false, 'a printed report from different evidence must fail');
+  assert.match(stdoutMismatch.results[0].reason, /gate output report differs from re-evaluation at "[a-zA-Z]+"/, 'a mismatched printed report should name the differing key');
+} finally {
+  fs.rmSync(gateDir, { recursive: true, force: true });
+}
+
+console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence + json-only final output + tdd quality gate).`);

package/code-review/SKILL.md

@@ -4,7 +4,7 @@ description: "Independently review a PR, diff or finished implementation for cor
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.40.0
+  version: 0.47.5
 ---
 
 # Code Review

package/code-review/references/review-playbook.md

@@ -19,6 +19,15 @@ Use this checklist for the applicable risks; do not force irrelevant categories.
 - [ ] Follow each important diagnostic/fact through every consumer and ancestor. Check whether required facts are computed before derived authority, indexes, or output.
 - [ ] Compare new rules with previously accepted rules and controls. Separate display severity from safety prerequisites.
 
+## Check evidence a gate trusts
+
+Apply these when the change grades, gates or labels evidence. Authors SHOULD run them as a self-review before push.
+
+- [ ] **Claimed vs observed.** Separate what the subject reports about itself (a tool's `completed` status, a result marker, an agent-supplied expected value, a status label) from what the checker observes independently (exit code, output, a recomputed result, the quoted source). A self-report alone never passes; when nothing independent is observable, the result is inconclusive.
+- [ ] **Select, then judge.** When the last, first or any item of a sequence decides (last run, final message, latest attempt), select from every attempt in order before filtering by outcome. Test both orders: failure then success, and success then failure or no result.
+- [ ] **One witness per rule.** Give each detection rule a negative case that triggers only that rule; a case that also trips an exit code or another pattern cannot show the rule works. Probe positive cases at structural boundaries (inside a token, string or quote), not only at safe separators.
+- [ ] **Comparable subjects, honest labels.** A comparison or conflict needs both sides to describe the same target. A status name claims only what was checked: a consistent quote is not a validated diagnosis.
+
 ## Check tests and output
 
 - [ ] Ask whether each test still passes if the behavior is disabled, bypassed, or broken (mutation thought experiment). Confirm the test is included in CI.

package/opencode-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "harness-enforcement",
   "description": "Repository inventory for the opencode enforcement plugin and its standalone helper",
-  "version": "0.47.4",
+  "version": "0.47.5",
   "runtime": "index.mjs",
   "hooks": {
     "verify": "hooks/verify.js"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-everything",
-  "version": "0.47.4",
+  "version": "0.47.5",
   "description": "Harness OS: An Orchestrated Agent Operating System for AI-driven Development",
   "main": "harness-everything/scripts/bootstrap.js",
   "bin": {

package/plugins/harness-everything/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-everything",
-  "version": "0.47.4",
+  "version": "0.47.5",
   "description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
   "author": {
     "name": "Miya Daniel",

package/plugins/harness-everything/plugin.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
   "name": "harness-everything",
-  "version": "0.47.4",
+  "version": "0.47.5",
   "description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
   "author": {
     "name": "Miya Daniel",

package/plugins/harness-everything/skills/code-review/SKILL.md

@@ -4,7 +4,7 @@ description: "Independently review a PR, diff or finished implementation for cor
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.40.0
+  version: 0.47.5
 ---
 
 # Code Review

package/plugins/harness-everything/skills/code-review/references/review-playbook.md

@@ -19,6 +19,15 @@ Use this checklist for the applicable risks; do not force irrelevant categories.
 - [ ] Follow each important diagnostic/fact through every consumer and ancestor. Check whether required facts are computed before derived authority, indexes, or output.
 - [ ] Compare new rules with previously accepted rules and controls. Separate display severity from safety prerequisites.
 
+## Check evidence a gate trusts
+
+Apply these when the change grades, gates or labels evidence. Authors SHOULD run them as a self-review before push.
+
+- [ ] **Claimed vs observed.** Separate what the subject reports about itself (a tool's `completed` status, a result marker, an agent-supplied expected value, a status label) from what the checker observes independently (exit code, output, a recomputed result, the quoted source). A self-report alone never passes; when nothing independent is observable, the result is inconclusive.
+- [ ] **Select, then judge.** When the last, first or any item of a sequence decides (last run, final message, latest attempt), select from every attempt in order before filtering by outcome. Test both orders: failure then success, and success then failure or no result.
+- [ ] **One witness per rule.** Give each detection rule a negative case that triggers only that rule; a case that also trips an exit code or another pattern cannot show the rule works. Probe positive cases at structural boundaries (inside a token, string or quote), not only at safe separators.
+- [ ] **Comparable subjects, honest labels.** A comparison or conflict needs both sides to describe the same target. A status name claims only what was checked: a consistent quote is not a validated diagnosis.
+
 ## Check tests and output
 
 - [ ] Ask whether each test still passes if the behavior is disabled, bypassed, or broken (mutation thought experiment). Confirm the test is included in CI.

package/plugins/harness-everything/skills/tdd/SKILL.md

@@ -4,7 +4,7 @@ description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GR
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.13.0
+  version: 0.47.5
 ---
 
 # Test-Driven Development

package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json

@@ -8,8 +8,16 @@
     { "name": "weighted integration failure", "mutation": "the 0.15 positive dimension scores zero", "expectedScore": 85, "expectedResult": "FAIL" },
     { "name": "weighted unit failure", "mutation": "unit determinism is N/A and the 0.15 positive dimension scores zero", "expectedScore": 83.33, "expectedResult": "FAIL" },
     { "name": "NON_CONFORMANT", "mutation": "sourceConformance = NON_CONFORMANT", "expectedResult": "FAIL" },
-    { "name": "SOURCE_DEFECT", "mutation": "sourceValidity = SOURCE_DEFECT and sourceNotes is present", "expectedResult": "PASS" },
-    { "name": "SOURCE_CONFLICT", "mutation": "sourceValidity = SOURCE_CONFLICT with a traced additional source", "expectedResult": "FAIL" },
+    { "name": "SOURCE_DEFECT", "mutation": "sourceValidity = SOURCE_DEFECT, sourceNotes is present, and sourceEvidence quotes the defective assertion against the observed value; sourceEvidenceStatus is REVIEWER_REQUIRED, never VERIFIED, because a single-source mismatch does not prove the source is wrong", "expectedResult": "PASS" },
+    { "name": "SOURCE_DEFECT without sourceEvidence", "mutation": "sourceValidity = SOURCE_DEFECT with sourceNotes but no sourceEvidence", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT", "mutation": "sourceValidity = SOURCE_CONFLICT with a traced additional source and sourceEvidence quoting both sources", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT without sourceEvidence", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources but no sourceEvidence", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT node-versus-value", "mutation": "sourceValidity = SOURCE_CONFLICT citing only expect(list.head.value).toEqual(1) from one spec, observed value is an Element object", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT agreeing expectations", "mutation": "sourceValidity = SOURCE_CONFLICT with two sources expecting the same value and a failing observed value; the mismatch is implementation behaviour", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT two-source evidence", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting disagreeing expectations for the same subject list.head and a failing observed value; one quote (toBeInstanceOf) is not mechanically checkable, so sourceEvidenceStatus is REVIEWER_REQUIRED and the unresolved conflict gate stays closed", "expectedEvidence": "ACCEPTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT relabelled expected", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting the identical assertion expect(list.head).toEqual(1) but expected values 1 and 3; each expected must match its quoted literal, so the second entry is rejected", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT verified literal disagreement", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting expect(list.head.value).toBe(1) and expect(list.head.value).toBe('1') with matching expected values 1 and '1' and a failing observed value; sourceEvidenceStatus is VERIFIED and the unresolved conflict gate stays closed", "expectedEvidence": "ACCEPTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT unrelated assertions", "mutation": "sourceValidity = SOURCE_CONFLICT quoting expect(foo).toEqual(1) from one source and expect(bar).toEqual(3) from another; unequal expectations about different subjects", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
     { "name": "malformed input", "mutation": "requirements is not an array", "expectedResult": "FAIL" },
     { "name": "unexpected input", "mutation": "one unexpectedInput check is FAIL", "expectedResult": "FAIL" },
     { "name": "error handling", "mutation": "one errorHandling check is SKIPPED", "expectedResult": "FAIL" },

package/plugins/harness-everything/skills/tdd/references/quality-model.md

@@ -2,6 +2,8 @@
 
 This model turns each source requirement into auditable evidence. The authoritative source remains the contract even when it appears wrong. Record a suspected documentation problem as `SOURCE_DEFECT`; record disagreement between traced sources as `SOURCE_CONFLICT`. Never reinterpret either into a passing implementation. If no authoritative source exists, use `INFERRED` and state the inference basis.
 
+Before declaring either outcome, `sourceEvidence` quotes the exact conflicting assertions, names each declared source by path, and compares each `expected` value with the `observed` value from the run. A `SOURCE_CONFLICT` needs at least two distinct sources whose expected values disagree about the same `subject` — the contract target each quoted assertion constrains, written as it appears in the quote (`list.head` is not `list.head.value`). Unequal expectations about unrelated targets are not a conflict. If the quoted expectations agree and the run still fails, the mismatch is in the implementation: record `NON_CONFORMANT`, not `SOURCE_CONFLICT`. Each `expected` must match a mechanically readable literal in the quoted assertion (`toEqual`, `toStrictEqual`, `toBe`, or `assert.*` with a JSON or single-quoted literal); otherwise the claim is marked `REVIEWER_REQUIRED` and the gate does not treat it as an established disagreement. `VERIFIED` applies only to a `SOURCE_CONFLICT` whose two quoted literals disagree. A `SOURCE_DEFECT` is always `REVIEWER_REQUIRED`, because a mismatch with the observed value does not prove the source is wrong; the implementation may be the one at fault.
+
 ## Profile contract
 
 Every requirement declares exactly one `testProfile`: `unit` or `integration`. The selected profile defines which case classes are applicable; the common scoring and source rules do not make every test use identical evidence.

package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json

@@ -149,6 +149,18 @@
         }
       ]
     },
+    "sourceEvidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["source", "assertion", "expected", "observed"],
+      "properties": {
+        "source": { "type": "string", "minLength": 1 },
+        "subject": { "type": "string", "minLength": 1 },
+        "assertion": { "type": "string", "minLength": 1 },
+        "expected": {},
+        "observed": {}
+      }
+    },
     "requirement": {
       "type": "object",
       "additionalProperties": false,
@@ -161,6 +173,7 @@
         "sourceConformance": { "enum": ["PASS", "NON_CONFORMANT"] },
         "sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT"] },
         "sourceNotes": { "type": "string", "minLength": 1 },
+        "sourceEvidence": { "type": "array", "items": { "$ref": "#/$defs/sourceEvidence" } },
         "skippedTests": { "type": "array" },
         "flakyTests": { "type": "array" },
         "naDeclarations": { "type": "array" },
@@ -186,6 +199,14 @@
         }
       },
       "allOf": [
+        {
+          "if": { "required": ["sourceValidity"], "properties": { "sourceValidity": { "enum": ["SOURCE_DEFECT", "SOURCE_CONFLICT"] } } },
+          "then": { "required": ["sourceEvidence"], "properties": { "sourceEvidence": { "minItems": 1 } } }
+        },
+        {
+          "if": { "required": ["sourceValidity"], "properties": { "sourceValidity": { "const": "SOURCE_CONFLICT" } } },
+          "then": { "properties": { "sourceEvidence": { "items": { "required": ["subject"] } } } }
+        },
         {
           "if": {
             "required": ["testProfile"],

package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json

@@ -96,6 +96,8 @@
         "sourceConformance": { "enum": ["PASS", "NON_CONFORMANT", null] },
         "sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT", null] },
         "sourceNotes": { "type": "string" },
+        "sourceEvidence": { "type": "array" },
+        "sourceEvidenceStatus": { "enum": ["VERIFIED", "REVIEWER_REQUIRED"], "description": "VERIFIED only for a SOURCE_CONFLICT whose two quoted literals disagree; SOURCE_DEFECT is always REVIEWER_REQUIRED." },
         "dimensions": { "$ref": "#/$defs/dimensions" },
         "skippedTests": { "type": "array" },
         "flakyTests": { "type": "array" },

package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js

@@ -239,6 +239,163 @@ function evaluateSource(source, errors, prefix) {
   return { source: clone(source), traced: valid ? 1 : 0, total: 1, percentage: valid ? 100 : 0 };
 }
 
+// SOURCE_DEFECT and SOURCE_CONFLICT must quote the assertions they blame and
+// compare each with the observed value from the run. Sources are identified by
+// their declared authoritative path.
+const EXPECT_ASSERTION = /^expect\((.*?)\)\s*\.\s*(?:toEqual|toStrictEqual|toBe)\s*\((.*)\)$/;
+const ASSERT_ASSERTION = /^assert\s*\.\s*(?:equal|strictEqual|deepEqual|deepStrictEqual)\s*\((.*)\)$/;
+
+function jsonLiteral(text) {
+  const literal = text.trim();
+  const single = /^'([^'"\\]*)'$/.exec(literal);
+  try {
+    return { checkable: true, value: JSON.parse(single ? `"${single[1]}"` : literal) };
+  } catch (error) {
+    return { checkable: false };
+  }
+}
+
+// Splits an argument list on commas at depth 0, outside string literals.
+// Returns null when brackets or strings are unbalanced. Not a general parser.
+function splitTopLevelArguments(text) {
+  const parts = [];
+  let depth = 0;
+  let quote = null;
+  let current = '';
+  for (let index = 0; index < text.length; index += 1) {
+    const ch = text[index];
+    if (quote) {
+      current += ch;
+      if (ch === '\\') {
+        current += text[index + 1] ?? '';
+        index += 1;
+      } else if (ch === quote) {
+        quote = null;
+      }
+      continue;
+    }
+    if (ch === '"' || ch === "'" || ch === '`') {
+      quote = ch;
+    } else if ('([{'.includes(ch)) {
+      depth += 1;
+    } else if (')]}'.includes(ch)) {
+      depth -= 1;
+      if (depth < 0) return null;
+    } else if (ch === ',' && depth === 0) {
+      parts.push(current);
+      current = '';
+      continue;
+    }
+    current += ch;
+  }
+  if (quote !== null || depth !== 0) return null;
+  parts.push(current);
+  return parts;
+}
+
+// The literal expected value pinned by a quoted assertion, for the recognised
+// shapes only. Anything else (identifiers, constructors, prose, negated or
+// resolved modifiers) is not checkable. One trailing ';' is tolerated.
+function quotedExpectedValue(assertion) {
+  if (typeof assertion !== 'string') return { checkable: false };
+  const text = assertion.trim().replace(/;$/, '').trim();
+  const expectCall = EXPECT_ASSERTION.exec(text);
+  if (expectCall) return jsonLiteral(expectCall[2]);
+  const assertCall = ASSERT_ASSERTION.exec(text);
+  if (!assertCall) return { checkable: false };
+  // assert.equal(actual, expected[, message]): the expected value is the second argument.
+  const args = splitTopLevelArguments(assertCall[1]);
+  if (!args || (args.length !== 2 && args.length !== 3)) return { checkable: false };
+  return jsonLiteral(args[1]);
+}
+
+// Returns the sourceEvidence status. VERIFIED means two quoted literals from
+// different sources disagree (SOURCE_CONFLICT only). A quote/observed mismatch
+// alone shows the implementation differs from the source, not that the source
+// is defective, so SOURCE_DEFECT is always REVIEWER_REQUIRED.
+function evaluateSourceEvidence(requirement, declaredSources, errors, prefix) {
+  const label = requirement.sourceValidity;
+  const evidence = requirement.sourceEvidence;
+  if (!Array.isArray(evidence) || evidence.length === 0) {
+    errors.push(`${prefix}.sourceEvidence must quote the assertions for ${label}`);
+    return undefined;
+  }
+  const declaredPaths = declaredSources.filter(isObject).map(source => source.path);
+  const entries = [];
+  const checkable = new Map();
+  evidence.forEach((entry, index) => {
+    const entryPrefix = `${prefix}.sourceEvidence[${index}]`;
+    if (!isObject(entry)) {
+      errors.push(`${entryPrefix} must be an object`);
+      return;
+    }
+    if (!declaredPaths.includes(entry.source)) errors.push(`${entryPrefix}.source must name a declared authoritative source path`);
+    if (!nonEmpty(entry.assertion)) errors.push(`${entryPrefix}.assertion must quote the exact assertion text`);
+    if (!('expected' in entry)) errors.push(`${entryPrefix}.expected is required`);
+    if (!('observed' in entry)) errors.push(`${entryPrefix}.observed is required`);
+    if (declaredPaths.includes(entry.source) && nonEmpty(entry.assertion) && 'expected' in entry && 'observed' in entry) {
+      const quoted = quotedExpectedValue(entry.assertion);
+      if (quoted.checkable && !equal(quoted.value, entry.expected)) {
+        errors.push(`${entryPrefix}.expected does not match the value in the quoted assertion`);
+        return;
+      }
+      checkable.set(entry, quoted.checkable);
+      entries.push(entry);
+    }
+  });
+  let status;
+  const mismatched = entries.filter(entry => !equal(entry.expected, entry.observed));
+  if (mismatched.length === 0) {
+    errors.push(`${prefix}.sourceEvidence must compare a quoted assertion with a failing observed value before blaming the source`);
+  } else if (label === 'SOURCE_DEFECT') {
+    // No independent corroboration exists for a single source, so the reviewer decides.
+    status = 'REVIEWER_REQUIRED';
+  }
+  if (label === 'SOURCE_CONFLICT') {
+    if (new Set(entries.map(entry => entry.source)).size < 2) {
+      errors.push(`${prefix}.sourceEvidence must quote assertions from at least two distinct authoritative sources`);
+    }
+    // A conflict needs two sources constraining the same contract subject;
+    // unequal expectations about unrelated targets are not a conflict.
+    const subjects = new Map();
+    evidence.forEach((entry, index) => {
+      if (!entries.includes(entry)) return;
+      const entryPrefix = `${prefix}.sourceEvidence[${index}]`;
+      if (!nonEmpty(entry.subject)) {
+        errors.push(`${entryPrefix}.subject must name the contract target the assertion constrains`);
+      } else if (!assertionNamesSubject(entry.assertion, entry.subject.trim())) {
+        errors.push(`${entryPrefix}.subject must appear as a whole target in the quoted assertion`);
+      } else {
+        subjects.set(entry, entry.subject.trim());
+      }
+    });
+    const sameSubjectPairs = [];
+    for (const a of subjects.keys()) {
+      for (const b of subjects.keys()) {
+        if (a !== b && a.source !== b.source && subjects.get(a) === subjects.get(b)) sameSubjectPairs.push([a, b]);
+      }
+    }
+    const disagreeing = sameSubjectPairs.filter(([a, b]) => !equal(a.expected, b.expected));
+    if (subjects.size >= 2 && sameSubjectPairs.length === 0) {
+      errors.push(`${prefix}.sourceEvidence must quote assertions from two sources about the same subject; unequal expectations about unrelated targets are not a SOURCE_CONFLICT`);
+    } else if (sameSubjectPairs.length && disagreeing.length === 0) {
+      errors.push(`${prefix}.sourceEvidence expected values must disagree across sources; agreeing expectations with a failing observed value are NON_CONFORMANT`);
+    } else if (disagreeing.length) {
+      // Only a disagreement between two literal quotes is verified; a quote that
+      // cannot be read mechanically goes to the reviewer without an error.
+      status = disagreeing.some(([a, b]) => checkable.get(a) && checkable.get(b)) ? 'VERIFIED' : 'REVIEWER_REQUIRED';
+    }
+  }
+  return status;
+}
+
+// True when `subject` appears in `assertion` as a whole target: `list.head`
+// matches `expect(list.head)` but not `expect(list.head.value)`.
+function assertionNamesSubject(assertion, subject) {
+  const escaped = subject.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  return new RegExp(`(?<![\\w$.])${escaped}(?![\\w$.\\[])`).test(assertion);
+}
+
 function scoreDimensions(dimensions) {
   let weighted = 0;
   let appliedWeight = 0;
@@ -323,6 +480,10 @@ function evaluateRequirement(requirement, index, errors) {
   if (requirement.sourceValidity === 'SOURCE_CONFLICT' && declaredSources.some(source => !isObject(source) || source.status !== 'AUTHORITATIVE')) {
     errors.push(`${prefix}.SOURCE_CONFLICT may only identify authoritative sources`);
   }
+  let sourceEvidenceStatus;
+  if (['SOURCE_DEFECT', 'SOURCE_CONFLICT'].includes(requirement.sourceValidity)) {
+    sourceEvidenceStatus = evaluateSourceEvidence(requirement, declaredSources, errors, prefix);
+  }
 
   const scoring = scoringDetails(dimensions);
   const score = scoreDimensions(dimensions);
@@ -347,6 +508,8 @@ function evaluateRequirement(requirement, index, errors) {
     sourceConformance: requirement.sourceConformance || null,
     sourceValidity: requirement.sourceValidity || null,
     ...(requirement.sourceNotes ? { sourceNotes: requirement.sourceNotes } : {}),
+    ...(requirement.sourceEvidence ? { sourceEvidence: clone(requirement.sourceEvidence) } : {}),
+    ...(sourceEvidenceStatus ? { sourceEvidenceStatus } : {}),
     dimensions,
     skippedTests,
     flakyTests: Array.isArray(requirement.flakyTests) ? clone(requirement.flakyTests) : [],
@@ -432,6 +595,7 @@ function printSummary(report) {
   console.log('Requirements:');
   for (const requirement of report.requirements) {
     console.log(`- ${requirement.requirementId} [${requirement.testProfile}]: ${requirement.result} ${requirement.score}%`);
+    if (requirement.sourceEvidenceStatus) console.log(`- ${requirement.requirementId} sourceEvidence: ${requirement.sourceEvidenceStatus}`);
   }
   console.log('Dimensions:');
   for (const name of DIMENSIONS) {
@@ -472,4 +636,4 @@ function runCli(argv) {
 
 if (require.main === module) process.exit(runCli(process.argv));
 
-module.exports = { DIMENSIONS, REQUIRED_CLASSES, VERSION, evaluateEvidence, runCli };
+module.exports = { DIMENSIONS, REQUIRED_CLASSES, VERSION, evaluateEvidence, quotedExpectedValue, runCli };

package/plugins/harness-everything/skills/zoom-out/SKILL.md

@@ -4,7 +4,7 @@ description: "Reflect-first circuit breaker - after 3 failures, stop edits, rebu
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.40.0
+  version: 0.47.5
 ---
 
 # Zoom Out (Circuit Breaker)