fraim 2.0.290 → 2.0.291

package/dist/src/core/test-evidence-contract.js

@@ -0,0 +1,119 @@
+"use strict";
+/**
+ * Test Evidence Contract Validation (Issue #1419, Change 4)
+ *
+ * The #1419 spike found that a written "never report a result you have not
+ * read" instruction is not reliably obeyed even by an agent that has just
+ * stated, in its own text, that the work is outstanding
+ * (docs/evidence/1419-spike-findings.md, Round 2). This adds a mechanical
+ * backstop that does not depend on the model remembering the instruction: a
+ * phase whose job-frontmatter declares `requiresTestEvidence: true` must
+ * supply `findings.testEvidence` — the actual observed result (exit code,
+ * duration, and either a timestamp or the log path read) — before that phase
+ * can be marked `complete`.
+ *
+ * This is a schema-shape check only. It cannot verify the numbers are
+ * truthful, only that the agent committed to a concrete, falsifiable claim
+ * instead of a bare "passed" — the same "operationalize the existing
+ * Supervision Contract line as a resume prompt rather than leave it to the
+ * agent to remember" pattern the RFC uses for Change 3's continuation message.
+ *
+ * Lives in src/core so both the local MCP proxy and evals can import the same
+ * contract without cross-layer dependencies — same pattern as
+ * quality-evidence.ts and handoff-contracts.ts.
+ *
+ * Primary call site: src/local-mcp-server/stdio-server.ts seekMentoring
+ * handler, after the existing handoff-contract enforcement block.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isTestEvidencePhase = isTestEvidencePhase;
+exports.validateTestEvidence = validateTestEvidence;
+exports.validateTestEvidenceContract = validateTestEvidenceContract;
+exports.buildTestEvidenceRejectionMessage = buildTestEvidenceRejectionMessage;
+/**
+ * Returns true when `currentPhase`'s job-frontmatter declares
+ * `requiresTestEvidence: true`. No hardcoded phase-name list: any job/phase
+ * combination opts in by declaring the flag on its own `phases` entry.
+ */
+function isTestEvidencePhase(currentPhase, phases) {
+    return phases?.[currentPhase]?.requiresTestEvidence === true;
+}
+/**
+ * Validates findings.testEvidence. Returns null if valid, or an array of
+ * human-readable error strings describing what is wrong.
+ *
+ * Required minimum shape:
+ * {
+ *   exitCode: number,
+ *   durationMs: number,
+ *   timestamp: string,   // either this...
+ *   logPath: string,     // ...or this
+ * }
+ */
+function validateTestEvidence(value) {
+    if (value === undefined || value === null) {
+        return ['findings.testEvidence is missing'];
+    }
+    if (typeof value !== 'object' || Array.isArray(value)) {
+        return ['findings.testEvidence must be an object'];
+    }
+    const obj = value;
+    const errors = [];
+    if (typeof obj.exitCode !== 'number' || !Number.isFinite(obj.exitCode)) {
+        errors.push(`findings.testEvidence.exitCode must be a number (got ${obj.exitCode === undefined ? 'missing' : typeof obj.exitCode})`);
+    }
+    if (typeof obj.durationMs !== 'number' || !Number.isFinite(obj.durationMs) || obj.durationMs < 0) {
+        errors.push(`findings.testEvidence.durationMs must be a non-negative number (got ${obj.durationMs === undefined ? 'missing' : typeof obj.durationMs})`);
+    }
+    const hasTimestamp = typeof obj.timestamp === 'string' && obj.timestamp.trim().length > 0;
+    const hasLogPath = typeof obj.logPath === 'string' && obj.logPath.trim().length > 0;
+    if (!hasTimestamp && !hasLogPath) {
+        errors.push('findings.testEvidence must include a non-empty timestamp or logPath — the point of this evidence is proving the result was actually observed, not merely narrated');
+    }
+    return errors.length > 0 ? errors : null;
+}
+/**
+ * Orchestrator: called from the seekMentoring handler. Returns an empty
+ * array when no contract is violated for this call (phase doesn't require
+ * test evidence, or the call is not a completion).
+ */
+function validateTestEvidenceContract(args) {
+    const phase = args.currentPhase ?? '';
+    if (args.status !== 'complete')
+        return [];
+    if (!isTestEvidencePhase(phase, args.phases))
+        return [];
+    return validateTestEvidence((args.findings ?? {}).testEvidence) ?? [];
+}
+const TEST_EVIDENCE_SCHEMA = `\`\`\`javascript
+findings: {
+  testEvidence: {
+    exitCode: 0,                          // the real exit code you read, not assumed
+    durationMs: 12345,                    // how long the run actually took
+    timestamp: "2026-08-29T18:00:00Z",    // when you observed it...
+    logPath: "path/to/run.log"            // ...and/or where the log lives
+  }
+}
+\`\`\``;
+/**
+ * Builds the rejection message returned to the agent when the test evidence
+ * contract is violated.
+ */
+function buildTestEvidenceRejectionMessage(currentPhase, errors) {
+    const errorBullets = errors.map((e) => `- ${e}`).join('\n');
+    return [
+        `❌ **seekMentoring rejected** at phase \`${currentPhase}\`.`,
+        '',
+        `This phase's Outcome depends on a test/validation run. Report the actual observed result, ` +
+            `not a narrated success, before it can complete. The following problems were found:`,
+        '',
+        errorBullets,
+        '',
+        'Required minimum schema:',
+        '',
+        TEST_EVIDENCE_SCHEMA,
+        '',
+        'The job is **not** marked complete. Run the validation, read its real exit code and duration, ' +
+            'and resubmit with `findings.testEvidence` populated.',
+    ].join('\n');
+}

package/dist/src/local-mcp-server/stdio-server.js

@@ -66,6 +66,7 @@ const local_registry_resolver_1 = require("../core/utils/local-registry-resolver
 const ai_mentor_1 = require("../core/ai-mentor");
 const quality_evidence_1 = require("../core/quality-evidence");
 const handoff_contracts_1 = require("../core/handoff-contracts");
+const test_evidence_contract_1 = require("../core/test-evidence-contract");
 const project_fraim_paths_1 = require("../core/utils/project-fraim-paths");
 const usage_collector_js_1 = require("./usage-collector.js");
 const otlp_metrics_receiver_js_1 = require("./otlp-metrics-receiver.js");
@@ -2298,6 +2299,27 @@ class FraimLocalMCPServer {
                             return await this.finalizeLocalToolTextResponse(request, requestSessionId, requestId, rejection);
                         }
                     }
+                    // Test evidence contract enforcement (Issue #1419, Change 4).
+                    //
+                    // A phase whose job-frontmatter declares requiresTestEvidence: true
+                    // (e.g. implement-validate, implement-regression) must supply the
+                    // actual observed test/validation result — not a narrated success —
+                    // before it can be marked complete. This is the mechanical backstop
+                    // the #1419 spike found necessary: a written policy note alone was
+                    // not obeyed even by an agent that had just stated the work was
+                    // outstanding. Reuses handoffPhaseMap (same mentor.getJobPhaseMap
+                    // call already made above) rather than re-fetching the job's phases.
+                    const testEvidenceErrors = (0, test_evidence_contract_1.validateTestEvidenceContract)({
+                        currentPhase: args.currentPhase,
+                        status: args.status,
+                        findings: args.findings,
+                        phases: handoffPhaseMap,
+                    });
+                    if (testEvidenceErrors.length > 0) {
+                        this.log(`⚠️ Test evidence contract rejected seekMentoring for ${args.jobName}:${args.currentPhase}: ${testEvidenceErrors.join('; ')}`);
+                        const rejection = (0, test_evidence_contract_1.buildTestEvidenceRejectionMessage)(args.currentPhase, testEvidenceErrors);
+                        return await this.finalizeLocalToolTextResponse(request, requestSessionId, requestId, rejection);
+                    }
                     return await this.finalizeLocalToolTextResponse(request, requestSessionId, requestId, tutoringResponse.message);
                 }
                 catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fraim",
-  "version": "2.0.290",
+  "version": "2.0.291",
   "description": "FRAIM core CLI and MCP package.",
   "main": "index.js",
   "bin": {