@securecode-ai/mcp 0.5.5 → 0.5.7

package/dist/tooling/agentRegression.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Agent regression gate — compares a new eval result against a stored
+ * baseline and fails when precision, recall, or completion rate regresses
+ * beyond strict thresholds.
+ *
+ * Gate rules (per the user's decision: "Strict regression gate"):
+ *   - precision drop > 0.02  → FAIL
+ *   - recall drop > 0.02     → FAIL
+ *   - completion-rate drop > 0.03 → FAIL
+ *   - fixture/schema mismatch → FAIL
+ *   - cache version mismatch → FAIL (result was produced by an old prompt)
+ *   - commit SHA mismatch → FAIL (result was produced on a different commit)
+ *
+ * Usage:
+ *   npx tsx tooling/checkAgentRegression.ts --baseline <path> --result <path>
+ *
+ * The baseline is stored in tooling/agent-testset/baseline.json. It's
+ * regenerated when a deliberate quality change is made (run the eval,
+ * inspect the new metrics, if they're better or equal, update the baseline).
+ */
+import type { AgentEvalMetrics } from './agentEvalScoring';
+export interface AgentEvalReport {
+    timestamp: string;
+    manifest_version: number;
+    fixtureCount: number;
+    metrics: AgentEvalMetrics;
+    results: any[];
+    targets: {
+        recall_min: number;
+        precision_min: number;
+        fpr_max: number;
+        completion_rate_min: number;
+    };
+    pass: boolean;
+    cacheVersion?: number;
+    commitSha?: string;
+}
+export interface RegressionResult {
+    passed: boolean;
+    reasons: string[];
+    deltas: {
+        precisionDelta: number;
+        recallDelta: number;
+        completionRateDelta: number;
+        fprDelta: number;
+    };
+    baseline: AgentEvalMetrics;
+    candidate: AgentEvalMetrics;
+}
+export declare function checkRegression(baseline: AgentEvalReport, candidate: AgentEvalReport, options?: {
+    currentSha?: string;
+}): RegressionResult;
+export declare function formatRegressionResult(result: RegressionResult): string;

package/dist/tooling/agentRegression.js

@@ -0,0 +1,99 @@
+"use strict";
+/**
+ * Agent regression gate — compares a new eval result against a stored
+ * baseline and fails when precision, recall, or completion rate regresses
+ * beyond strict thresholds.
+ *
+ * Gate rules (per the user's decision: "Strict regression gate"):
+ *   - precision drop > 0.02  → FAIL
+ *   - recall drop > 0.02     → FAIL
+ *   - completion-rate drop > 0.03 → FAIL
+ *   - fixture/schema mismatch → FAIL
+ *   - cache version mismatch → FAIL (result was produced by an old prompt)
+ *   - commit SHA mismatch → FAIL (result was produced on a different commit)
+ *
+ * Usage:
+ *   npx tsx tooling/checkAgentRegression.ts --baseline <path> --result <path>
+ *
+ * The baseline is stored in tooling/agent-testset/baseline.json. It's
+ * regenerated when a deliberate quality change is made (run the eval,
+ * inspect the new metrics, if they're better or equal, update the baseline).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkRegression = checkRegression;
+exports.formatRegressionResult = formatRegressionResult;
+const child_process_1 = require("child_process");
+const scanCache_1 = require("../project-map/scanCache");
+const THRESHOLDS = {
+    maxPrecisionDrop: 0.02,
+    maxRecallDrop: 0.02,
+    maxCompletionRateDrop: 0.03,
+};
+function getCommitSha() {
+    try {
+        return (0, child_process_1.execSync)('git rev-parse HEAD', { encoding: 'utf8', windowsHide: true }).trim();
+    }
+    catch {
+        return 'unknown';
+    }
+}
+function checkRegression(baseline, candidate, options = {}) {
+    const reasons = [];
+    const b = baseline.metrics;
+    const c = candidate.metrics;
+    const precisionDelta = c.precision - b.precision;
+    const recallDelta = c.recall - b.recall;
+    const completionRateDelta = c.completionRate - b.completionRate;
+    const fprDelta = c.fpr - b.fpr;
+    if (precisionDelta < -THRESHOLDS.maxPrecisionDrop) {
+        reasons.push(`Precision dropped by ${Math.abs(precisionDelta).toFixed(4)} (max allowed: ${THRESHOLDS.maxPrecisionDrop}). ${b.precision.toFixed(4)} → ${c.precision.toFixed(4)}`);
+    }
+    if (recallDelta < -THRESHOLDS.maxRecallDrop) {
+        reasons.push(`Recall dropped by ${Math.abs(recallDelta).toFixed(4)} (max allowed: ${THRESHOLDS.maxRecallDrop}). ${b.recall.toFixed(4)} → ${c.recall.toFixed(4)}`);
+    }
+    if (completionRateDelta < -THRESHOLDS.maxCompletionRateDrop) {
+        reasons.push(`Completion rate dropped by ${Math.abs(completionRateDelta).toFixed(4)} (max allowed: ${THRESHOLDS.maxCompletionRateDrop}). ${b.completionRate.toFixed(4)} → ${c.completionRate.toFixed(4)}`);
+    }
+    if (baseline.fixtureCount !== candidate.fixtureCount) {
+        reasons.push(`Fixture count mismatch: baseline has ${baseline.fixtureCount}, candidate has ${candidate.fixtureCount}. Run the eval with the same testset.`);
+    }
+    if (baseline.manifest_version !== candidate.manifest_version) {
+        reasons.push(`Manifest version mismatch: baseline is v${baseline.manifest_version}, candidate is v${candidate.manifest_version}. Update the baseline.`);
+    }
+    if (candidate.cacheVersion !== undefined && candidate.cacheVersion !== scanCache_1.AGENT_SCAN_CACHE_VERSION) {
+        reasons.push(`Cache version mismatch: candidate result was produced with cache version ${candidate.cacheVersion}, but current AGENT_SCAN_CACHE_VERSION is ${scanCache_1.AGENT_SCAN_CACHE_VERSION}. The result is stale — re-run the eval.`);
+    }
+    const candidateSha = candidate.commitSha;
+    const expectedSha = options.currentSha ?? getCommitSha();
+    if (candidateSha && candidateSha !== 'unknown' && candidateSha !== expectedSha) {
+        reasons.push(`Commit SHA mismatch: candidate result was produced on ${candidateSha.slice(0, 8)}, but current HEAD is ${expectedSha.slice(0, 8)}. Re-run the eval on the current commit.`);
+    }
+    return {
+        passed: reasons.length === 0,
+        reasons,
+        deltas: { precisionDelta, recallDelta, completionRateDelta, fprDelta },
+        baseline: b,
+        candidate: c,
+    };
+}
+function formatRegressionResult(result) {
+    const lines = [];
+    const status = result.passed ? 'PASS' : 'FAIL';
+    lines.push(`## Regression Gate: ${status}`);
+    lines.push('');
+    lines.push('| Metric | Baseline | Candidate | Delta | Threshold |');
+    lines.push('|--------|----------|-----------|-------|-----------|');
+    lines.push(`| Precision | ${result.baseline.precision.toFixed(4)} | ${result.candidate.precision.toFixed(4)} | ${result.deltas.precisionDelta >= 0 ? '+' : ''}${result.deltas.precisionDelta.toFixed(4)} | ≥ -${THRESHOLDS.maxPrecisionDrop} |`);
+    lines.push(`| Recall | ${result.baseline.recall.toFixed(4)} | ${result.candidate.recall.toFixed(4)} | ${result.deltas.recallDelta >= 0 ? '+' : ''}${result.deltas.recallDelta.toFixed(4)} | ≥ -${THRESHOLDS.maxRecallDrop} |`);
+    lines.push(`| Completion Rate | ${result.baseline.completionRate.toFixed(4)} | ${result.candidate.completionRate.toFixed(4)} | ${result.deltas.completionRateDelta >= 0 ? '+' : ''}${result.deltas.completionRateDelta.toFixed(4)} | ≥ -${THRESHOLDS.maxCompletionRateDrop} |`);
+    lines.push(`| FPR | ${result.baseline.fpr.toFixed(4)} | ${result.candidate.fpr.toFixed(4)} | ${result.deltas.fprDelta >= 0 ? '+' : ''}${result.deltas.fprDelta.toFixed(4)} | (info) |`);
+    if (result.reasons.length > 0) {
+        lines.push('');
+        lines.push('### Failure Reasons:');
+        for (const r of result.reasons) {
+            lines.push(`- ${r}`);
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=agentRegression.js.map

package/dist/tooling/agentRegression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentRegression.js","sourceRoot":"","sources":["../../src/tooling/agentRegression.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;GAmBG;;AA+CH,0CAmDC;AAED,wDAqBC;AArHD,iDAAyC;AACzC,wDAAoE;AA4BpE,MAAM,UAAU,GAAG;IACf,gBAAgB,EAAE,IAAI;IACtB,aAAa,EAAE,IAAI;IACnB,qBAAqB,EAAE,IAAI;CACrB,CAAC;AAEX,SAAS,YAAY;IACjB,IAAI,CAAC;QACD,OAAO,IAAA,wBAAQ,EAAC,oBAAoB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1F,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC;AAED,SAAgB,eAAe,CAC3B,QAAyB,EACzB,SAA0B,EAC1B,UAAmC,EAAE;IAErC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC;IAC3B,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC;IAE5B,MAAM,cAAc,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;IACjD,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,mBAAmB,GAAG,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC;IAChE,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;IAE/B,IAAI,cAAc,GAAG,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,UAAU,CAAC,gBAAgB,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrL,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,qBAAqB,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,UAAU,CAAC,aAAa,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACtK,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,UAAU,CAAC,qBAAqB,EAAE,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,UAAU,CAAC,qBAAqB,MAAM,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/M,CAAC;IAED,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;QACnD,OAAO,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,YAAY,mBAAmB,SAAS,CAAC,YAAY,uCAAuC,CAAC,CAAC;IAChK,CAAC;IAED,IAAI,QAAQ,CAAC,gBAAgB,KAAK,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,2CAA2C,QAAQ,CAAC,gBAAgB,mBAAmB,SAAS,CAAC,gBAAgB,wBAAwB,CAAC,CAAC;IAC5J,CAAC;IAED,IAAI,SAAS,CAAC,YAAY,KAAK,SAAS,IAAI,SAAS,CAAC,YAAY,KAAK,oCAAwB,EAAE,CAAC;QAC9F,OAAO,CAAC,IAAI,CAAC,4EAA4E,SAAS,CAAC,YAAY,6CAA6C,oCAAwB,0CAA0C,CAAC,CAAC;IACpO,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,YAAY,EAAE,CAAC;IACzD,IAAI,YAAY,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;QAC7E,OAAO,CAAC,IAAI,CAAC,yDAAyD,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,yBAAyB,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,0CAA0C,CAAC,CAAC;IAC9L,CAAC;IAED,OAAO;QACH,MAAM,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;QAC5B,OAAO;QACP,MAAM,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE;QACtE,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,CAAC;KACf,CAAC;AACN,CAAC;AAED,SAAgB,sBAAsB,CAAC,MAAwB;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACjP,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,aAAa,IAAI,CAAC,CAAC;IAC/N,KAAK,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,mBAAmB,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,qBAAqB,IAAI,CAAC,CAAC;IAChR,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IAExL,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC"}

package/dist/tools/agentScan.d.ts

@@ -16,4 +16,73 @@
  *   6. Return result with proven stamps
  */
 import type { ServerContext } from '../mcp/types';
+import type { AgentScanFinding, VerificationLevel } from '../attack/agentScanProtocol';
+interface ProvenFinding extends AgentScanFinding {
+    proven: 'PROVEN' | 'UNPROVEN' | 'INCONCLUSIVE' | 'NOT_REPRODUCIBLE' | 'SKIPPED';
+    provenReason?: string;
+    evidenceLevel?: string;
+    originalConfidence?: number;
+    verificationLevel?: VerificationLevel;
+    fixStatus?: 'fix-generated' | 'fix-denied' | 'fix-error' | 'fix-verified-closed' | 'fix-still-vulnerable' | 'fix-verification-inconclusive' | 'fix-syntax-invalid';
+    fixDeniedReason?: string;
+    fixApprovalId?: string;
+    /** Sub-verdict from the verify loop — used to map to a review reason. */
+    verifySubVerdict?: string;
+    /** Human review queue status for INCONCLUSIVE findings. */
+    reviewStatus?: 'pending' | 'confirmed' | 'rejected' | 'deferred';
+    reviewId?: string;
+    /** Result of re-verifying the exploit against the fixed code. */
+    fixVerification?: FixVerificationResult;
+    /** Structured proof evidence from the proof gate, if the finding was verified. */
+    proofEvidence?: import('../attack/proofTypes').ProofEvidence;
+    /** Result of the deterministic proof gate evaluation. */
+    proofGateResult?: import('../attack/proofTypes').ProofGateResult;
+    /** Whether human review is required before acting on this finding. */
+    humanReviewRequired?: boolean;
+}
+export interface FixVerificationResult {
+    status: 'not-run' | 'closed' | 'still-vulnerable' | 'inconclusive' | 'syntax-invalid' | 'sandbox-unavailable' | 'cancelled';
+    reason: string;
+    originalVerdict: 'PROVEN' | 'UNPROVEN' | 'INCONCLUSIVE' | 'SKIPPED';
+    fixedVerdict?: 'PROVEN' | 'UNPROVEN' | 'INCONCLUSIVE';
+    roundsUsed: number;
+    fixedCodeHash: string;
+}
 export declare function toolAgentScan(ctx: ServerContext, args: any): Promise<unknown>;
+/**
+ * Explicit confidence-ceiling policy for a finding, based on:
+ *   - the verify verdict (PROVEN / UNPROVEN / INCONCLUSIVE /
+ *     NOT_REPRODUCIBLE / SKIPPED)
+ *   - the language capability tier (deep / standard / fallback)
+ *   - how many structural evidence tools the agent actually used
+ *     (trace_flow, check_guard, check_policy)
+ *
+ * Returns `undefined` for PROVEN (no ceiling — the proof overrules the
+ * LLM's belief). For everything else, returns the maximum allowed
+ * confidence. The caller takes `Math.min(confidence, ceiling)` so multiple
+ * applicable bounds compose correctly — last-write-wins is impossible.
+ *
+ * Verdict semantics:
+ *   - PROVEN: exploit test ran and reproduced the vulnerability. Strong
+ *     evidence. No ceiling; floor at 80.
+ *   - UNPROVEN: exploit test ran and did NOT reproduce. The finding is
+ *     likely a false positive. Hard cap at 25.
+ *   - NOT_REPRODUCIBLE: exploit test ran, but the test setup couldn't
+ *     trigger the vulnerability (e.g. the path requires a live DB). This
+ *     is NOT the same as UNPROVEN (which actively disproved) and NOT the
+ *     same as INCONCLUSIVE (which couldn't even run a test). Treat it as
+ *     "we tried and couldn't confirm" — cap at 35, tighter than
+ *     INCONCLUSIVE but looser than UNPROVEN.
+ *   - INCONCLUSIVE: no test could be generated or the sandbox was
+ *     unavailable. The finding is unverified — keep the LLM's confidence
+ *     but cap it based on structural evidence (no tools → ≤40, ≥2 tools
+ *     → ≤75). Fallback-tier languages get ≤55 across the board.
+ *   - SKIPPED: low severity, we didn't even try. Same as INCONCLUSIVE.
+ */
+export declare function confidenceCeilingForFinding(verdict: ProvenFinding['proven'], capabilityTier: 'deep' | 'standard' | 'fallback', evidenceTools: number): number | undefined;
+/**
+ * Apply the confidence clamp to a single finding. Exported for testing
+ * so the test imports the real policy, not a copy that can drift.
+ */
+export declare function applyConfidenceClamp(original: number, verdict: ProvenFinding['proven'], capabilityTier: 'deep' | 'standard' | 'fallback', evidenceTools: number): number;
+export {};