nodebench-mcp 2.58.0 → 2.60.0

package/dist/benchmarks/llmJudgeEval.d.ts

@@ -0,0 +1,114 @@
+#!/usr/bin/env npx tsx
+/**
+ * llmJudgeEval.ts — LLM-judged boolean-metric eval harness for NodeBench MCP
+ *
+ * Architecture:
+ *   1. Query Corpus — 500+ typed test queries across 11 personas × 8 scenarios
+ *   2. Tool Executor — loads preset, runs discover_tools + tool chain, captures outputs
+ *   3. LLM Judge — Gemini Flash Lite boolean evaluation per criterion
+ *   4. Boolean Metrics — precision, recall, forbidden violations, criteria pass rate
+ *   5. Regression Detection — SQLite-backed diff between runs
+ *
+ * Usage:
+ *   cd packages/mcp-local
+ *   npx tsx src/benchmarks/llmJudgeEval.ts [--queries N] [--persona X] [--baseline RUN_ID] [--flywheel]
+ */
+export type Persona = "founder" | "banker" | "ceo" | "researcher" | "student" | "operator" | "legal" | "pm" | "contractor" | "investor" | "content";
+export type Scenario = "weekly_reset" | "company_search" | "competitor_brief" | "delegation" | "important_change" | "memo_export" | "packet_diff" | "role_switch";
+export interface BooleanCriterion {
+    criterion: string;
+    weight: number;
+}
+export interface EvalQuery {
+    id: string;
+    query: string;
+    persona: Persona;
+    scenario: Scenario;
+    expectedTools: string[];
+    forbiddenTools: string[];
+    booleanCriteria: BooleanCriterion[];
+}
+export interface CriterionResult {
+    criterion: string;
+    pass: boolean;
+    evidence: string;
+}
+export interface JudgeResponse {
+    criteria: CriterionResult[];
+    overallPass: boolean;
+}
+export interface QueryResult {
+    queryId: string;
+    pass: boolean;
+    criteriaResults: CriterionResult[];
+    toolsFired: string[];
+    toolPrecision: number;
+    toolRecall: number;
+    forbiddenViolations: number;
+    criteriaPassRate: number;
+    judgeResponse: string;
+    ms: number;
+}
+export interface RunSummary {
+    runId: string;
+    timestamp: string;
+    queryCount: number;
+    passRate: number;
+    avgToolPrecision: number;
+    avgToolRecall: number;
+    totalForbiddenViolations: number;
+    avgCriteriaPassRate: number;
+    byPersona: Record<string, {
+        pass: number;
+        total: number;
+        rate: number;
+    }>;
+    byScenario: Record<string, {
+        pass: number;
+        total: number;
+        rate: number;
+    }>;
+    byCriterion: Record<string, {
+        pass: number;
+        total: number;
+        rate: number;
+    }>;
+}
+export interface RegressionItem {
+    queryId: string;
+    criterion: string;
+    baselinePass: boolean;
+    currentPass: boolean;
+}
+/** Build the full 500-query corpus */
+export declare function generateQueryCorpus(): EvalQuery[];
+export declare function detectRegressions(currentRunId: string, baselineRunId: string): RegressionItem[];
+export declare function detectImprovements(currentRunId: string, baselineRunId: string): RegressionItem[];
+export interface RunOptions {
+    queryLimit: number;
+    persona?: Persona;
+    scenario?: Scenario;
+    baselineRunId?: string;
+    /** If true, only generate corpus and print stats without executing */
+    dryRun?: boolean;
+    /** If true, run self-improving flywheel loop: eval → diagnose → grow → re-eval */
+    flywheel?: boolean;
+}
+export declare function runLlmJudgeEval(options: RunOptions): Promise<RunSummary>;
+export type FailureRootCause = "tool_not_found" | "tool_error" | "empty_output" | "criteria_mismatch" | "heuristic_too_strict";
+export interface DiagnosisEntry {
+    queryId: string;
+    rootCause: FailureRootCause;
+    detail: string;
+    suggestedFix: string;
+}
+export interface DiagnosisReport {
+    runId: string;
+    totalFails: number;
+    byCause: Record<FailureRootCause, DiagnosisEntry[]>;
+    topSuggestions: string[];
+}
+/** Diagnose all FAIL results from a given run, grouping by root cause */
+export declare function diagnoseFailures(runId: string): Promise<DiagnosisReport>;
+/** Generate new corpus queries from a diagnosis report to cover gaps */
+export declare function growCorpus(diagnosis: DiagnosisReport): EvalQuery[];