nodebench-mcp 2.67.0 → 2.68.0

package/dist/benchmarks/pipelineEval.d.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env npx tsx
+/**
+ * pipelineEval.ts — Multi-step pipeline eval harness for NodeBench MCP
+ *
+ * Unlike llmJudgeEval.ts which tests tools independently, this harness tests
+ * realistic multi-step chains where Tool A's output feeds Tool B's input.
+ *
+ * Architecture:
+ *   1. Pipeline Definitions — 6 canonical pipelines modeling real agent workflows
+ *   2. Chaining Engine — executes steps sequentially, extracts fields from output
+ *   3. LLM Judge — Gemini Flash Lite evaluates 5 boolean criteria on full trace
+ *   4. Per-step tracking — tool name, args, output size, duration, pass/fail
+ *   5. SQLite persistence — pipeline_eval_runs + pipeline_eval_steps tables
+ *
+ * Usage:
+ *   cd packages/mcp-local
+ *   npx tsx src/benchmarks/pipelineEval.ts [--pipeline NAME] [--all]
+ */
+export type PipelineName = "founder_weekly_reset" | "company_intelligence" | "competitor_brief" | "pre_delegation" | "important_change_review" | "session_memory_cycle";
+export interface PipelineStep {
+    /** Tool name to call */
+    tool: string;
+    /** Static args merged with dynamic args from previous steps */
+    staticArgs: Record<string, unknown>;
+    /**
+     * Functions that extract values from the previous step's output to build
+     * dynamic args. Key = arg name, value = extractor function.
+     */
+    dynamicArgs?: Record<string, (prevOutput: unknown) => unknown>;
+    /** Human-readable description of what this step does */
+    description: string;
+}
+export interface PipelineDefinition {
+    name: PipelineName;
+    description: string;
+    steps: PipelineStep[];
+}
+export interface StepResult {
+    stepIndex: number;
+    tool: string;
+    description: string;
+    args: Record<string, unknown>;
+    ok: boolean;
+    outputSize: number;
+    outputPreview: string;
+    error?: string;
+    ms: number;
+}
+export interface PipelineCriterion {
+    criterion: string;
+    weight: number;
+    pass: boolean;
+    evidence: string;
+}
+export interface PipelineRunResult {
+    pipelineName: PipelineName;
+    runId: string;
+    steps: StepResult[];
+    criteria: PipelineCriterion[];
+    overallPass: boolean;
+    totalMs: number;
+    timestamp: string;
+}