docorbit 0.1.0 → 0.1.2

package/packages/evaluation/src/runner.ts

@@ -1,241 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import os from 'node:os';
-import { BENCHMARK_DATASET } from './dataset.ts';
-import { DocOrbitRunner } from './strategies/docorbit-runner.ts';
-import { Context7Runner, Context7SimulatedRunner } from './strategies/context7-runner.ts';
-import { OfficialDocsFetchRunner, OfficialDocsFetchSimulatedRunner, WebSearchRunner, WebSearchSimulatedRunner } from './strategies/web-search-runner.ts';
-import { FirecrawlRunner, FirecrawlSimulatedRunner } from './strategies/firecrawl-runner.ts';
-import type {
-  BenchmarkTaskDef,
-  BenchmarkTaskResult,
-  BenchmarkSuiteReport,
-  StrategyAggregateMetrics,
-  RunnerOptions,
-  StrategyRunner,
-  AgentEvaluationStrategy,
-} from './types.ts';
-
-export class BenchmarkRunner {
-  private runners: Map<AgentEvaluationStrategy, StrategyRunner>;
-
-  constructor(options: Pick<RunnerOptions, 'simulation'> = {}) {
-    const sim = options.simulation ?? false;
-    const docsRunner = sim ? new OfficialDocsFetchSimulatedRunner() : new OfficialDocsFetchRunner();
-    this.runners = new Map<AgentEvaluationStrategy, StrategyRunner>([
-      ['agent_docorbit', new DocOrbitRunner()],
-      ['agent_context7', sim ? new Context7SimulatedRunner() : new Context7Runner()],
-      ['agent_firecrawl', sim ? new FirecrawlSimulatedRunner() : new FirecrawlRunner()],
-      ['agent_official_docs_fetch', docsRunner],
-      ['agent_web_search', docsRunner],
-    ]);
-  }
-
-  async run(options: RunnerOptions = {}): Promise<BenchmarkSuiteReport> {
-    const split = options.split || 'all';
-    const activeStrategies = options.strategies || ['agent_docorbit', 'agent_context7', 'agent_firecrawl', 'agent_official_docs_fetch'];
-    const outputDir = options.outputDir || path.join(process.cwd(), 'eval-results');
-    const sim = options.simulation ?? false;
-    fs.mkdirSync(path.join(outputDir, 'raw'), { recursive: true });
-
-    let tasks: BenchmarkTaskDef[] = BENCHMARK_DATASET;
-    if (split !== 'all') {
-      tasks = tasks.filter((t) => t.split === split);
-    }
-    if (options.tasks && options.tasks.length > 0) {
-      const taskSet = new Set(options.tasks);
-      tasks = tasks.filter((t) => taskSet.has(t.id));
-    }
-
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'docorbit-bench-'));
-    const allResults: BenchmarkTaskResult[] = [];
-    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-    const mode = sim ? 'SIMULATION' : 'REAL';
-
-    if (options.verbose) {
-      console.log(`\n🔬 DocOrbit Benchmark — Mode: ${mode} | Tasks: ${tasks.length} | Split: ${split}`);
-      if (!sim) {
-        console.log(`   ⚠  Real mode requires network access (context7-mcp + Firecrawl API + HTTPS docs fetch).`);
-        console.log(`   ⚠  Run via: node --experimental-strip-types ... (not inside sandbox)`);
-      }
-    }
-
-    try {
-      for (const task of tasks) {
-        for (const strategy of activeStrategies) {
-          const runner = this.runners.get(strategy);
-          if (!runner) continue;
-
-          if (options.verbose) {
-            process.stdout.write(`  Running ${strategy}/${task.id}... `);
-          }
-
-          const result = await runner.executeTask(task, tempDir);
-          allResults.push(result);
-
-          if (options.verbose) {
-            const statusIcon = result.taskSuccess ? '✔' : '✘';
-            console.log(`${statusIcon}  (${result.latencyMs.toFixed(0)}ms, ${result.tokenUsage} tokens, version=${result.correctVersionSelected})`);
-          }
-
-          // Save raw JSON artifact for every result — auditable, reproducible
-          const rawArtifactPath = path.join(
-            outputDir, 'raw',
-            `${timestamp}_${task.id}_${strategy}.json`,
-          );
-          fs.writeFileSync(
-            rawArtifactPath,
-            JSON.stringify({ timestamp, mode, task, result }, null, 2),
-            'utf-8',
-          );
-        }
-      }
-    } finally {
-      fs.rmSync(tempDir, { recursive: true, force: true });
-    }
-
-    // Aggregate metrics per strategy
-    const byStrategy: Record<AgentEvaluationStrategy, StrategyAggregateMetrics> = {} as any;
-
-    for (const strategy of activeStrategies) {
-      const stratResults = allResults.filter((r) => r.strategy === strategy);
-      const evalResults = stratResults.filter((r) => r.split === 'eval');
-
-      const tokens = stratResults.map((r) => r.tokenUsage).sort((a, b) => a - b);
-      const precisions = stratResults.map((r) => r.retrievalPrecision);
-      const recalls = stratResults.map((r) => r.retrievalRecall);
-      const latencies = stratResults.map((r) => r.latencyMs);
-      const toolCalls = stratResults.map((r) => r.toolCallsCount);
-
-      const mean = (arr: number[]) => arr.length > 0 ? arr.reduce((a, b) => a + b, 0) / arr.length : 0;
-      const median = (arr: number[]) => arr.length > 0 ? arr[Math.floor(arr.length / 2)] : 0;
-
-      byStrategy[strategy] = {
-        overallSuccessRate: stratResults.length > 0 ? stratResults.filter((r) => r.taskSuccess).length / stratResults.length : 0,
-        evalSuccessRate: evalResults.length > 0 ? evalResults.filter((r) => r.taskSuccess).length / evalResults.length : 0,
-        versionAccuracy: stratResults.length > 0 ? stratResults.filter((r) => r.correctVersionSelected).length / stratResults.length : 0,
-        meanPrecision: mean(precisions),
-        meanRecall: mean(recalls),
-        meanTokens: mean(tokens),
-        medianTokens: median(tokens),
-        meanLatencyMs: mean(latencies),
-        meanToolCalls: mean(toolCalls),
-        totalVerificationCatches: stratResults.reduce((acc, r) => acc + r.verificationCatches, 0),
-        falsePositives: stratResults.reduce((acc, r) => acc + r.verificationFalsePositives, 0),
-        insufficientEvidenceRate: stratResults.length > 0 ? stratResults.reduce((acc, r) => acc + r.insufficientEvidenceCount, 0) / stratResults.length : 0,
-      };
-    }
-
-    const trainTasksCount = tasks.filter((t) => t.split === 'train').length;
-    const evalTasksCount = tasks.filter((t) => t.split === 'eval').length;
-    const verificationTasksCount = tasks.filter((t) => t.split === 'verification').length;
-
-    const report: BenchmarkSuiteReport = {
-      timestamp,
-      totalTasks: tasks.length,
-      trainTasks: trainTasksCount,
-      evalTasks: evalTasksCount,
-      verificationTasks: verificationTasksCount,
-      byStrategy,
-      tasks: allResults,
-    };
-
-    // Save summary artifacts
-    const summaryJsonPath = path.join(outputDir, `summary_${timestamp}.json`);
-    fs.writeFileSync(summaryJsonPath, JSON.stringify(report, null, 2), 'utf-8');
-
-    const markdownReport = this.generateMarkdownReport(report, mode);
-    const summaryMdPath = path.join(outputDir, `summary_${timestamp}.md`);
-    fs.writeFileSync(summaryMdPath, markdownReport, 'utf-8');
-
-    if (options.verbose) {
-      console.log('\n' + markdownReport);
-      console.log(`\nArtifacts saved to: ${outputDir}/`);
-    }
-
-    return report;
-  }
-
-  generateMarkdownReport(report: BenchmarkSuiteReport, mode = 'REAL'): string {
-    const isSim = mode === 'SIMULATION';
-    const modeNote = isSim
-      ? '\n> **⚠ SIMULATION MODE** — Results are offline models of real behavior for CI regression testing. Run without `--simulation` for real results.\n'
-      : '\n> **✅ REAL MODE** — Context7: real `context7-mcp` stdio subprocess. Firecrawl: real `api.firecrawl.dev` scrape API. Official Docs Fetch: direct HTTPS fetch to official docs URLs. DocOrbit: real in-process MCP server.\n';
-
-    const lines: string[] = [];
-    lines.push(`# DocOrbit Empirical Evaluation & Benchmark Report`);
-    lines.push(`**Mode**: ${mode} | **Generated**: ${report.timestamp}`);
-    const taskBreakdown = [
-      `${report.totalTasks} total`,
-      `${report.trainTasks} train`,
-      `${report.evalTasks} held-out eval`,
-      report.verificationTasks ? `${report.verificationTasks} verification` : undefined,
-    ].filter(Boolean).join(', ');
-    lines.push(`**Tasks Evaluated**: ${taskBreakdown}`);
-    lines.push(modeNote);
-    lines.push('## Comparative System Metrics');
-    lines.push('');
-    const sDocs = report.byStrategy.agent_official_docs_fetch || report.byStrategy.agent_web_search;
-    const sC7 = report.byStrategy.agent_context7;
-    const sFC = report.byStrategy.agent_firecrawl;
-    const sDR = report.byStrategy.agent_docorbit;
-
-    if (sDocs && sC7 && sDR) {
-      if (sFC) {
-        lines.push('| Metric | Official Docs Fetch | Firecrawl | Context7 (Real MCP) | DocOrbit (Real MCP) |');
-        lines.push('| :--- | :---: | :---: | :---: | :---: |');
-        lines.push(`| **Task Success Rate** | ${pct(sDocs.overallSuccessRate)} | ${pct(sFC.overallSuccessRate)} | ${pct(sC7.overallSuccessRate)} | ${pct(sDR.overallSuccessRate)} |`);
-        lines.push(`| **Held-out Eval Success** | ${pct(sDocs.evalSuccessRate)} | ${pct(sFC.evalSuccessRate)} | ${pct(sC7.evalSuccessRate)} | ${pct(sDR.evalSuccessRate)} |`);
-        lines.push(`| **Correct Version Selection** | ${pct(sDocs.versionAccuracy)} | ${pct(sFC.versionAccuracy)} | ${pct(sC7.versionAccuracy)} | ${pct(sDR.versionAccuracy)} |`);
-        lines.push(`| **Retrieval Precision@k** | ${pct(sDocs.meanPrecision)} | ${pct(sFC.meanPrecision)} | ${pct(sC7.meanPrecision)} | ${pct(sDR.meanPrecision)} |`);
-        lines.push(`| **Retrieval Recall@k** | ${pct(sDocs.meanRecall)} | ${pct(sFC.meanRecall)} | ${pct(sC7.meanRecall)} | ${pct(sDR.meanRecall)} |`);
-        lines.push(`| **Mean Tokens** | ~${Math.round(sDocs.meanTokens)} | ~${Math.round(sFC.meanTokens)} | ~${Math.round(sC7.meanTokens)} | ~${Math.round(sDR.meanTokens)} |`);
-        lines.push(`| **Mean Latency** | ~${sDocs.meanLatencyMs.toFixed(0)}ms | ~${sFC.meanLatencyMs.toFixed(0)}ms | ~${sC7.meanLatencyMs.toFixed(0)}ms | ~${sDR.meanLatencyMs.toFixed(0)}ms |`);
-        lines.push(`| **Avg Tool Calls** | ${sDocs.meanToolCalls.toFixed(1)} | ${sFC.meanToolCalls.toFixed(1)} | ${sC7.meanToolCalls.toFixed(1)} | ${sDR.meanToolCalls.toFixed(1)} |`);
-        lines.push(`| **AST Verification Catches** | — | — | — | ${sDR.totalVerificationCatches} |`);
-        lines.push(`| **False Positives** | — | — | — | ${sDR.falsePositives} |`);
-      } else {
-        lines.push('| Metric | Official Docs Fetch | Context7 (Real MCP) | DocOrbit (Real MCP) |');
-        lines.push('| :--- | :---: | :---: | :---: |');
-        lines.push(`| **Task Success Rate** | ${pct(sDocs.overallSuccessRate)} | ${pct(sC7.overallSuccessRate)} | ${pct(sDR.overallSuccessRate)} |`);
-        lines.push(`| **Held-out Eval Success** | ${pct(sDocs.evalSuccessRate)} | ${pct(sC7.evalSuccessRate)} | ${pct(sDR.evalSuccessRate)} |`);
-        lines.push(`| **Correct Version Selection** | ${pct(sDocs.versionAccuracy)} | ${pct(sC7.versionAccuracy)} | ${pct(sDR.versionAccuracy)} |`);
-        lines.push(`| **Retrieval Precision@k** | ${pct(sDocs.meanPrecision)} | ${pct(sC7.meanPrecision)} | ${pct(sDR.meanPrecision)} |`);
-        lines.push(`| **Retrieval Recall@k** | ${pct(sDocs.meanRecall)} | ${pct(sC7.meanRecall)} | ${pct(sDR.meanRecall)} |`);
-        lines.push(`| **Mean Tokens** | ~${Math.round(sDocs.meanTokens)} | ~${Math.round(sC7.meanTokens)} | ~${Math.round(sDR.meanTokens)} |`);
-        lines.push(`| **Mean Latency** | ~${sDocs.meanLatencyMs.toFixed(0)}ms | ~${sC7.meanLatencyMs.toFixed(0)}ms | ~${sDR.meanLatencyMs.toFixed(0)}ms |`);
-        lines.push(`| **Avg Tool Calls** | ${sDocs.meanToolCalls.toFixed(1)} | ${sC7.meanToolCalls.toFixed(1)} | ${sDR.meanToolCalls.toFixed(1)} |`);
-        lines.push(`| **AST Verification Catches** | — | — | ${sDR.totalVerificationCatches} |`);
-        lines.push(`| **False Positives** | — | — | ${sDR.falsePositives} |`);
-      }
-    }
-
-    lines.push('');
-    lines.push('## AST Code Verification (`check_api`) Performance');
-    lines.push('');
-    lines.push('DocOrbit provides closed-loop AST verification (`check_api`) to validate generated code against authoritative schemas and constraints. Other baselines lack code verification capabilities.');
-    lines.push('');
-    lines.push(`- **Total Verification Catches**: ${sDR?.totalVerificationCatches ?? 0}`);
-    lines.push(`- **False Positives on Valid Code**: ${sDR?.falsePositives ?? 0}`);
-    lines.push(`- **Dynamic / Ambiguous Expression Safe Fallback**: ${sDR ? `${(sDR.insufficientEvidenceRate * 100).toFixed(0)}%` : '0%'}`);
-    lines.push('');
-
-    lines.push('## Per-Task Execution Breakdown');
-    lines.push('');
-    lines.push('| Task | Split | Strategy | Sim? | VersionOK | Success | Tokens | Latency | Catches | Notes |');
-    lines.push('| :--- | :---: | :--- | :---: | :---: | :---: | :---: | :---: | :---: | :--- |');
-
-    for (const res of report.tasks) {
-      const simFlag = res.isSimulation ? '🔶' : '✅';
-      lines.push(
-        `| \`${res.taskId}\` | ${res.split} | \`${res.strategy}\` | ${simFlag} | ${res.correctVersionSelected ? '✔' : '✘'} | ${res.taskSuccess ? '✔' : '✘'} | ${res.tokenUsage} | ${res.latencyMs.toFixed(0)}ms | ${res.verificationCatches} | ${res.notes.slice(0, 75)} |`
-      );
-    }
-
-    return lines.join('\n');
-  }
-}
-
-function pct(n: number): string {
-  return `${(n * 100).toFixed(1)}%`;
-}

package/packages/evaluation/src/strategies/context7-runner.ts

@@ -1,269 +0,0 @@
-/**
- * Context7 REAL runner — spawns the real `context7-mcp` binary via stdio JSON-RPC.
- *
- * Tool flow per task:
- *   1. initialize
- *   2. resolve-library-id  → gets Context7 library ID
- *   3. query-docs          → fetches real documentation
- *
- * Context7 retrieves docs from its cloud index without consulting workspace lockfiles,
- * so it defaults to the highest-quality (usually latest) version it has indexed.
- * This is the real behavior — not a model of it.
- *
- * Requires: `context7-mcp` in PATH (install: npm install -g @upstash/context7-mcp)
- * Requires: network access to context7.com
- */
-
-import fs from 'node:fs';
-import path from 'node:path';
-import { spawn } from 'node:child_process';
-import { performance } from 'node:perf_hooks';
-import type { StrategyRunner, BenchmarkTaskDef, BenchmarkTaskResult } from '../types.ts';
-import { isVersionContentMatch } from '../version-matcher.ts';
-
-interface JsonRpcResponse {
-  jsonrpc: '2.0';
-  id: number;
-  result?: unknown;
-  error?: { code: number; message: string };
-}
-
-/** Send one JSON-RPC message and get the response back over stdio. */
-async function mcpCall(
-  proc: ReturnType<typeof spawn>,
-  id: number,
-  method: string,
-  params: Record<string, unknown>,
-  timeoutMs = 30_000,
-): Promise<{ response: JsonRpcResponse; elapsedMs: number }> {
-  const msg = JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n';
-  const t0 = performance.now();
-
-  return new Promise((resolve, reject) => {
-    let buffer = '';
-    const timer = setTimeout(() => reject(new Error(`Timeout waiting for id=${id} method=${method}`)), timeoutMs);
-
-    const onData = (chunk: Buffer) => {
-      buffer += chunk.toString();
-      const lines = buffer.split('\n');
-      buffer = lines.pop() ?? '';
-      for (const line of lines) {
-        if (!line.trim()) continue;
-        try {
-          const parsed = JSON.parse(line) as JsonRpcResponse;
-          if (parsed.id === id) {
-            clearTimeout(timer);
-            proc.stdout!.off('data', onData);
-            resolve({ response: parsed, elapsedMs: performance.now() - t0 });
-          }
-        } catch {
-          // incomplete JSON — keep buffering
-        }
-      }
-    };
-
-    proc.stdout!.on('data', onData);
-    proc.stdin!.write(msg);
-  });
-}
-
-export class Context7Runner implements StrategyRunner {
-  readonly strategy = 'agent_context7' as const;
-
-  async executeTask(task: BenchmarkTaskDef, tempDir: string): Promise<BenchmarkTaskResult> {
-    const taskDir = path.join(tempDir, `context7_${task.id}`);
-    fs.mkdirSync(taskDir, { recursive: true });
-
-    for (const [filename, content] of Object.entries(task.workspaceFiles)) {
-      const filePath = path.join(taskDir, filename);
-      fs.mkdirSync(path.dirname(filePath), { recursive: true });
-      fs.writeFileSync(filePath, content, 'utf-8');
-    }
-
-    // Resolve the Context7-native library name (e.g. "next" → "Next.js", "stripe" → "Stripe")
-    const libraryDisplayNames: Record<string, string> = {
-      next: 'Next.js',
-      stripe: 'Stripe',
-      pydantic: 'Pydantic',
-      fastapi: 'FastAPI',
-      'tokio-postgres': 'tokio-postgres',
-      'github.com/gin-gonic/gin': 'Gin',
-    };
-    const libraryName = libraryDisplayNames[task.library] ?? task.library;
-
-    let toolCallsCount = 0;
-    let totalLatencyMs = 0;
-    let rawRetrievedContent = '';
-    let resolvedLibraryId = '';
-    let errorNotes = '';
-
-    const proc = spawn('context7-mcp', [], {
-      stdio: ['pipe', 'pipe', 'pipe'],
-    });
-
-    try {
-      // 1. Initialize
-      await mcpCall(proc, 1, 'initialize', {
-        protocolVersion: '2024-11-05',
-        clientInfo: { name: 'docorbit-eval', version: '1.0' },
-        capabilities: {},
-      });
-
-      // 2. resolve-library-id
-      toolCallsCount++;
-      const { response: resolveResp, elapsedMs: resolveMs } = await mcpCall(proc, 2, 'tools/call', {
-        name: 'resolve-library-id',
-        arguments: { libraryName, query: task.taskPrompt },
-      });
-      totalLatencyMs += resolveMs;
-
-      if (resolveResp.error) {
-        throw new Error(`resolve-library-id failed: ${resolveResp.error.message}`);
-      }
-
-      // Extract the best library ID from the response text
-      const resolveText = extractTextContent(resolveResp.result);
-      // Format: /org/project or /org/project/version — first match wins
-      const idMatch = resolveText.match(/\/[a-z0-9_\-\.]+\/[a-z0-9_\-\.]+(?:\/[^\s,)]+)?/i);
-      resolvedLibraryId = idMatch?.[0] ?? `/${task.library}`;
-
-      // 3. query-docs
-      toolCallsCount++;
-      const { response: docsResp, elapsedMs: docsMs } = await mcpCall(proc, 3, 'tools/call', {
-        name: 'query-docs',
-        arguments: {
-          libraryId: resolvedLibraryId,
-          query: task.taskPrompt,
-          tokens: 3000,
-        },
-      });
-      totalLatencyMs += docsMs;
-
-      if (docsResp.error) {
-        throw new Error(`query-docs failed: ${docsResp.error.message}`);
-      }
-
-      rawRetrievedContent = extractTextContent(docsResp.result).slice(0, 2000);
-    } catch (err) {
-      errorNotes = ` | ERROR: ${String(err)}`;
-    } finally {
-      proc.stdin?.end();
-      proc.kill();
-    }    const content = rawRetrievedContent;
-
-    // Version correctness: evaluated semantically against the task's version requirements.
-    // Context7 retrieves latest docs globally from its cloud index without local project lockfile awareness.
-    const correctVersionSelected = isVersionContentMatch(task, content);
-
-
-
-    let correctApiSelected = false;
-    const { path: apiPath, symbol } = task.groundTruth.expectedApi;
-    if (symbol && content.includes(symbol)) {
-      correctApiSelected = true;
-    } else if (apiPath && content.toLowerCase().includes(apiPath.toLowerCase())) {
-      correctApiSelected = true;
-    }
-
-    const tokenUsage = Math.ceil(content.length / 4);
-
-    // Precision: fraction of retrieved content relevant to the task
-    // (approximated by checking if ground-truth API symbol/path appears in retrieved content)
-    const precision = correctApiSelected ? (correctVersionSelected ? 0.85 : 0.6) : 0.2;
-    const recall = correctApiSelected ? (correctVersionSelected ? 0.85 : 0.65) : 0.25;
-    const taskSuccess = correctVersionSelected && correctApiSelected;
-
-    return {
-      strategy: this.strategy,
-      taskId: task.id,
-      split: task.split,
-      taskSuccess,
-      correctApiSelected,
-      correctVersionSelected,
-      retrievalPrecision: precision,
-      retrievalRecall: recall,
-      tokenUsage,
-      latencyMs: totalLatencyMs,
-      toolCallsCount,
-      verificationCatches: 0,
-      verificationFalsePositives: 0,
-      insufficientEvidenceCount: 0,
-      isSimulation: false,
-      rawRetrievedContent,
-      notes: `Context7 REAL MCP (context7-mcp v4). resolve-library-id → query-docs. resolvedId=${resolvedLibraryId}. No workspace lockfile awareness.${errorNotes}`,
-    };
-  }
-}
-
-/** Extract text content from an MCP tool call result (handles nested content arrays). */
-function extractTextContent(result: unknown): string {
-  if (typeof result === 'string') return result;
-  if (result && typeof result === 'object') {
-    const r = result as Record<string, unknown>;
-    if (Array.isArray(r.content)) {
-      return r.content
-        .map((c: unknown) => {
-          if (c && typeof c === 'object' && 'text' in (c as object)) {
-            return (c as { text: string }).text;
-          }
-          return String(c);
-        })
-        .join('\n');
-    }
-    if (typeof r.text === 'string') return r.text;
-  }
-  return JSON.stringify(result);
-}
-
-// ─── Simulation fallback (offline CI mode) ────────────────────────────────────
-
-export class Context7SimulatedRunner implements StrategyRunner {
-  readonly strategy = 'agent_context7' as const;
-
-  async executeTask(task: BenchmarkTaskDef, tempDir: string): Promise<BenchmarkTaskResult> {
-    const taskDir = path.join(tempDir, `context7sim_${task.id}`);
-    fs.mkdirSync(taskDir, { recursive: true });
-
-    for (const [filename, content] of Object.entries(task.workspaceFiles)) {
-      const filePath = path.join(taskDir, filename);
-      fs.mkdirSync(path.dirname(filePath), { recursive: true });
-      fs.writeFileSync(filePath, content, 'utf-8');
-    }
-
-    // Simulate: Context7 returns the latest doc it has (no version pinning)
-    const latestDoc = task.docs[task.docs.length - 1];
-    const content = latestDoc?.content ?? '';
-    const returnedVersion = latestDoc?.version ?? 'latest';
-    const correctVersionSelected = isVersionContentMatch(task, content);
-
-
-
-    let correctApiSelected = false;
-    const { path: apiPath, symbol } = task.groundTruth.expectedApi;
-    if (symbol && content.includes(symbol)) {
-      correctApiSelected = true;
-    } else if (apiPath && content.toLowerCase().includes(apiPath.toLowerCase())) {
-      correctApiSelected = true;
-    }
-
-    return {
-      strategy: this.strategy,
-      taskId: task.id,
-      split: task.split,
-      taskSuccess: correctVersionSelected && correctApiSelected,
-      correctApiSelected,
-      correctVersionSelected,
-      retrievalPrecision: correctVersionSelected ? 0.75 : 0.25,
-      retrievalRecall: correctVersionSelected ? 0.8 : 0.4,
-      tokenUsage: Math.ceil(content.length / 4),
-      latencyMs: 50,
-      toolCallsCount: 2,
-      verificationCatches: 0,
-      verificationFalsePositives: 0,
-      insufficientEvidenceCount: 0,
-      isSimulation: true,
-      rawRetrievedContent: '',
-      notes: `[SIMULATION MODE — no network] Context7 protocol model. returnedVersion=${returnedVersion}.`,
-    };
-  }
-}