contextos-agents 1.6.1 → 2.0.0-beta.2

package/benchmarks/v2/analysis/statistics.js

@@ -0,0 +1,140 @@
+/**
+ * benchmarks/v2/analysis/statistics.js
+ * ContextOS Benchmark v2 — Statistical Analysis Engine
+ *
+ * Implements Section 24 of CONTEXTOS_IMPLEMENTATION_PLAN.md:
+ *   - Primary metric: cost per independently verified successful task
+ *   - Wilson score 95% confidence intervals for binomial success proportions
+ *   - Pairwise delta comparison between Arm C/D and Arm B (Concise Checklist comparator)
+ *   - Structured tabular summary generation
+ */
+
+'use strict';
+
+/**
+ * Calculates Wilson score 95% confidence interval for a proportion.
+ *
+ * @param {number} successes
+ * @param {number} total
+ * @param {number} [z=1.96] - 95% confidence z-score
+ * @returns {[number, number]} [lower, upper] as percentages [0, 100]
+ */
+function calculateWilsonInterval(successes, total, z = 1.96) {
+  if (total === 0) return [0, 0];
+  const p = successes / total;
+  const z2 = z * z;
+  const denominator = 1 + z2 / total;
+  const center = (p + z2 / (2 * total)) / denominator;
+  const margin = (z * Math.sqrt((p * (1 - p)) / total + z2 / (4 * total * total))) / denominator;
+
+  return [
+    Math.max(0, parseFloat(((center - margin) * 100).toFixed(1))),
+    Math.min(100, parseFloat(((center + margin) * 100).toFixed(1))),
+  ];
+}
+
+class BenchmarkStatistics {
+  /**
+   * Analyzes an array of run outcomes across experimental arms.
+   *
+   * @param {Array<Object>} runs - List of run records: { armId, success, totalCost, durationMs }
+   * @returns {Object} Comprehensive statistical summary
+   */
+  static analyze(runs) {
+    const armsMap = {};
+
+    for (const run of runs) {
+      const { armId, success, totalCost = 0, durationMs = 0 } = run;
+      if (!armsMap[armId]) {
+        armsMap[armId] = {
+          armId,
+          totalRuns: 0,
+          successfulRuns: 0,
+          totalCost: 0,
+          totalDurationMs: 0,
+        };
+      }
+
+      const item = armsMap[armId];
+      item.totalRuns++;
+      if (success) item.successfulRuns++;
+      item.totalCost += totalCost;
+      item.totalDurationMs += durationMs;
+    }
+
+    const armStats = {};
+    for (const [armId, d] of Object.entries(armsMap)) {
+      const successRate = d.totalRuns > 0 ? (d.successfulRuns / d.totalRuns) * 100 : 0;
+      const ci95 = calculateWilsonInterval(d.successfulRuns, d.totalRuns);
+      const costPerSuccess = d.successfulRuns > 0 ? d.totalCost / d.successfulRuns : null;
+      const avgDurationMs = d.totalRuns > 0 ? d.totalDurationMs / d.totalRuns : 0;
+
+      armStats[armId] = {
+        armId,
+        totalRuns: d.totalRuns,
+        successfulRuns: d.successfulRuns,
+        successRate: parseFloat(successRate.toFixed(1)),
+        ci95,
+        totalCost: parseFloat(d.totalCost.toFixed(4)),
+        costPerVerifiedSuccess: costPerSuccess !== null ? parseFloat(costPerSuccess.toFixed(4)) : null,
+        avgDurationMs: Math.round(avgDurationMs),
+      };
+    }
+
+    // Pairwise comparisons against Arm B (Concise Checklist)
+    const comparator = armStats['arm-b-concise-checklist'];
+    const comparisons = {};
+
+    if (comparator) {
+      for (const [armId, stat] of Object.entries(armStats)) {
+        if (armId === 'arm-b-concise-checklist') continue;
+
+        const rateDelta = parseFloat((stat.successRate - comparator.successRate).toFixed(1));
+        let costRatio = null;
+        if (comparator.costPerVerifiedSuccess && stat.costPerVerifiedSuccess) {
+          costRatio = parseFloat((stat.costPerVerifiedSuccess / comparator.costPerVerifiedSuccess).toFixed(2));
+        }
+
+        comparisons[armId] = {
+          vsComparator: 'arm-b-concise-checklist',
+          successRateDelta: rateDelta,
+          costRatio,
+        };
+      }
+    }
+
+    return {
+      timestamp: Date.now(),
+      totalRunsAnalyzed: runs.length,
+      arms: armStats,
+      comparisons,
+    };
+  }
+
+  /**
+   * Formats statistical analysis into an aligned markdown summary table.
+   *
+   * @param {Object} analysis
+   * @returns {string}
+   */
+  static formatTable(analysis) {
+    const lines = [];
+    lines.push('| Arm ID | Runs | Successes | Success Rate (95% CI) | Cost / Verified Success | Avg Latency |');
+    lines.push('|---|---:|---:|---:|---:|---:|');
+
+    for (const arm of Object.values(analysis.arms)) {
+      const ciStr = `[${arm.ci95[0]}%, ${arm.ci95[1]}%]`;
+      const costStr = arm.costPerVerifiedSuccess !== null ? `$${arm.costPerVerifiedSuccess}` : 'N/A';
+      lines.push(
+        `| **${arm.armId}** | ${arm.totalRuns} | ${arm.successfulRuns} | ${arm.successRate}% ${ciStr} | ${costStr} | ${arm.avgDurationMs}ms |`
+      );
+    }
+
+    return lines.join('\n');
+  }
+}
+
+module.exports = {
+  calculateWilsonInterval,
+  BenchmarkStatistics,
+};

package/benchmarks/v2/analysis/stats.js

@@ -0,0 +1,69 @@
+'use strict';
+
+/**
+ * Calculates 95% Confidence Interval for a proportion using the Wald method.
+ * @param {number} p - sample proportion (success rate)
+ * @param {number} n - sample size
+ * @returns {number} Margin of Error
+ */
+function calculate95CI(p, n) {
+  if (n === 0) return 0;
+  // Z-value for 95% confidence is 1.96
+  const z = 1.96;
+  const standardError = Math.sqrt((p * (1 - p)) / n);
+  return z * standardError;
+}
+
+/**
+ * Analyzes the results of a benchmark run.
+ * @param {Array} results Array of execution result objects from the runner
+ * @returns {Object} Statistical summary
+ */
+function analyzeResults(results) {
+  const statsByArm = {};
+
+  // Group by arm
+  for (const res of results) {
+    if (!statsByArm[res.armId]) {
+      statsByArm[res.armId] = {
+        total: 0,
+        successes: 0,
+        failures: 0,
+        totalTokens: 0,
+        totalDurationMs: 0
+      };
+    }
+    
+    const stats = statsByArm[res.armId];
+    stats.total++;
+    if (res.success) {
+      stats.successes++;
+    } else {
+      stats.failures++;
+    }
+    stats.totalTokens += res.usage.totalTokens;
+    stats.totalDurationMs += res.durationMs;
+  }
+
+  // Calculate rates and CI
+  const finalStats = {};
+  for (const [armId, stats] of Object.entries(statsByArm)) {
+    const successRate = stats.total > 0 ? stats.successes / stats.total : 0;
+    const marginOfError = calculate95CI(successRate, stats.total);
+    
+    finalStats[armId] = {
+      ...stats,
+      successRate: parseFloat((successRate * 100).toFixed(2)),
+      confidenceInterval95: `±${(marginOfError * 100).toFixed(2)}%`,
+      avgTokensPerTask: stats.total > 0 ? Math.round(stats.totalTokens / stats.total) : 0,
+      avgDurationMs: stats.total > 0 ? Math.round(stats.totalDurationMs / stats.total) : 0
+    };
+  }
+
+  return finalStats;
+}
+
+module.exports = {
+  analyzeResults,
+  calculate95CI
+};

package/benchmarks/v2/arms/arm-definitions.js

@@ -0,0 +1,79 @@
+/**
+ * benchmarks/v2/arms/arm-definitions.js
+ * ContextOS Benchmark v2 — Evaluation Arms Specification
+ *
+ * Implements Section 24.2 of CONTEXTOS_IMPLEMENTATION_PLAN.md:
+ *   - Arm A (Vanilla): Neutral baseline system prompt without artificial debuffing
+ *   - Arm B (Concise Checklist): 10-15 universal engineering rules (~600 tokens) — Primary Comparator
+ *   - Arm C (ContextOS Core): Dynamic canonical skill resolver without ceremony
+ *   - Arm D (Full ContextOS): Resolver + risk workflow + isolated runtime verification + review
+ */
+
+'use strict';
+
+const ARMS = {
+  ARM_A_VANILLA: {
+    id: 'arm-a-vanilla',
+    name: 'Vanilla Baseline',
+    description: 'Neutral baseline system prompt without ContextOS rules or checklists.',
+    tokenBudgetEstimate: 120,
+    buildSystemPrompt: () => {
+      return 'You are an expert software engineer. Write clean, complete, working production code that solves the user request.';
+    },
+  },
+
+  ARM_B_CONCISE_CHECKLIST: {
+    id: 'arm-b-concise-checklist',
+    name: 'Concise Checklist',
+    description: 'High-density 12-rule engineering checklist (~600 tokens). Primary comparator.',
+    tokenBudgetEstimate: 580,
+    buildSystemPrompt: () => {
+      return [
+        'You are a Senior Staff Engineer.',
+        'Follow this strict engineering checklist:',
+        '1. Inspect existing files before editing.',
+        '2. Never use placeholders, stubs, or TODO comments.',
+        '3. Maintain existing codebase naming conventions and architectural boundaries.',
+        '4. Minimize blast radius — modify only files required for the task.',
+        '5. Validate all user input and sanitize data paths.',
+        '6. Use parameterized queries for database operations.',
+        '7. Handle all asynchronous error boundaries explicitly.',
+        '8. Write comprehensive unit and integration test assertions.',
+        '9. Ensure clean TypeScript typing without any unsafe casts.',
+        '10. Verify backward compatibility with existing public APIs.',
+        '11. No secrets or credentials in code or commits.',
+        '12. Ensure code compiles and all tests pass.',
+      ].join('\n');
+    },
+  },
+
+  ARM_C_CONTEXTOS_CORE: {
+    id: 'arm-c-contextos-core',
+    name: 'ContextOS Core (Dynamic Context Selection)',
+    description: 'Dynamic canonical resolver selecting exact skills and rules without ceremony.',
+    tokenBudgetEstimate: 1400,
+    buildSystemPrompt: (resolvedSkills = []) => {
+      const skillsHeader = resolvedSkills.length > 0
+        ? `[ContextOS Resolved Skills: ${resolvedSkills.join(', ')}]`
+        : '[ContextOS Core]';
+      return `${skillsHeader}\nExecute task adhering to compiled workspace rules and exact skill invariants.`;
+    },
+  },
+
+  ARM_D_FULL_CONTEXTOS: {
+    id: 'arm-d-full-contextos',
+    name: 'Full ContextOS (Core + Runtime Verification)',
+    description: 'Dynamic resolver + risk-based workflow + isolated runtime verification + reviewer pipeline.',
+    tokenBudgetEstimate: 2200,
+    buildSystemPrompt: (resolvedSkills = [], riskLevel = 'STANDARD') => {
+      return [
+        `[ContextOS Full Runtime] [RISK: ${riskLevel}] [Skills: ${resolvedSkills.join(', ')}]`,
+        'Execution gated by isolated worktree and mandatory verification attestations before merge readiness.',
+      ].join('\n');
+    },
+  },
+};
+
+module.exports = {
+  ARMS,
+};

package/benchmarks/v2/dataset.schema.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "ContextOS Benchmark v2 Task Schema",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": "string",
+      "description": "Unique immutable identifier for the task"
+    },
+    "description": {
+      "type": "string",
+      "description": "The actual prompt provided to the LLM"
+    },
+    "expectedState": {
+      "type": "object",
+      "description": "The expected state of the filesystem or output after execution",
+      "properties": {
+        "filesToExist": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "filesToContain": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        }
+      }
+    },
+    "hash": {
+      "type": "string",
+      "description": "SHA-256 hash of the task for immutability verification"
+    }
+  },
+  "required": ["id", "description", "hash"]
+}

package/benchmarks/v2/evaluators/index.js

@@ -0,0 +1,25 @@
+'use strict';
+
+/**
+ * Evaluates the result of a task execution against the expected state.
+ * Since we are using a Mock Provider, the evaluation logic is simplified
+ * to just trust the provider's mocked success status. In a real system,
+ * this would run ESLint, execute the generated code in a sandbox,
+ * and verify the exact AST or output.
+ *
+ * @param {Object} task The benchmark task definition
+ * @param {Object} llmResult The result from the LLM/Mock Provider
+ * @returns {Object} { passed: boolean, error: string|null }
+ */
+function evaluateTask(task, llmResult) {
+  if (llmResult.success) {
+    return { passed: true, error: null };
+  } else {
+    return { 
+      passed: false, 
+      error: `Failed to meet expected state for task ${task.id}: ${llmResult.mockedOutput}` 
+    };
+  }
+}
+
+module.exports = evaluateTask;

package/benchmarks/v2/evaluators/verified-success.js

@@ -0,0 +1,116 @@
+/**
+ * benchmarks/v2/evaluators/verified-success.js
+ * ContextOS Benchmark v2 — Primary Outcome Evaluator
+ *
+ * Implements Section 24.8 of CONTEXTOS_IMPLEMENTATION_PLAN.md:
+ *   - Primary metric: independently_verified_success
+ *   - Evaluates:
+ *     1. Patch application / syntax correctness
+ *     2. Typecheck / build status
+ *     3. Public unit test pass rate
+ *     4. Hidden test suite pass rate (isolated oracle)
+ *     5. Zero regression on baseline suites
+ *     6. Zero P0/P1 security findings or leaked credentials
+ *     7. Zero placeholder stubs (TODO, mock placeholders)
+ *     8. Budget constraints (tokens, time, turns)
+ */
+
+'use strict';
+
+const PLACEHOLDER_PATTERNS = [
+  /\/\/\s*TODO:\s*implement\b/i,
+  /\/\/\s*\.\.\.\s*rest of code\b/i,
+  /\bthrow new Error\(["']Not implemented["']\)/i,
+  /\bpass\s*#\s*TODO\b/i,
+];
+
+class BenchmarkEvaluator {
+  /**
+   * Evaluates task run evidence against rigorous quality gates.
+   *
+   * @param {Object} runEvidence
+   * @param {boolean} runEvidence.patchApplied
+   * @param {boolean} runEvidence.buildPass
+   * @param {number} runEvidence.publicTestsTotal
+   * @param {number} runEvidence.publicTestsPassed
+   * @param {number} runEvidence.hiddenTestsTotal
+   * @param {number} runEvidence.hiddenTestsPassed
+   * @param {boolean} [runEvidence.regressions=false]
+   * @param {Array<string>} [runEvidence.securityFindings=[]]
+   * @param {string} [runEvidence.generatedCode='']
+   * @param {Object} [runEvidence.budget]
+   * @param {number} [runEvidence.budget.tokensUsed=0]
+   * @param {number} [runEvidence.budget.tokenLimit=50000]
+   * @param {number} [runEvidence.budget.durationMs=0]
+   * @param {number} [runEvidence.budget.timeoutMs=60000]
+   * @returns {Object} Evaluation report
+   */
+  static evaluate(runEvidence) {
+    const {
+      patchApplied = false,
+      buildPass = false,
+      publicTestsTotal = 0,
+      publicTestsPassed = 0,
+      hiddenTestsTotal = 0,
+      hiddenTestsPassed = 0,
+      regressions = false,
+      securityFindings = [],
+      generatedCode = '',
+      budget = {},
+    } = runEvidence;
+
+    const failures = [];
+
+    // 1. Patch & build
+    if (!patchApplied) failures.push('Patch was not successfully applied');
+    if (!buildPass) failures.push('Compilation or typecheck failed');
+
+    // 2. Tests
+    if (publicTestsTotal > 0 && publicTestsPassed < publicTestsTotal) {
+      failures.push(`Public tests failed: ${publicTestsPassed}/${publicTestsTotal}`);
+    }
+    if (hiddenTestsTotal > 0 && hiddenTestsPassed < hiddenTestsTotal) {
+      failures.push(`Hidden test oracle failed: ${hiddenTestsPassed}/${hiddenTestsTotal}`);
+    }
+    if (regressions) {
+      failures.push('Regression detected in existing test baseline');
+    }
+
+    // 3. Security
+    if (Array.isArray(securityFindings) && securityFindings.length > 0) {
+      failures.push(`Security vulnerabilities detected: ${securityFindings.join(', ')}`);
+    }
+
+    // 4. Zero placeholders
+    if (generatedCode) {
+      for (const pat of PLACEHOLDER_PATTERNS) {
+        if (pat.test(generatedCode)) {
+          failures.push(`Lazy placeholder detected matching pattern: ${pat.source}`);
+          break;
+        }
+      }
+    }
+
+    // 5. Budget constraints
+    if (budget.tokensUsed && budget.tokenLimit && budget.tokensUsed > budget.tokenLimit) {
+      failures.push(`Token budget exceeded: ${budget.tokensUsed} > ${budget.tokenLimit}`);
+    }
+    if (budget.durationMs && budget.timeoutMs && budget.durationMs > budget.timeoutMs) {
+      failures.push(`Time budget exceeded: ${budget.durationMs}ms > ${budget.timeoutMs}ms`);
+    }
+
+    const isSuccess = failures.length === 0;
+
+    return {
+      independently_verified_success: isSuccess,
+      publicTestRate: publicTestsTotal > 0 ? publicTestsPassed / publicTestsTotal : 1.0,
+      hiddenTestRate: hiddenTestsTotal > 0 ? hiddenTestsPassed / hiddenTestsTotal : 1.0,
+      failureCount: failures.length,
+      failures,
+    };
+  }
+}
+
+module.exports = {
+  BenchmarkEvaluator,
+};

package/benchmarks/v2/harness/runner.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const crypto = require('crypto');
+const { ARMS } = require('../arms/arm-definitions');
+const evaluateTask = require('../evaluators/index');
+
+/**
+ * Mock LLM Provider used when real API keys are unavailable.
+ * Deterministically simulates success/failure rates based on the arm's capability.
+ */
+class MockProvider {
+  /**
+   * Probability of success for each arm to simulate real-world capability differences.
+   */
+  static getSuccessProbability(armId) {
+    switch (armId) {
+      case ARMS.ARM_A_VANILLA.id: return 0.40; // 40% success
+      case ARMS.ARM_B_CONCISE_CHECKLIST.id: return 0.65; // 65% success
+      case ARMS.ARM_C_CONTEXTOS_CORE.id: return 0.85; // 85% success
+      case ARMS.ARM_D_FULL_CONTEXTOS.id: return 0.98; // 98% success
+      default: return 0.0;
+    }
+  }
+
+  static async execute(task, arm) {
+    const probability = this.getSuccessProbability(arm.id);
+    // Use hash to deterministically seed pseudo-randomness for the mock run
+    const hashInt = parseInt(task.hash.substring(7, 15), 16);
+    const successThreshold = probability * 0xffffffff;
+    
+    // Slight artificial delay to simulate API request
+    await new Promise(resolve => setTimeout(resolve, 50));
+    
+    const isSuccess = hashInt <= successThreshold;
+    
+    return {
+      success: isSuccess,
+      mockedOutput: isSuccess 
+        ? `Successfully generated code for ${task.id}` 
+        : `Failed or hallucinated output for ${task.id}`,
+      usage: {
+        promptTokens: arm.tokenBudgetEstimate,
+        completionTokens: 300,
+        totalTokens: arm.tokenBudgetEstimate + 300
+      }
+    };
+  }
+}
+
+/**
+ * Executes a single task against a single arm.
+ */
+async function runTask(task, armId) {
+  const arm = Object.values(ARMS).find(a => a.id === armId);
+  if (!arm) throw new Error(`Unknown arm: ${armId}`);
+
+  const startTime = Date.now();
+  const requestId = crypto.randomUUID();
+
+  // Execute using Mock Provider (replace with real LLM client when API keys are available)
+  const llmResult = await MockProvider.execute(task, arm);
+  
+  const durationMs = Date.now() - startTime;
+
+  // Evaluate the output
+  const evaluation = evaluateTask(task, llmResult);
+
+  return {
+    taskId: task.id,
+    armId: arm.id,
+    requestId,
+    timestamp: new Date().toISOString(),
+    durationMs,
+    success: evaluation.passed,
+    error: evaluation.error || null,
+    usage: llmResult.usage,
+    environmentProvenance: {
+      nodeVersion: process.version,
+      platform: process.platform,
+      engine: 'mock-provider-v1'
+    }
+  };
+}
+
+module.exports = {
+  runTask,
+  MockProvider
+};