contextos-agents 2.0.0-beta.3 → 2.1.0

package/benchmarks/v2/analysis/stats.js

@@ -1,69 +0,0 @@
-'use strict';
-
-/**
- * Calculates 95% Confidence Interval for a proportion using the Wald method.
- * @param {number} p - sample proportion (success rate)
- * @param {number} n - sample size
- * @returns {number} Margin of Error
- */
-function calculate95CI(p, n) {
-  if (n === 0) return 0;
-  // Z-value for 95% confidence is 1.96
-  const z = 1.96;
-  const standardError = Math.sqrt((p * (1 - p)) / n);
-  return z * standardError;
-}
-
-/**
- * Analyzes the results of a benchmark run.
- * @param {Array} results Array of execution result objects from the runner
- * @returns {Object} Statistical summary
- */
-function analyzeResults(results) {
-  const statsByArm = {};
-
-  // Group by arm
-  for (const res of results) {
-    if (!statsByArm[res.armId]) {
-      statsByArm[res.armId] = {
-        total: 0,
-        successes: 0,
-        failures: 0,
-        totalTokens: 0,
-        totalDurationMs: 0
-      };
-    }
-    
-    const stats = statsByArm[res.armId];
-    stats.total++;
-    if (res.success) {
-      stats.successes++;
-    } else {
-      stats.failures++;
-    }
-    stats.totalTokens += res.usage.totalTokens;
-    stats.totalDurationMs += res.durationMs;
-  }
-
-  // Calculate rates and CI
-  const finalStats = {};
-  for (const [armId, stats] of Object.entries(statsByArm)) {
-    const successRate = stats.total > 0 ? stats.successes / stats.total : 0;
-    const marginOfError = calculate95CI(successRate, stats.total);
-    
-    finalStats[armId] = {
-      ...stats,
-      successRate: parseFloat((successRate * 100).toFixed(2)),
-      confidenceInterval95: `±${(marginOfError * 100).toFixed(2)}%`,
-      avgTokensPerTask: stats.total > 0 ? Math.round(stats.totalTokens / stats.total) : 0,
-      avgDurationMs: stats.total > 0 ? Math.round(stats.totalDurationMs / stats.total) : 0
-    };
-  }
-
-  return finalStats;
-}
-
-module.exports = {
-  analyzeResults,
-  calculate95CI
-};

package/benchmarks/v2/arms/arm-definitions.js

@@ -1,79 +0,0 @@
-/**
- * benchmarks/v2/arms/arm-definitions.js
- * ContextOS Benchmark v2 — Evaluation Arms Specification
- *
- * Implements Section 24.2 of CONTEXTOS_IMPLEMENTATION_PLAN.md:
- *   - Arm A (Vanilla): Neutral baseline system prompt without artificial debuffing
- *   - Arm B (Concise Checklist): 10-15 universal engineering rules (~600 tokens) — Primary Comparator
- *   - Arm C (ContextOS Core): Dynamic canonical skill resolver without ceremony
- *   - Arm D (Full ContextOS): Resolver + risk workflow + isolated runtime verification + review
- */
-
-'use strict';
-
-const ARMS = {
-  ARM_A_VANILLA: {
-    id: 'arm-a-vanilla',
-    name: 'Vanilla Baseline',
-    description: 'Neutral baseline system prompt without ContextOS rules or checklists.',
-    tokenBudgetEstimate: 120,
-    buildSystemPrompt: () => {
-      return 'You are an expert software engineer. Write clean, complete, working production code that solves the user request.';
-    },
-  },
-
-  ARM_B_CONCISE_CHECKLIST: {
-    id: 'arm-b-concise-checklist',
-    name: 'Concise Checklist',
-    description: 'High-density 12-rule engineering checklist (~600 tokens). Primary comparator.',
-    tokenBudgetEstimate: 580,
-    buildSystemPrompt: () => {
-      return [
-        'You are a Senior Staff Engineer.',
-        'Follow this strict engineering checklist:',
-        '1. Inspect existing files before editing.',
-        '2. Never use placeholders, stubs, or TODO comments.',
-        '3. Maintain existing codebase naming conventions and architectural boundaries.',
-        '4. Minimize blast radius — modify only files required for the task.',
-        '5. Validate all user input and sanitize data paths.',
-        '6. Use parameterized queries for database operations.',
-        '7. Handle all asynchronous error boundaries explicitly.',
-        '8. Write comprehensive unit and integration test assertions.',
-        '9. Ensure clean TypeScript typing without any unsafe casts.',
-        '10. Verify backward compatibility with existing public APIs.',
-        '11. No secrets or credentials in code or commits.',
-        '12. Ensure code compiles and all tests pass.',
-      ].join('\n');
-    },
-  },
-
-  ARM_C_CONTEXTOS_CORE: {
-    id: 'arm-c-contextos-core',
-    name: 'ContextOS Core (Dynamic Context Selection)',
-    description: 'Dynamic canonical resolver selecting exact skills and rules without ceremony.',
-    tokenBudgetEstimate: 1400,
-    buildSystemPrompt: (resolvedSkills = []) => {
-      const skillsHeader = resolvedSkills.length > 0
-        ? `[ContextOS Resolved Skills: ${resolvedSkills.join(', ')}]`
-        : '[ContextOS Core]';
-      return `${skillsHeader}\nExecute task adhering to compiled workspace rules and exact skill invariants.`;
-    },
-  },
-
-  ARM_D_FULL_CONTEXTOS: {
-    id: 'arm-d-full-contextos',
-    name: 'Full ContextOS (Core + Runtime Verification)',
-    description: 'Dynamic resolver + risk-based workflow + isolated runtime verification + reviewer pipeline.',
-    tokenBudgetEstimate: 2200,
-    buildSystemPrompt: (resolvedSkills = [], riskLevel = 'STANDARD') => {
-      return [
-        `[ContextOS Full Runtime] [RISK: ${riskLevel}] [Skills: ${resolvedSkills.join(', ')}]`,
-        'Execution gated by isolated worktree and mandatory verification attestations before merge readiness.',
-      ].join('\n');
-    },
-  },
-};
-
-module.exports = {
-  ARMS,
-};

package/benchmarks/v2/dataset.schema.json

@@ -1,34 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "ContextOS Benchmark v2 Task Schema",
-  "type": "object",
-  "properties": {
-    "id": {
-      "type": "string",
-      "description": "Unique immutable identifier for the task"
-    },
-    "description": {
-      "type": "string",
-      "description": "The actual prompt provided to the LLM"
-    },
-    "expectedState": {
-      "type": "object",
-      "description": "The expected state of the filesystem or output after execution",
-      "properties": {
-        "filesToExist": {
-          "type": "array",
-          "items": { "type": "string" }
-        },
-        "filesToContain": {
-          "type": "object",
-          "additionalProperties": { "type": "string" }
-        }
-      }
-    },
-    "hash": {
-      "type": "string",
-      "description": "SHA-256 hash of the task for immutability verification"
-    }
-  },
-  "required": ["id", "description", "hash"]
-}

package/benchmarks/v2/evaluators/index.js

@@ -1,25 +0,0 @@
-'use strict';
-
-/**
- * Evaluates the result of a task execution against the expected state.
- * Since we are using a Mock Provider, the evaluation logic is simplified
- * to just trust the provider's mocked success status. In a real system,
- * this would run ESLint, execute the generated code in a sandbox,
- * and verify the exact AST or output.
- *
- * @param {Object} task The benchmark task definition
- * @param {Object} llmResult The result from the LLM/Mock Provider
- * @returns {Object} { passed: boolean, error: string|null }
- */
-function evaluateTask(task, llmResult) {
-  if (llmResult.success) {
-    return { passed: true, error: null };
-  } else {
-    return { 
-      passed: false, 
-      error: `Failed to meet expected state for task ${task.id}: ${llmResult.mockedOutput}` 
-    };
-  }
-}
-
-module.exports = evaluateTask;

package/benchmarks/v2/evaluators/verified-success.js

@@ -1,116 +0,0 @@
-/**
- * benchmarks/v2/evaluators/verified-success.js
- * ContextOS Benchmark v2 — Primary Outcome Evaluator
- *
- * Implements Section 24.8 of CONTEXTOS_IMPLEMENTATION_PLAN.md:
- *   - Primary metric: independently_verified_success
- *   - Evaluates:
- *     1. Patch application / syntax correctness
- *     2. Typecheck / build status
- *     3. Public unit test pass rate
- *     4. Hidden test suite pass rate (isolated oracle)
- *     5. Zero regression on baseline suites
- *     6. Zero P0/P1 security findings or leaked credentials
- *     7. Zero placeholder stubs (TODO, mock placeholders)
- *     8. Budget constraints (tokens, time, turns)
- */
-
-'use strict';
-
-const PLACEHOLDER_PATTERNS = [
-  /\/\/\s*TODO:\s*implement\b/i,
-  /\/\/\s*\.\.\.\s*rest of code\b/i,
-  /\bthrow new Error\(["']Not implemented["']\)/i,
-  /\bpass\s*#\s*TODO\b/i,
-];
-
-class BenchmarkEvaluator {
-  /**
-   * Evaluates task run evidence against rigorous quality gates.
-   *
-   * @param {Object} runEvidence
-   * @param {boolean} runEvidence.patchApplied
-   * @param {boolean} runEvidence.buildPass
-   * @param {number} runEvidence.publicTestsTotal
-   * @param {number} runEvidence.publicTestsPassed
-   * @param {number} runEvidence.hiddenTestsTotal
-   * @param {number} runEvidence.hiddenTestsPassed
-   * @param {boolean} [runEvidence.regressions=false]
-   * @param {Array<string>} [runEvidence.securityFindings=[]]
-   * @param {string} [runEvidence.generatedCode='']
-   * @param {Object} [runEvidence.budget]
-   * @param {number} [runEvidence.budget.tokensUsed=0]
-   * @param {number} [runEvidence.budget.tokenLimit=50000]
-   * @param {number} [runEvidence.budget.durationMs=0]
-   * @param {number} [runEvidence.budget.timeoutMs=60000]
-   * @returns {Object} Evaluation report
-   */
-  static evaluate(runEvidence) {
-    const {
-      patchApplied = false,
-      buildPass = false,
-      publicTestsTotal = 0,
-      publicTestsPassed = 0,
-      hiddenTestsTotal = 0,
-      hiddenTestsPassed = 0,
-      regressions = false,
-      securityFindings = [],
-      generatedCode = '',
-      budget = {},
-    } = runEvidence;
-
-    const failures = [];
-
-    // 1. Patch & build
-    if (!patchApplied) failures.push('Patch was not successfully applied');
-    if (!buildPass) failures.push('Compilation or typecheck failed');
-
-    // 2. Tests
-    if (publicTestsTotal > 0 && publicTestsPassed < publicTestsTotal) {
-      failures.push(`Public tests failed: ${publicTestsPassed}/${publicTestsTotal}`);
-    }
-    if (hiddenTestsTotal > 0 && hiddenTestsPassed < hiddenTestsTotal) {
-      failures.push(`Hidden test oracle failed: ${hiddenTestsPassed}/${hiddenTestsTotal}`);
-    }
-    if (regressions) {
-      failures.push('Regression detected in existing test baseline');
-    }
-
-    // 3. Security
-    if (Array.isArray(securityFindings) && securityFindings.length > 0) {
-      failures.push(`Security vulnerabilities detected: ${securityFindings.join(', ')}`);
-    }
-
-    // 4. Zero placeholders
-    if (generatedCode) {
-      for (const pat of PLACEHOLDER_PATTERNS) {
-        if (pat.test(generatedCode)) {
-          failures.push(`Lazy placeholder detected matching pattern: ${pat.source}`);
-          break;
-        }
-      }
-    }
-
-    // 5. Budget constraints
-    if (budget.tokensUsed && budget.tokenLimit && budget.tokensUsed > budget.tokenLimit) {
-      failures.push(`Token budget exceeded: ${budget.tokensUsed} > ${budget.tokenLimit}`);
-    }
-    if (budget.durationMs && budget.timeoutMs && budget.durationMs > budget.timeoutMs) {
-      failures.push(`Time budget exceeded: ${budget.durationMs}ms > ${budget.timeoutMs}ms`);
-    }
-
-    const isSuccess = failures.length === 0;
-
-    return {
-      independently_verified_success: isSuccess,
-      publicTestRate: publicTestsTotal > 0 ? publicTestsPassed / publicTestsTotal : 1.0,
-      hiddenTestRate: hiddenTestsTotal > 0 ? hiddenTestsPassed / hiddenTestsTotal : 1.0,
-      failureCount: failures.length,
-      failures,
-    };
-  }
-}
-
-module.exports = {
-  BenchmarkEvaluator,
-};

package/benchmarks/v2/harness/runner.js

@@ -1,88 +0,0 @@
-'use strict';
-
-const crypto = require('crypto');
-const { ARMS } = require('../arms/arm-definitions');
-const evaluateTask = require('../evaluators/index');
-
-/**
- * Mock LLM Provider used when real API keys are unavailable.
- * Deterministically simulates success/failure rates based on the arm's capability.
- */
-class MockProvider {
-  /**
-   * Probability of success for each arm to simulate real-world capability differences.
-   */
-  static getSuccessProbability(armId) {
-    switch (armId) {
-      case ARMS.ARM_A_VANILLA.id: return 0.40; // 40% success
-      case ARMS.ARM_B_CONCISE_CHECKLIST.id: return 0.65; // 65% success
-      case ARMS.ARM_C_CONTEXTOS_CORE.id: return 0.85; // 85% success
-      case ARMS.ARM_D_FULL_CONTEXTOS.id: return 0.98; // 98% success
-      default: return 0.0;
-    }
-  }
-
-  static async execute(task, arm) {
-    const probability = this.getSuccessProbability(arm.id);
-    // Use hash to deterministically seed pseudo-randomness for the mock run
-    const hashInt = parseInt(task.hash.substring(7, 15), 16);
-    const successThreshold = probability * 0xffffffff;
-    
-    // Slight artificial delay to simulate API request
-    await new Promise(resolve => setTimeout(resolve, 50));
-    
-    const isSuccess = hashInt <= successThreshold;
-    
-    return {
-      success: isSuccess,
-      mockedOutput: isSuccess 
-        ? `Successfully generated code for ${task.id}` 
-        : `Failed or hallucinated output for ${task.id}`,
-      usage: {
-        promptTokens: arm.tokenBudgetEstimate,
-        completionTokens: 300,
-        totalTokens: arm.tokenBudgetEstimate + 300
-      }
-    };
-  }
-}
-
-/**
- * Executes a single task against a single arm.
- */
-async function runTask(task, armId) {
-  const arm = Object.values(ARMS).find(a => a.id === armId);
-  if (!arm) throw new Error(`Unknown arm: ${armId}`);
-
-  const startTime = Date.now();
-  const requestId = crypto.randomUUID();
-
-  // Execute using Mock Provider (replace with real LLM client when API keys are available)
-  const llmResult = await MockProvider.execute(task, arm);
-  
-  const durationMs = Date.now() - startTime;
-
-  // Evaluate the output
-  const evaluation = evaluateTask(task, llmResult);
-
-  return {
-    taskId: task.id,
-    armId: arm.id,
-    requestId,
-    timestamp: new Date().toISOString(),
-    durationMs,
-    success: evaluation.passed,
-    error: evaluation.error || null,
-    usage: llmResult.usage,
-    environmentProvenance: {
-      nodeVersion: process.version,
-      platform: process.platform,
-      engine: 'mock-provider-v1'
-    }
-  };
-}
-
-module.exports = {
-  runTask,
-  MockProvider
-};