jfl 0.8.1 → 0.9.1

package/scripts/test-planning-loop-e2e.ts

@@ -0,0 +1,181 @@
+/**
+ * End-to-end test for the Planning Loop
+ *
+ * Exercises: PolicyHead v2 → DynamicsModel rollouts → InvariantMonitor → Action Selection
+ *
+ * Run: npx tsx scripts/test-planning-loop-e2e.ts
+ */
+
+import { createPlanningLoop } from "../src/lib/planning-loop.js"
+import { PolicyHeadInference } from "../src/lib/policy-head.js"
+import type { PlanningResult, EvaluatedAction } from "../src/lib/planning-loop.js"
+
+const projectRoot = process.cwd()
+
+// ============================================================================
+// Test scenarios — represent real situations Peter Parker faces
+// ============================================================================
+
+const SCENARIOS = [
+  {
+    name: "Bug reported — tests failing",
+    agentId: "error-fixer",
+    goal: "Tests are failing in planning-loop.test.ts — TypeError on undefined property. Fix the failing test.",
+    expectedTypes: ["fix"],
+  },
+  {
+    name: "Feature request — add new capability",
+    agentId: "feature-builder",
+    goal: "Add multi-step rollout support to the planning loop for deeper lookahead.",
+    expectedTypes: ["feature"],
+  },
+  {
+    name: "Performance optimization",
+    agentId: "optimizer",
+    goal: "Reduce PolicyHead inference latency from 2s to under 500ms for interactive use.",
+    expectedTypes: ["experiment", "refactor"],
+  },
+  {
+    name: "Test coverage gap",
+    agentId: "test-coverage",
+    goal: "Add unit tests for the counterfactual training bridge — currently 0% coverage.",
+    expectedTypes: ["test"],
+  },
+  {
+    name: "Config change needed",
+    agentId: "config-updater",
+    goal: "Update the nightly pipeline schedule to run at 2am MST instead of midnight.",
+    expectedTypes: ["config"],
+  },
+]
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function formatAction(ea: EvaluatedAction): string {
+  const pred = ea.prediction
+  const delta = pred.outcome?.immediate?.evalScoreChange
+  const deltaStr = delta !== undefined ? `Δ=${delta > 0 ? "+" : ""}${delta.toFixed(4)}` : "Δ=N/A"
+  const violations = ea.invariantViolations.length > 0 ? ` ⚠️${ea.invariantViolations.length} violations` : ""
+  const filtered = ea.filtered ? ` [FILTERED: ${ea.filterReason}]` : ""
+
+  return `  ${ea.action.actionType.padEnd(12)} conf=${ea.phConfidence.toFixed(3)} score=${ea.combinedScore.toFixed(3)} ${deltaStr} src=${pred.source}${violations}${filtered}`
+}
+
+function logResult(scenario: typeof SCENARIOS[0], result: PlanningResult) {
+  console.log(`\n${"═".repeat(70)}`)
+  console.log(`  Scenario: ${scenario.name}`)
+  console.log(`  Agent:    ${scenario.agentId}`)
+  console.log(`  Goal:     ${scenario.goal.slice(0, 70)}...`)
+  console.log(`${"─".repeat(70)}`)
+  console.log(`  Time:     ${result.planningTimeMs}ms`)
+  console.log(`  Rollouts: ${result.rolloutsPerformed}`)
+  console.log(`  Complete: ${result.completed}`)
+  console.log()
+
+  // All actions
+  console.log(`  Actions evaluated (${result.allActions.length}):`)
+  for (const ea of result.allActions) {
+    const selected = result.selectedAction === ea ? " ← SELECTED" : ""
+    console.log(`${formatAction(ea)}${selected}`)
+  }
+
+  // Selected action
+  if (result.selectedAction) {
+    const sel = result.selectedAction
+    const correctType = scenario.expectedTypes.includes(sel.action.actionType)
+    const check = correctType ? "✅" : "❌"
+    console.log(`\n  ${check} Selected: ${sel.action.actionType} (expected: ${scenario.expectedTypes.join("|")})`)
+  } else {
+    console.log(`\n  ❌ No action selected: ${result.noSelectionReason}`)
+  }
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+async function main() {
+  console.log("╔══════════════════════════════════════════════════════════════════════╗")
+  console.log("║  Planning Loop — End-to-End Test                                    ║")
+  console.log("║  PH (v2 transformer) → DM (rollouts) → IM (invariants) → Select    ║")
+  console.log("╚══════════════════════════════════════════════════════════════════════╝")
+
+  // Check prerequisites
+  const ph = new PolicyHeadInference(projectRoot)
+  console.log(`\n  PolicyHead loaded: ${ph.isLoaded} (version: ${ph.version})`)
+  if (ph.stats) {
+    console.log(`  Trained on: ${ph.stats.trained_on} examples`)
+    const valAcc = ph.stats.val_accuracy ?? ph.stats.direction_accuracy
+    console.log(`  Val accuracy: ${(valAcc > 1 ? valAcc : valAcc * 100).toFixed(1)}%`)
+  }
+
+  if (!ph.isLoaded) {
+    console.error("\n  ❌ PolicyHead not loaded — cannot run e2e test")
+    console.error("  Copy checkpoint to .jfl/checkpoints/best_policy_head.pt")
+    process.exit(1)
+  }
+
+  // Create planning loop
+  const planner = createPlanningLoop(projectRoot, {
+    topK: 5,
+    verbose: true,
+    maxPlanningTimeMs: 30000, // 30s for test
+    checkInvariants: true,
+    recordTransitions: false, // Don't pollute real data during test
+    writeTrainingTuples: false,
+  })
+
+  console.log(`  Planning loop ready: ${planner.isReady}`)
+
+  // Run each scenario
+  let passed = 0
+  let failed = 0
+  const results: Array<{ scenario: string; result: PlanningResult; correct: boolean }> = []
+
+  for (const scenario of SCENARIOS) {
+    try {
+      console.log(`\n  Running: ${scenario.name}...`)
+      const result = await planner.plan(scenario.agentId, scenario.goal)
+      logResult(scenario, result)
+
+      const correct = result.selectedAction
+        ? scenario.expectedTypes.includes(result.selectedAction.action.actionType)
+        : false
+
+      if (correct) passed++
+      else failed++
+
+      results.push({ scenario: scenario.name, result, correct })
+    } catch (err: any) {
+      console.error(`\n  ❌ ${scenario.name} THREW: ${err.message}`)
+      failed++
+      results.push({
+        scenario: scenario.name,
+        result: null as any,
+        correct: false,
+      })
+    }
+  }
+
+  // Summary
+  console.log(`\n${"═".repeat(70)}`)
+  console.log(`  SUMMARY`)
+  console.log(`${"─".repeat(70)}`)
+  for (const r of results) {
+    const check = r.correct ? "✅" : "❌"
+    const time = r.result ? `${r.result.planningTimeMs}ms` : "ERRORED"
+    const selected = r.result?.selectedAction?.action.actionType ?? "none"
+    console.log(`  ${check} ${r.scenario.padEnd(35)} → ${selected.padEnd(12)} (${time})`)
+  }
+  console.log(`\n  Passed: ${passed}/${SCENARIOS.length}  Failed: ${failed}/${SCENARIOS.length}`)
+  console.log(`${"═".repeat(70)}`)
+
+  process.exit(failed > 0 ? 1 : 0)
+}
+
+main().catch(err => {
+  console.error("Fatal error:", err)
+  process.exit(1)
+})

package/scripts/test-server-inference.ts

@@ -0,0 +1,49 @@
+#!/usr/bin/env npx tsx
+/**
+ * Test the inference server path in PolicyHead
+ */
+import { PolicyHeadInference } from "../src/lib/policy-head.js"
+
+async function main() {
+  const ph = new PolicyHeadInference(".")
+  console.log("PH loaded:", ph.isLoaded, "version:", ph.version)
+
+  if (!ph.isLoaded || ph.version !== 2) {
+    console.log("v2 not loaded, aborting")
+    process.exit(1)
+  }
+
+  const state = {
+    composite_score: 0.72,
+    dimension_scores: { test_pass_rate: 0.85, build_health: 0.9, code_quality: 0.8 },
+    tests_passing: 17,
+    tests_total: 20,
+    trajectory_length: 3,
+    recent_deltas: [-0.03, 0.01],
+    agent: "error-fixer",
+  }
+
+  const goals = [
+    "Fix failing tests in planning-loop.test.ts — TypeError on undefined property",
+    "Add multi-step rollout support to the planning loop",
+    "Add unit tests for the counterfactual training bridge",
+    "Update the nightly pipeline schedule to run at 2am MST",
+    "Reduce PolicyHead inference latency from 20s to under 5s",
+  ]
+
+  for (const goal of goals) {
+    const t0 = Date.now()
+    const result = await ph.selectAction(state, goal)
+    const elapsed = Date.now() - t0
+    console.log(`[${elapsed}ms] ${result.action} (${(result.confidence * 100).toFixed(1)}%) ← ${goal.slice(0, 60)}`)
+  }
+
+  console.log("\nStopping server...")
+  ph.stopServer()
+  console.log("Done")
+}
+
+main().catch(err => {
+  console.error(err)
+  process.exit(1)
+})

package/scripts/test-state-sensitivity.ts

@@ -0,0 +1,32 @@
+#!/usr/bin/env npx tsx
+/**
+ * Test how state values affect PolicyHead predictions
+ */
+import { PolicyHeadInference } from "../src/lib/policy-head.js"
+
+async function main() {
+  const ph = new PolicyHeadInference(".")
+  if (!ph.isLoaded || ph.version !== 2) { process.exit(1) }
+
+  const goal = "Add multi-step rollout support to the planning loop"
+
+  const states = [
+    { label: "healthy (all high)", composite_score: 0.95, dimension_scores: { test_pass_rate: 1.0, build_health: 1.0, code_quality: 0.95 }, tests_passing: 25, tests_total: 25, trajectory_length: 5, recent_deltas: [0.02, 0.01], agent: "feature-builder" },
+    { label: "degraded tests", composite_score: 0.72, dimension_scores: { test_pass_rate: 0.85, build_health: 0.9, code_quality: 0.8 }, tests_passing: 17, tests_total: 20, trajectory_length: 3, recent_deltas: [-0.03], agent: "error-fixer" },
+    { label: "low coverage", composite_score: 0.55, dimension_scores: { test_pass_rate: 1.0, build_health: 0.9, code_quality: 0.5 }, tests_passing: 15, tests_total: 15, trajectory_length: 2, recent_deltas: [0.01], agent: "test-coverage" },
+    { label: "fresh start", composite_score: 0.3, dimension_scores: { test_pass_rate: 0.5, build_health: 0.5, code_quality: 0.5 }, tests_passing: 5, tests_total: 10, trajectory_length: 0, recent_deltas: [], agent: "onboarding" },
+    { label: "near perfect", composite_score: 0.98, dimension_scores: { test_pass_rate: 1.0, build_health: 1.0, code_quality: 1.0, hub_health: 1.0 }, tests_passing: 30, tests_total: 30, trajectory_length: 10, recent_deltas: [0.005, 0.003], agent: "optimizer" },
+  ]
+
+  for (const s of states) {
+    const { label, ...state } = s
+    const t0 = Date.now()
+    const result = await ph.selectAction(state as any, goal)
+    const elapsed = Date.now() - t0
+    const alts = result.alternatives?.map(a => `${a.action}(${(a.confidence*100).toFixed(0)}%)`).join(", ") || ""
+    console.log(`[${elapsed}ms] ${label}: ${result.action} (${(result.confidence*100).toFixed(1)}%) | alts: ${alts}`)
+  }
+
+  ph.stopServer()
+}
+main().catch(err => { console.error(err); process.exit(1) })