@bradtaylorsf/alpha-loop 1.14.0 → 1.14.2

package/dist/lib/pipeline.js

@@ -5,7 +5,8 @@ import { mkdirSync, readFileSync, writeFileSync, unlinkSync, existsSync } from '
 import { join } from 'node:path';
 import { log } from './logger.js';
 import { exec } from './shell.js';
-import { spawnAgent } from './agent.js';
+import { spawnAgent, buildEndpointEnv } from './agent.js';
+import { classifyToolError, classifyToolErrors, EscalationTracker, defaultEscalationStatePath, appendEscalationEventToTrace, } from './escalation.js';
 import { setupWorktree, cleanupWorktree } from './worktree.js';
 import { assignIssue, labelIssue, commentIssue, createPR, mergePR, updateProjectStatus, getIssueComments, } from './github.js';
 import { buildImplementPrompt, buildReviewPrompt, buildAssumptionsPrompt, buildBatchPlanPrompt, buildBatchImplementPrompt, buildBatchReviewPrompt, } from './prompts.js';
@@ -13,8 +14,9 @@ import { runTests } from './testing.js';
 import { runVerify } from './verify.js';
 import { extractLearnings, getLearningContext } from './learning.js';
 import { saveResult, getPreviousResult } from './session.js';
-import { writeTrace, writeTraceMetadata, writeTraceToSubdir, writeConfigSnapshot, writeScores, writeCosts, computeScores, computeCosts, } from './traces.js';
-import { estimateCost } from './config.js';
+import { writeTrace, writeTraceMetadata, writeTraceToSubdir, writeConfigSnapshot, writeScores, writeCosts, computeScores, computeCosts, runDir, } from './traces.js';
+import { estimateCost, getFallbackPolicy, resolveRoutingStage, selectRoutingProfile } from './config.js';
+import { buildStageTelemetry, writeStageTelemetry } from './telemetry.js';
 /** Max diff size to include in learning analysis. */
 const MAX_DIFF_CHARS = 10_000;
 /**
@@ -212,16 +214,184 @@ export function formatGateFindings(gate, gateType) {
     }
     return lines.join('\n');
 }
+/**
+ * Build a StageTelemetry entry from the agent result and append it to the
+ * run's stages.jsonl file. Called alongside each stepCosts.push site so that
+ * every stage invocation — not just the top-level session — emits telemetry.
+ */
+function recordStageTelemetry(session, issueNum, stage, agentResult, config, ctx) {
+    try {
+        const entry = buildStageTelemetry(agentResult, stage, config, {
+            endpoint: ctx?.endpointName,
+            endpointType: ctx?.endpointType,
+            profile: ctx?.profile,
+            issueNum,
+        });
+        writeStageTelemetry(runDir(session.name), entry);
+    }
+    catch {
+        /* telemetry is best-effort */
+    }
+}
+/**
+ * Invoke an agent with retry-with-escalation and the rolling-rate guardrail.
+ *
+ * Resolves the stage's routing (model + endpoint), runs the agent once, and
+ * if the output contains two or more classified tool-call errors escalates
+ * the turn to the configured fallback model. Escalation is scoped to the
+ * current turn: the caller's next invocation reverts to the primary model.
+ *
+ * Also honors the guardrail — when the stage is pinned to fallback because
+ * the recent error rate exceeded the threshold, the primary is skipped
+ * entirely and the fallback runs up-front.
+ */
+async function spawnStageAgent(options, config, ctx) {
+    const policy = getFallbackPolicy(config);
+    const primary = resolveRoutingStage(config, ctx.stage);
+    const primaryModel = primary?.model ?? options.model;
+    const primaryEndpoint = primary?.endpoint;
+    const primaryEnv = primaryEndpoint ? buildEndpointEnv(primaryEndpoint, primaryModel) : undefined;
+    // Expose endpoint info back to the caller so it can tag telemetry.
+    const primaryEndpointName = config.routing?.stages?.[ctx.stage]?.endpoint;
+    ctx.endpointName = primaryEndpointName ?? 'default';
+    ctx.endpointType = primaryEndpoint?.type;
+    ctx.profile = selectRoutingProfile(config, ctx.issueNum);
+    const hasEscalateTarget = policy?.escalate_to !== undefined;
+    const escalateTo = policy?.escalate_to;
+    const escalateEndpoint = escalateTo
+        ? config.routing?.endpoints?.[escalateTo.endpoint]
+        : undefined;
+    const escalateEnv = escalateTo && escalateEndpoint
+        ? buildEndpointEnv(escalateEndpoint, escalateTo.model)
+        : undefined;
+    const nowIso = () => new Date().toISOString();
+    const recordEvent = (event) => {
+        ctx.events.push(event);
+        try {
+            appendEscalationEventToTrace(runDir(ctx.session.name), event);
+        }
+        catch { /* non-fatal */ }
+    };
+    // Guardrail: stage pinned to fallback — use fallback up-front.
+    const reverted = policy?.on_tool_error === 'escalate'
+        && hasEscalateTarget
+        && ctx.tracker.isStageReverted(ctx.stage);
+    if (reverted && escalateTo) {
+        recordEvent({
+            type: 'stage_revert_active',
+            stage: ctx.stage,
+            from_model: primaryModel,
+            to_model: escalateTo.model,
+            reason: 'rolling_error_rate_above_threshold',
+            turn_index: ctx.turnIndex,
+            issue: ctx.issueNum,
+            ts: nowIso(),
+        });
+        ctx.endpointName = escalateTo.endpoint;
+        ctx.endpointType = escalateEndpoint?.type;
+        const result = await spawnAgent({
+            ...options,
+            model: escalateTo.model,
+            env: escalateEnv,
+        });
+        ctx.tracker.recordTurn({
+            stage: ctx.stage,
+            errored: classifyToolError(result.output) !== null,
+            escalated: true,
+            windowSize: policy.escalation_window_issues,
+        });
+        return result;
+    }
+    // Normal primary invocation.
+    const firstResult = await spawnAgent({
+        ...options,
+        model: primaryModel,
+        env: primaryEnv,
+    });
+    const classified = classifyToolErrors(firstResult.output);
+    const errored = classified.length > 0;
+    let escalated = false;
+    let finalResult = firstResult;
+    if (policy?.on_tool_error === 'escalate'
+        && hasEscalateTarget
+        && escalateTo
+        && classified.length >= 2) {
+        const last = classified[classified.length - 1];
+        recordEvent({
+            type: 'escalation',
+            stage: ctx.stage,
+            from_model: primaryModel,
+            to_model: escalateTo.model,
+            reason: last.kind,
+            turn_index: ctx.turnIndex,
+            issue: ctx.issueNum,
+            ts: nowIso(),
+        });
+        ctx.endpointName = escalateTo.endpoint;
+        ctx.endpointType = escalateEndpoint?.type;
+        finalResult = await spawnAgent({
+            ...options,
+            model: escalateTo.model,
+            env: escalateEnv,
+        });
+        escalated = true;
+    }
+    if (policy) {
+        ctx.tracker.recordTurn({
+            stage: ctx.stage,
+            errored,
+            escalated,
+            windowSize: policy.escalation_window_issues,
+        });
+        if (ctx.tracker.maybeTriggerRevert(ctx.stage, policy)) {
+            const untilMs = ctx.tracker.revertUntil(ctx.stage) ?? 0;
+            recordEvent({
+                type: 'stage_revert',
+                stage: ctx.stage,
+                from_model: primaryModel,
+                to_model: escalateTo?.model ?? primaryModel,
+                reason: `rolling_error_rate_above_${policy.escalation_error_threshold}`,
+                turn_index: ctx.turnIndex,
+                issue: ctx.issueNum,
+                ts: nowIso(),
+            });
+            recordEvent({
+                type: 'needs_human_input',
+                stage: ctx.stage,
+                from_model: primaryModel,
+                to_model: escalateTo?.model ?? primaryModel,
+                reason: `stage_pinned_to_fallback_until_${new Date(untilMs).toISOString()}`,
+                turn_index: ctx.turnIndex,
+                issue: ctx.issueNum,
+                ts: nowIso(),
+            });
+        }
+    }
+    return finalResult;
+}
 /**
  * Process a single issue through the full pipeline.
  * Steps: status → worktree → plan → implement → test+retry → verify+retry →
  *        review → PR → learnings → update → auto-merge → cleanup
  */
-export async function processIssue(issueNum, title, body, config, session) {
+export async function processIssue(issueNum, title, body, config, session, trackerOverride) {
     const startTime = Date.now();
     const projectDir = process.cwd();
     const stepCosts = [];
     const stepsCompleted = [];
+    const escalationEvents = [];
+    const tracker = trackerOverride ?? new EscalationTracker({
+        statePath: defaultEscalationStatePath(projectDir),
+    });
+    let turnCounter = 0;
+    const stageCtx = (stage) => ({
+        stage,
+        issueNum,
+        turnIndex: ++turnCounter,
+        session,
+        tracker,
+        events: escalationEvents,
+    });
     // Setup logging
     mkdirSync(session.logsDir, { recursive: true });
     const logFile = join(session.logsDir, `issue-${issueNum}.log`);
@@ -309,7 +479,8 @@ Rules:
 - Write ONLY the JSON file. Do not create any other files or make any code changes.`;
             // Trace the plan prompt
             tracePrompt(session.name, issueNum, 'plan', planPrompt);
-            const planResult = await spawnAgent({
+            const planCtx = stageCtx('plan');
+            const planResult = await spawnStageAgent({
                 agent: config.agent,
                 model: config.model,
                 prompt: planPrompt,
@@ -317,10 +488,11 @@ Rules:
                 logFile: join(session.logsDir, `issue-${issueNum}-plan.log`),
                 verbose: config.verbose,
                 timeout: config.agentTimeout * 1000,
-            });
+            }, config, planCtx);
             // Trace the plan output and costs
             traceOutput(session.name, issueNum, 'plan', planResult.output);
             stepCosts.push(buildStepCost('plan', issueNum, planResult, config));
+            recordStageTelemetry(session, issueNum, 'plan', planResult, config, planCtx);
             // Detect transient errors (usage limits) during planning
             if (planResult.exitCode !== 0 && isTransientError(planResult.output)) {
                 log.warn(`Agent hit a transient error during planning for #${issueNum} — re-queuing`);
@@ -381,7 +553,8 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
         });
         // Trace the implement prompt
         tracePrompt(session.name, issueNum, 'implement', implementPrompt);
-        const implResult = await spawnAgent({
+        const implCtx = stageCtx('build');
+        const implResult = await spawnStageAgent({
             agent: config.agent,
             model: config.model,
             prompt: implementPrompt,
@@ -389,10 +562,11 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
             logFile: join(session.logsDir, `issue-${issueNum}-implement.log`),
             verbose: config.verbose,
             timeout: config.agentTimeout * 1000,
-        });
+        }, config, implCtx);
         // Trace the implement output and costs
         traceOutput(session.name, issueNum, 'implement', implResult.output);
         stepCosts.push(buildStepCost('implement', issueNum, implResult, config));
+        recordStageTelemetry(session, issueNum, 'implement', implResult, config, implCtx);
         if (implResult.exitCode !== 0) {
             // Auto-commit any uncommitted work before deciding on cleanup
             const dirtyCheck = exec('git status --porcelain', { cwd: worktreePath });
@@ -459,7 +633,8 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                 const fixPrompt = `Tests are failing for issue #${issueNum} (attempt ${attempt} of ${config.maxTestRetries}). Fix the failing tests.\n\nTest output:\n${testOutput}\n\nInstructions:\n1. Read the failing test output carefully and identify the ROOT CAUSE\n2. Fix ONLY code related to issue #${issueNum} — do NOT modify test infrastructure, build scripts, or unrelated files\n3. If tests fail due to environment issues (missing venv, wrong port, missing deps), fix only YOUR code — do NOT rewrite the test runner or package.json scripts\n4. Run the tests again to verify\n5. Commit your fixes with a DESCRIPTIVE message that explains WHAT you fixed and WHY it failed.\n   Format: fix(#${issueNum}): <what you changed> — <why it was failing>\n   Example: fix(#${issueNum}): use port 5435 for postgres — default 5432 conflicts with host service\n   DO NOT use generic messages like "fix: resolve test failures"`;
                 // Trace fix prompt
                 tracePrompt(session.name, issueNum, `fix-${attempt}`, fixPrompt);
-                const fixResult = await spawnAgent({
+                const fixCtx = stageCtx('test_write');
+                const fixResult = await spawnStageAgent({
                     agent: config.agent,
                     model: config.model,
                     prompt: fixPrompt,
@@ -468,10 +643,11 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                     logFile: join(session.logsDir, `issue-${issueNum}-fix-${attempt}.log`),
                     verbose: config.verbose,
                     timeout: config.agentTimeout * 1000,
-                });
+                }, config, fixCtx);
                 // Trace fix output and costs
                 traceOutput(session.name, issueNum, `fix-${attempt}`, fixResult.output);
                 stepCosts.push(buildStepCost('test_fix', issueNum, fixResult, config));
+                recordStageTelemetry(session, issueNum, 'test_fix', fixResult, config, fixCtx);
                 stepsCompleted.push(`fix-${attempt}`);
                 // Auto-commit fixes
                 const fixStatus = exec('git status --porcelain', { cwd: worktreePath });
@@ -519,7 +695,8 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                 });
                 // Trace review prompt
                 tracePrompt(session.name, issueNum, `review${attempt > 1 ? `-${attempt}` : ''}`, reviewPrompt);
-                const reviewResult = await spawnAgent({
+                const reviewCtx = stageCtx('review');
+                const reviewResult = await spawnStageAgent({
                     agent: config.agent,
                     model: config.reviewModel,
                     prompt: reviewPrompt,
@@ -527,10 +704,11 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                     logFile: join(session.logsDir, `issue-${issueNum}-review${attempt > 1 ? `-${attempt}` : ''}.log`),
                     verbose: config.verbose,
                     timeout: config.agentTimeout * 1000,
-                });
+                }, config, reviewCtx);
                 // Trace review output and costs
                 traceOutput(session.name, issueNum, `review${attempt > 1 ? `-${attempt}` : ''}`, reviewResult.output);
                 stepCosts.push(buildStepCost('review', issueNum, reviewResult, config));
+                recordStageTelemetry(session, issueNum, 'review', reviewResult, config, reviewCtx);
                 reviewOutput = reviewResult.output;
             }
             catch {
@@ -554,7 +732,8 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                 const fixPrompt = `The code review for issue #${issueNum} found problems that need to be fixed.\n\n${findings}\n\nInstructions:\n1. Address each finding listed above\n2. Run tests to make sure nothing is broken\n3. Commit your fixes with: git commit -m "fix(#${issueNum}): address review findings"`;
                 // Trace review-fix prompt
                 tracePrompt(session.name, issueNum, `review-fix-${attempt}`, fixPrompt);
-                const reviewFixResult = await spawnAgent({
+                const reviewFixCtx = stageCtx('build');
+                const reviewFixResult = await spawnStageAgent({
                     agent: config.agent,
                     model: config.model,
                     prompt: fixPrompt,
@@ -563,10 +742,11 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                     logFile: join(session.logsDir, `issue-${issueNum}-review-fix-${attempt}.log`),
                     verbose: config.verbose,
                     timeout: config.agentTimeout * 1000,
-                });
+                }, config, reviewFixCtx);
                 // Trace review-fix output and costs
                 traceOutput(session.name, issueNum, `review-fix-${attempt}`, reviewFixResult.output);
                 stepCosts.push(buildStepCost('review', issueNum, reviewFixResult, config));
+                recordStageTelemetry(session, issueNum, 'review_fix', reviewFixResult, config, reviewFixCtx);
                 // Auto-commit if agent didn't
                 const fixStatus = exec('git status --porcelain', { cwd: worktreePath });
                 if (fixStatus.stdout.trim()) {
@@ -647,7 +827,8 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                     const fixPrompt = `Live verification failed for issue #${issueNum} (attempt ${attempt} of ${config.maxTestRetries}).\n\n${findings}\n\nInstructions:\n1. Read the verification findings and identify the ROOT CAUSE\n2. Fix ONLY code related to issue #${issueNum}\n3. Run tests to make sure nothing is broken\n4. Commit your fixes with: git commit -m "fix(#${issueNum}): address verification findings"`;
                     // Trace verify-fix prompt
                     tracePrompt(session.name, issueNum, `verify-fix-${attempt}`, fixPrompt);
-                    const verifyFixResult = await spawnAgent({
+                    const verifyFixCtx = stageCtx('test_exec');
+                    const verifyFixResult = await spawnStageAgent({
                         agent: config.agent,
                         model: config.model,
                         prompt: fixPrompt,
@@ -656,10 +837,11 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                         logFile: join(session.logsDir, `issue-${issueNum}-verify-fix-${attempt}.log`),
                         verbose: config.verbose,
                         timeout: config.agentTimeout * 1000,
-                    });
+                    }, config, verifyFixCtx);
                     // Trace verify-fix output and costs
                     traceOutput(session.name, issueNum, `verify-fix-${attempt}`, verifyFixResult.output);
                     stepCosts.push(buildStepCost('verify', issueNum, verifyFixResult, config));
+                    recordStageTelemetry(session, issueNum, 'verify_fix', verifyFixResult, config, verifyFixCtx);
                     // Auto-commit if agent didn't
                     const fixStatus = exec('git status --porcelain', { cwd: worktreePath });
                     if (fixStatus.stdout.trim()) {
@@ -749,6 +931,9 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
             });
             traceOutput(session.name, issueNum, 'assumptions', assumptionsResult.output);
             stepCosts.push(buildStepCost('assumptions', issueNum, assumptionsResult, config));
+            recordStageTelemetry(session, issueNum, 'assumptions', assumptionsResult, config, {
+                profile: selectRoutingProfile(config, issueNum),
+            });
             if (assumptionsResult.exitCode === 0 && assumptionsResult.output.trim()) {
                 commentIssue(config.repo, issueNum, `## AI Implementation Notes\n\n${assumptionsResult.output.trim()}\n\n---\n_Posted by alpha-loop for user validation._`);
                 log.success('Posted assumptions/decisions comment');
@@ -912,6 +1097,7 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
         verifySkipped,
         duration,
         filesChanged,
+        escalationEvents: escalationEvents.length > 0 ? escalationEvents : undefined,
     };
     // Save result to session
     saveResult(session, result);
@@ -1019,6 +1205,9 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
             });
             traceOutput(session.name, issues[0].number, 'batch-plan', planResult.output);
             stepCosts.push(buildStepCost('plan', issues[0].number, planResult, config));
+            recordStageTelemetry(session, issues[0].number, 'batch-plan', planResult, config, {
+                profile: selectRoutingProfile(config, issues[0].number),
+            });
             if (planResult.exitCode !== 0 && isTransientError(planResult.output)) {
                 log.warn('Agent hit a transient error during batch planning — re-queuing all issues');
                 for (const issue of issues)
@@ -1073,6 +1262,9 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
         });
         traceOutput(session.name, issues[0].number, 'batch-implement', implResult.output);
         stepCosts.push(buildStepCost('implement', issues[0].number, implResult, config));
+        recordStageTelemetry(session, issues[0].number, 'batch-implement', implResult, config, {
+            profile: selectRoutingProfile(config, issues[0].number),
+        });
         if (implResult.exitCode !== 0) {
             // Auto-commit any uncommitted work before deciding on cleanup
             const dirtyCheck = exec('git status --porcelain', { cwd: worktreePath });
@@ -1152,6 +1344,9 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                 });
                 traceOutput(session.name, issues[0].number, `batch-fix-${attempt}`, fixResult.output);
                 stepCosts.push(buildStepCost('test_fix', issues[0].number, fixResult, config));
+                recordStageTelemetry(session, issues[0].number, 'batch-test_fix', fixResult, config, {
+                    profile: selectRoutingProfile(config, issues[0].number),
+                });
                 const fixStatus = exec('git status --porcelain', { cwd: worktreePath });
                 if (fixStatus.stdout.trim()) {
                     exec('git add -A', { cwd: worktreePath });
@@ -1193,6 +1388,9 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                 });
                 traceOutput(session.name, issues[0].number, `batch-review${attempt > 1 ? `-${attempt}` : ''}`, reviewResult.output);
                 stepCosts.push(buildStepCost('review', issues[0].number, reviewResult, config));
+                recordStageTelemetry(session, issues[0].number, 'batch-review', reviewResult, config, {
+                    profile: selectRoutingProfile(config, issues[0].number),
+                });
                 reviewOutput = reviewResult.output;
             }
             catch {
@@ -1224,6 +1422,9 @@ Do NOT redo work that is already committed. Build on top of existing progress.\n
                 });
                 traceOutput(session.name, issues[0].number, `batch-review-fix-${attempt}`, reviewFixResult.output);
                 stepCosts.push(buildStepCost('review', issues[0].number, reviewFixResult, config));
+                recordStageTelemetry(session, issues[0].number, 'batch-review_fix', reviewFixResult, config, {
+                    profile: selectRoutingProfile(config, issues[0].number),
+                });
                 const fixStatus = exec('git status --porcelain', { cwd: worktreePath });
                 if (fixStatus.stdout.trim()) {
                     exec('git add -A', { cwd: worktreePath });