ai-foreman 1.7.11 → 1.7.13

package/dist/diagnostics.js

@@ -0,0 +1,428 @@
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { hostname } from "node:os";
+import { join, resolve } from "node:path";
+import { readBuildRuns } from "./buildRuns.js";
+import { diagnosticDigest, ObservabilityReader, sanitizeDiagnosticValue } from "./observability.js";
+import { WorkflowReader } from "./workflowReader.js";
+import { isTicketsInitialized, loadTicketsConfig, resolveTicketPaths } from "./tickets/config.js";
+import { loadTickets } from "./tickets/ticketLoader.js";
+const WAIT_KINDS = new Set(["user_wait", "dependency_wait", "external_ci_wait"]);
+const ALLOWED_GIT = new Set(["status", "rev-parse", "rev-list", "diff"]);
+export function selectDiagnosticRun(projectDir, requestedRunId) {
+    const runs = readBuildRuns(projectDir);
+    if (requestedRunId) {
+        const exact = runs.find(run => run.runId === requestedRunId);
+        if (!exact)
+            throw new Error(`build run not found: ${requestedRunId}`);
+        return exact;
+    }
+    const workflow = new WorkflowReader(projectDir);
+    try {
+        const lease = workflow.currentLease();
+        const leased = lease && evaluateLease(lease, new Date()).live ? runs.find(run => run.runId === lease.runId) : undefined;
+        if (leased)
+            return leased;
+    }
+    finally {
+        workflow.close();
+    }
+    const latest = runs[0];
+    if (!latest)
+        throw new Error("no build runs found for this project");
+    return latest;
+}
+/** Collects and calculates a canonical report. The provider receives this report, never raw database rows. */
+export function collectManagerDiagnostics(projectDir, options = {}) {
+    const root = resolve(projectDir);
+    const now = options.now ?? new Date();
+    const run = selectDiagnosticRun(root, options.runId);
+    const observable = new ObservabilityReader(root);
+    const workflow = new WorkflowReader(root);
+    const sources = {};
+    const evidence = [];
+    try {
+        const schema = observable.schemaVersion();
+        sources.observability = source("observability", schema === 1 || schema === 2 ? "available" : observable.available() ? "partial" : "unavailable", now, schema === 1 || schema === 2 ? undefined : observable.available() ? `unsupported schema ${schema ?? "unknown"}` : "observability database is absent");
+        sources.recovery = source("recovery", workflow.available() ? "available" : "unavailable", now, workflow.available() ? undefined : "recovery database is absent");
+        sources.build_snapshot = source("build_snapshot", "available", now);
+        sources.ticket_definitions = source("ticket_definitions", existsSync(join(root, ".tickets", "tickets.yaml")) ? "available" : "not_applicable", now);
+        const spans = observable.spans(run.runId);
+        const retainedSpanCount = spans.length;
+        const currentState = reconcileCurrentState(observable.currentState(run.runId), spans, now);
+        const executions = observable.executions(run.runId);
+        const workflowRun = workflow.getRun(run.runId);
+        const workflowEvents = workflow.events(run.runId);
+        const issues = workflow.issues(run.runId);
+        const operations = workflow.operations(run.runId);
+        const lease = workflow.currentLease();
+        const leaseEvidence = evaluateLease(lease?.runId === run.runId ? lease : undefined, now);
+        const continuity = workflow.continuityHeads(run.runId);
+        sources.continuity = source("continuity", continuity.length ? "available" : workflow.available() ? "partial" : "unavailable", now, continuity.length ? undefined : "no continuity head is recorded for this run");
+        for (const head of continuity)
+            evidence.push({ evidenceId: `continuity-${head.role}`, source: "continuity", kind: "continuity_head",
+                summary: `${head.role} continuity is ${head.state} at sequence ${head.sequence}`, observedAt: head.updatedAt });
+        if (!spans.length) {
+            const legacy = readLegacyLifecycle(root, run, options.maxDetailSpans ?? 50);
+            spans.push(...legacy.spans);
+            sources.legacy_jsonl = source("legacy_jsonl", legacy.found ? "partial" : "unavailable", now, legacy.found ? "legacy lifecycle timing is incomplete" : "no correlated legacy lifecycle records found");
+        }
+        else
+            sources.legacy_jsonl = source("legacy_jsonl", "not_applicable", now);
+        const storedSummary = observable.runSummaries({ runIds: [run.runId], limit: 1 })[0];
+        if (leaseEvidence.live) {
+            const git = collectGit(root, run.repository.worktree, options.commandRunner ?? defaultCommandRunner);
+            sources.git = source("git", git.state, now, git.detail);
+            for (const item of git.evidence)
+                evidence.push(item);
+        }
+        else if (storedSummary?.git) {
+            sources.git = source("git", "available", now, "saved terminal Git summary; no unrelated current repository state was queried");
+            evidence.push({ evidenceId: "git-summary", source: "git", kind: "git", summary: `saved branch=${storedSummary.git.branch ?? "unknown"}, head=${storedSummary.git.terminalHead?.slice(0, 12) ?? "unknown"}, commits=${storedSummary.git.commitCount ?? "unavailable"}, changedPaths=${storedSummary.git.changedPathCount ?? "unavailable"}`, observedAt: storedSummary.git.observedAt });
+            if (storedSummary.git.pullRequest)
+                evidence.push({ evidenceId: `external-${storedSummary.git.pullRequest.provider}-${storedSummary.git.pullRequest.id}`, source: "external", kind: storedSummary.git.pullRequest.provider === "github" ? "pull_request" : "merge_request", summary: `saved ${storedSummary.git.pullRequest.provider} review ${storedSummary.git.pullRequest.id}, state=${storedSummary.git.pullRequest.state ?? "unknown"}`, observedAt: storedSummary.git.observedAt });
+        }
+        else
+            sources.git = source("git", "partial", now, "no saved terminal Git summary; current repository state was not attributed to this historical run");
+        const shouldExternal = options.external === "on" || (options.external !== "off" && /\b(?:pr|pull request|mr|merge request|ci|pipeline|check|dependency|waiting)\b/i.test(`${options.question ?? ""} ${run.checkpoint}`));
+        const external = shouldExternal && leaseEvidence.live
+            ? collectExternal(run.repository.worktree, options.commandRunner ?? defaultCommandRunner, now)
+            : shouldExternal && storedSummary?.git?.pullRequest
+                ? { state: "available", detail: "saved historical external evidence", evidence: [] }
+                : { state: shouldExternal ? "partial" : "not_applicable", detail: shouldExternal ? "live external refresh is limited to a verified active run; saved historical evidence is unavailable" : undefined, evidence: [] };
+        sources.external = source("external", external.state, now, external.detail);
+        evidence.push(...external.evidence);
+        const terminalEnd = run.completedAt ?? (["completed", "failed", "superseded"].includes(run.status) ? run.updatedAt : undefined);
+        const observationEnd = leaseEvidence.live ? now : new Date(terminalEnd ?? run.updatedAt);
+        const calendarAgeMs = Math.max(0, observationEnd.getTime() - new Date(run.createdAt).getTime());
+        const executionIntervals = executions.length
+            ? executions.map(item => interval(item.startedAt, item.endedAt ?? observationEnd.toISOString(), observationEnd))
+            : leaseEvidence.live ? [interval(run.createdAt, undefined, now)] : [interval(run.createdAt, terminalEnd ?? run.updatedAt, observationEnd)];
+        const activeExecutionMs = unionDuration(executionIntervals);
+        const explicitWaitMs = unionDuration(spans.filter(span => WAIT_KINDS.has(span.kind)).map(span => spanInterval(span, now)));
+        const attributedMs = unionDuration(spans.map(span => spanInterval(span, now)));
+        const inclusiveByKind = durationsByKind(spans, now);
+        const exclusiveByKind = exclusiveDurationsByKind(spans, now);
+        const byKind = exclusiveByKind;
+        const observedRetryMs = inclusiveByKind.retry ?? 0;
+        const reportedRetryDelayMs = spans.filter(span => span.kind === "retry").reduce((sum, span) => sum + numericAttribute(span, "reportedDelayMs"), 0);
+        const unattributedMs = Math.max(0, activeExecutionMs - attributedMs);
+        const pausedOfflineMs = Math.max(0, calendarAgeMs - activeExecutionMs);
+        const topContributors = Object.entries(byKind).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 3).map(([kind, durationMs]) => ({ kind, durationMs }));
+        evidence.push({ evidenceId: "timing-active", source: "observability", kind: "active_execution", summary: `Active execution accounted for ${activeExecutionMs} ms`, durationMs: activeExecutionMs });
+        for (const [index, contributor] of topContributors.entries())
+            evidence.push({ evidenceId: `timing-${index + 1}`, source: "observability", kind: contributor.kind, summary: `${contributor.kind} accounted for ${contributor.durationMs} ms`, durationMs: contributor.durationMs });
+        const significantSpans = spans.filter(span => WAIT_KINDS.has(span.kind) || ["retry", "qa_attempt", "qa_fix", "fix"].includes(span.kind)
+            || ["failed", "error", "blocked"].includes(span.outcome ?? ""))
+            .sort((left, right) => new Date(right.startedAt).getTime() - new Date(left.startedAt).getTime()).slice(0, 30);
+        for (const span of significantSpans)
+            evidence.push({ evidenceId: `span-${span.spanId}`, source: "observability", kind: span.kind,
+                summary: `${span.name}: ${span.outcome ?? (span.endedAt ? "completed" : "open")}`, observedAt: span.endedAt ?? span.startedAt,
+                durationMs: span.durationMs ?? liveDuration(span, now), spanId: span.spanId });
+        const longOperations = spans.filter(span => span.kind === "tool" || span.kind === "provider_turn")
+            .sort((left, right) => (right.durationMs ?? liveDuration(right, now)) - (left.durationMs ?? liveDuration(left, now))).slice(0, 6);
+        for (const span of longOperations)
+            evidence.push({ evidenceId: `long-${span.spanId}`, source: "observability", kind: span.kind,
+                summary: `${span.endedAt ? "completed" : "open"} ${span.kind}: ${span.name}`, observedAt: span.endedAt ?? span.startedAt,
+                durationMs: span.durationMs ?? liveDuration(span, now), spanId: span.spanId });
+        sources.process_health = source("process_health", run.status !== "running" ? "not_applicable" : leaseEvidence.live || leaseEvidence.failureObserved ? "available" : "partial", now, leaseEvidence.summary);
+        evidence.push({ evidenceId: "lease-health", source: "recovery", kind: "lease", summary: leaseEvidence.summary, observedAt: lease?.heartbeatAt });
+        for (const issue of issues.slice(-10))
+            evidence.push({ evidenceId: `issue-${evidence.length + 1}`, source: "recovery", kind: issue.code, summary: safeSummary(issue.detail, 500), observedAt: issue.occurred_at });
+        for (const operation of operations.filter(item => item.status === "failed" || item.status === "uncertain").slice(-10))
+            evidence.push({ evidenceId: `operation-${evidence.length + 1}`, source: "recovery", kind: operation.kind, summary: safeSummary(`${operation.kind}: ${operation.status}${operation.error ? ` (${operation.error.slice(0, 300)})` : ""}`, 500), observedAt: operation.updatedAt });
+        const counts = {
+            qaAttempts: spans.filter(span => span.kind === "qa_attempt").length,
+            qaFailures: spans.filter(span => span.kind === "qa_attempt" && span.outcome && !["passed", "completed"].includes(span.outcome)).length,
+            fixes: spans.filter(span => span.kind === "qa_fix" || span.kind === "fix").length,
+            retries: spans.filter(span => span.kind === "retry").length + workflowEvents.filter(item => item.type.includes("retry")).length,
+            waits: spans.filter(span => WAIT_KINDS.has(span.kind)).length,
+        };
+        const comparisons = chooseComparison(observable.rollups(), run, ticketSizeBucketForRun(root, run.tickets), now);
+        const findings = buildFindings({ run, now, spans, currentState, evidence, topContributors, activeExecutionMs, explicitWaitMs, unattributedMs, byKind, counts, comparisons, leaseEvidence });
+        if (findings.every((finding) => finding.code === "largest_contributors"))
+            findings.push({ code: "progressing", title: "Run appears to be progressing", summary: `No supported anomaly is present; the longest current phase is ${topContributors[0]?.kind ?? run.checkpoint}.`, confidence: spans.length ? "derived" : "limited", evidenceIds: topContributors[0] ? ["timing-1", "lease-health"] : ["lease-health"] });
+        const max = Math.max(1, options.maxDetailSpans ?? 100);
+        const sortedDetail = [...spans].sort((a, b) => (b.durationMs ?? liveDuration(b, now)) - (a.durationMs ?? liveDuration(a, now)) || a.spanId.localeCompare(b.spanId));
+        const capability = observable.capability(run.runId);
+        if (observable.storage().summaryOnly)
+            sources.observability = source("observability", "partial", now, "detail hard limit reached; summary-only mode is active");
+        const withoutDigest = {
+            version: 1, observabilitySchemaVersion: schema === 2 ? 2 : 1, generatedAt: now.toISOString(), runId: run.runId,
+            runStatus: workflowRun?.status ?? run.status, legacy: Boolean(workflowRun?.legacy || !observable.available() || (!retainedSpanCount && !executions.length && !storedSummary)),
+            capabilities: { version: 1, ...(capability?.rafiVersion ? { rafiVersion: capability.rafiVersion } : {}), sources: { ...capability?.sources, ...sources } },
+            currentState, timing: { calendarAgeMs, activeExecutionMs, pausedOfflineMs, explicitWaitMs, attributedMs, unattributedMs, byKind, inclusiveByKind, exclusiveByKind, observedRetryMs, reportedRetryDelayMs, topContributors },
+            counts, ...(comparisons ? { comparisons } : {}), findings, evidence,
+            detail: { spans: sortedDetail.slice(0, max), omittedSpans: Math.max(0, sortedDetail.length - max) },
+        };
+        return { ...withoutDigest, digest: diagnosticDigest(withoutDigest) };
+    }
+    finally {
+        observable.close();
+        workflow.close();
+    }
+}
+function buildFindings(input) {
+    const findings = [];
+    if (input.topContributors.length)
+        findings.push({ code: "largest_contributors", title: "Largest measured contributors", summary: input.topContributors.map(item => `${item.kind} (${formatDuration(item.durationMs)})`).join(", "), confidence: input.spans.length ? "derived" : "limited", evidenceIds: input.topContributors.map((_, index) => `timing-${index + 1}`) });
+    if (input.comparisons && (input.activeExecutionMs > input.comparisons.p90Ms || input.activeExecutionMs >= input.comparisons.medianMs * 1.5))
+        findings.push({ code: "duration_abnormal", title: "Duration exceeds comparable runs", summary: `Active execution is ${formatDuration(input.activeExecutionMs)} versus a ${formatDuration(input.comparisons.medianMs)} cohort median and ${formatDuration(input.comparisons.p90Ms)} p90.`, confidence: "derived", evidenceIds: ["timing-active"] });
+    const qaMs = (input.byKind.qa_attempt ?? 0) + (input.byKind.qa_fix ?? 0) + (input.byKind.fix ?? 0);
+    if (input.counts.qaFailures > 1 || (input.activeExecutionMs > 0 && qaMs / input.activeExecutionMs > 0.3))
+        findings.push({ code: "qa_rework", title: "QA rework is a material contributor", summary: `${input.counts.qaFailures} failed review(s), ${input.counts.fixes} fix pass(es), and ${formatDuration(qaMs)} of QA/fix time were observed.`, confidence: "derived", evidenceIds: evidenceForKinds(input.evidence, ["qa_attempt", "qa_fix", "fix"]) });
+    const retryMs = input.byKind.retry ?? 0;
+    const reportedRetryMs = input.spans.filter(span => span.kind === "retry").reduce((sum, span) => sum + numericAttribute(span, "reportedDelayMs"), 0);
+    if (retryMs > 60_000 || reportedRetryMs > 60_000 || (input.activeExecutionMs > 0 && Math.max(retryMs, reportedRetryMs) / input.activeExecutionMs > 0.1))
+        findings.push({ code: "retry_overhead", title: "Retry overhead is material", summary: `${formatDuration(retryMs)} observed and ${formatDuration(reportedRetryMs)} provider-reported delay across ${input.counts.retries} retry record(s).`, confidence: "derived", evidenceIds: evidenceForKinds(input.evidence, ["retry"]) });
+    if (input.unattributedMs > 60_000 && input.activeExecutionMs > 0 && input.unattributedMs / input.activeExecutionMs > 0.1)
+        findings.push({ code: "unattributed_time", title: "Material time is unattributed", summary: `${formatDuration(input.unattributedMs)} of active execution has no correlated span.`, confidence: "limited", evidenceIds: ["timing-active", "lease-health"] });
+    if (input.explicitWaitMs > 0)
+        findings.push({ code: "explicit_wait", title: "The run includes an explicit wait", summary: `${formatDuration(input.explicitWaitMs)} is attributed to user, dependency, or CI waiting.`, confidence: "observed", evidenceIds: evidenceForKinds(input.evidence, [...WAIT_KINDS]) });
+    const latestProgress = input.currentState.map(state => state.lastSemanticProgressAt).filter((value) => Boolean(value)).sort().at(-1);
+    const staleMs = latestProgress ? Math.max(0, input.now.getTime() - new Date(latestProgress).getTime()) : 0;
+    const activeWait = input.spans.some(span => WAIT_KINDS.has(span.kind) && !span.endedAt);
+    if (!activeWait && input.run.status === "running" && staleMs >= 300_000 && input.leaseEvidence.live)
+        findings.push({ code: "possibly_stalled", title: "Execution is possibly stalled", summary: `No semantic progress has been observed for ${formatDuration(staleMs)} while the lease/process evidence remains live.`, confidence: "derived", evidenceIds: ["lease-health"] });
+    if (input.leaseEvidence.failureObserved)
+        findings.push({ code: "lease_process_failure", title: "Lease or process failure observed", summary: input.leaseEvidence.summary, confidence: "observed", evidenceIds: ["lease-health"] });
+    return findings;
+}
+function reconcileCurrentState(states, spans, now) {
+    const open = new Map();
+    for (const span of spans.filter(item => !item.endedAt)) {
+        const role = span.role ?? "host";
+        const stream = span.stream ?? role;
+        const key = `${role}\u0000${stream}`;
+        const current = open.get(key);
+        if (!current || new Date(span.startedAt).getTime() > new Date(current.startedAt).getTime())
+            open.set(key, span);
+    }
+    const seen = new Set();
+    const reconciled = states.map((state) => {
+        const key = `${state.role}\u0000${state.stream}`;
+        seen.add(key);
+        const active = open.get(key);
+        const { activeSpanId: _activeSpanId, activeSpanKind: _activeSpanKind, ...base } = state;
+        return active ? { ...base, phase: active.name, activeSpanId: active.spanId, activeSpanKind: active.kind } : base;
+    });
+    for (const [key, span] of open) {
+        if (seen.has(key))
+            continue;
+        reconciled.push({ version: 1, runId: span.runId, role: span.role ?? "host", stream: span.stream ?? span.role ?? "host",
+            ...(span.executionId ? { executionId: span.executionId } : {}), ...(span.ticketId ? { ticketId: span.ticketId } : {}),
+            ...(span.deliveryUnitId ? { deliveryUnitId: span.deliveryUnitId } : {}), ...(span.providerSessionId ? { providerSessionId: span.providerSessionId } : {}),
+            phase: span.name, activeSpanId: span.spanId, activeSpanKind: span.kind, updatedAt: now.toISOString() });
+    }
+    return reconciled;
+}
+function chooseComparison(rollups, run, sizeBucket, now) {
+    const cutoff = now.getTime() - 90 * 86_400_000;
+    const candidates = rollups.filter(item => item.runId !== run.runId && item.status === "completed" && item.completedAt && new Date(item.completedAt).getTime() >= cutoff).slice(0, 50);
+    const provider = run.builder?.settings.make;
+    const qa = Boolean(run.qa);
+    const countBucket = ticketCountBucket(run.tickets.length);
+    const levels = [
+        { cohort: { branchMode: run.branchMode, qaEnabled: qa, primaryProvider: provider ?? "unknown", ticketCountBucket: countBucket, ticketSizeBucket: sizeBucket ?? "unknown" }, match: item => item.branchMode === run.branchMode && item.qaEnabled === qa && item.primaryProvider === provider && item.ticketCountBucket === countBucket && item.ticketSizeBucket === sizeBucket },
+        { cohort: { branchMode: run.branchMode, qaEnabled: qa, primaryProvider: provider ?? "unknown" }, match: item => item.branchMode === run.branchMode && item.qaEnabled === qa && item.primaryProvider === provider },
+        { cohort: { branchMode: run.branchMode, qaEnabled: qa }, match: item => item.branchMode === run.branchMode && item.qaEnabled === qa },
+    ];
+    for (const level of levels) {
+        const sample = candidates.filter(level.match).map(item => item.activeExecutionMs).filter(value => value >= 0).sort((a, b) => a - b);
+        if (sample.length < 5)
+            continue;
+        return { cohort: level.cohort, sampleSize: sample.length, medianMs: percentile(sample, 0.5), p75Ms: percentile(sample, 0.75), p90Ms: percentile(sample, 0.9) };
+    }
+    return undefined;
+}
+function durationsByKind(spans, now) {
+    const grouped = new Map();
+    for (const span of spans)
+        grouped.set(span.kind, [...(grouped.get(span.kind) ?? []), spanInterval(span, now)]);
+    return Object.fromEntries([...grouped].map(([kind, intervals]) => [kind, unionDuration(intervals)]));
+}
+function exclusiveDurationsByKind(spans, now) {
+    const byParent = new Map();
+    for (const span of spans)
+        if (span.parentSpanId)
+            byParent.set(span.parentSpanId, [...(byParent.get(span.parentSpanId) ?? []), span]);
+    const grouped = new Map();
+    for (const span of spans) {
+        const base = spanInterval(span, now);
+        const children = (byParent.get(span.spanId) ?? []).map(child => intersect(base, spanInterval(child, now))).filter((value) => Boolean(value));
+        const segments = subtractIntervals(base, children);
+        grouped.set(span.kind, [...(grouped.get(span.kind) ?? []), ...segments]);
+    }
+    return Object.fromEntries([...grouped].map(([kind, intervals]) => [kind, unionDuration(intervals)]));
+}
+function subtractIntervals(base, cuts) {
+    const merged = mergeIntervals(cuts);
+    const result = [];
+    let cursor = base[0];
+    for (const [start, end] of merged) {
+        if (start > cursor)
+            result.push([cursor, start]);
+        cursor = Math.max(cursor, end);
+    }
+    if (cursor < base[1])
+        result.push([cursor, base[1]]);
+    return result;
+}
+function mergeIntervals(intervals) {
+    const sorted = intervals.sort((a, b) => a[0] - b[0]);
+    const out = [];
+    for (const item of sorted) {
+        const last = out.at(-1);
+        if (!last || item[0] > last[1])
+            out.push([...item]);
+        else
+            last[1] = Math.max(last[1], item[1]);
+    }
+    return out;
+}
+function intersect(left, right) { const start = Math.max(left[0], right[0]); const end = Math.min(left[1], right[1]); return end >= start ? [start, end] : undefined; }
+function spanInterval(span, now) { return interval(span.startedAt, span.endedAt, now); }
+function interval(start, end, now) { const a = new Date(start).getTime(); const b = end ? new Date(end).getTime() : now.getTime(); return [a, Math.max(a, b)]; }
+export function unionDuration(intervals) {
+    const sorted = intervals.filter(([a, b]) => Number.isFinite(a) && Number.isFinite(b) && b >= a).map(([a, b]) => [a, b]).sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+    let total = 0;
+    let start;
+    let end = 0;
+    for (const [a, b] of sorted) {
+        if (start === undefined) {
+            start = a;
+            end = b;
+        }
+        else if (a <= end)
+            end = Math.max(end, b);
+        else {
+            total += end - start;
+            start = a;
+            end = b;
+        }
+    }
+    return total + (start === undefined ? 0 : end - start);
+}
+function evaluateLease(lease, now) {
+    if (!lease)
+        return { summary: "No current project lease exists.", live: false, failureObserved: false };
+    const age = Math.max(0, now.getTime() - new Date(lease.heartbeatAt).getTime());
+    if (lease.host !== hostname())
+        return { summary: `Remote lease heartbeat is ${formatDuration(age)} old; matching remote process identity cannot be verified.`, live: false, failureObserved: false };
+    let exists = false;
+    try {
+        process.kill(lease.pid, 0);
+        exists = true;
+    }
+    catch {
+        exists = false;
+    }
+    let sameStart = false;
+    try {
+        sameStart = readFileSync(`/proc/${lease.pid}/stat`, "utf8").split(" ")[21] === lease.processStart;
+    }
+    catch {
+        sameStart = false;
+    }
+    const live = age <= 45_000 && exists && sameStart;
+    return { summary: live ? `Lease heartbeat is fresh (${formatDuration(age)} old) and PID/start identity match.` : `Lease health failed: heartbeat age ${formatDuration(age)}, process exists=${exists}, process-start matches=${sameStart}.`, live, failureObserved: !live && (!exists || !sameStart) };
+}
+function collectGit(root, cwd, runner) {
+    const evidence = [];
+    try {
+        const head = runGit(runner, cwd, ["rev-parse", "HEAD"]);
+        const branch = runGit(runner, cwd, ["rev-parse", "--abbrev-ref", "HEAD"]);
+        const status = runGit(runner, cwd, ["status", "--porcelain=v1"]);
+        const diff = runGit(runner, cwd, ["diff", "--shortstat"]);
+        evidence.push({ evidenceId: "git-summary", source: "git", kind: "git", summary: `branch=${branch.stdout.trim() || "unknown"}, head=${head.stdout.trim().slice(0, 12) || "unknown"}, changedPaths=${status.stdout.split("\n").filter(Boolean).length}, diff=${diff.stdout.trim() || "none"}` });
+        return { state: [head, branch, status, diff].every(item => item.ok) ? "available" : "partial", evidence };
+    }
+    catch (error) {
+        return { state: "unavailable", detail: String(error).slice(0, 300), evidence };
+    }
+    finally {
+        void root;
+    }
+}
+function runGit(runner, cwd, args) {
+    if (!ALLOWED_GIT.has(args[0] ?? ""))
+        throw new Error("diagnostic git command is not allowlisted");
+    return runner("git", args, { cwd, timeoutMs: 5_000 });
+}
+function collectExternal(cwd, runner, now) {
+    const remote = runner("git", ["remote", "get-url", "origin"], { cwd, timeoutMs: 5_000 });
+    if (remote.timedOut)
+        return { state: "timed_out", detail: "Git remote inspection exceeded five seconds", evidence: [] };
+    const gitlab = /gitlab/i.test(remote.stdout);
+    const command = gitlab ? "glab" : "gh";
+    const args = gitlab ? ["mr", "view", "--output", "json"] : ["pr", "view", "--json", "number,state,statusCheckRollup,url"];
+    const result = runner(command, args, { cwd, timeoutMs: 5_000 });
+    if (result.timedOut)
+        return { state: "timed_out", detail: `${gitlab ? "GitLab" : "GitHub"} CLI check exceeded five seconds`, evidence: [] };
+    if (!result.ok)
+        return { state: "unavailable", detail: safeSummary(result.stderr || `${gitlab ? "GitLab" : "GitHub"} CLI unavailable or unauthenticated`, 300), evidence: [] };
+    return { state: "available", evidence: [{ evidenceId: gitlab ? "external-mr" : "external-pr", source: "external", kind: gitlab ? "merge_request" : "pull_request", summary: safeSummary(result.stdout, 2000), observedAt: now.toISOString() }] };
+}
+function defaultCommandRunner(command, args, options) {
+    if (command !== "git" && command !== "gh" && command !== "glab")
+        return { ok: false, stdout: "", stderr: "command is not allowlisted" };
+    try {
+        return { ok: true, stdout: execFileSync(command, [...args], { cwd: options.cwd, timeout: options.timeoutMs, encoding: "utf8", maxBuffer: 64 * 1024, stdio: ["ignore", "pipe", "pipe"] }) };
+    }
+    catch (error) {
+        const item = error;
+        return { ok: false, stdout: String(item.stdout ?? ""), stderr: String(item.stderr ?? error), timedOut: Boolean(item.killed || item.signal === "SIGTERM") };
+    }
+}
+function readLegacyLifecycle(root, run, limit) {
+    const dir = join(root, ".foreman");
+    if (!existsSync(dir))
+        return { found: false, spans: [] };
+    const files = readdirSync(dir).filter(name => name.endsWith(".jsonl")).sort().slice(-5);
+    const spans = [];
+    for (const file of files) {
+        let rows = [];
+        try {
+            rows = readFileSync(join(dir, file), "utf8").split("\n").filter(Boolean).map(line => JSON.parse(line));
+        }
+        catch {
+            continue;
+        }
+        for (const [index, row] of rows.entries()) {
+            const at = typeof row.timestamp === "string" ? row.timestamp : typeof row.at === "string" ? row.at : undefined;
+            if (!at || new Date(at).getTime() < new Date(run.createdAt).getTime() || new Date(at).getTime() > new Date(run.updatedAt).getTime() + 60_000)
+                continue;
+            if (!["step", "branch-complete", "branch-issue", "retry", "escalation"].includes(String(row.event)))
+                continue;
+            spans.push({ version: 1, spanId: `legacy:${file}:${index}`, runId: run.runId, kind: `legacy_${String(row.event)}`, name: String(row.event), startedAt: at, endedAt: at, durationMs: 0, outcome: typeof row.outcome === "string" ? row.outcome : undefined, completionKnown: false });
+            if (spans.length >= limit)
+                return { found: true, spans };
+        }
+    }
+    return { found: spans.length > 0, spans };
+}
+function source(name, state, now, detail) { return { source: name, state, observedAt: now.toISOString(), ...(detail ? { detail } : {}) }; }
+function evidenceForKinds(evidence, kinds) { const ids = evidence.filter(item => kinds.includes(item.kind)).map(item => item.evidenceId); return ids.length ? ids : ["timing-1"].filter(id => evidence.some(item => item.evidenceId === id)); }
+function liveDuration(span, now) { return Math.max(0, now.getTime() - new Date(span.startedAt).getTime()); }
+function numericAttribute(span, key) { const value = Number(span.attributes?.[key]); return Number.isFinite(value) && value >= 0 ? value : 0; }
+function percentile(sorted, p) { if (!sorted.length)
+    return 0; const index = (sorted.length - 1) * p; const lower = Math.floor(index); const upper = Math.ceil(index); return Math.round(sorted[lower] + (sorted[upper] - sorted[lower]) * (index - lower)); }
+function ticketCountBucket(count) { return count <= 1 ? "1" : count <= 3 ? "2-3" : count <= 7 ? "4-7" : "8+"; }
+export function ticketSizeBucketForRun(projectDir, ticketIds) {
+    if (!ticketIds.length || !isTicketsInitialized(projectDir))
+        return undefined;
+    try {
+        const paths = resolveTicketPaths(loadTicketsConfig(projectDir), projectDir);
+        const selected = loadTickets(paths.tickets).filter(ticket => ticketIds.includes(ticket.id));
+        const order = ["XS", "S", "M", "L", "XL"];
+        return selected.length ? selected.map(ticket => ticket.size).sort((a, b) => order.indexOf(b) - order.indexOf(a))[0] : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function formatDuration(ms) { if (ms < 1000)
+    return `${Math.round(ms)}ms`; if (ms < 60_000)
+    return `${(ms / 1000).toFixed(1)}s`; if (ms < 3_600_000)
+    return `${(ms / 60_000).toFixed(1)}m`; return `${(ms / 3_600_000).toFixed(1)}h`; }
+function safeSummary(value, maximum) { return String(sanitizeDiagnosticValue(value)).slice(0, maximum); }

package/dist/foreman.d.ts

@@ -2,8 +2,10 @@ import type { BuilderAdapter, PermissionHandler, PermissionRequest, TurnResult }
 import type { PermissionPolicy } from "./permissions/policy.js";
 import { type AnsweredProviderQuestion } from "./providerQuestions.js";
 import type { Log } from "./log.js";
-import { type QaNonconvergenceContext, type QaNonconvergenceDecision } from "./qaReview.js";
+import { type QaNonconvergenceContext, type QaNonconvergenceDecision, type QaReportRecoveryHandler, type QaSessionBoundaryRecovery, type QaSessionHandle } from "./qaReview.js";
 import type { SessionStrategy } from "rafi-spec";
+import type { RunObserver } from "./observability.js";
+import type { QaRecoveryPacket } from "./qaRecovery.js";
 /** Parsed STEP_STATUS marker from a builder's turn. */
 export interface StepStatus {
     kind: "done" | "blocked" | "plan_complete" | "needs_input" | "qa_pass" | "qa_fail" | "unknown";
@@ -48,15 +50,17 @@ export interface BatchResult {
     outcome: "all-done" | "plan-complete" | "blocked" | "needs-human";
     detail?: string;
 }
+export interface ForemanNotificationOptions {
+    desktop: boolean;
+    terminalBell: boolean;
+}
 /** Drives one builder through a batch of N steps via the STEP_STATUS protocol. */
 export declare class Foreman {
     private builder;
     private readonly log;
-    private readonly notificationsEnabled;
     private readonly qaEnabled;
     private readonly qaMaxCycles;
     private readonly projectDir?;
-    private readonly qaReviewer?;
     private readonly qaFactory?;
     private readonly qaSessionStrategy;
     private readonly builderFactory?;
@@ -68,12 +72,23 @@ export declare class Foreman {
     private readonly observeQaNativeCompactions?;
     /** Persist a Builder's provider-native event immediately after every turn. */
     private readonly observeBuilderNativeCompactions?;
+    private readonly observer?;
+    private readonly qaRuntimeContext?;
+    private readonly qaContinuityManaged;
+    private readonly qaReportRecovery?;
+    private qaResumedRecovery?;
     private readonly ticketsEnabled;
+    private readonly notificationsEnabled;
+    private readonly terminalBellEnabled;
     private readonly qaStream;
     private builderWorkSessions;
-    constructor(builder: BuilderAdapter, log: Log, notificationsEnabled?: boolean, qaEnabled?: boolean, qaMaxCycles?: number, projectDir?: string | undefined, qaReviewer?: BuilderAdapter | undefined, qaFactory?: ((cwd: string, sessionId?: string) => Promise<BuilderAdapter>) | undefined, qaSessionStrategy?: SessionStrategy, builderFactory?: ((cwd: string, sessionId?: string) => Promise<BuilderAdapter>) | undefined, builderSessionStrategy?: SessionStrategy, qaNonconvergence?: ((context: QaNonconvergenceContext) => Promise<QaNonconvergenceDecision>) | undefined, beforeBuilderTurn?: ((adapter: BuilderAdapter, frozenAction: string) => Promise<BuilderAdapter>) | undefined, builderSessionBoundary?: ((adapter: BuilderAdapter, frozenAction: string, strategy: SessionStrategy) => Promise<BuilderAdapter>) | undefined, qaSessionBoundary?: ((adapter: BuilderAdapter, frozenAction: string, strategy: SessionStrategy, cwd: string) => Promise<BuilderAdapter>) | undefined, observeQaNativeCompactions?: ((adapter: BuilderAdapter) => Promise<void>) | undefined, 
+    private readonly fallbackQaRunId;
+    constructor(builder: BuilderAdapter, log: Log, notifications?: boolean | ForemanNotificationOptions, qaEnabled?: boolean, qaMaxCycles?: number, projectDir?: string | undefined, 
+    /** @deprecated QA must be created by qaFactory in a disposable snapshot. */
+    _deprecatedSameSessionReviewer?: BuilderAdapter, qaFactory?: ((cwd: string, sessionId?: string) => Promise<BuilderAdapter | QaSessionHandle>) | undefined, qaSessionStrategy?: SessionStrategy, builderFactory?: ((cwd: string, sessionId?: string) => Promise<BuilderAdapter>) | undefined, builderSessionStrategy?: SessionStrategy, qaNonconvergence?: ((context: QaNonconvergenceContext) => Promise<QaNonconvergenceDecision>) | undefined, beforeBuilderTurn?: ((adapter: BuilderAdapter, frozenAction: string) => Promise<BuilderAdapter>) | undefined, builderSessionBoundary?: ((adapter: BuilderAdapter, frozenAction: string, strategy: SessionStrategy) => Promise<BuilderAdapter>) | undefined, qaSessionBoundary?: ((adapter: BuilderAdapter, frozenAction: string, strategy: SessionStrategy, cwd: string, recovery?: QaSessionBoundaryRecovery) => Promise<BuilderAdapter>) | undefined, observeQaNativeCompactions?: ((adapter: BuilderAdapter) => Promise<void>) | undefined, 
     /** Persist a Builder's provider-native event immediately after every turn. */
-    observeBuilderNativeCompactions?: ((adapter: BuilderAdapter) => Promise<void>) | undefined);
+    observeBuilderNativeCompactions?: ((adapter: BuilderAdapter) => Promise<void>) | undefined, observer?: RunObserver | undefined, qaRuntimeContext?: unknown | undefined, qaContinuityManaged?: boolean, qaReportRecovery?: QaReportRecoveryHandler | undefined, qaResumedRecovery?: QaRecoveryPacket | undefined);
+    private waitForUserInput;
     /**
      * Send one instruction and resolve any needs_input exchanges before returning.
      * The returned result and status always reflect the final (non-needs_input) turn.
@@ -90,6 +105,11 @@ export declare class Foreman {
         result: TurnResult;
         status: StepStatus;
     }>;
+    /** Ask an already-active role session to propose choices for its reported blocker. */
+    resolveBlocker(adapter: BuilderAdapter, reason: string, mode?: "builder" | "qa"): Promise<{
+        result: TurnResult;
+        status: StepStatus;
+    }>;
     private observeBuilderNative;
     /**
      * Run a QA review pass on the ticket the builder just completed.
@@ -97,7 +117,7 @@ export declare class Foreman {
      * QA turns are free — they do not advance the step counter.
      */
     private runQa;
-    runQaReview(stepIndex: number): Promise<{
+    runQaReview(stepIndex: number, builderResult: string): Promise<{
         outcome: "passed" | "blocked" | "needs-human" | "waived";
         detail?: string;
         summary?: string;