merge-steward 0.26.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import type { DiscoveredRepoSettings } from "./github-repo-discovery.ts";
2
+ import type { QueueRuntimeStatus } from "./types.ts";
2
3
  export type RepoRuntimeState = "initializing" | "ready" | "failed";
3
4
  export interface RepoRuntimeStatus {
4
5
  repoId: string;
@@ -9,6 +10,7 @@ export interface RepoRuntimeStatus {
9
10
  readyAt?: string | undefined;
10
11
  failedAt?: string | undefined;
11
12
  lastError?: string | undefined;
13
+ runtime?: QueueRuntimeStatus | undefined;
12
14
  }
13
15
  export interface ServiceHealthResponse {
14
16
  ok: boolean;
@@ -1,4 +1,4 @@
1
- import type { QueueEntry } from "../../types.ts";
1
+ import type { QueueEntry, QueueEventSummary, QueueRuntimeStatus, QueueWatchSnapshot } from "../../types.ts";
2
2
  import type { Output, ParsedArgs } from "../types.ts";
3
3
  import { type ResolveCommandRunner } from "../resolve.ts";
4
4
  import { type PrGitHubOverview } from "./pr-github.ts";
@@ -14,6 +14,9 @@ export interface PrStatusReport {
14
14
  exitCode: number;
15
15
  reason?: string;
16
16
  queueEntry?: QueueEntry;
17
+ queueSource?: "service" | "database";
18
+ queueRuntime?: QueueRuntimeStatus;
19
+ queueLatestEvent?: QueueEventSummary;
17
20
  github?: PrGitHubOverview;
18
21
  checkedAt: string;
19
22
  }
@@ -22,14 +25,19 @@ export declare function classifyGitHubOverview(overview: PrGitHubOverview): {
22
25
  kind: PrStatusReport["kind"];
23
26
  reason?: string;
24
27
  };
28
+ export declare function latestEventForPr(snapshot: QueueWatchSnapshot, prNumber: number): QueueEventSummary | undefined;
25
29
  export interface BuildReportOptions {
26
30
  repoId: string;
27
31
  repoFullName: string;
28
32
  prNumber: number;
29
33
  queueEntry?: QueueEntry | undefined;
34
+ queueSource?: "service" | "database" | undefined;
35
+ queueRuntime?: QueueRuntimeStatus | undefined;
36
+ queueLatestEvent?: QueueEventSummary | undefined;
30
37
  github?: PrGitHubOverview | undefined;
31
38
  }
32
39
  export declare function buildPrStatusReport(options: BuildReportOptions): PrStatusReport;
40
+ export declare function formatReportText(report: PrStatusReport): string;
33
41
  export interface HandlePrStatusOptions {
34
42
  parsed: ParsedArgs;
35
43
  stdout: Output;
@@ -4,6 +4,7 @@ import { ServiceApiError, fetchLocalJson, loadRepoConfigById } from "../system.j
4
4
  import { defaultResolveRunner, resolvePrNumber, resolveRepo } from "../resolve.js";
5
5
  import { parseIntegerFlag } from "../args.js";
6
6
  import { fetchPrGitHubOverview } from "./pr-github.js";
7
+ import { formatRuntimeActivity } from "../../runtime-format.js";
7
8
  export function classifyQueueEntry(entry) {
8
9
  switch (entry.status) {
9
10
  case "merged": return "merged";
@@ -79,9 +80,18 @@ function findEntryForPr(snapshot, prNumber) {
79
80
  }
80
81
  async function loadQueueEntry(config, prNumber) {
81
82
  try {
82
- const snapshot = await fetchLocalJson(config.repoId, "/queue/watch?eventLimit=1");
83
+ const snapshot = await fetchLocalJson(config.repoId, "/queue/watch?eventLimit=40");
83
84
  const entry = findEntryForPr(snapshot, prNumber);
84
- return entry ? { kind: "found", entry, source: "service" } : { kind: "not_found" };
85
+ if (!entry)
86
+ return { kind: "not_found" };
87
+ const latestEvent = latestEventForPr(snapshot, prNumber);
88
+ return {
89
+ kind: "found",
90
+ entry,
91
+ source: "service",
92
+ runtime: snapshot.runtime,
93
+ ...(latestEvent ? { latestEvent } : {}),
94
+ };
85
95
  }
86
96
  catch (error) {
87
97
  if (error instanceof ServiceApiError) {
@@ -106,6 +116,14 @@ async function loadQueueEntry(config, prNumber) {
106
116
  }
107
117
  }
108
118
  }
119
+ export function latestEventForPr(snapshot, prNumber) {
120
+ for (let index = snapshot.recentEvents.length - 1; index >= 0; index -= 1) {
121
+ const event = snapshot.recentEvents[index];
122
+ if (event?.prNumber === prNumber)
123
+ return event;
124
+ }
125
+ return undefined;
126
+ }
109
127
  export function buildPrStatusReport(options) {
110
128
  const checkedAt = new Date().toISOString();
111
129
  if (options.queueEntry) {
@@ -119,6 +137,9 @@ export function buildPrStatusReport(options) {
119
137
  terminal: isTerminalKind(kind),
120
138
  exitCode: exitCodeForKind(kind),
121
139
  queueEntry: options.queueEntry,
140
+ ...(options.queueSource ? { queueSource: options.queueSource } : {}),
141
+ ...(options.queueRuntime ? { queueRuntime: options.queueRuntime } : {}),
142
+ ...(options.queueLatestEvent ? { queueLatestEvent: options.queueLatestEvent } : {}),
122
143
  checkedAt,
123
144
  };
124
145
  }
@@ -139,7 +160,7 @@ export function buildPrStatusReport(options) {
139
160
  }
140
161
  throw new Error("buildPrStatusReport requires either queueEntry or github overview");
141
162
  }
142
- function formatReportText(report) {
163
+ export function formatReportText(report) {
143
164
  const lines = [
144
165
  `Repo: ${report.repoFullName} (${report.repoId})`,
145
166
  `PR: #${report.prNumber}`,
@@ -156,6 +177,12 @@ function formatReportText(report) {
156
177
  lines.push(`Head SHA: ${entry.headSha}`);
157
178
  if (entry.waitDetail)
158
179
  lines.push(`Wait detail: ${entry.waitDetail}`);
180
+ if (report.queueRuntime) {
181
+ lines.push(...formatRuntimeActivity(report.queueRuntime));
182
+ }
183
+ if (report.queueLatestEvent) {
184
+ lines.push(`Latest event for this PR: ${report.queueLatestEvent.toStatus}${report.queueLatestEvent.detail ? ` (${report.queueLatestEvent.detail})` : ""} at ${report.queueLatestEvent.at}`);
185
+ }
159
186
  }
160
187
  if (report.github) {
161
188
  const gh = report.github;
@@ -232,6 +259,9 @@ export async function handlePrStatus(options) {
232
259
  repoFullName: resolvedRepo.repoFullName,
233
260
  prNumber: resolvedPr.prNumber,
234
261
  queueEntry: queueResult.entry,
262
+ queueSource: queueResult.source,
263
+ ...(queueResult.runtime ? { queueRuntime: queueResult.runtime } : {}),
264
+ ...(queueResult.latestEvent ? { queueLatestEvent: queueResult.latestEvent } : {}),
235
265
  });
236
266
  }
237
267
  const overview = await fetchGh(resolvedRepo.repoFullName, resolvedPr.prNumber);
@@ -1,5 +1,6 @@
1
- import { type QueueWatchSnapshot } from "../../types.ts";
1
+ import { type QueueReconcileResult, type QueueWatchSnapshot } from "../../types.ts";
2
2
  import type { ParsedArgs, Output } from "../types.ts";
3
3
  import { type ResolveCommandRunner } from "../resolve.ts";
4
4
  export declare function formatQueueStatusText(source: "service" | "database", snapshot: QueueWatchSnapshot): string;
5
5
  export declare function handleQueue(parsed: ParsedArgs, stdout: Output, resolveCommand?: ResolveCommandRunner): Promise<number>;
6
+ export declare function formatReconcileRequestText(result: QueueReconcileResult): string;
@@ -6,6 +6,7 @@ import { formatJson, writeOutput } from "../output.js";
6
6
  import { ServiceApiError, loadRepoConfigById, fetchLocalJson } from "../system.js";
7
7
  import { resolveRepo } from "../resolve.js";
8
8
  import { buildQueueSummary } from "../../watch/dashboard-model.js";
9
+ import { formatDurationMs, formatRuntimeActivity } from "../../runtime-format.js";
9
10
  async function resolveRepoIdWithCwdFallback(parsed, positionalIndex, resolveCommand) {
10
11
  const positional = parsed.positionals[positionalIndex];
11
12
  if (positional)
@@ -84,6 +85,7 @@ export function formatQueueStatusText(source, snapshot) {
84
85
  `Queued: ${snapshot.summary.queued} preparing: ${snapshot.summary.preparingHead} validating: ${snapshot.summary.validating} merging: ${snapshot.summary.merging}`,
85
86
  `Merged: ${snapshot.summary.merged} evicted: ${snapshot.summary.evicted} dequeued: ${snapshot.summary.dequeued}`,
86
87
  snapshot.summary.headPrNumber ? `Head PR: #${snapshot.summary.headPrNumber}` : "Head PR: none",
88
+ ...formatRuntimeActivity(snapshot.runtime),
87
89
  ...(snapshot.runtime.lastTickOutcome === "failed"
88
90
  ? [
89
91
  "Last tick: failed",
@@ -241,7 +243,7 @@ export async function handleQueue(parsed, stdout, resolveCommand) {
241
243
  else {
242
244
  writeOutput(stdout, [
243
245
  `Repo: ${repoId}`,
244
- result.started ? "Reconcile started." : "Reconcile request accepted; a tick was already in progress.",
246
+ formatReconcileRequestText(result),
245
247
  `Last outcome: ${result.runtime.lastTickOutcome}`,
246
248
  ].join("\n") + "\n");
247
249
  }
@@ -253,3 +255,18 @@ export async function handleQueue(parsed, stdout, resolveCommand) {
253
255
  }
254
256
  throw new UsageError(`Unknown queue command: ${subcommand}`, "queue");
255
257
  }
258
+ export function formatReconcileRequestText(result) {
259
+ if (result.started)
260
+ return "Reconcile started.";
261
+ if (result.reason === "already_running") {
262
+ const age = formatDurationMs(result.runtime.tickAgeMs);
263
+ const latest = result.runtime.lastReconcileEvent
264
+ ? `; latest action ${result.runtime.lastReconcileEvent.action} PR #${result.runtime.lastReconcileEvent.prNumber}`
265
+ : "";
266
+ const guidance = result.runtime.staleTick
267
+ ? "inspect logs before restarting"
268
+ : "wait for the current tick before restarting";
269
+ return `Reconcile already running for ${age}${latest}; ${guidance}.`;
270
+ }
271
+ return "Reconcile request accepted; a tick was already in progress.";
272
+ }
@@ -3,6 +3,7 @@ import { UsageError } from "../types.js";
3
3
  import { parseIntegerFlag } from "../args.js";
4
4
  import { formatJson, writeOutput } from "../output.js";
5
5
  import { fetchServiceHealthStatus, formatCommandFailure, parseSystemctlShowOutput, runSystemctl } from "../system.js";
6
+ import { formatDurationMs } from "../../runtime-format.js";
6
7
  const UNIT_NAME = "merge-steward.service";
7
8
  export async function handleService(parsed, stdout, runCommand) {
8
9
  const subcommand = parsed.positionals[1];
@@ -71,6 +72,7 @@ export async function handleService(parsed, stdout, runCommand) {
71
72
  health.reachable
72
73
  ? `Health: ${health.ok ? "ok" : "unhealthy"} (HTTP ${health.status})`
73
74
  : `Health: not reachable (${health.error})`,
75
+ ...(health.reachable && health.health ? staleRuntimeLines(health.health.repos) : []),
74
76
  ]
75
77
  .filter(Boolean)
76
78
  .join("\n") + "\n");
@@ -94,3 +96,12 @@ export async function handleService(parsed, stdout, runCommand) {
94
96
  }
95
97
  throw new UsageError(`Unknown service command: ${subcommand}`, "service");
96
98
  }
99
+ function staleRuntimeLines(repos) {
100
+ return repos
101
+ .filter((repo) => repo.runtime?.staleTick)
102
+ .map((repo) => {
103
+ const event = repo.runtime?.lastReconcileEvent;
104
+ const latest = event ? `; latest action ${event.action} PR #${event.prNumber}` : "";
105
+ return `Warning: ${repo.repoId} reconcile tick is stale after ${formatDurationMs(repo.runtime?.tickAgeMs)}${latest}`;
106
+ });
107
+ }
package/dist/config.d.ts CHANGED
@@ -10,6 +10,7 @@ export declare const stewardConfigSchema: z.ZodObject<{
10
10
  flakyRetries: z.ZodDefault<z.ZodNumber>;
11
11
  speculativeDepth: z.ZodDefault<z.ZodNumber>;
12
12
  pollIntervalMs: z.ZodDefault<z.ZodNumber>;
13
+ reconcileStaleAfterMs: z.ZodDefault<z.ZodNumber>;
13
14
  server: z.ZodDefault<z.ZodObject<{
14
15
  bind: z.ZodDefault<z.ZodString>;
15
16
  port: z.ZodDefault<z.ZodNumber>;
package/dist/config.js CHANGED
@@ -16,6 +16,7 @@ export const stewardConfigSchema = z.object({
16
16
  /** Max speculative branches to maintain in parallel. 1 = serial mode. */
17
17
  speculativeDepth: z.number().int().min(1).default(10),
18
18
  pollIntervalMs: z.number().int().min(1000).default(30_000),
19
+ reconcileStaleAfterMs: z.number().int().min(1000).default(5 * 60_000),
19
20
  server: z.object({
20
21
  bind: z.string().default("127.0.0.1"),
21
22
  port: z.number().int().default(8790),
@@ -48,6 +48,7 @@ export async function buildMultiRepoHttpServer(options) {
48
48
  ...(record.readyAt ? { readyAt: record.readyAt } : {}),
49
49
  ...(record.failedAt ? { failedAt: record.failedAt } : {}),
50
50
  ...(record.lastError ? { lastError: record.lastError } : {}),
51
+ ...(record.instance ? { runtime: record.instance.service.getRuntimeStatus() } : {}),
51
52
  })),
52
53
  }));
53
54
  app.get("/admin/runtime/auth", async () => githubAdmin.getStatus());
@@ -32,11 +32,20 @@ export async function evictEntry(ctx, entry, failureClass, extra) {
32
32
  const retryHistory = [];
33
33
  for (const event of events) {
34
34
  const eventBaseSha = event.baseSha || "unknown";
35
+ const detail = event.detail ?? "";
35
36
  if (event.fromStatus === "preparing_head" && event.toStatus === "validating") {
36
37
  retryHistory.push({ at: event.at, baseSha: eventBaseSha, outcome: "passed_to_validation" });
37
38
  }
38
39
  else if (event.fromStatus === "validating" && event.toStatus === "preparing_head") {
39
- retryHistory.push({ at: event.at, baseSha: eventBaseSha, outcome: "ci_failed_retry" });
40
+ const outcome = detail.startsWith("invalidated:")
41
+ ? "invalidated"
42
+ : detail.toLowerCase().includes("ci failed")
43
+ ? "ci_failed_retry"
44
+ : "validation_reset";
45
+ retryHistory.push({ at: event.at, baseSha: eventBaseSha, outcome });
46
+ }
47
+ else if (event.fromStatus === "merging" && event.toStatus === "preparing_head") {
48
+ retryHistory.push({ at: event.at, baseSha: eventBaseSha, outcome: "push_failed_retry" });
40
49
  }
41
50
  else if (event.fromStatus === "preparing_head" && event.toStatus === "preparing_head") {
42
51
  retryHistory.push({ at: event.at, baseSha: eventBaseSha, outcome: "conflict_retry" });
@@ -7,6 +7,136 @@ const DEFAULT_PR_MERGED_POLL_DELAY_MS = 2_000;
7
7
  function delay(ms) {
8
8
  return new Promise((resolve) => setTimeout(resolve, ms));
9
9
  }
10
+ function shortSha(sha) {
11
+ return sha ? sha.slice(0, 8) : "unknown";
12
+ }
13
+ function sanitizeCommandOutput(value) {
14
+ return value
15
+ .replace(/x-access-token:[^@\s]+@/g, "x-access-token:[redacted]@")
16
+ .replace(/\bgh[psu]_[A-Za-z0-9_]+\b/g, "[redacted-token]")
17
+ .replace(/\s+/g, " ")
18
+ .trim()
19
+ .slice(0, 1200);
20
+ }
21
+ function describeError(error) {
22
+ if (!(error instanceof Error)) {
23
+ return sanitizeCommandOutput(String(error));
24
+ }
25
+ const failure = error;
26
+ const parts = [
27
+ failure.stderr,
28
+ failure.stdout,
29
+ error.message,
30
+ typeof failure.exitCode === "number" ? `exit code ${failure.exitCode}` : undefined,
31
+ failure.signal ? `signal ${failure.signal}` : undefined,
32
+ ].filter((part) => Boolean(part && part.trim()));
33
+ return sanitizeCommandOutput(parts.join(" "));
34
+ }
35
+ function classifyPushFailure(error, detail) {
36
+ const failure = error instanceof Error ? error : undefined;
37
+ const lower = detail.toLowerCase();
38
+ if (failure?.timedOut || lower.includes("timed out")) {
39
+ return "timeout";
40
+ }
41
+ if (lower.includes("refusing to allow a github app to create or update workflow")) {
42
+ return "workflow_permission";
43
+ }
44
+ if (lower.includes("non-fast-forward")
45
+ || lower.includes("fetch first")
46
+ || lower.includes("stale info")
47
+ || lower.includes("cannot lock ref")) {
48
+ return "non_fast_forward";
49
+ }
50
+ if (lower.includes("protected branch")
51
+ || lower.includes("gh006")
52
+ || lower.includes("required status check")
53
+ || lower.includes("required approving review")
54
+ || lower.includes("changes must be made through a pull request")) {
55
+ return "protected_branch";
56
+ }
57
+ if (lower.includes("authentication failed")
58
+ || lower.includes("permission denied")
59
+ || lower.includes("write access")
60
+ || lower.includes("403")
61
+ || lower.includes("not authorized")) {
62
+ return "auth_or_permission";
63
+ }
64
+ return "github_push_rejected";
65
+ }
66
+ function normalizeCheckName(name) {
67
+ return name.trim().toLowerCase();
68
+ }
69
+ function getMissingRequiredChecks(requiredChecks, checks) {
70
+ if (requiredChecks.length === 0) {
71
+ return [];
72
+ }
73
+ const available = new Set(checks.map((check) => normalizeCheckName(check.name)).filter(Boolean));
74
+ return requiredChecks.filter((check) => !available.has(normalizeCheckName(check)));
75
+ }
76
+ function summarizeChecks(checks, missingRequiredChecks) {
77
+ const visible = checks
78
+ .filter((check) => check.name.trim())
79
+ .map((check) => `${check.name}=${check.conclusion}`);
80
+ const parts = [];
81
+ if (visible.length > 0) {
82
+ parts.push(`spec checks ${visible.slice(0, 5).join(", ")}`);
83
+ if (visible.length > 5) {
84
+ parts.push(`+${visible.length - 5} more`);
85
+ }
86
+ }
87
+ else {
88
+ parts.push("no spec checks visible");
89
+ }
90
+ if (missingRequiredChecks.length > 0) {
91
+ parts.push(`missing required ${missingRequiredChecks.join(", ")}`);
92
+ }
93
+ return parts.join("; ");
94
+ }
95
+ async function inspectSpecChecks(ctx, specSha) {
96
+ if (!specSha) {
97
+ return {
98
+ detail: "spec checks unavailable: no spec SHA",
99
+ failingChecks: [],
100
+ pendingChecks: [],
101
+ missingRequiredChecks: ctx.policy.getRequiredChecks(),
102
+ };
103
+ }
104
+ try {
105
+ const checks = await ctx.github.listChecksForRef(specSha);
106
+ const failingChecks = checks.filter((check) => check.conclusion === "failure");
107
+ const pendingChecks = checks.filter((check) => check.conclusion === "pending");
108
+ const missingRequiredChecks = getMissingRequiredChecks(ctx.policy.getRequiredChecks(), checks);
109
+ return {
110
+ detail: summarizeChecks(checks, missingRequiredChecks),
111
+ failingChecks,
112
+ pendingChecks,
113
+ missingRequiredChecks,
114
+ };
115
+ }
116
+ catch (error) {
117
+ return {
118
+ detail: `spec checks unavailable: ${describeError(error)}`,
119
+ failingChecks: [],
120
+ pendingChecks: [],
121
+ missingRequiredChecks: ctx.policy.getRequiredChecks(),
122
+ };
123
+ }
124
+ }
125
+ async function verifySpecStillFastForwards(ctx, specSha) {
126
+ try {
127
+ await ctx.git.fetch();
128
+ const currentBase = await ctx.git.headSha(ref(ctx, ctx.baseBranch));
129
+ const isFastForward = await ctx.git.isAncestor(currentBase, specSha);
130
+ return { currentBase, isFastForward };
131
+ }
132
+ catch (error) {
133
+ return {
134
+ currentBase: null,
135
+ isFastForward: null,
136
+ detail: `fast-forward verification unavailable: ${describeError(error)}`,
137
+ };
138
+ }
139
+ }
10
140
  export async function mergeHead(ctx, entry) {
11
141
  emit(ctx, entry, "merge_revalidating");
12
142
  const prStatus = await ctx.github.getStatus(entry.prNumber);
@@ -93,7 +223,12 @@ export async function mergeHead(ctx, entry) {
93
223
  try {
94
224
  await ctx.git.push(entry.specBranch, false, ctx.baseBranch);
95
225
  }
96
- catch {
226
+ catch (error) {
227
+ const pushErrorDetail = describeError(error);
228
+ const pushFailureKind = classifyPushFailure(error, pushErrorDetail);
229
+ const fastForward = entry.specSha
230
+ ? await verifySpecStillFastForwards(ctx, entry.specSha)
231
+ : { currentBase: null, isFastForward: null, detail: "fast-forward verification unavailable: no spec SHA" };
97
232
  try {
98
233
  const refresh = await ctx.policy.refreshOnIssue("merge_push_rejected");
99
234
  if (refresh.attempted && refresh.changed) {
@@ -109,20 +244,52 @@ export async function mergeHead(ctx, entry) {
109
244
  catch {
110
245
  // Fall through to the normal push failure handling when policy refresh is unavailable.
111
246
  }
112
- emit(ctx, entry, "merge_rejected", { detail: "push to main failed" });
113
- const allActive = ctx.store.listActive(ctx.repoId);
114
- if (isBudgetExhausted(entry)) {
115
- emit(ctx, entry, "budget_exhausted");
116
- await evictEntry(ctx, entry, "integration_conflict");
247
+ const checkState = await inspectSpecChecks(ctx, entry.specSha);
248
+ const detail = [
249
+ `push to ${ctx.baseBranch} failed (${pushFailureKind})`,
250
+ `spec ${shortSha(entry.specSha)}`,
251
+ `main ${shortSha(fastForward.currentBase ?? currentBase)}`,
252
+ fastForward.isFastForward === null
253
+ ? fastForward.detail
254
+ : `spec fast-forward ${fastForward.isFastForward ? "yes" : "no"}`,
255
+ checkState.detail,
256
+ pushErrorDetail,
257
+ ].filter((part) => Boolean(part && part.trim())).join("; ");
258
+ emit(ctx, entry, "merge_rejected", {
259
+ detail,
260
+ baseSha: fastForward.currentBase ?? currentBase ?? undefined,
261
+ failingChecks: checkState.failingChecks,
262
+ pendingChecks: checkState.pendingChecks,
263
+ missingRequiredChecks: checkState.missingRequiredChecks,
264
+ });
265
+ const mustRebuild = pushFailureKind === "non_fast_forward" || fastForward.isFastForward === false;
266
+ if (mustRebuild) {
267
+ const allActive = ctx.store.listActive(ctx.repoId);
268
+ if (isBudgetExhausted(entry)) {
269
+ emit(ctx, entry, "budget_exhausted", {
270
+ detail: "push retry budget exhausted after non-fast-forward rejection",
271
+ });
272
+ await evictEntry(ctx, entry, "integration_conflict");
273
+ }
274
+ else {
275
+ ctx.store.transition(entry.id, "preparing_head", {
276
+ retryAttempts: entry.retryAttempts + 1,
277
+ ...CLEAN_CI,
278
+ ...CLEAN_SPEC,
279
+ }, `push failed, retry ${entry.retryAttempts + 1}/${entry.maxRetries}`);
280
+ }
281
+ await invalidateDownstream(ctx, allActive, 0);
282
+ return;
117
283
  }
118
- else {
119
- ctx.store.transition(entry.id, "preparing_head", {
120
- retryAttempts: entry.retryAttempts + 1,
121
- ...CLEAN_CI,
122
- ...CLEAN_SPEC,
123
- }, `push failed, retry ${entry.retryAttempts + 1}/${entry.maxRetries}`);
284
+ if (isBudgetExhausted(entry)) {
285
+ emit(ctx, entry, "budget_exhausted", {
286
+ detail: "push retry budget exhausted; keeping validated spec for GitHub recovery",
287
+ });
124
288
  }
125
- await invalidateDownstream(ctx, allActive, 0);
289
+ ctx.store.transition(entry.id, "merging", {
290
+ retryAttempts: Math.min(entry.retryAttempts + 1, entry.maxRetries),
291
+ waitDetail: detail,
292
+ }, `push failed, keeping validated spec: ${detail}`);
126
293
  return;
127
294
  }
128
295
  emit(ctx, entry, "merge_succeeded");
@@ -0,0 +1,4 @@
1
+ import type { QueueRuntimeStatus, ReconcileEventSummary } from "./types.ts";
2
+ export declare function formatDurationMs(ms: number | null | undefined): string;
3
+ export declare function formatReconcileEvent(event: ReconcileEventSummary | null | undefined): string | null;
4
+ export declare function formatRuntimeActivity(runtime: QueueRuntimeStatus): string[];
@@ -0,0 +1,35 @@
1
+ export function formatDurationMs(ms) {
2
+ if (ms === null || ms === undefined || !Number.isFinite(ms))
3
+ return "unknown duration";
4
+ const seconds = Math.max(0, Math.round(ms / 1000));
5
+ if (seconds < 60)
6
+ return `${seconds}s`;
7
+ const minutes = Math.floor(seconds / 60);
8
+ const remainder = seconds % 60;
9
+ if (minutes < 60)
10
+ return remainder === 0 ? `${minutes}m` : `${minutes}m${remainder}s`;
11
+ const hours = Math.floor(minutes / 60);
12
+ const minuteRemainder = minutes % 60;
13
+ return minuteRemainder === 0 ? `${hours}h` : `${hours}h${minuteRemainder}m`;
14
+ }
15
+ export function formatReconcileEvent(event) {
16
+ if (!event)
17
+ return null;
18
+ return `${event.action} PR #${event.prNumber}${event.detail ? ` (${event.detail})` : ""} at ${event.at}`;
19
+ }
20
+ export function formatRuntimeActivity(runtime) {
21
+ if (!runtime.tickInProgress)
22
+ return [];
23
+ const age = formatDurationMs(runtime.tickAgeMs);
24
+ const started = runtime.lastTickStartedAt ? `, started ${runtime.lastTickStartedAt}` : "";
25
+ const lines = [
26
+ `Reconcile: ${runtime.staleTick ? "stale" : "running"} for ${age}${started}`,
27
+ ];
28
+ const latest = formatReconcileEvent(runtime.lastReconcileEvent);
29
+ if (latest)
30
+ lines.push(`Latest action: ${latest}`);
31
+ if (runtime.staleTick) {
32
+ lines.push(`Warning: reconcile tick appears stale; threshold ${formatDurationMs(runtime.staleTickThresholdMs)}.`);
33
+ }
34
+ return lines;
35
+ }
@@ -1,7 +1,7 @@
1
1
  import type { Logger } from "pino";
2
2
  import type { GitOperations, CIRunner, GitHubPRApi, EvictionReporter, SpeculativeBranchBuilder } from "./interfaces.ts";
3
3
  import type { QueueStore } from "./store.ts";
4
- import type { QueueBlockState, QueueRuntimeStatus } from "./types.ts";
4
+ import type { QueueBlockState, QueueReconcileResult, QueueRuntimeStatus } from "./types.ts";
5
5
  import type { StewardConfig } from "./config.ts";
6
6
  import type { GitHubPolicyCache } from "./github-policy.ts";
7
7
  export declare class MergeStewardRuntime {
@@ -16,25 +16,27 @@ export declare class MergeStewardRuntime {
16
16
  private readonly logger;
17
17
  private readonly beforeTick?;
18
18
  private tickTimer;
19
+ private staleTickTimer;
19
20
  private tickInProgress;
20
21
  private lastTickStartedAt;
21
22
  private lastTickCompletedAt;
22
23
  private lastTickOutcome;
23
24
  private lastTickError;
25
+ private lastReconcileEvent;
24
26
  private currentQueueBlock;
25
27
  private readonly waitingMainInfoAt;
26
28
  private static readonly WAITING_MAIN_INFO_INTERVAL_MS;
27
29
  constructor(config: StewardConfig, policy: GitHubPolicyCache, store: QueueStore, git: GitOperations, ci: CIRunner, github: GitHubPRApi, eviction: EvictionReporter, specBuilder: SpeculativeBranchBuilder, logger: Logger, beforeTick?: (() => Promise<void>) | undefined);
28
30
  start(): Promise<void>;
29
31
  stop(): Promise<void>;
30
- triggerReconcile(): Promise<{
31
- started: boolean;
32
- runtime: QueueRuntimeStatus;
33
- }>;
32
+ triggerReconcile(): Promise<QueueReconcileResult>;
34
33
  getRuntimeStatus(): QueueRuntimeStatus;
35
34
  getCurrentQueueBlock(): QueueBlockState | null;
36
35
  getGitHubPolicy(): import("./github-policy.ts").GitHubPolicySnapshot;
37
36
  private scheduleNextTick;
37
+ private getTickAgeMs;
38
+ private scheduleStaleTickWarning;
39
+ private clearStaleTickTimer;
38
40
  private runTick;
39
41
  private describeMainBroken;
40
42
  private getMissingRequiredChecks;
@@ -11,11 +11,13 @@ export class MergeStewardRuntime {
11
11
  logger;
12
12
  beforeTick;
13
13
  tickTimer;
14
+ staleTickTimer;
14
15
  tickInProgress = false;
15
16
  lastTickStartedAt = null;
16
17
  lastTickCompletedAt = null;
17
18
  lastTickOutcome = "idle";
18
19
  lastTickError = null;
20
+ lastReconcileEvent = null;
19
21
  currentQueueBlock = null;
20
22
  // Per-entry timestamp of the last info-level `merge_waiting_main` log.
21
23
  // The event fires every tick while the gate waits, so subsequent ticks
@@ -44,6 +46,7 @@ export class MergeStewardRuntime {
44
46
  clearTimeout(this.tickTimer);
45
47
  this.tickTimer = undefined;
46
48
  }
49
+ this.clearStaleTickTimer();
47
50
  const deadline = Date.now() + 10_000;
48
51
  while (this.tickInProgress && Date.now() < deadline) {
49
52
  await new Promise((resolve) => setTimeout(resolve, 100));
@@ -54,16 +57,23 @@ export class MergeStewardRuntime {
54
57
  const started = await this.runTick();
55
58
  return {
56
59
  started,
60
+ ...(!started ? { reason: "already_running" } : {}),
57
61
  runtime: this.getRuntimeStatus(),
58
62
  };
59
63
  }
60
64
  getRuntimeStatus() {
65
+ const tickAgeMs = this.getTickAgeMs();
66
+ const staleTickThresholdMs = this.config.reconcileStaleAfterMs;
61
67
  return {
62
68
  tickInProgress: this.tickInProgress,
63
69
  lastTickStartedAt: this.lastTickStartedAt,
64
70
  lastTickCompletedAt: this.lastTickCompletedAt,
65
71
  lastTickOutcome: this.lastTickOutcome,
66
72
  lastTickError: this.lastTickError,
73
+ tickAgeMs,
74
+ staleTickThresholdMs,
75
+ staleTick: tickAgeMs !== null && tickAgeMs >= staleTickThresholdMs,
76
+ lastReconcileEvent: this.lastReconcileEvent,
67
77
  };
68
78
  }
69
79
  getCurrentQueueBlock() {
@@ -79,6 +89,36 @@ export class MergeStewardRuntime {
79
89
  this.tickTimer = setTimeout(() => void this.runTick(), this.config.pollIntervalMs);
80
90
  this.tickTimer.unref?.();
81
91
  }
92
+ getTickAgeMs() {
93
+ if (!this.tickInProgress || !this.lastTickStartedAt)
94
+ return null;
95
+ const startedMs = Date.parse(this.lastTickStartedAt);
96
+ if (!Number.isFinite(startedMs))
97
+ return null;
98
+ return Math.max(0, Date.now() - startedMs);
99
+ }
100
+ scheduleStaleTickWarning(startedAt) {
101
+ this.clearStaleTickTimer();
102
+ const timer = setTimeout(() => {
103
+ if (!this.tickInProgress || this.lastTickStartedAt !== startedAt)
104
+ return;
105
+ const runtime = this.getRuntimeStatus();
106
+ this.logger.warn({
107
+ startedAt,
108
+ tickAgeMs: runtime.tickAgeMs,
109
+ staleTickThresholdMs: runtime.staleTickThresholdMs,
110
+ lastReconcileEvent: runtime.lastReconcileEvent,
111
+ }, "Reconcile tick appears stale");
112
+ }, this.config.reconcileStaleAfterMs);
113
+ timer.unref?.();
114
+ this.staleTickTimer = timer;
115
+ }
116
+ clearStaleTickTimer() {
117
+ if (this.staleTickTimer) {
118
+ clearTimeout(this.staleTickTimer);
119
+ this.staleTickTimer = undefined;
120
+ }
121
+ }
82
122
  async runTick() {
83
123
  if (this.tickInProgress)
84
124
  return false;
@@ -86,6 +126,8 @@ export class MergeStewardRuntime {
86
126
  this.lastTickStartedAt = new Date().toISOString();
87
127
  this.lastTickOutcome = "running";
88
128
  this.lastTickError = null;
129
+ this.lastReconcileEvent = null;
130
+ this.scheduleStaleTickWarning(this.lastTickStartedAt);
89
131
  try {
90
132
  await this.beforeTick?.();
91
133
  let tickQueueBlockEvent = null;
@@ -103,6 +145,7 @@ export class MergeStewardRuntime {
103
145
  flakyRetries: this.config.flakyRetries,
104
146
  policy: this.policy,
105
147
  onEvent: (event) => {
148
+ this.lastReconcileEvent = summarizeReconcileEvent(event);
106
149
  const isWarn = event.action === "evicted" || event.action === "spec_build_conflict"
107
150
  || event.action === "ci_failed"
108
151
  || event.action === "merge_rejected" || event.action === "budget_exhausted";
@@ -142,6 +185,7 @@ export class MergeStewardRuntime {
142
185
  this.logger.error({ err: error instanceof Error ? { message: error.message, stack: error.stack } : error }, "Reconcile tick failed");
143
186
  }
144
187
  finally {
188
+ this.clearStaleTickTimer();
145
189
  this.tickInProgress = false;
146
190
  this.lastTickCompletedAt = new Date().toISOString();
147
191
  this.scheduleNextTick();
@@ -191,3 +235,14 @@ export class MergeStewardRuntime {
191
235
  return requiredChecks.filter((check) => !available.has(check.trim().toLowerCase()));
192
236
  }
193
237
  }
238
+ function summarizeReconcileEvent(event) {
239
+ return {
240
+ at: event.at,
241
+ entryId: event.entryId,
242
+ prNumber: event.prNumber,
243
+ action: event.action,
244
+ ...(event.detail ? { detail: event.detail } : {}),
245
+ ...(event.ciRunId ? { ciRunId: event.ciRunId } : {}),
246
+ ...(event.specBranch ? { specBranch: event.specBranch } : {}),
247
+ };
248
+ }
package/dist/service.d.ts CHANGED
@@ -1,10 +1,9 @@
1
1
  import type { Logger } from "pino";
2
2
  import type { GitOperations, CIRunner, GitHubPRApi, EvictionReporter, SpeculativeBranchBuilder } from "./interfaces.ts";
3
- import type { QueueEntry, QueueEntryDetail, IncidentRecord, QueueRuntimeStatus, QueueWatchSnapshot } from "./types.ts";
3
+ import type { QueueEntry, QueueEntryDetail, IncidentRecord, QueueReconcileResult, QueueRuntimeStatus, QueueWatchSnapshot } from "./types.ts";
4
4
  import type { StewardConfig } from "./config.ts";
5
5
  import type { GitHubPolicyCache, GitHubPolicyRefreshResult, GitHubPolicySnapshot } from "./github-policy.ts";
6
6
  import type { QueueStore } from "./store.ts";
7
- import { MergeStewardRuntime } from "./service-runtime.ts";
8
7
  /**
9
8
  * Merge steward service. The public shell stays thin and delegates
10
9
  * lifecycle, queue mutations, and query shaping to focused helpers.
@@ -51,10 +50,7 @@ export declare class MergeStewardService {
51
50
  getEntryDetail(entryId: string, options?: {
52
51
  eventLimit?: number;
53
52
  }): QueueEntryDetail | undefined;
54
- triggerReconcile(): Promise<{
55
- started: boolean;
56
- runtime: ReturnType<MergeStewardRuntime["getRuntimeStatus"]>;
57
- }>;
53
+ triggerReconcile(): Promise<QueueReconcileResult>;
58
54
  getGitHubPolicy(): GitHubPolicySnapshot;
59
55
  refreshGitHubPolicyFromWebhook(reason: string): Promise<GitHubPolicyRefreshResult>;
60
56
  }
package/dist/types.d.ts CHANGED
@@ -141,6 +141,24 @@ export interface QueueRuntimeStatus {
141
141
  lastTickCompletedAt: string | null;
142
142
  lastTickOutcome: "idle" | "running" | "succeeded" | "failed";
143
143
  lastTickError: string | null;
144
+ tickAgeMs?: number | null;
145
+ staleTickThresholdMs?: number;
146
+ staleTick?: boolean;
147
+ lastReconcileEvent?: ReconcileEventSummary | null;
148
+ }
149
+ export interface ReconcileEventSummary {
150
+ at: string;
151
+ entryId: string;
152
+ prNumber: number;
153
+ action: ReconcileAction;
154
+ detail?: string | undefined;
155
+ ciRunId?: string | undefined;
156
+ specBranch?: string | undefined;
157
+ }
158
+ export interface QueueReconcileResult {
159
+ started: boolean;
160
+ reason?: "already_running" | undefined;
161
+ runtime: QueueRuntimeStatus;
144
162
  }
145
163
  export interface QueueStatusSummary {
146
164
  total: number;
package/dist/watch/App.js CHANGED
@@ -2,6 +2,7 @@ import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
2
2
  import { useEffect, useMemo, useState } from "react";
3
3
  import { Box, Text, useApp, useInput, useStdout } from "ink";
4
4
  import { fetchGatewayHealth, fetchSnapshot, triggerReconcile } from "./api.js";
5
+ import { formatDurationMs } from "../runtime-format.js";
5
6
  import { buildDashboard, matchRepoRef, repoSelector, stepRepo } from "./dashboard-model.js";
6
7
  import { HelpBar } from "./HelpBar.js";
7
8
  import { OverviewView } from "./OverviewView.js";
@@ -147,7 +148,9 @@ export function App({ gatewayBaseUrl, repos, initialRepoRef, initialPrNumber })
147
148
  return;
148
149
  try {
149
150
  const result = await triggerReconcile(gatewayBaseUrl, repoId);
150
- setFlashMessage(result.started ? `reconcile tick completed for ${repoId}` : `reconcile already running for ${repoId}`);
151
+ setFlashMessage(result.started
152
+ ? `reconcile tick completed for ${repoId}`
153
+ : `reconcile already running for ${formatDurationMs(result.runtime.tickAgeMs)} in ${repoId}`);
151
154
  }
152
155
  catch (error) {
153
156
  setFlashMessage(error instanceof Error ? error.message : String(error));
@@ -1,5 +1,5 @@
1
1
  import type { ServiceHealthResponse } from "../admin-types.ts";
2
- import type { QueueEntryDetail, QueueWatchSnapshot } from "../types.ts";
2
+ import type { QueueEntryDetail, QueueReconcileResult, QueueWatchSnapshot } from "../types.ts";
3
3
  export declare class ServiceApiError extends Error {
4
4
  readonly options?: {
5
5
  status?: number;
@@ -15,6 +15,5 @@ export declare function fetchGatewayHealth(gatewayBaseUrl: string): Promise<Serv
15
15
  export declare function fetchEntryDetail(gatewayBaseUrl: string, repoId: string, entryId: string): Promise<QueueEntryDetail>;
16
16
  export declare function triggerReconcile(gatewayBaseUrl: string, repoId: string): Promise<{
17
17
  ok: true;
18
- started: boolean;
19
- }>;
18
+ } & QueueReconcileResult>;
20
19
  export declare function dequeueEntry(gatewayBaseUrl: string, repoId: string, entryId: string): Promise<void>;
@@ -164,7 +164,7 @@ export function specChainLabel(entry, allEntries) {
164
164
  }
165
165
  export function runtimeLabel(runtime) {
166
166
  if (runtime.tickInProgress) {
167
- return "running";
167
+ return runtime.staleTick ? "stale" : "running";
168
168
  }
169
169
  if (runtime.lastTickOutcome === "idle") {
170
170
  return "idle";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "merge-steward",
3
- "version": "0.26.1",
3
+ "version": "0.27.0",
4
4
  "description": "Serial merge queue for GitHub — rebase, CI-gate, and merge PRs one at a time",
5
5
  "type": "module",
6
6
  "repository": {