merge-steward 0.34.1 → 0.34.2

package/README.md

@@ -66,6 +66,8 @@ merge-steward queue reconcile --repo <id>       # force one reconcile tick
 merge-steward service logs --lines 100
 ```
 
+Each repository reconcile tick is bounded by `reconcileStaleAfterMs` (five minutes by default). If a tick exceeds that threshold, Merge Steward records the failed runtime state, performs bounded service cleanup, and exits unsuccessfully so its `Restart=always` systemd unit restarts from the durable queue. It never starts a second reconciler beside the stuck tick.
+
 `pr status`, `queue status`, `queue show`, and `queue reconcile` auto-resolve `--repo` and `--pr` from the current git checkout. `pr status` supports `--wait --timeout <s> --poll <s>` for blocking until a terminal state. Exit codes:
 
 | Code | Meaning |

package/dist/cli/commands/queue.js

@@ -253,7 +253,7 @@ export function formatReconcileRequestText(result) {
             ? `; latest action ${result.runtime.lastReconcileEvent.action} PR #${result.runtime.lastReconcileEvent.prNumber}`
             : "";
         const guidance = result.runtime.staleTick
-            ? "inspect logs before restarting"
+            ? "the reconcile watchdog is restarting the service"
             : "wait for the current tick before restarting";
         return `Reconcile already running for ${age}${latest}; ${guidance}.`;
     }

package/dist/cli/commands/service.js

@@ -102,6 +102,6 @@ function staleRuntimeLines(repos) {
         .map((repo) => {
         const event = repo.runtime?.lastReconcileEvent;
         const latest = event ? `; latest action ${event.action} PR #${event.prNumber}` : "";
-        return `Warning: ${repo.repoId} reconcile tick is stale after ${formatDurationMs(repo.runtime?.tickAgeMs)}${latest}`;
+        return `Warning: ${repo.repoId} reconcile watchdog is restarting the service after ${formatDurationMs(repo.runtime?.tickAgeMs)}${latest}`;
     });
 }

package/dist/graceful-shutdown.d.ts

@@ -8,4 +8,5 @@ export declare function createGracefulShutdown(options: {
     logger: ShutdownLogger;
     cleanup: () => Promise<void>;
     terminate?: (code: number) => void;
-}): (trigger: string) => Promise<void>;
+    forceTerminateAfterMs?: number;
+}): (trigger: string, exitCode?: number) => Promise<void>;

package/dist/graceful-shutdown.js

@@ -1,28 +1,49 @@
 export function createGracefulShutdown(options) {
     let shutdownPromise;
     let firstTrigger;
+    let terminated = false;
     const terminate = options.terminate ?? ((code) => {
         process.exitCode = code;
         setImmediate(() => process.exit(code));
     });
-    return (trigger) => {
+    const terminateOnce = (code) => {
+        if (terminated)
+            return;
+        terminated = true;
+        terminate(code);
+    };
+    return (trigger, exitCode) => {
         if (shutdownPromise) {
             options.logger.warn({ service: options.service, trigger, firstTrigger }, "Shutdown already in progress");
             return shutdownPromise;
         }
         firstTrigger = trigger;
         options.logger.info({ service: options.service, trigger }, "Shutdown requested");
+        const forcedTermination = exitCode !== undefined && options.forceTerminateAfterMs !== undefined
+            ? setTimeout(() => {
+                options.logger.error({ service: options.service, trigger, forceTerminateAfterMs: options.forceTerminateAfterMs }, "Shutdown deadline exceeded; terminating");
+                terminateOnce(exitCode);
+            }, options.forceTerminateAfterMs)
+            : undefined;
+        forcedTermination?.unref?.();
         shutdownPromise = options.cleanup()
             .then(() => {
+            if (forcedTermination)
+                clearTimeout(forcedTermination);
             options.logger.info({ service: options.service, trigger }, "Shutdown complete");
+            if (exitCode !== undefined) {
+                terminateOnce(exitCode);
+            }
         })
             .catch((error) => {
+            if (forcedTermination)
+                clearTimeout(forcedTermination);
             options.logger.error({
                 service: options.service,
                 trigger,
                 error: error instanceof Error ? error.message : String(error),
             }, "Shutdown failed");
-            terminate(1);
+            terminateOnce(1);
         });
         return shutdownPromise;
     };

package/dist/runtime-format.js

@@ -29,7 +29,7 @@ export function formatRuntimeActivity(runtime) {
     if (latest)
         lines.push(`Latest action: ${latest}`);
     if (runtime.staleTick) {
-        lines.push(`Warning: reconcile tick appears stale; threshold ${formatDurationMs(runtime.staleTickThresholdMs)}.`);
+        lines.push(`Warning: reconcile watchdog is restarting the service after threshold ${formatDurationMs(runtime.staleTickThresholdMs)}.`);
     }
     return lines;
 }

package/dist/server.js

@@ -17,7 +17,7 @@ import { discoverRepoSettings } from "./github-repo-discovery.js";
 import { resolveSecretWithSource } from "./resolve-secret.js";
 import { setRuntimeGitHubAuthProvider } from "./exec.js";
 import { readFileSync, existsSync } from "node:fs";
-async function createRepoInstance(config, policy, logger, botIdentity) {
+async function createRepoInstance(config, policy, logger, botIdentity, onReconcileWatchdog) {
     const repoUrl = `https://github.com/${config.repoFullName}.git`;
     const clone = new CloneManager(config.clonePath, repoUrl, config.repoFullName, config.gitBin, logger);
     await clone.ensureClone();
@@ -31,7 +31,7 @@ async function createRepoInstance(config, policy, logger, botIdentity) {
     const ci = new GitHubActionsRunner(config.repoFullName, () => policy.getRequiredCheckRules(), () => policy.shouldRequireAllChecksOnEmptyRequiredSet());
     const github = new GitHubPRClient(config.repoFullName);
     const eviction = new GitHubCheckRunReporter(config.repoFullName, config.server.bind, config.server.port, config.server.publicBaseUrl, config.admissionLabel, config.mergeQueueCheckName);
-    const service = new MergeStewardService(config, policy, store, git, ci, github, eviction, git, logger);
+    const service = new MergeStewardService(config, policy, store, git, ci, github, eviction, git, logger, onReconcileWatchdog);
     return { config, service, store };
 }
 function githubHeaders(token) {
@@ -211,6 +211,7 @@ export async function startMultiServer() {
             }
             await app.close();
         },
+        forceTerminateAfterMs: 15_000,
     });
     process.once("SIGTERM", () => void shutdown("SIGTERM"));
     process.once("SIGINT", () => void shutdown("SIGINT"));
@@ -258,7 +259,10 @@ export async function startMultiServer() {
                 githubRequiredChecks: policy.getRequiredChecks(),
                 requireAllChecksOnEmptyRequiredSet: policy.shouldRequireAllChecksOnEmptyRequiredSet(),
             }, "Resolved GitHub protection requirements");
-            const instance = await createRepoInstance(config, policy, logger.child({ repoId: config.repoId }), botIdentity);
+            const instance = await createRepoInstance(config, policy, logger.child({ repoId: config.repoId }), botIdentity, async (runtime) => {
+                logger.error({ repoId: config.repoId, runtime }, "Reconcile watchdog is restarting merge-steward");
+                await shutdown(`reconcile_watchdog:${config.repoId}`, 1);
+            });
             if (shuttingDown) {
                 instance.store.close();
                 return;

package/dist/service-runtime.d.ts

@@ -15,6 +15,7 @@ export declare class MergeStewardRuntime {
     private readonly specBuilder;
     private readonly logger;
     private readonly beforeTick?;
+    private readonly onReconcileWatchdog?;
     private tickTimer;
     private staleTickTimer;
     private tickInProgress;
@@ -23,7 +24,8 @@ export declare class MergeStewardRuntime {
     private lastTickOutcome;
     private lastTickError;
     private lastReconcileEvent;
-    constructor(config: StewardConfig, policy: GitHubPolicyCache, store: QueueStore, git: GitOperations, ci: CIRunner, github: GitHubPRApi, eviction: EvictionReporter, specBuilder: SpeculativeBranchBuilder, logger: Logger, beforeTick?: (() => Promise<void>) | undefined);
+    private watchdogTickStartedAt;
+    constructor(config: StewardConfig, policy: GitHubPolicyCache, store: QueueStore, git: GitOperations, ci: CIRunner, github: GitHubPRApi, eviction: EvictionReporter, specBuilder: SpeculativeBranchBuilder, logger: Logger, beforeTick?: (() => Promise<void>) | undefined, onReconcileWatchdog?: ((runtime: QueueRuntimeStatus) => Promise<void> | void) | undefined);
     start(): Promise<void>;
     stop(): Promise<void>;
     triggerReconcile(): Promise<QueueReconcileResult>;
@@ -32,7 +34,7 @@ export declare class MergeStewardRuntime {
     private recoverTerminalQueueLabels;
     private scheduleNextTick;
     private getTickAgeMs;
-    private scheduleStaleTickWarning;
+    private scheduleReconcileWatchdog;
     private clearStaleTickTimer;
     private runTick;
 }

package/dist/service-runtime.js

@@ -10,6 +10,7 @@ export class MergeStewardRuntime {
     specBuilder;
     logger;
     beforeTick;
+    onReconcileWatchdog;
     tickTimer;
     staleTickTimer;
     tickInProgress = false;
@@ -18,7 +19,8 @@ export class MergeStewardRuntime {
     lastTickOutcome = "idle";
     lastTickError = null;
     lastReconcileEvent = null;
-    constructor(config, policy, store, git, ci, github, eviction, specBuilder, logger, beforeTick) {
+    watchdogTickStartedAt = null;
+    constructor(config, policy, store, git, ci, github, eviction, specBuilder, logger, beforeTick, onReconcileWatchdog) {
         this.config = config;
         this.policy = policy;
         this.store = store;
@@ -29,6 +31,7 @@ export class MergeStewardRuntime {
         this.specBuilder = specBuilder;
         this.logger = logger;
         this.beforeTick = beforeTick;
+        this.onReconcileWatchdog = onReconcileWatchdog;
     }
     async start() {
         this.logger.info({ pollIntervalMs: this.config.pollIntervalMs }, "Steward service starting");
@@ -45,6 +48,9 @@ export class MergeStewardRuntime {
         while (this.tickInProgress && Date.now() < deadline) {
             await new Promise((resolve) => setTimeout(resolve, 100));
         }
+        if (this.tickInProgress) {
+            this.logger.warn({ runtime: this.getRuntimeStatus() }, "Stopping while a reconcile tick is still active; process restart will recover from durable queue state");
+        }
         this.logger.info("Steward service stopped");
     }
     async triggerReconcile() {
@@ -118,18 +124,27 @@ export class MergeStewardRuntime {
             return null;
         return Math.max(0, Date.now() - startedMs);
     }
-    scheduleStaleTickWarning(startedAt) {
+    scheduleReconcileWatchdog(startedAt) {
         this.clearStaleTickTimer();
         const timer = setTimeout(() => {
             if (!this.tickInProgress || this.lastTickStartedAt !== startedAt)
                 return;
+            this.staleTickTimer = undefined;
+            this.watchdogTickStartedAt = startedAt;
+            this.lastTickOutcome = "failed";
+            this.lastTickError = `Reconcile tick exceeded the ${this.config.reconcileStaleAfterMs}ms stale threshold; service restart requested.`;
             const runtime = this.getRuntimeStatus();
-            this.logger.warn({
+            this.logger.error({
                 startedAt,
                 tickAgeMs: runtime.tickAgeMs,
                 staleTickThresholdMs: runtime.staleTickThresholdMs,
                 lastReconcileEvent: runtime.lastReconcileEvent,
-            }, "Reconcile tick appears stale");
+            }, "Reconcile watchdog detected a stuck tick; requesting service restart");
+            void (async () => {
+                await this.onReconcileWatchdog?.(runtime);
+            })().catch((error) => {
+                this.logger.error({ startedAt, error: error instanceof Error ? error.message : String(error) }, "Reconcile watchdog could not restart the service");
+            });
         }, this.config.reconcileStaleAfterMs);
         timer.unref?.();
         this.staleTickTimer = timer;
@@ -148,7 +163,9 @@ export class MergeStewardRuntime {
         this.lastTickOutcome = "running";
         this.lastTickError = null;
         this.lastReconcileEvent = null;
-        this.scheduleStaleTickWarning(this.lastTickStartedAt);
+        this.watchdogTickStartedAt = null;
+        const startedAt = this.lastTickStartedAt;
+        this.scheduleReconcileWatchdog(startedAt);
         try {
             await this.beforeTick?.();
             await reconcile({
@@ -179,7 +196,9 @@ export class MergeStewardRuntime {
                     this.logger[level]({ ...event }, `Queue: ${event.action} PR #${event.prNumber}`);
                 },
             });
-            this.lastTickOutcome = "succeeded";
+            if (this.watchdogTickStartedAt !== startedAt) {
+                this.lastTickOutcome = "succeeded";
+            }
         }
         catch (error) {
             this.lastTickOutcome = "failed";

package/dist/service.d.ts

@@ -21,7 +21,7 @@ export declare class MergeStewardService {
     private readonly runtime;
     private readonly queueCommands;
     private readonly watchQueries;
-    constructor(config: StewardConfig, policy: GitHubPolicyCache, store: QueueStore, git: GitOperations, ci: CIRunner, github: GitHubPRApi, eviction: EvictionReporter, specBuilder: SpeculativeBranchBuilder, logger: Logger);
+    constructor(config: StewardConfig, policy: GitHubPolicyCache, store: QueueStore, git: GitOperations, ci: CIRunner, github: GitHubPRApi, eviction: EvictionReporter, specBuilder: SpeculativeBranchBuilder, logger: Logger, onReconcileWatchdog?: ((runtime: QueueRuntimeStatus) => Promise<void> | void) | undefined);
     /** Expose the GitHub client for webhook handler branch→PR lookups. */
     get githubApi(): GitHubPRApi;
     start(): Promise<void>;

package/dist/service.js

@@ -18,7 +18,7 @@ export class MergeStewardService {
     runtime;
     queueCommands;
     watchQueries;
-    constructor(config, policy, store, git, ci, github, eviction, specBuilder, logger) {
+    constructor(config, policy, store, git, ci, github, eviction, specBuilder, logger, onReconcileWatchdog) {
         this.config = config;
         this.policy = policy;
         this.store = store;
@@ -31,7 +31,7 @@ export class MergeStewardService {
         this.queueCommands = new MergeStewardQueueCommands(config, policy, store, github, specBuilder, logger);
         this.runtime = new MergeStewardRuntime(config, policy, store, git, ci, github, eviction, specBuilder, logger, async () => {
             await this.queueCommands.scanEligibleOpenPrs();
-        });
+        }, onReconcileWatchdog);
         this.watchQueries = new MergeStewardWatchQueries(config, store, this.runtime);
     }
     /** Expose the GitHub client for webhook handler branch→PR lookups. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "merge-steward",
-  "version": "0.34.1",
+  "version": "0.34.2",
   "description": "Serial merge queue for GitHub — rebase, CI-gate, and merge PRs one at a time",
   "type": "module",
   "repository": {