@celilo/cli 4.0.0 → 5.0.0

package/src/services/build-bus/hook-dispatch-executor.test.ts

@@ -121,7 +121,7 @@ describe('the dispatcher dispatches on_upstream_publish through the ordinary exe
           id: 'upstream-fixture',
           name: 'upstream-fixture',
           version: '1.0.0',
-          celilo_contract: '1.0',
+          celilo_contract: '2.0',
           provides: { capabilities: [] },
           requires: { capabilities: [] },
           hooks: { on_upstream_publish: [hookEntry] },
@@ -178,7 +178,7 @@ describe('the dispatcher dispatches on_upstream_publish through the ordinary exe
           id: 'upstream-fixture',
           name: 'upstream-fixture',
           version: '1.0.0',
-          celilo_contract: '1.0',
+          celilo_contract: '2.0',
           provides: { capabilities: [] },
           requires: { capabilities: [] },
           hooks: { on_upstream_publish: [hookEntry] },
@@ -232,7 +232,7 @@ test('a failing hook run lands in the build_bus_hook_runs ledger', async () => {
         id: 'upstream-fixture',
         name: 'upstream-fixture',
         version: '1.0.0',
-        celilo_contract: '1.0',
+        celilo_contract: '2.0',
         provides: { capabilities: [] },
         requires: { capabilities: [] },
         hooks: { on_upstream_publish: [hookEntry] },

package/src/services/bus-interview-park.test.ts

@@ -71,7 +71,7 @@ beforeEach(() => {
       name: 'iptables',
       sourcePath: join(dir, 'installed'),
       version: '1.0.2+9',
-      manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
+      manifestData: { celilo_contract: '2.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
     })
     .run();
 

package/src/services/bus-secret-flow.test.ts

@@ -396,7 +396,7 @@ describe('bus-mediated interviewForMissingSecrets', () => {
     writeFileSync(
       join(tempDir, 'manifest.yml'),
       `
-celilo_contract: "1.0"
+celilo_contract: "2.0"
 id: testmod
 name: Test Module
 version: 1.0.0

package/src/services/capability-compat.test.ts

@@ -24,6 +24,28 @@ function provider(
 }
 
 describe('unservedCapabilityRequirements', () => {
+  /**
+   * celilo#1384. Optional in PRESENCE, not in contract: with no provider the
+   * module is fine, but an installed provider on another major is called by
+   * the module's hooks and fails there — as a warning, silently.
+   */
+  test('an optional capability with no provider is not a blocker', () => {
+    expect(
+      unservedCapabilityRequirements([], [], [{ name: 'external_web', version: '1.0.0' }]),
+    ).toEqual([]);
+  });
+
+  test('an optional capability whose installed provider is another major IS a blocker', () => {
+    const result = unservedCapabilityRequirements(
+      [],
+      [provider('external_web', 'generic-cpanel', '2.0.0')],
+      [{ name: 'external_web', version: '1.0.0' }],
+    );
+    expect(result).toHaveLength(1);
+    expect(result[0]?.capability).toBe('external_web');
+    expect(result[0]?.providerModuleId).toBe('generic-cpanel');
+  });
+
   test('a requirement any deployed provider serves is not a blocker', () => {
     const providers = [provider('public_web', 'caddy', '4.0.0')];
     expect(

package/src/services/capability-compat.ts

@@ -53,15 +53,27 @@ export interface UnservedCapability {
 export function unservedCapabilityRequirements(
   requirements: CapabilityRequirement[] | undefined,
   providers: DeployedCapabilityProvider[],
+  /**
+   * Capabilities the module uses only if present. Their provider may be
+   * absent, but an installed provider on another major breaks at runtime
+   * exactly as a required one would (celilo#1384), so they take the version
+   * check and skip the presence check.
+   */
+  optional: CapabilityRequirement[] | undefined = undefined,
 ): UnservedCapability[] {
   const unserved: UnservedCapability[] = [];
-  for (const cap of requirements ?? []) {
+  const consumed = [
+    ...(requirements ?? []).map((cap) => ({ cap, required: true })),
+    ...(optional ?? []).map((cap) => ({ cap, required: false })),
+  ];
+  for (const { cap, required } of consumed) {
     // Framework-granted privileges (e.g. cross_module_read) are not
     // provider-backed — the same skip `deploy-preflight` makes.
     if (isPrivilegedCapability(cap.name)) continue;
 
     const installed = providers.filter((p) => p.capabilityName === cap.name);
     if (installed.length === 0) {
+      if (!required) continue;
       unserved.push({
         capability: cap.name,
         required: cap.version,
@@ -119,10 +131,11 @@ export function readDeployedProviders(db: DbClient): DeployedCapabilityProvider[
  */
 export function capabilityBlockersForManifest(
   db: DbClient,
-  manifest: Pick<ModuleManifest, 'requires' | 'id'>,
+  manifest: Pick<ModuleManifest, 'requires' | 'optional' | 'id'>,
 ): UnservedCapability[] {
   return unservedCapabilityRequirements(
     manifest.requires?.capabilities as CapabilityRequirement[] | undefined,
     readDeployedProviders(db),
+    manifest.optional?.capabilities as CapabilityRequirement[] | undefined,
   );
 }

package/src/services/capability-table-rows.test.ts

@@ -18,7 +18,14 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { allDeclaredTables } from '@celilo/capabilities';
 import type { DbClient } from '../db/client';
-import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
+import {
+  dnsInternalRecords,
+  modules,
+  portForwards,
+  privateRoutes,
+  trustedSources,
+  webRoutes,
+} from '../db/schema';
 import { setupTestDatabaseAt } from '../test-utils/database';
 import { resetTestDbPath } from '../test-utils/db-path';
 import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
@@ -89,6 +96,15 @@ describe('deleteClaimedRows', () => {
         hostname,
       })
       .run();
+    db.insert(privateRoutes)
+      .values({
+        slug: `${consumer}-private-slug`,
+        moduleId: consumer,
+        type: 'reverse_proxy',
+        path: `/${consumer}`,
+        hostname: `private-${hostname}`,
+      })
+      .run();
     db.insert(portForwards)
       .values({
         firewallIp: FW,
@@ -109,7 +125,7 @@ describe('deleteClaimedRows', () => {
       .run();
   }
 
-  it('clears the departing consumer from all four declared tables at once', () => {
+  it('clears the departing consumer from every declared table at once', () => {
     seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
 
     const cleared = deleteClaimedRows(db, 'caddy');
@@ -117,10 +133,12 @@ describe('deleteClaimedRows', () => {
     expect(cleared.map((c) => c.table).sort()).toEqual([
       'dns_internal_records',
       'port_forwards',
+      'private_routes',
       'trusted_sources',
       'web_routes',
     ]);
     expect(db.select().from(webRoutes).all()).toHaveLength(0);
+    expect(db.select().from(privateRoutes).all()).toHaveLength(0);
     expect(db.select().from(portForwards).all()).toHaveLength(0);
     expect(db.select().from(trustedSources).all()).toHaveLength(0);
     expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);

package/src/services/celilo-events.test.ts

@@ -16,6 +16,7 @@ import {
   emitWebRoutesChanged,
   emitWebRoutesChangedAndWait,
   routesChangedHighWater,
+  subscriberBudgetMs,
   waitForRouteReconcile,
 } from './celilo-events';
 
@@ -171,12 +172,13 @@ describe('celilo lifecycle events', () => {
   });
 
   describe('deploy-waits for web-route reconcile (ISS-0035)', () => {
-    function subscribeReconciler(): void {
+    function subscribeReconciler(budget?: { maxAttempts: number; timeoutMs: number }): void {
       const bus = openBus({ dbPath, events: defineEvents({}) });
       bus.subscribe({
         name: 'caddy.reconcile-web-routes',
         pattern: 'public_web.routes_changed',
         handler: 'unused',
+        ...budget,
       });
       bus.close();
     }
@@ -208,6 +210,30 @@ describe('celilo lifecycle events', () => {
       bus.close();
     }
 
+    function settleAllDeliveries(): void {
+      const bus = openBus({ dbPath, events: defineEvents({}) });
+      const sub = bus.getSubscriberByName('caddy.reconcile-web-routes');
+      if (!sub) throw new Error('expected a subscriber');
+      for (const event of bus.recentEvents({ type: 'public_web.routes_changed', limit: 100 })) {
+        bus.markSucceeded({ eventId: event.id, subscriberId: sub.id });
+      }
+      bus.close();
+    }
+
+    /** One failed attempt that will be retried — the delivery stays pending. */
+    function recordRetryingFailure(message: string): void {
+      const bus = openBus({ dbPath, events: defineEvents({}) });
+      const sub = bus.getSubscriberByName('caddy.reconcile-web-routes');
+      const event = bus.recentEvents({ type: 'public_web.routes_changed', limit: 1 })[0];
+      if (!sub || !event) {
+        throw new Error('expected a subscriber and an emitted event');
+      }
+      bus.markFailed({ eventId: event.id, subscriberId: sub.id }, message, {
+        retryAfter: Date.now() + 60_000,
+      });
+      bus.close();
+    }
+
     it('routesChangedHighWater is 0 with no events, then the latest id', () => {
       expect(routesChangedHighWater()).toBe(0);
       emitWebRoutesChanged('celilo-website');
@@ -217,11 +243,12 @@ describe('celilo lifecycle events', () => {
     it('returns immediately when the deploy changed no routes', async () => {
       const since = routesChangedHighWater();
       const result = await waitForRouteReconcile(since, { timeoutMs: 1000 });
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         events: 0,
         succeeded: 0,
         failed: 0,
         timedOut: false,
+        lastError: null,
         noDispatcher: false,
       });
     });
@@ -232,11 +259,12 @@ describe('celilo lifecycle events', () => {
       emitWebRoutesChanged('celilo-website');
       // No heartbeat stamped → health() is no_dispatcher.
       const result = await waitForRouteReconcile(since, { timeoutMs: 5000, pollMs: 50 });
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         events: 1,
         succeeded: 0,
         failed: 0,
         timedOut: false,
+        lastError: null,
         noDispatcher: true,
       });
     });
@@ -246,11 +274,12 @@ describe('celilo lifecycle events', () => {
       const since = routesChangedHighWater();
       emitWebRoutesChanged('celilo-website');
       const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         events: 1,
         succeeded: 0,
         failed: 0,
         timedOut: false,
+        lastError: null,
         noDispatcher: false,
       });
     });
@@ -265,6 +294,26 @@ describe('celilo lifecycle events', () => {
       expect(result.timedOut).toBe(true);
     });
 
+    /**
+     * celilo#1401. A handler that CRASHES is retried, so between attempts its
+     * delivery is `pending` with the error already recorded. The deadline hits
+     * in that window and the caller used to report a bare timeout — `0 ok, 0
+     * failed of 1` — which is what hid a caddy crash for days.
+     */
+    it('carries the crashing handler error out of a timeout', async () => {
+      markDispatcherAlive();
+      subscribeReconciler();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      // Failed but NOT abandoned: it goes back to pending for the next attempt.
+      recordRetryingFailure('{} is not iterable');
+
+      const result = await waitForRouteReconcile(since, { timeoutMs: 300, pollMs: 50 });
+      expect(result.timedOut).toBe(true);
+      expect(result.failed).toBe(0);
+      expect(result.lastError).toContain('{} is not iterable');
+    });
+
     it('completes once the provider delivery succeeds', async () => {
       markDispatcherAlive();
       subscribeReconciler();
@@ -273,11 +322,12 @@ describe('celilo lifecycle events', () => {
       settleDelivery('succeed');
 
       const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         events: 1,
         succeeded: 1,
         failed: 0,
         timedOut: false,
+        lastError: null,
         noDispatcher: false,
       });
     });
@@ -295,6 +345,68 @@ describe('celilo lifecycle events', () => {
       expect(result.timedOut).toBe(false);
     });
 
+    /**
+     * celilo#1416. The waiter's deadline was the constant 90s while caddy's
+     * subscription is `timeout_ms: 90000, max_attempts: 3` — so the ENTIRE wait
+     * equalled ONE attempt and a successful retry could never be observed. The
+     * budget must come from the subscriber's own registration.
+     */
+    it('budgets for the subscriber whole retry allowance, not one attempt', async () => {
+      markDispatcherAlive();
+      subscribeReconciler({ maxAttempts: 3, timeoutMs: 90_000 });
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      settleDelivery('succeed');
+
+      const result = await waitForRouteReconcile(since, { pollMs: 50 });
+      expect(result.succeeded).toBe(1);
+      expect(result.budgetMs).toBeGreaterThanOrEqual(subscriberBudgetMs(3, 90_000));
+    });
+
+    /** A deploy emitting N change events waits for N of them, not for one. */
+    it('scales the budget with the number of change events', async () => {
+      markDispatcherAlive();
+      subscribeReconciler({ maxAttempts: 2, timeoutMs: 60_000 });
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      emitWebRoutesChanged('celilo-website');
+      settleAllDeliveries();
+
+      const result = await waitForRouteReconcile(since, { pollMs: 50 });
+      expect(result.events).toBe(2);
+      expect(result.budgetMs).toBeGreaterThanOrEqual(2 * subscriberBudgetMs(2, 60_000));
+    });
+
+    /** The end-to-end shape: one failed attempt, then a success, reports SUCCESS. */
+    it('reports success when the provider fails once and then succeeds', async () => {
+      markDispatcherAlive();
+      subscribeReconciler({ maxAttempts: 3, timeoutMs: 400 });
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      recordRetryingFailure('transient boom');
+
+      const pending = waitForRouteReconcile(since, { pollMs: 50 });
+      await new Promise((r) => setTimeout(r, 120));
+      settleDelivery('succeed');
+
+      const result = await pending;
+      expect(result.timedOut).toBe(false);
+      expect(result.succeeded).toBe(1);
+    });
+
+    /** An expiry with deliveries in flight is not "nobody consumed it". */
+    it('separates deliveries still running from settled failures on expiry', async () => {
+      markDispatcherAlive();
+      subscribeReconciler();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+
+      const result = await waitForRouteReconcile(since, { timeoutMs: 300, pollMs: 50 });
+      expect(result.timedOut).toBe(true);
+      expect(result.stillRunning).toBe(1);
+      expect(result.failed).toBe(0);
+    });
+
     it('emitWebRoutesChangedAndWait emits then waits for the reconcile to settle', async () => {
       markDispatcherAlive();
       subscribeReconciler();
@@ -308,11 +420,12 @@ describe('celilo lifecycle events', () => {
       settleDelivery('succeed');
 
       const result = await pending;
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         events: 1,
         succeeded: 1,
         failed: 0,
         timedOut: false,
+        lastError: null,
         noDispatcher: false,
       });
     });

package/src/services/celilo-events.ts

@@ -184,12 +184,36 @@ export function emitSystemDestroyed(payload: SystemDestroyedPayload): void {
  * Best-effort — a failed emit never breaks register_route; the route is already
  * persisted in web_routes and the provider's next deploy reconciles it anyway.
  */
-export function emitWebRoutesChanged(triggeredBy: string): void {
-  emitBest('public_web.routes_changed', { triggeredBy });
+export function emitWebRoutesChanged(
+  triggeredBy: string,
+  type: RoutesChangedType = ROUTES_CHANGED_TYPE,
+): void {
+  emitBest(type, { triggeredBy });
 }
 
 const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
 
+/**
+ * The private ingress's twin of {@link ROUTES_CHANGED_TYPE}
+ * (`providers-converge-declared-state` slice 2). The two ingresses are
+ * deliberately SEPARATE event types rather than one shared signal: a private
+ * route lives in `private_routes`, a public one in `web_routes` (celilo#846),
+ * and waking the wrong provider would make it re-render from a table it does
+ * not serve.
+ */
+const PRIVATE_ROUTES_CHANGED_TYPE = 'private_web.routes_changed';
+
+/**
+ * Which ingress's route table changed. Every function below defaults to the
+ * public type, so existing callers are unaffected by construction.
+ */
+export type RoutesChangedType = typeof ROUTES_CHANGED_TYPE | typeof PRIVATE_ROUTES_CHANGED_TYPE;
+
+/** Emit `private_web.routes_changed` — the private ingress's coarse signal. */
+export function emitPrivateRoutesChanged(triggeredBy: string): void {
+  emitWebRoutesChanged(triggeredBy, PRIVATE_ROUTES_CHANGED_TYPE);
+}
+
 /**
  * Emit `public_web.routes_changed` and block until the provider's reconcile
  * delivery settles (ISS-0035). This is what `register_route` / `unregister_routes`
@@ -205,10 +229,30 @@ const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
 export async function emitWebRoutesChangedAndWait(
   triggeredBy: string,
   opts?: { timeoutMs?: number; pollMs?: number },
+  type: RoutesChangedType = ROUTES_CHANGED_TYPE,
 ): Promise<RouteReconcileWaitResult> {
-  const since = routesChangedHighWater();
-  emitWebRoutesChanged(triggeredBy);
-  return waitForRouteReconcile(since, opts);
+  const since = routesChangedHighWater(type);
+  emitWebRoutesChanged(triggeredBy, type);
+  return waitForRouteReconcile(since, opts, type);
+}
+
+/**
+ * The private ingress's twin of {@link emitWebRoutesChangedAndWait}
+ * (`providers-converge-declared-state` slice 2, D7).
+ *
+ * `registerReverseProxy` awaits THIS, so it still returns only once the route
+ * is actually live — the synchronous guarantee the in-call reconcile used to
+ * give, now provider-agnostic and on one path with the deploy.
+ *
+ * ⚠️ Callers must honour `noDispatcher` (ISS-0081): it means the event was
+ * persisted but never DELIVERED, so the route is NOT live. Treating it as a
+ * benign zero is the silent-success shape this change exists to remove.
+ */
+export async function emitPrivateRoutesChangedAndWait(
+  triggeredBy: string,
+  opts?: { timeoutMs?: number; pollMs?: number },
+): Promise<RouteReconcileWaitResult> {
+  return emitWebRoutesChangedAndWait(triggeredBy, opts, PRIVATE_ROUTES_CHANGED_TYPE);
 }
 
 /**
@@ -222,6 +266,17 @@ export interface RouteReconcileWaitResult {
   succeeded: number;
   failed: number;
   timedOut: boolean;
+  /**
+   * The handler's own error text from the most recent settled-or-retrying
+   * delivery, or null when no delivery has reported one.
+   *
+   * A handler that CRASHES is retried, so for `max_attempts` retries its
+   * delivery is back in `pending` with an error already recorded. The deadline
+   * can hit in that window, and the caller then reported a timeout for what was
+   * a crash — `0 ok, 0 failed of 1` with the real cause nowhere in sight
+   * (celilo#1401, which hid a caddy crash for days behind a deadline message).
+   */
+  lastError: string | null;
   /**
    * True when no event dispatcher was running, so the `routes_changed` event
    * was persisted but never DELIVERED to the provider (caddy). The route is
@@ -229,6 +284,19 @@ export interface RouteReconcileWaitResult {
    * failure, not a benign "0 deliveries" success.
    */
   noDispatcher: boolean;
+  /**
+   * Deliveries still `pending`/`running` when the deadline hit. Non-zero means
+   * the provider had NOT given up — its retries were still in flight — which is
+   * a different thing from a failure and must not be reported as one
+   * (celilo#1416: "0 ok, 0 failed of 1" read as "nobody consumed it").
+   */
+  stillRunning: number;
+  /**
+   * The deadline actually used, in ms. Derived from the subscribers' own retry
+   * budgets unless the caller passed `timeoutMs`, so the message can say what
+   * it waited for rather than quoting a constant.
+   */
+  budgetMs: number;
 }
 
 /**
@@ -236,11 +304,11 @@ export interface RouteReconcileWaitResult {
  * exist yet. A deploy captures this before it runs so {@link waitForRouteReconcile}
  * can tell which route-change events the deploy itself produced.
  */
-export function routesChangedHighWater(): number {
+export function routesChangedHighWater(type: RoutesChangedType = ROUTES_CHANGED_TYPE): number {
   let bus: ReturnType<typeof openBus> | undefined;
   try {
     bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
-    return bus.recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 1 })[0]?.id ?? 0;
+    return bus.recentEvents({ type, limit: 1 })[0]?.id ?? 0;
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     console.warn(`[celilo] failed to read routes-changed high-water: ${msg}`);
@@ -250,6 +318,27 @@ export function routesChangedHighWater(): number {
   }
 }
 
+/**
+ * A subscriber's whole retry budget: every attempt's handler timeout plus the
+ * dispatcher's exponential backoff between them (1s, 2s, 4s..., capped at 5m —
+ * `backoffMs` in the event bus's dispatcher).
+ *
+ * Exported so the manifest gate that keeps consumer hook timeouts above this
+ * can compute the same number from the same formula rather than a second copy
+ * of it (celilo#1416).
+ */
+export function subscriberBudgetMs(maxAttempts: number, timeoutMs: number): number {
+  const attempts = Math.max(1, maxAttempts);
+  let budget = attempts * timeoutMs;
+  for (let attempt = 1; attempt < attempts; attempt++) {
+    budget += Math.min(1000 * 2 ** (attempt - 1), 5 * 60 * 1000);
+  }
+  return budget;
+}
+
+/** Slack over the derived budget, so the wait outlives the last attempt. */
+const RECONCILE_BUDGET_SLACK_MS = 5_000;
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -273,10 +362,15 @@ function sleep(ms: number): Promise<void> {
 export async function waitForRouteReconcile(
   sinceEventId: number,
   opts: { timeoutMs?: number; pollMs?: number } = {},
+  type: RoutesChangedType = ROUTES_CHANGED_TYPE,
 ): Promise<RouteReconcileWaitResult> {
-  const timeoutMs = opts.timeoutMs ?? 90_000;
   const pollMs = opts.pollMs ?? 250;
-  const deadline = Date.now() + timeoutMs;
+  // The deadline is DERIVED below, once we know which subscribers are on the
+  // hook and what retry budget each was registered with. A constant here is
+  // what celilo#1416 was: 90s, exactly ONE attempt of a subscriber configured
+  // for three, so a successful retry could never be observed.
+  let budgetMs = opts.timeoutMs ?? 0;
+  let deadline = opts.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : 0;
 
   let bus: ReturnType<typeof openBus> | undefined;
   try {
@@ -285,24 +379,44 @@ export async function waitForRouteReconcile(
 
     // The deploy's emits already happened, so the set of route-change events
     // is fixed; only their deliveries transition as the dispatcher works.
-    const events = b
-      .recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 100 })
-      .filter((e) => e.id > sinceEventId);
+    const events = b.recentEvents({ type, limit: 100 }).filter((e) => e.id > sinceEventId);
 
     if (events.length === 0) {
-      return { events: 0, succeeded: 0, failed: 0, timedOut: false, noDispatcher: false };
+      return {
+        events: 0,
+        succeeded: 0,
+        failed: 0,
+        timedOut: false,
+        lastError: null,
+        noDispatcher: false,
+        stillRunning: 0,
+        budgetMs,
+      };
     }
 
-    if (b.health().status === 'no_dispatcher') {
+    // Confirmed, not instant: a dispatcher starved by the deploy's own load looks
+    // exactly like a dead one for a single sample, and abandoning the reconcile on
+    // that reading is celilo#1399 — it failed consumers' re-registration during the
+    // one operation that generates route changes. `lagging` waits like any healthy
+    // bus; the deadline below still bounds it.
+    const health = await b.healthConfirmed();
+    if (health.status === 'no_dispatcher' || health.status === 'wedged') {
+      const why =
+        health.status === 'wedged'
+          ? `dispatcher heartbeat stale with nothing running (wedged, ${Math.round((health.lastHeartbeatAgeMs ?? 0) / 1000)}s)`
+          : 'no dispatcher running';
       console.warn(
-        `[celilo] route-reconcile: no dispatcher running — not waiting; ${events.length} change event(s) persisted, the provider will reconcile on its next run`,
+        `[celilo] route-reconcile: ${why} — not waiting; ${events.length} change event(s) persisted, the provider will reconcile on its next run`,
       );
       return {
         events: events.length,
         succeeded: 0,
         failed: 0,
         timedOut: false,
+        lastError: null,
         noDispatcher: true,
+        stillRunning: 0,
+        budgetMs,
       };
     }
 
@@ -313,13 +427,30 @@ export async function waitForRouteReconcile(
         (d) => d.status === 'pending' || d.status === 'running',
       ).length;
 
+      if (deadline === 0) {
+        // Every subscriber gets its whole registered retry budget, and a deploy
+        // that emitted N change events is waiting on N of them — budgeting for
+        // one is how a two-event deploy charged both against a single 90s wall.
+        const subscriberIds = [...new Set(deliveries.map((d) => d.subscriberId))];
+        const worst = subscriberIds.reduce((max, id) => {
+          const sub = b.getSubscriber(id);
+          if (!sub) return max;
+          return Math.max(max, subscriberBudgetMs(sub.maxAttempts, sub.timeoutMs));
+        }, 0);
+        budgetMs = worst * events.length + RECONCILE_BUDGET_SLACK_MS;
+        deadline = Date.now() + budgetMs;
+      }
+
       if (pending === 0 || Date.now() >= deadline) {
         return {
           events: events.length,
           succeeded: settled('succeeded'),
           failed: settled('failed') + settled('abandoned'),
           timedOut: pending > 0,
+          lastError: deliveries.find((d) => d.lastError)?.lastError ?? null,
           noDispatcher: false,
+          stillRunning: pending,
+          budgetMs,
         };
       }
 
@@ -328,7 +459,16 @@ export async function waitForRouteReconcile(
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     console.warn(`[celilo] route-reconcile wait errored: ${msg}`);
-    return { events: 0, succeeded: 0, failed: 0, timedOut: false, noDispatcher: false };
+    return {
+      events: 0,
+      succeeded: 0,
+      failed: 0,
+      timedOut: false,
+      lastError: null,
+      noDispatcher: false,
+      stillRunning: 0,
+      budgetMs,
+    };
   } finally {
     bus?.close();
   }