deepline 0.3.357 → 0.3.358

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.357',
+  version: '0.3.358',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/monitors/deactivation-reasons.ts

@@ -20,7 +20,7 @@ import { monitorBillingRequiresExplicitReactivation } from './billing/state';
  * this off" and "we did, and here is why" must not be the same state.
  */
 
-export type MonitorDeactivationOwner = 'policy' | 'fleet';
+export type MonitorDeactivationOwner = 'policy' | 'fleet' | 'admin';
 
 /**
  * The delivery window, in days, that `monitors health` reports against.
@@ -43,6 +43,13 @@ export type MonitorDeactivationReasonDefinition = {
 
 export const MONITOR_DEACTIVATION_REASONS: readonly MonitorDeactivationReasonDefinition[] =
   [
+    {
+      id: 'workspace_archived',
+      owner: 'admin',
+      summary: 'Deactivated because the workspace is archived.',
+      resolution:
+        'Contact a platform administrator. Credit changes never reactivate an archived workspace.',
+    },
     {
       id: 'insufficient_balance',
       owner: 'policy',
@@ -122,6 +129,12 @@ export function isPolicyDeactivation(id: string | undefined | null): boolean {
   return monitorDeactivationReason(id)?.owner === 'policy';
 }
 
+/** Both policy and admin shutdown retain the definition; only policy owns recovery. */
+export function isMonitorSuspension(id: string | undefined | null): boolean {
+  const owner = monitorDeactivationReason(id)?.owner;
+  return owner === 'policy' || owner === 'admin';
+}
+
 // ---------------------------------------------------------------------------
 // The balance policy
 // ---------------------------------------------------------------------------

package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts

@@ -954,6 +954,17 @@ function asyncOperationRateLimitPollDelayMs(input: {
   return Math.max(input.retryAfterMs ?? 0, localBackoffMs);
 }
 
+/** A paced read of an already accepted job must never restart its launch. */
+export class AsyncOperationCompanionPacingError extends Error {
+  constructor(
+    message: string,
+    readonly retryAfterMs: number,
+  ) {
+    super(message);
+    this.name = 'AsyncOperationCompanionPacingError';
+  }
+}
+
 /**
  * Runtime-neutral executor for a declared async lifecycle.
  *
@@ -1074,25 +1085,47 @@ export async function executeAsyncOperationLifecycle<Response>(input: {
   }): Promise<
     { kind: 'response'; response: Response } | { kind: 'timed_out' }
   > => {
-    const remainingTimeoutMs = Math.max(
-      0,
-      polling.timeoutMs - (now() - startedAt),
-    );
-    if (remainingTimeoutMs <= 0) return { kind: 'timed_out' };
-    const abortSignal = AbortSignal.timeout(Math.ceil(remainingTimeoutMs));
-    try {
-      const response = await input.executeAction({
-        ...request,
-        jobId,
-        timeoutMs: Math.ceil(remainingTimeoutMs),
-        abortSignal,
-      });
-      return now() - startedAt >= polling.timeoutMs || abortSignal.aborted
-        ? { kind: 'timed_out' }
-        : { kind: 'response', response };
-    } catch (error) {
-      if (abortSignal.aborted) return { kind: 'timed_out' };
-      throw error;
+    let pacingAttempt = 0;
+    while (true) {
+      const remainingTimeoutMs = Math.max(
+        0,
+        polling.timeoutMs - (now() - startedAt),
+      );
+      if (remainingTimeoutMs <= 0) return { kind: 'timed_out' };
+      const abortSignal = AbortSignal.timeout(Math.ceil(remainingTimeoutMs));
+      try {
+        const response = await input.executeAction({
+          ...request,
+          jobId,
+          timeoutMs: Math.ceil(remainingTimeoutMs),
+          abortSignal,
+        });
+        return now() - startedAt >= polling.timeoutMs || abortSignal.aborted
+          ? { kind: 'timed_out' }
+          : { kind: 'response', response };
+      } catch (error) {
+        if (abortSignal.aborted) return { kind: 'timed_out' };
+        if (!(error instanceof AsyncOperationCompanionPacingError)) throw error;
+        // This refusal applies only to the current companion read. The launch
+        // already owns its job and hold. Stay inside their original deadline
+        // and repeat this exact action/page, never the enclosing start request.
+        pacingAttempt += 1;
+        const delayMs = Math.max(
+          1,
+          asyncOperationRateLimitPollDelayMs({
+            contract: effectiveContract,
+            pollAttempt: request.pollAttempt,
+            rateLimitAttempt: pacingAttempt,
+            retryAfterMs: error.retryAfterMs,
+          }),
+        );
+        const remainingAfterRefusal = Math.max(
+          0,
+          polling.timeoutMs - (now() - startedAt),
+        );
+        if (remainingAfterRefusal <= 0) return { kind: 'timed_out' };
+        await sleep(Math.min(delayMs, remainingAfterRefusal));
+      }
     }
   };
   while (true) {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -17609,6 +17609,22 @@ export async function readPostgresSchedulerObservationAdmissions(
   }));
 }
 
+/** Admission authority: includes live runs whose Convex projection has not landed. */
+export async function listPostgresSchedulerActiveRunsForOrg(
+  client: PostgresSchedulerQueryClient,
+  orgId: string,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<string[]> {
+  if (!orgId.trim()) throw new Error('Workspace ID is required.');
+  const result = await client.query<{ run_id: string }>(
+    `SELECT run_id FROM ${tableName('runs', options)}
+     WHERE org_id = $1 AND status IN ('queued', 'running', 'waiting')
+     ORDER BY created_at, run_id LIMIT 5`,
+    [orgId],
+  );
+  return result.rows.map((row) => row.run_id);
+}
+
 export async function readPostgresSchedulerRunStatus(
   client: PostgresSchedulerQueryClient,
   runId: string,
@@ -18770,26 +18786,25 @@ async function claimPostgresSchedulerOutboxPrefix(
  */
 export async function readPostgresSchedulerOutboxNextDueAt(
   client: PostgresSchedulerQueryClient,
-  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+  options?: Pick<PostgresSchedulerOptions, 'schema'> & {
+    lane?: 'lifecycle' | 'observability';
+  },
 ): Promise<Date | null> {
+  const lanes =
+    options?.lane === 'lifecycle'
+      ? [false]
+      : options?.lane === 'observability'
+        ? [true]
+        : [false, true];
   const result = await client.query<{ due_at: Date | string | null }>(
     `
       -- Seek distinct run identities without reading their pending fact tails.
-      WITH RECURSIVE ${runLedgerPendingRunsSql(options)}, heads AS MATERIALIZED (
-        SELECT pending_run.run_id, false AS search_only, head.*
-        FROM pending_runs AS pending_run
-        CROSS JOIN LATERAL (
-          SELECT pending.created_at, pending.next_attempt_at,
-                 pending.failed_at, pending.lease_expires_at
-          FROM ${tableName('outbox', options)} AS pending
-          WHERE pending.run_id = pending_run.run_id
-            AND pending.delivered_at IS NULL
-            AND ${searchLaneSql('pending')} = false
-          ORDER BY ${runLedgerOrderSql('pending')}
-          LIMIT 1
-        ) AS head
-        UNION ALL
-        SELECT pending_run.run_id, true AS search_only, head.*
+      WITH RECURSIVE ${runLedgerPendingRunsSql(options, options?.lane === undefined ? undefined : lanes[0])}, heads AS MATERIALIZED (
+        ${lanes
+          .map(
+            (
+              searchOnly,
+            ) => `        SELECT pending_run.run_id, ${searchOnly} AS search_only, head.*
         FROM pending_runs AS pending_run
         CROSS JOIN LATERAL (
           SELECT pending.created_at, pending.next_attempt_at,
@@ -18797,10 +18812,12 @@ export async function readPostgresSchedulerOutboxNextDueAt(
           FROM ${tableName('outbox', options)} AS pending
           WHERE pending.run_id = pending_run.run_id
             AND pending.delivered_at IS NULL
-            AND ${searchLaneSql('pending')} = true
+            AND ${searchLaneSql('pending')} = ${searchOnly}
           ORDER BY ${runLedgerOrderSql('pending')}
           LIMIT 1
-        ) AS head
+        ) AS head`,
+          )
+          .join('\n        UNION ALL\n')}
       )
       SELECT min(greatest(
         CASE

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -100,7 +100,7 @@ export type RunLedgerOutboxRecoveryDeps = {
   ) => Promise<PostgresSchedulerOutboxProjectionResult>;
   observabilityParallelism?: number;
   /** Earliest pending retry time or active lease expiry in the durable outbox. */
-  readNextDueAt: () => Promise<Date | null>;
+  readNextDueAt: (lane?: 'lifecycle' | 'observability') => Promise<Date | null>;
   /** A direct LISTEN connection. It is a hint only; a null lease is safe. */
   acquireNotifyHub: () => PgNotifyHubLease | null;
   /** Bounded number of independent runs this replica may deliver at once. */
@@ -201,6 +201,15 @@ export function startRunLedgerOutboxRecovery(
   let requested = false;
   let failureCount = 0;
   let timer: RecoveryTimer | null = null;
+  let timerAtMs: number | null = null;
+  let dueClockGeneration = 0;
+  let dueClockRead: Promise<void> | null = null;
+  let dueClockRefreshRequested = false;
+  let dueClockReadGeneration = 0;
+  let dueClockKnownLane: 'both' | 'lifecycle' | 'observability' | null = null;
+  let producerHintGeneration = 0;
+  let dueClockReadHintGeneration = 0;
+  let dueClockKnownHintGeneration = -1;
   let drainPromise: Promise<void> | null = null;
   let wakeActiveDrain: (() => void) | null = null;
   let cooldownUntilMs: number | null = null;
@@ -208,23 +217,31 @@ export function startRunLedgerOutboxRecovery(
   let unsubscribeNotify: (() => void) | undefined;
   let unsubscribeState: (() => void) | undefined;
   let requestDrain: (reason: string, runId?: string) => void = () => undefined;
-  const lifecycleRunHints = new Set<string>();
-  const observabilityRunHints = new Set<string>();
+  // A producer hint may reveal a lease that the claim cannot acquire. A
+  // successful scoped claim's continuation does not force a metadata query
+  // while other deliveries keep the drain busy. Zero marks that continuation.
+  const lifecycleRunHints = new Map<string, number>();
+  const observabilityRunHints = new Map<string, number>();
   const hasRequests = () =>
     requested ||
     observabilityRequested ||
     lifecycleRunHints.size > 0 ||
     observabilityRunHints.size > 0;
-  const addRunHint = (hints: Set<string>, runId: string) => {
-    hints.add(runId);
+  const addRunHint = (
+    hints: Map<string, number>,
+    runId: string,
+    discoveryGeneration = producerHintGeneration,
+  ) => {
+    hints.set(runId, Math.max(discoveryGeneration, hints.get(runId) ?? 0));
     if (hints.size <= MAX_NOTIFICATION_RUN_HINTS) return true;
     hints.clear();
     return false;
   };
-  const takeRunHint = (hints: Set<string>): string | undefined => {
-    const runId = hints.values().next().value;
-    if (runId !== undefined) hints.delete(runId);
-    return runId;
+  const takeRunHint = (hints: Map<string, number>) => {
+    const hint = hints.entries().next().value;
+    if (!hint) return undefined;
+    hints.delete(hint[0]);
+    return { runId: hint[0], discoveryGeneration: hint[1] };
   };
   const inFlightStartedAt = new Map<number, number>();
   let nextDispatchId = 0;
@@ -261,6 +278,7 @@ export function startRunLedgerOutboxRecovery(
     if (!timer) return;
     clearTimer(timer);
     timer = null;
+    timerAtMs = null;
   };
 
   const listenerIsAvailable = () => hubLease?.hub.state() === 'listening';
@@ -285,8 +303,8 @@ export function startRunLedgerOutboxRecovery(
 
   const startDrain = (reason: string) => {
     if (closed) return;
-    if (cooldownUntilMs !== null && now() < cooldownUntilMs) return;
-    clearWakeTimer();
+    if (cooldownUntilMs !== null) return;
+    // Scoped producer hints cannot cancel another run's retry/lease clock.
     if (drainPromise) {
       wakeActiveDrain?.();
       return;
@@ -308,6 +326,10 @@ export function startRunLedgerOutboxRecovery(
 
   requestDrain = (reason: string, runId?: string) => {
     if (closed) return;
+    // Tag producer hints so a later missed claim can reject an older empty
+    // snapshot. Productive hints need no metadata refresh while work is busy.
+    producerHintGeneration += 1;
+    dueClockKnownLane = null;
     if (runId === undefined) {
       requested = true;
       observabilityRequested = !!deps.dispatchObservabilityOnce;
@@ -354,27 +376,32 @@ export function startRunLedgerOutboxRecovery(
       error: error instanceof Error ? error.message : String(error),
     });
     if (closed) return;
+    dueClockGeneration += 1;
+    dueClockKnownLane = null;
     clearWakeTimer();
     cooldownUntilMs = now() + delayMs;
+    timerAtMs = cooldownUntilMs;
     timer = setTimer(() => {
       timer = null;
+      timerAtMs = null;
       cooldownUntilMs = null;
       requestDrain('recovery_error_retry');
     }, delayMs);
     timer.unref?.();
   };
 
-  const scheduleNextDue = async (): Promise<void> => {
-    if (closed) return;
-    let dueAt: Date | null;
-    try {
-      dueAt = await deps.readNextDueAt();
-      failureCount = 0;
-    } catch (error) {
-      scheduleFailureRetry('next_due', error);
-      return;
+  // A retained global probe already owns discovery for its lane. Read only
+  // the other lane, so a full lane's ready head cannot create a zero-delay loop.
+  const dueClockLane = (): 'both' | 'lifecycle' | 'observability' | null => {
+    if (requested) {
+      return deps.dispatchObservabilityOnce && !observabilityRequested
+        ? 'observability'
+        : null;
     }
-    if (closed || hasRequests()) return;
+    return observabilityRequested ? 'lifecycle' : 'both';
+  };
+
+  const scheduleDueWake = (dueAt: Date | null, provisional = false) => {
     // When LISTEN is healthy the exact durable due time is sufficient. If the
     // optional listener is absent, down, or lost an election, an indexed
     // short safety wake discovers a fresh row that otherwise had no
@@ -383,24 +410,101 @@ export function startRunLedgerOutboxRecovery(
     const delayMs = listenerIsAvailable()
       ? exactDueMs
       : Math.min(exactDueMs ?? Infinity, notifyUnavailableSafetyWakeMs);
-    if (delayMs === null || !Number.isFinite(delayMs)) return;
+    if (provisional && (delayMs === null || !Number.isFinite(delayMs))) return;
+    const boundedDelayMs = Math.min(MAX_TIMEOUT_MS, delayMs ?? Infinity);
+    const wakeAtMs = now() + boundedDelayMs;
+    if (provisional && timerAtMs !== null && timerAtMs <= wakeAtMs) return;
     clearWakeTimer();
-    timer = setTimer(
-      () => {
-        timer = null;
-        if (!listenerIsAvailable()) attachNotifyHub();
-        requestDrain(dueAt ? 'next_due' : 'listener_safety_wake');
-      },
-      Math.min(MAX_TIMEOUT_MS, delayMs),
-    );
+    if (delayMs === null || !Number.isFinite(delayMs)) return;
+    timerAtMs = wakeAtMs;
+    timer = setTimer(() => {
+      timer = null;
+      timerAtMs = null;
+      dueClockGeneration += 1;
+      if (!listenerIsAvailable()) attachNotifyHub();
+      requestDrain(dueAt ? 'next_due' : 'listener_safety_wake');
+    }, boundedDelayMs);
     timer.unref?.();
   };
 
+  const readNextDueClock = async (
+    generation: number,
+    lane: Exclude<ReturnType<typeof dueClockLane>, null>,
+    hintGeneration: number,
+  ): Promise<void> => {
+    let dueAt: Date | null;
+    try {
+      dueAt = await deps.readNextDueAt(lane === 'both' ? undefined : lane);
+    } catch (error) {
+      if (
+        closed ||
+        cooldownUntilMs !== null ||
+        generation !== dueClockGeneration
+      ) {
+        logger.warn('[run-ledger-outbox.recovery_retry]', {
+          stage: 'next_due',
+          superseded: true,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        return;
+      }
+      scheduleFailureRetry('next_due', error);
+      return;
+    }
+    if (closed || cooldownUntilMs !== null) return;
+    const currentLane = dueClockLane();
+    if (
+      generation !== dueClockGeneration ||
+      lane !== currentLane ||
+      dueClockRefreshRequested
+    ) {
+      dueClockRefreshRequested = true;
+      // Hints and retry completions invalidate freshness, not a known due
+      // time. Only the currently discoverable lane may supply a provisional
+      // wake, which advances the timer without caching a stale snapshot.
+      if (lane === currentLane && dueAt) scheduleDueWake(dueAt, true);
+      return;
+    }
+    failureCount = 0;
+    dueClockKnownLane = lane;
+    dueClockKnownHintGeneration = hintGeneration;
+    scheduleDueWake(dueAt);
+  };
+
+  const scheduleNextDue = (refresh = false): Promise<void> => {
+    if (closed || cooldownUntilMs !== null) return Promise.resolve();
+    if (dueClockRead) {
+      // A fired clock invalidates even an unchanged coalesced read. Retain
+      // that discovery debt until the old read settles.
+      dueClockRefreshRequested ||=
+        refresh || dueClockReadGeneration !== dueClockGeneration;
+      return dueClockRead;
+    }
+    const lane = dueClockLane();
+    if (lane === null || (dueClockKnownLane === lane && !refresh))
+      return Promise.resolve();
+    dueClockReadGeneration = dueClockGeneration;
+    dueClockReadHintGeneration = producerHintGeneration;
+    dueClockRead = readNextDueClock(
+      dueClockReadGeneration,
+      lane,
+      dueClockReadHintGeneration,
+    ).finally(() => {
+      dueClockRead = null;
+      if (dueClockRefreshRequested) {
+        dueClockRefreshRequested = false;
+        if (!closed && cooldownUntilMs === null) void scheduleNextDue(true);
+      }
+    });
+    return dueClockRead;
+  };
+
   const drain = async (reason: string): Promise<void> => {
     type Delivery = {
       id: symbol;
       observability: boolean;
       runId?: string;
+      discoveryGeneration: number;
       result:
         | PromiseSettledResult<PostgresSchedulerOutboxProjectionResult>
         | { status: 'skipped' };
@@ -435,7 +539,11 @@ export function startRunLedgerOutboxRecovery(
       completed = attempted = projected = retrying = blocked = 0;
     };
 
-    const launch = (observability = false, runId?: string) => {
+    const launch = (
+      observability = false,
+      runId?: string,
+      discoveryGeneration = producerHintGeneration,
+    ) => {
       const id = Symbol();
       pending.add(id);
       if (observability) pendingObservability.add(id);
@@ -444,13 +552,19 @@ export function startRunLedgerOutboxRecovery(
           dispatchFailed = true;
           scheduleFailureRetry('dispatch', result.reason);
         }
-        deliveries.push({ id, observability, runId, result });
+        deliveries.push({
+          id,
+          observability,
+          runId,
+          discoveryGeneration,
+          result,
+        });
         wakeActiveDrain?.();
       };
       // Subscribe once per delivery. Repeated Promise.race calls would retain
       // a new reaction on a slow callback every time a faster slot completes.
       void Promise.resolve().then(async () => {
-        if (closed || dispatchFailed) {
+        if (closed || dispatchFailed || cooldownUntilMs !== null) {
           settle({ status: 'skipped' });
           return;
         }
@@ -468,32 +582,34 @@ export function startRunLedgerOutboxRecovery(
     const canProbeLifecycle = () =>
       !closed &&
       !dispatchFailed &&
+      cooldownUntilMs === null &&
       (requested || lifecycleRunHints.size > 0) &&
       pending.size < parallelism &&
       pending.size - pendingObservability.size < lifecycleParallelism;
     const canProbeObservability = () =>
       !closed &&
       !dispatchFailed &&
+      cooldownUntilMs === null &&
       (observabilityRequested || observabilityRunHints.size > 0) &&
       pending.size < parallelism &&
       pendingObservability.size < observabilityParallelism;
 
     for (;;) {
-      if (!closed && !dispatchFailed) {
+      if (!closed && !dispatchFailed && cooldownUntilMs === null) {
         // A wake grants one indexed probe, including when other slots are
         // still delivering. Keep it pending when full so an empty result
         // cannot consume a notification that arrived after its query began.
         if (canProbeObservability()) {
-          const runId = observabilityRequested
+          const hint = observabilityRequested
             ? undefined
             : takeRunHint(observabilityRunHints);
           observabilityRequested = false;
-          launch(true, runId);
+          launch(true, hint?.runId, hint?.discoveryGeneration);
         }
         if (canProbeLifecycle()) {
-          const runId = requested ? undefined : takeRunHint(lifecycleRunHints);
+          const hint = requested ? undefined : takeRunHint(lifecycleRunHints);
           requested = false;
-          launch(false, runId);
+          launch(false, hint?.runId, hint?.discoveryGeneration);
         }
         // Productive completion permits immediate use of every free slot.
         // Empty results grant no refill, so idle work cannot spin on scans.
@@ -516,7 +632,9 @@ export function startRunLedgerOutboxRecovery(
       }
       if (pending.size === 0) {
         reportDeliveries();
-        if (!closed && !dispatchFailed) await scheduleNextDue();
+        // Metadata discovery is single-flight, but never owns a claim slot.
+        // A slow snapshot must not delay a producer's next scoped claim.
+        if (!closed && !dispatchFailed) void scheduleNextDue();
         return;
       }
 
@@ -541,12 +659,46 @@ export function startRunLedgerOutboxRecovery(
         continue;
       }
       const result = delivery.result.value;
+      const hintLane = delivery.observability ? 'observability' : 'lifecycle';
+      const missedProducerHint =
+        delivery.discoveryGeneration > 0 &&
+        result.attempted === 0 &&
+        (dueClockKnownHintGeneration < delivery.discoveryGeneration ||
+          (dueClockKnownLane !== 'both' && dueClockKnownLane !== hintLane));
+      if (result.attempted > 0 || missedProducerHint) dueClockKnownLane = null;
+      // A claim returning empty may have lost to an inline owner. If its
+      // notification followed the pending metadata snapshot, retain discovery
+      // debt so that snapshot cannot cache an obsolete empty result.
+      if (
+        missedProducerHint &&
+        dueClockRead &&
+        dueClockReadHintGeneration < delivery.discoveryGeneration
+      )
+        dueClockRefreshRequested = true;
+      if (result.retrying > 0) dueClockGeneration += 1;
       completed += 1;
       attempted += result.attempted;
       projected += result.projected;
       retrying += result.retrying;
       blocked += result.failed;
       if (completed >= parallelism) reportDeliveries();
+      // Discovery cannot wait for every independent HTTP delivery to finish.
+      // A retry commits a new clock; an empty global probe can expose that
+      // clock while another slot remains busy. A missed producer hint may
+      // expose an inline owner's lease; one later snapshot covers all earlier
+      // hints. Scoped success keeps its cheap continuation, and the idle drain
+      // still refreshes metadata after those continuations settle.
+      if (
+        !closed &&
+        !dispatchFailed &&
+        pending.size > deliveries.length &&
+        (result.retrying > 0 ||
+          missedProducerHint ||
+          ((timer === null || dueClockKnownLane !== dueClockLane()) &&
+            delivery.runId === undefined &&
+            result.attempted === 0))
+      )
+        void scheduleNextDue(result.retrying > 0);
       // Retryable rows own durable retry clocks. The SQL claim keeps their
       // run ordered while successful slots keep unrelated runs moving.
       if (delivery.runId !== undefined) {
@@ -556,12 +708,12 @@ export function startRunLedgerOutboxRecovery(
           if (delivery.observability) {
             if (
               !observabilityRequested &&
-              !addRunHint(observabilityRunHints, delivery.runId)
+              !addRunHint(observabilityRunHints, delivery.runId, 0)
             )
               observabilityRequested = true;
           } else if (
             !requested &&
-            !addRunHint(lifecycleRunHints, delivery.runId)
+            !addRunHint(lifecycleRunHints, delivery.runId, 0)
           )
             requested = true;
         }
@@ -594,6 +746,7 @@ export function startRunLedgerOutboxRecovery(
     close: async () => {
       if (closed) return;
       closed = true;
+      dueClockGeneration += 1;
       wakeActiveDrain?.();
       clearWakeTimer();
       cooldownUntilMs = null;
@@ -602,6 +755,7 @@ export function startRunLedgerOutboxRecovery(
       unsubscribeNotify?.();
       unsubscribeState?.();
       await drainPromise;
+      await dueClockRead;
       hubLease?.release();
     },
   };
@@ -859,10 +1013,11 @@ export function startRunLedgerOutboxRecoveryController(input: {
     runId?: string,
   ) =>
     dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
-      (operation) =>
+      (operation, trace) =>
         withRuntimeSchedulerClient(
           searchOnly ? observabilityPoolOptions : input.poolOptions,
           operation,
+          trace,
         ),
       {
         tokenAuthority: 'gateway',
@@ -901,9 +1056,12 @@ export function startRunLedgerOutboxRecoveryController(input: {
     notifyUnavailableSafetyWakeMs: runLedgerOutboxNotifySafetyWakeMs(
       schedulerOptions.schema,
     ),
-    readNextDueAt: () =>
+    readNextDueAt: (lane) =>
       withRuntimeSchedulerClient(input.poolOptions, (client) =>
-        readPostgresSchedulerOutboxNextDueAt(client, schedulerOptions),
+        readPostgresSchedulerOutboxNextDueAt(client, {
+          ...schedulerOptions,
+          lane,
+        }),
       ),
   });
 }

package/dist/cli/{auth-5Y2YV5LY.mjs → auth-QSEB4OI6.mjs}

@@ -1,5 +1,6 @@
 import {
   EXIT_WORKSPACE_MISMATCH,
+  authStatusLine,
   clearPendingClaim,
   clearSatisfiedPendingClaim,
   handleRegister,
@@ -9,16 +10,17 @@ import {
   registerAuthCommands,
   reportSetupCompleted,
   resolvePendingAuthClaimForResume
-} from "./chunk-VXH4225Y.mjs";
-import "./chunk-PJYCAGKK.mjs";
-import "./chunk-VG5WMYSF.mjs";
+} from "./chunk-HGLESMAC.mjs";
+import "./chunk-EL62MLRD.mjs";
+import "./chunk-VZBNBCEB.mjs";
 import "./chunk-DLIRSHV6.mjs";
-import "./chunk-N47R7S3E.mjs";
 import "./chunk-NXGCO4JB.mjs";
+import "./chunk-SJY4SUCH.mjs";
 import "./chunk-WZYKRD7T.mjs";
 import "./chunk-KRD6H754.mjs";
 export {
   EXIT_WORKSPACE_MISMATCH,
+  authStatusLine,
   clearPendingClaim,
   clearSatisfiedPendingClaim,
   handleRegister,