deepline 0.3.296 → 0.3.298

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -8,6 +8,7 @@
  * as a wake hint, and one exact timer for the next due row. There is
  * deliberately no worker fleet polling cadence.
  */
+import { isIsolatedRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
 import {
   dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope,
   postgresSchedulerNotifyChannel,
@@ -36,7 +37,17 @@ const RECOVERY_FAILURE_RETRY_MAX_MS = 60_000;
 // seconds of worker deferrals. Only the dedicated projector performs this
 // fallback; the worker fleet still never polls the outbox.
 export const NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS = 1_000;
+/** Isolated Preview/dev schemas have no LISTEN session, so halve discovery latency. */
+export const ISOLATED_NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS = 500;
 const MAX_TIMEOUT_MS = 2_147_483_647;
+
+export function runLedgerOutboxNotifySafetyWakeMs(
+  schedulerSchema: string | null | undefined,
+): number {
+  return isIsolatedRuntimeSchedulerSchema(schedulerSchema)
+    ? ISOLATED_NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS
+    : NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS;
+}
 /**
  * One projector can keep this many independent Run Ledger deliveries in
  * flight after it has found durable work. Each delivery owns only short
@@ -63,6 +74,7 @@ export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 32;
 // The higher exported ceiling remains available to isolated recovery callers,
 // but it is not the steady-state production setting.
 const RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT = 8;
+const RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT = 4;
 // Admissions retain three of four claim opportunities. The fourth selects
 // the oldest eligible causal head across all runs, preventing terminal work
 // from starving behind continuous new admissions. An empty class never costs
@@ -78,12 +90,17 @@ export type RunLedgerOutboxRecoveryDeps = {
   dispatchOnce: (
     headPriority: RunLedgerOutboxHeadPriority,
   ) => Promise<PostgresSchedulerOutboxProjectionResult>;
+  /** Independent index deliveries share this controller and notification source. */
+  dispatchObservabilityOnce?: () => Promise<PostgresSchedulerOutboxProjectionResult>;
+  observabilityParallelism?: number;
   /** Earliest pending retry time or active lease expiry in the durable outbox. */
   readNextDueAt: () => Promise<Date | null>;
   /** A direct LISTEN connection. It is a hint only; a null lease is safe. */
   acquireNotifyHub: () => PgNotifyHubLease | null;
   /** Bounded number of independent runs this replica may deliver at once. */
   parallelism?: number;
+  /** Fast polling is used only when the topology intentionally has no LISTEN session. */
+  notifyUnavailableSafetyWakeMs?: number;
   now?: () => number;
   setTimer?: (callback: () => void, delayMs: number) => RecoveryTimer;
   clearTimer?: (timer: RecoveryTimer) => void;
@@ -135,6 +152,8 @@ export function startRunLedgerOutboxRecovery(
   const clearTimer = deps.clearTimer ?? clearTimeout;
   const logger = deps.logger ?? console;
   const requestedParallelism = Math.floor(deps.parallelism ?? 1);
+  const notifyUnavailableSafetyWakeMs =
+    deps.notifyUnavailableSafetyWakeMs ?? NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS;
   const parallelism = Number.isFinite(requestedParallelism)
     ? Math.max(
         1,
@@ -144,6 +163,22 @@ export function startRunLedgerOutboxRecovery(
         ),
       )
     : 1;
+  if (deps.dispatchObservabilityOnce && parallelism < 2) {
+    throw new Error(
+      'Independent observability delivery requires at least two slots.',
+    );
+  }
+  const observabilityParallelism = deps.dispatchObservabilityOnce
+    ? Math.min(
+        Math.max(1, deps.observabilityParallelism ?? 1),
+        Math.max(1, parallelism - 1),
+      )
+    : 0;
+  const lifecycleParallelism = Math.max(
+    1,
+    parallelism - observabilityParallelism,
+  );
+  let observabilityRequested = false;
   let closed = false;
   let requested = false;
   let failureCount = 0;
@@ -159,22 +194,24 @@ export function startRunLedgerOutboxRecovery(
   let nextDispatchId = 0;
   let lastDispatchSettledAt: number | null = null;
   let admissionTurns = 0;
-  const dispatchOnce =
-    async (): Promise<PostgresSchedulerOutboxProjectionResult> => {
-      const oldestReady =
-        admissionTurns === ADMISSION_TURNS_PER_OLDEST_READY_TURN;
-      admissionTurns = oldestReady ? 0 : admissionTurns + 1;
-      const id = nextDispatchId++;
-      inFlightStartedAt.set(id, now());
-      try {
-        return await deps.dispatchOnce(
-          oldestReady ? 'oldest_ready' : 'admission_first',
-        );
-      } finally {
-        inFlightStartedAt.delete(id);
-        lastDispatchSettledAt = now();
-      }
-    };
+  const dispatchOnce = async (
+    observability = false,
+  ): Promise<PostgresSchedulerOutboxProjectionResult> => {
+    const oldestReady =
+      admissionTurns === ADMISSION_TURNS_PER_OLDEST_READY_TURN;
+    if (!observability) admissionTurns = oldestReady ? 0 : admissionTurns + 1;
+    const id = nextDispatchId++;
+    inFlightStartedAt.set(id, now());
+    try {
+      if (observability) return await deps.dispatchObservabilityOnce!();
+      return await deps.dispatchOnce(
+        oldestReady ? 'oldest_ready' : 'admission_first',
+      );
+    } finally {
+      inFlightStartedAt.delete(id);
+      lastDispatchSettledAt = now();
+    }
+  };
 
   const clearWakeTimer = () => {
     if (!timer) return;
@@ -205,6 +242,7 @@ export function startRunLedgerOutboxRecovery(
   requestDrain = (reason: string) => {
     if (closed) return;
     requested = true;
+    observabilityRequested = !!deps.dispatchObservabilityOnce;
     if (cooldownUntilMs !== null && now() < cooldownUntilMs) return;
     clearWakeTimer();
     if (drainPromise) {
@@ -240,13 +278,13 @@ export function startRunLedgerOutboxRecovery(
     failureCount += 1;
     // A rejected pooled query is normally worth exponential backoff. The
     // exception is an unavailable LISTEN session: this dedicated projector is
-    // the one durable poller, and its one-second safety wake is what keeps
+    // the one durable poller, and its topology-specific safety wake keeps
     // `run.created` delivery live while a direct PostgreSQL connection is
     // refused (for example, during a connection-capacity spike). Do not let a
     // process-local exponential cooldown silently disable that fallback.
     const delayMs = listenerIsAvailable()
       ? boundedFailureDelayMs(failureCount)
-      : NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS;
+      : notifyUnavailableSafetyWakeMs;
     logger.warn('[run-ledger-outbox.recovery_retry]', {
       stage,
       delayMs,
@@ -281,7 +319,7 @@ export function startRunLedgerOutboxRecovery(
     const exactDueMs = dueAt ? Math.max(0, dueAt.getTime() - now()) : null;
     const delayMs = listenerIsAvailable()
       ? exactDueMs
-      : Math.min(exactDueMs ?? Infinity, NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS);
+      : Math.min(exactDueMs ?? Infinity, notifyUnavailableSafetyWakeMs);
     if (delayMs === null || !Number.isFinite(delayMs)) return;
     clearWakeTimer();
     timer = setTimer(
@@ -298,11 +336,13 @@ export function startRunLedgerOutboxRecovery(
   const drain = async (reason: string): Promise<void> => {
     type Delivery = {
       id: symbol;
+      observability: boolean;
       result:
         | PromiseSettledResult<PostgresSchedulerOutboxProjectionResult>
         | { status: 'skipped' };
     };
     const pending = new Set<symbol>();
+    const pendingObservability = new Set<symbol>();
     const deliveries: Delivery[] = [];
     let refill = false;
     let dispatchFailed = false;
@@ -330,15 +370,16 @@ export function startRunLedgerOutboxRecovery(
       completed = attempted = projected = retrying = blocked = 0;
     };
 
-    const launch = () => {
+    const launch = (observability = false) => {
       const id = Symbol();
       pending.add(id);
+      if (observability) pendingObservability.add(id);
       const settle = (result: Delivery['result']) => {
         if (result.status === 'rejected' && !dispatchFailed) {
           dispatchFailed = true;
           scheduleFailureRetry('dispatch', result.reason);
         }
-        deliveries.push({ id, result });
+        deliveries.push({ id, observability, result });
         wakeActiveDrain?.();
       };
       // Subscribe once per delivery. Repeated Promise.race calls would retain
@@ -349,26 +390,54 @@ export function startRunLedgerOutboxRecovery(
           return;
         }
         try {
-          settle({ status: 'fulfilled', value: await dispatchOnce() });
+          settle({
+            status: 'fulfilled',
+            value: await dispatchOnce(observability),
+          });
         } catch (reason) {
           settle({ status: 'rejected', reason });
         }
       });
     };
 
+    const canProbeLifecycle = () =>
+      !closed &&
+      !dispatchFailed &&
+      requested &&
+      pending.size < parallelism &&
+      pending.size - pendingObservability.size < lifecycleParallelism;
+    const canProbeObservability = () =>
+      !closed &&
+      !dispatchFailed &&
+      observabilityRequested &&
+      pending.size < parallelism &&
+      pendingObservability.size < observabilityParallelism;
+
     for (;;) {
       if (!closed && !dispatchFailed) {
         // A wake grants one indexed probe, including when other slots are
         // still delivering. Keep it pending when full so an empty result
         // cannot consume a notification that arrived after its query began.
-        if (requested && pending.size < parallelism) {
+        if (canProbeObservability()) {
+          observabilityRequested = false;
+          while (
+            pendingObservability.size < observabilityParallelism &&
+            pending.size < parallelism
+          )
+            launch(true);
+        }
+        if (canProbeLifecycle()) {
           requested = false;
           launch();
         }
         // Productive completion permits immediate use of every free slot.
         // Empty results grant no refill, so idle work cannot spin on scans.
         if (refill) {
-          while (pending.size < parallelism) launch();
+          while (
+            pending.size < parallelism &&
+            pending.size - pendingObservability.size < lifecycleParallelism
+          )
+            launch();
           refill = false;
         }
       }
@@ -383,10 +452,8 @@ export function startRunLedgerOutboxRecovery(
         wakeActiveDrain = resolve;
         if (
           deliveries.length > 0 ||
-          (requested &&
-            !closed &&
-            !dispatchFailed &&
-            pending.size < parallelism)
+          canProbeLifecycle() ||
+          canProbeObservability()
         )
           resolve();
       });
@@ -394,6 +461,7 @@ export function startRunLedgerOutboxRecovery(
       const delivery = deliveries.shift();
       if (!delivery) continue;
       pending.delete(delivery.id);
+      pendingObservability.delete(delivery.id);
       if (delivery.result.status !== 'fulfilled') {
         // Stop new claims but settle every already-started delivery before
         // cooldown or close can hand ownership to another drain.
@@ -408,7 +476,9 @@ export function startRunLedgerOutboxRecovery(
       if (completed >= parallelism) reportDeliveries();
       // Retryable rows own durable retry clocks. The SQL claim keeps their
       // run ordered while successful slots keep unrelated runs moving.
-      refill = result.attempted > 0;
+      if (delivery.observability) {
+        if (result.attempted > 0) observabilityRequested = true;
+      } else refill = result.attempted > 0;
     }
   };
 
@@ -583,12 +653,37 @@ export function startRunLedgerOutboxRecoveryController(input: {
   onDelivered?: (deliveries: readonly RunLedgerOutboxDeliveredRow[]) => void;
 }): RunLedgerOutboxRecoveryController {
   const schedulerOptions = input.schedulerOptions ?? {};
-  let dispatchOrdinal = 0;
-  let searchInFlight = 0;
-  const maxSearchInFlight = Math.max(
-    1,
-    Math.floor(RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT / 2),
-  );
+  const observabilityPoolOptions: RuntimeSchedulerPoolOptions = {
+    ...input.poolOptions,
+    purpose: 'observability',
+    applicationName: 'deepline-runtime-observability',
+    maxClients: RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT,
+  };
+  const dispatch = (
+    headPriority: RunLedgerOutboxHeadPriority,
+    searchOnly: boolean,
+  ) =>
+    dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
+      (operation) =>
+        withRuntimeSchedulerClient(
+          searchOnly ? observabilityPoolOptions : input.poolOptions,
+          operation,
+        ),
+      {
+        tokenAuthority: 'gateway',
+        materializeObservability: true,
+        searchOnly,
+        withObservabilityClient: (operation) =>
+          withRuntimeSchedulerClient(observabilityPoolOptions, operation),
+        limit: 8,
+        headPriority,
+        // The lease holder delivers a run's terminal committed during its
+        // append, instead of leaving it for the next wave.
+        continueThroughTerminal: true,
+        ...(input.onDelivered ? { onDelivered: input.onDelivered } : {}),
+      },
+      schedulerOptions,
+    );
   return startRunLedgerOutboxRecovery({
     acquireNotifyHub: () =>
       input.useDirectNotify === false
@@ -599,38 +694,17 @@ export function startRunLedgerOutboxRecoveryController(input: {
               schedulerOptions,
             }),
           ),
-    dispatchOnce: async (headPriority) => {
-      const searchOnly =
-        dispatchOrdinal++ % 2 === 1 && searchInFlight < maxSearchInFlight;
-      if (searchOnly) searchInFlight += 1;
-      try {
-        return await dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
-          (operation) =>
-            withRuntimeSchedulerClient(input.poolOptions, operation),
-          {
-            tokenAuthority: 'gateway',
-            materializeObservability: true,
-            // One durable controller, with half its capacity reserved for
-            // lifecycle delivery even when every search operation is blocked.
-            searchOnly,
-            limit: 8,
-            headPriority,
-            // The lease holder delivers a run's terminal committed during its
-            // append, instead of leaving it for the next wave.
-            continueThroughTerminal: true,
-            ...(input.onDelivered ? { onDelivered: input.onDelivered } : {}),
-          },
-          schedulerOptions,
-        );
-      } finally {
-        if (searchOnly) searchInFlight -= 1;
-      }
-    },
+    dispatchOnce: (headPriority) => dispatch(headPriority, false),
+    dispatchObservabilityOnce: () => dispatch('oldest_ready', true),
+    observabilityParallelism: RUN_LEDGER_OUTBOX_RECOVERY_INDEX_IN_FLIGHT,
     // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
     // short scheduler checkout before it calls the Vercel gateway. The two
     // replicas therefore drain up to sixteen independent runs concurrently
     // without turning one outbox notification into sixty-four head scans.
     parallelism: RUN_LEDGER_OUTBOX_RECOVERY_PRODUCTION_IN_FLIGHT,
+    notifyUnavailableSafetyWakeMs: runLedgerOutboxNotifySafetyWakeMs(
+      schedulerOptions.schema,
+    ),
     readNextDueAt: () =>
       withRuntimeSchedulerClient(input.poolOptions, (client) =>
         readPostgresSchedulerOutboxNextDueAt(client, schedulerOptions),

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts

@@ -61,7 +61,7 @@ const RUNTIME_EVENT_LOG_POSITION_EXPRESSION = `jsonb_set(
 // Migration readiness tracks this projection body independently of scheduler
 // wire and schema versions. Bump when a changed trigger must be reinstalled.
 export const RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER =
-  'runtime-event-search-origin-batches-log-metadata';
+  'runtime-event-search-origin-batches-log-metadata-point-replay';
 /** Trusted trigger-authored state; absent on already-projected legacy rows. */
 export const RUNTIME_EVENT_DEFERRED_PROJECTION_FIELD =
   '_runtimeSearchProjectionPending';
@@ -1310,8 +1310,16 @@ export async function attachRuntimeEventSearchTriggers(
       -- Tombstones retain digest/identity: retry never restores an expired body.
       SELECT count(*) INTO mismatches
       FROM jsonb_array_elements(item->'lines') WITH ORDINALITY AS entries(body, ordinal)
-      LEFT JOIN runtime_observability.events event ON event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
-        AND event.event_id = 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinal-1)::int)
+      -- Keep each replay proof on the journal primary key. An unrestricted
+      -- join can hash the whole shared org history for a small JSON batch.
+      LEFT JOIN LATERAL (
+        SELECT event.run_id, event.kind, event.source_digest
+        FROM runtime_observability.events event
+        WHERE event.runtime_scope = ${quoteLiteral(scope)} AND event.org_id = run_org
+          AND event.event_id = 'log-origin:' || observation.run_id || ':' || (item->'lineIds'->>(ordinal-1)::int)
+        -- The primary key admits one row; retain a parameterized point lookup.
+        LIMIT 1
+      ) event ON true
       WHERE event.run_id IS DISTINCT FROM observation.run_id OR event.kind IS DISTINCT FROM ${RUN_EVENT_KIND_SQL_LITERAL.logAppended}
         OR event.source_digest IS DISTINCT FROM CASE WHEN NULLIF(item->'lineContexts'->(ordinal-1)::int, 'null'::jsonb) IS NULL
           THEN encode(sha256(convert_to(body #>> '{}','UTF8')),'hex')
@@ -1982,11 +1990,15 @@ export async function attachRuntimeEventSearchTriggers(
         DO UPDATE SET last_seq = runtime_observability.run_log_cursors.last_seq + EXCLUDED.last_seq,
                       updated_at = clock_timestamp()
         RETURNING last_seq INTO assigned_seq;
+        -- Bound sequence assignment before joining the variable-size ID list.
+        -- Shared-journal history from other runs is never a candidate.
         UPDATE runtime_observability.events AS event
            SET log_seq = assigned_seq - line_count + fresh.ordinality
           FROM unnest(fresh_event_ids) WITH ORDINALITY AS fresh(event_id, ordinality)
          WHERE event.runtime_scope = ${quoteLiteral(scope)}
            AND event.org_id = run_org
+           AND event.run_id = observation.run_id
+           AND event.event_id = ANY(fresh_event_ids)
            AND event.event_id = fresh.event_id;
       END IF;
       RETURN;

package/dist/cli/auth-AQCORSG5.js

@@ -0,0 +1,31 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true});
+
+
+
+
+
+
+
+
+
+
+var _chunkBXYKLY5Tjs = require('./chunk-BXYKLY5T.js');
+require('./chunk-TOAXEXTC.js');
+require('./chunk-2Q4LGPVP.js');
+require('./chunk-XNYIZMNQ.js');
+require('./chunk-BUKTFUM7.js');
+require('./chunk-7P5AA3SN.js');
+require('./chunk-OG64RBAC.js');
+require('./chunk-COGTRVWW.js');
+
+
+
+
+
+
+
+
+
+
+
+exports.EXIT_WORKSPACE_MISMATCH = _chunkBXYKLY5Tjs.EXIT_WORKSPACE_MISMATCH; exports.clearPendingClaim = _chunkBXYKLY5Tjs.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkBXYKLY5Tjs.clearSatisfiedPendingClaim; exports.handleRegister = _chunkBXYKLY5Tjs.handleRegister; exports.handleStatus = _chunkBXYKLY5Tjs.handleStatus; exports.handleWait = _chunkBXYKLY5Tjs.handleWait; exports.readPendingAuthClaim = _chunkBXYKLY5Tjs.readPendingAuthClaim; exports.registerAuthCommands = _chunkBXYKLY5Tjs.registerAuthCommands; exports.reportSetupCompleted = _chunkBXYKLY5Tjs.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkBXYKLY5Tjs.resolvePendingAuthClaimForResume;

package/dist/cli/{auth-JCGIDVDF.mjs → auth-TJGQB4RK.mjs}

@@ -9,11 +9,11 @@ import {
   registerAuthCommands,
   reportSetupCompleted,
   resolvePendingAuthClaimForResume
-} from "./chunk-BZ77OTRF.mjs";
-import "./chunk-NTEZFZLE.mjs";
-import "./chunk-NPPJX3BL.mjs";
+} from "./chunk-RES6RQXX.mjs";
+import "./chunk-U6G52KUV.mjs";
+import "./chunk-YE5PJMFK.mjs";
 import "./chunk-DLIRSHV6.mjs";
-import "./chunk-QSIO4TX2.mjs";
+import "./chunk-6IOQ43VG.mjs";
 import "./chunk-NXGCO4JB.mjs";
 import "./chunk-WZYKRD7T.mjs";
 import "./chunk-KRD6H754.mjs";

package/dist/cli/{billing-OQBSIXRG.mjs → billing-BUHGVESC.mjs}

@@ -1,7 +1,7 @@
 import {
   DeeplineClient,
   iterateCursorPages
-} from "./chunk-J2D3VDOU.mjs";
+} from "./chunk-FJTNI2TE.mjs";
 import "./chunk-CBVIJSHF.mjs";
 import {
   createTimestampedName,
@@ -9,13 +9,13 @@ import {
   openInBrowser,
   printCommandEnvelope,
   shouldEmitJson
-} from "./chunk-NTEZFZLE.mjs";
+} from "./chunk-U6G52KUV.mjs";
 import {
   AuthError,
   ConfigError
-} from "./chunk-NPPJX3BL.mjs";
+} from "./chunk-YE5PJMFK.mjs";
 import "./chunk-DLIRSHV6.mjs";
-import "./chunk-QSIO4TX2.mjs";
+import "./chunk-6IOQ43VG.mjs";
 import "./chunk-GVUJJJ5G.mjs";
 import {
   DeeplineError