deepline 0.3.403 → 0.3.404

package/dist/bundling-sources/sdk/src/release.ts

@@ -202,7 +202,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.403',
+  version: '0.3.404',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -20574,9 +20574,7 @@ export function postgresSchedulerCancellationOutboxLease(runId: string): {
 
 export type RunLedgerOutboxDispatchInput = {
   /** Dedicated bounded checkout for indexing; claims/ACKs use withClient. */
-  withObservabilityClient?: <T>(
-    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
-  ) => Promise<T>;
+  withObservabilityClient?: RunLedgerOutboxWithClient;
 
   /** Only the existing background projector claims the independent search prefix. */
   searchOnly?: boolean;
@@ -21435,6 +21433,29 @@ async function dispatchRunLedgerOutboxBatchWithScope(
   let appendResponse:
     | (RunLedgerDispatchResponseTrace & { roundTripMs: number | null })
     | null = null;
+  // Checkout spans include retries and overlap query/commit spans. Acquisition
+  // includes earlier callback attempts: it is neither CPU nor pure pool wait.
+  // These observations must not be summed. Null means the stage was not reached.
+  const searchPhaseTimingsMs = {
+    renewal: null as number | null,
+    materializeCheckout: null as number | null,
+    materializeAcquireAndRetry: null as number | null,
+    materializeQuery: null as number | null,
+    materializeCommit: null as number | null,
+    acknowledge: null as number | null,
+  };
+  const observeSearchPhase = async <T>(
+    phase: keyof typeof searchPhaseTimingsMs,
+    operation: () => Promise<T>,
+  ): Promise<T> => {
+    const startedAtMs = performance.now();
+    try {
+      return await operation();
+    } finally {
+      searchPhaseTimingsMs[phase] =
+        (searchPhaseTimingsMs[phase] ?? 0) + performance.now() - startedAtMs;
+    }
+  };
   const callerOnResponseMeta = input.runtimeApi?.onResponseMeta;
   // The last append attempt sent, named even when it never gets a response
   // (an append aborted at the caller's deadline has no response to read).
@@ -21481,35 +21502,50 @@ async function dispatchRunLedgerOutboxBatchWithScope(
           ) === true,
       );
       const materialize = async (prefix: typeof deferredEvents) => {
-        await withClient((client) =>
-          renewPostgresRunLedgerDelivery(
-            client,
+        await observeSearchPhase('renewal', () =>
+          withClient(
+            (client) =>
+              renewPostgresRunLedgerDelivery(
+                client,
+                {
+                  eventIds: events.map((event) => event.eventId),
+                  leaseOwner,
+                  leaseSeconds: 120,
+                },
+                options,
+              ),
             {
-              eventIds: events.map((event) => event.eventId),
-              leaseOwner,
-              leaseSeconds: 120,
+              runId: events[0]!.runId,
+              phase: 'run_ledger_outbox_search_renew',
             },
-            options,
           ),
         );
-        await (input.withObservabilityClient ?? withClient)(async (client) => {
-          await client.query('BEGIN');
-          try {
-            await client.query(
-              "SELECT set_config('statement_timeout', $1, true)",
-              [
-                `${Math.min(60_000, RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs - 1_000)}ms`,
-              ],
-            );
-            // The recursive dependency evaluates this already-bounded causal
-            // prefix in order, in one database round trip. Its eight-fact
-            // contract is unchanged. The timeout bounds the entire statement
-            // below the lease; the installed function still fences every fact.
-            const result = await client.query<{
-              ordinal: number;
-              payload: unknown;
-            }>(
-              `WITH RECURSIVE projected(ordinal, payload) AS (
+        const checkoutStartedAtMs = performance.now();
+        let materializeEnteredAtMs: number | null = null;
+        try {
+          await (input.withObservabilityClient ?? withClient)(
+            async (client) => {
+              materializeEnteredAtMs = performance.now();
+              await client.query('BEGIN');
+              try {
+                await client.query(
+                  "SELECT set_config('statement_timeout', $1, true)",
+                  [
+                    `${Math.min(60_000, RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs - 1_000)}ms`,
+                  ],
+                );
+                // The recursive dependency evaluates this already-bounded causal
+                // prefix in order, in one database round trip. Its eight-fact
+                // contract is unchanged. The timeout bounds the entire statement
+                // below the lease; the installed function still fences every fact.
+                const result = await observeSearchPhase(
+                  'materializeQuery',
+                  () =>
+                    client.query<{
+                      ordinal: number;
+                      payload: unknown;
+                    }>(
+                      `WITH RECURSIVE projected(ordinal, payload) AS (
                  SELECT 1, ${quoteIdent(schemaName(options))}.materialize_runtime_outbox_event(($1::text[])[1], $2)
                  UNION ALL
                  SELECT prior.ordinal + 1,
@@ -21518,18 +21554,33 @@ async function dispatchRunLedgerOutboxBatchWithScope(
                  WHERE prior.ordinal < cardinality($1::text[])
                )
                SELECT ordinal, payload FROM projected ORDER BY ordinal`,
-              [prefix.map((event) => event.eventId), leaseOwner],
-            );
-            await client.query('COMMIT');
-            for (const row of result.rows)
-              prefix[row.ordinal - 1]!.payload = row.payload;
-          } catch (error) {
-            // A query timeout may already have closed this connection. Keep
-            // the delivery failure for retry classification and diagnostics.
-            await client.query('ROLLBACK').catch(() => undefined);
-            throw error;
-          }
-        });
+                      [prefix.map((event) => event.eventId), leaseOwner],
+                    ),
+                );
+                await observeSearchPhase('materializeCommit', () =>
+                  client.query('COMMIT'),
+                );
+                for (const row of result.rows)
+                  prefix[row.ordinal - 1]!.payload = row.payload;
+              } catch (error) {
+                // A query timeout may already have closed this connection. Keep
+                // the delivery failure for retry classification and diagnostics.
+                await client.query('ROLLBACK').catch(() => undefined);
+                throw error;
+              }
+            },
+            { runId: events[0]!.runId, phase: 'run_ledger_outbox_materialize' },
+          );
+        } finally {
+          searchPhaseTimingsMs.materializeCheckout =
+            (searchPhaseTimingsMs.materializeCheckout ?? 0) +
+            performance.now() -
+            checkoutStartedAtMs;
+          searchPhaseTimingsMs.materializeAcquireAndRetry =
+            (searchPhaseTimingsMs.materializeAcquireAndRetry ?? 0) +
+            (materializeEnteredAtMs ?? performance.now()) -
+            checkoutStartedAtMs;
+        }
       };
       try {
         await materialize(deferredEvents);
@@ -21591,11 +21642,18 @@ async function dispatchRunLedgerOutboxBatchWithScope(
         )
       )
         throw new Error('Search lease includes a lifecycle fact.');
-      const delivered = await withClient((client) =>
-        markPostgresSchedulerOutboxDeliveredRows(
-          client,
-          events.map((event) => event.eventId),
-          { ...options, leaseOwner },
+      const delivered = await observeSearchPhase('acknowledge', () =>
+        withClient(
+          (client) =>
+            markPostgresSchedulerOutboxDeliveredRows(
+              client,
+              events.map((event) => event.eventId),
+              { ...options, leaseOwner },
+            ),
+          {
+            runId: events[0]!.runId,
+            phase: 'run_ledger_outbox_search_acknowledge',
+          },
         ),
       );
       if (delivered.length !== events.length)
@@ -22084,11 +22142,14 @@ async function dispatchRunLedgerOutboxBatchWithScope(
       input.signal?.aborted ||
       claimed.transportOwnershipUnconfirmed ||
       (error instanceof RunLedgerProjectionLeaseLostError &&
-        error.reason === 'head_lease_owner_mismatch')
+        (error.reason === 'head_lease_owner_mismatch' ||
+          (input.searchOnly === true &&
+            error.reason === 'delivery_lease_not_current')))
     ) {
-      // A skipped busy Run during prefix widening or a transferred head is a
-      // handoff, not a failed delivery. The owner fence also protects a newer
-      // cancellation lease. Caller deadline expiry uses the same handoff.
+      // A skipped busy Run during prefix widening or a transferred/settled
+      // lease is a handoff, not a failed delivery. The owner fence also
+      // protects a newer cancellation lease. Caller deadline expiry uses the
+      // same handoff.
       // Release the lease (fenced to this owner) without charging a
       // projection attempt or scheduling backoff, and wake the durable
       // projector. If the append already reached the ledger, the projector's
@@ -22135,7 +22196,9 @@ async function dispatchRunLedgerOutboxBatchWithScope(
       // even when this owner's fenced release cannot update that row.
       const handedOff =
         error instanceof RunLedgerProjectionLeaseLostError &&
-        error.reason === 'head_lease_owner_mismatch'
+        (error.reason === 'head_lease_owner_mismatch' ||
+          (input.searchOnly === true &&
+            error.reason === 'delivery_lease_not_current'))
           ? eventIds
           : released;
       return withTransportDelivered({
@@ -22319,6 +22382,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
         continuationMs:
           acknowledgedAtMs === null ? null : settledAtMs - acknowledgedAtMs,
         durationMs: settledAtMs - dispatchStartedAtMs,
+        ...(input.searchOnly ? { searchPhaseTimingsMs } : {}),
         // Which platform requests served the two app hops, so a slow
         // authority or append stage can be found in the app's request log.
         ...runLedgerSlowDeliveryResponseFields(

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -1042,8 +1042,12 @@ export function startRunLedgerOutboxRecoveryController(input: {
         materializeObservability: true,
         searchOnly,
         ...(runId === undefined ? {} : { runId }),
-        withObservabilityClient: (operation) =>
-          withRuntimeSchedulerClient(observabilityPoolOptions, operation),
+        withObservabilityClient: (operation, trace) =>
+          withRuntimeSchedulerClient(
+            observabilityPoolOptions,
+            operation,
+            trace,
+          ),
         limit: 8,
         headPriority,
         // The lease holder delivers a run's terminal committed during its