deepline 0.3.202 → 0.3.204

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.202',
+  version: '0.3.204',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts

@@ -1,16 +1,73 @@
 import { spawnSync } from 'node:child_process';
 
-function run(command: string, args: string[], quiet = false): boolean {
-  const result = spawnSync(command, args, {
+const FLY_REGISTRY_COMMAND_TIMEOUT_MS = 30_000;
+const RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS = 10_000;
+
+type FlyRegistryCommandResult = Readonly<{
+  error?: Error & { code?: string };
+  status: number | null;
+  stdout?: string | null;
+  stderr?: string | null;
+}>;
+
+type FlyRegistryCommandSpawn = (
+  command: string,
+  args: readonly string[],
+  options: Readonly<{
+    encoding: 'utf8';
+    killSignal: 'SIGKILL';
+    stdio: 'inherit' | 'pipe';
+    timeout: number;
+  }>,
+) => FlyRegistryCommandResult;
+
+function run(
+  command: string,
+  args: string[],
+  quiet = false,
+  timeoutMs = FLY_REGISTRY_COMMAND_TIMEOUT_MS,
+  spawnOverride?: FlyRegistryCommandSpawn,
+): boolean {
+  const spawnCommand: FlyRegistryCommandSpawn =
+    spawnOverride ??
+    ((executable, commandArgs, spawnOptions) =>
+      spawnSync(executable, [...commandArgs], spawnOptions));
+  const result = spawnCommand(command, args, {
     encoding: 'utf8',
+    killSignal: 'SIGKILL',
     stdio: quiet ? 'pipe' : 'inherit',
+    timeout: timeoutMs,
   });
+  if (result.error) {
+    const error = result.error as NodeJS.ErrnoException;
+    if (error.code === 'ETIMEDOUT') {
+      throw Object.assign(
+        new Error(
+          `[fly-registry] ${command} ${args.join(' ')} timed out after ${timeoutMs}ms.`,
+        ),
+        { code: error.code },
+      );
+    }
+    throw result.error;
+  }
   return result.status === 0;
 }
 
 /** Shared immutable-tag presence probe for image builds and rollback planning. */
-export function runtimeImageManifestExists(image: string): boolean {
-  return run('docker', ['manifest', 'inspect', image], true);
+export function runtimeImageManifestExists(
+  image: string,
+  options: {
+    timeoutMs?: number;
+    spawnCommand?: FlyRegistryCommandSpawn;
+  } = {},
+): boolean {
+  return run(
+    'docker',
+    ['manifest', 'inspect', image],
+    true,
+    options.timeoutMs ?? RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS,
+    options.spawnCommand,
+  );
 }
 
 export const FLY_REGISTRY_AUTH_ATTEMPTS = 3;
@@ -25,13 +82,37 @@ const FLY_REGISTRY_AUTH_RETRY_DELAY_MS = 1_500;
  */
 export function authenticateFlyRegistry(
   sleep: (ms: number) => void = sleepSync,
+  spawnOverride?: FlyRegistryCommandSpawn,
 ): void {
+  let lastTimeoutError: Error | undefined;
   for (let attempt = 1; attempt <= FLY_REGISTRY_AUTH_ATTEMPTS; attempt += 1) {
-    if (run('flyctl', ['auth', 'docker'])) return;
+    try {
+      if (
+        run(
+          'flyctl',
+          ['auth', 'docker'],
+          false,
+          FLY_REGISTRY_COMMAND_TIMEOUT_MS,
+          spawnOverride,
+        )
+      ) {
+        return;
+      }
+      lastTimeoutError = undefined;
+    } catch (error) {
+      if (
+        !(error instanceof Error) ||
+        (error as NodeJS.ErrnoException).code !== 'ETIMEDOUT'
+      ) {
+        throw error;
+      }
+      lastTimeoutError = error;
+    }
     if (attempt < FLY_REGISTRY_AUTH_ATTEMPTS) {
       sleep(FLY_REGISTRY_AUTH_RETRY_DELAY_MS * attempt);
     }
   }
+  if (lastTimeoutError) throw lastTimeoutError;
   throw new Error(
     `Fly registry authentication failed after ${FLY_REGISTRY_AUTH_ATTEMPTS} attempts.`,
   );

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -307,7 +307,7 @@ type BreakerState = {
   consecutiveFailures: number;
   openedAt: number | null;
   lastFailureMessage: string;
-  halfOpenInFlight: boolean;
+  halfOpenProbe: symbol | null;
 };
 
 export type RuntimeSchedulerPoolPriority = 'ordinary' | 'critical';
@@ -645,6 +645,9 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
       maxAcquireQueueDepth,
       maxAcquireQueueWaitMs,
     );
+    // A pool created by the old breaker used a boolean claim. The new token
+    // does not inherit a stranded half-open claim across a hot reload.
+    existing.breaker.halfOpenProbe ??= null;
     return existing;
   }
 
@@ -671,7 +674,7 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
       consecutiveFailures: 0,
       openedAt: null,
       lastFailureMessage: '',
-      halfOpenInFlight: false,
+      halfOpenProbe: null,
     },
     stats: {
       acquires: 0,
@@ -698,31 +701,39 @@ function getPoolEntry(options: RuntimeSchedulerPoolOptions): PoolEntry {
   return entry;
 }
 
-function breakerGate(entry: PoolEntry): void {
+function breakerGate(entry: PoolEntry): symbol | null {
   const { breaker } = entry;
-  if (breaker.openedAt === null) return;
+  if (breaker.openedAt === null) return null;
   const elapsed = Date.now() - breaker.openedAt;
   if (elapsed < RUNTIME_SCHEDULER_POOL_DEFAULTS.breaker.openMs) {
     entry.stats.breakerRejections += 1;
     throw new RuntimeSchedulerCircuitOpenError(breaker.lastFailureMessage);
   }
   // Half-open: let exactly one caller through as the trial.
-  if (breaker.halfOpenInFlight) {
+  if (breaker.halfOpenProbe !== null) {
     entry.stats.breakerRejections += 1;
     throw new RuntimeSchedulerCircuitOpenError(breaker.lastFailureMessage);
   }
-  breaker.halfOpenInFlight = true;
+  const probe = Symbol('runtime-scheduler-half-open-probe');
+  breaker.halfOpenProbe = probe;
+  return probe;
 }
 
 function recordBreakerSuccess(entry: PoolEntry): void {
   entry.breaker.consecutiveFailures = 0;
   entry.breaker.openedAt = null;
-  entry.breaker.halfOpenInFlight = false;
+  entry.breaker.halfOpenProbe = null;
 }
 
-function recordBreakerFailure(entry: PoolEntry, error: unknown): void {
+function recordBreakerFailure(
+  entry: PoolEntry,
+  error: unknown,
+  probe: symbol | null,
+): void {
   const { breaker } = entry;
-  breaker.halfOpenInFlight = false;
+  if (probe !== null && breaker.halfOpenProbe === probe) {
+    breaker.halfOpenProbe = null;
+  }
   breaker.consecutiveFailures += 1;
   breaker.lastFailureMessage =
     error instanceof Error ? error.message : String(error);
@@ -798,190 +809,46 @@ export async function withRuntimeSchedulerClient<T>(
     throwIfSchedulerAborted(signal);
     for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
       throwIfSchedulerAborted(signal);
-      breakerGate(entry);
-      if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
-        entry.stats.saturationRejections += 1;
-        throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
-      }
-      const acquireStart = performance.now();
-      entry.stats.peakTotalClients = Math.max(
-        entry.stats.peakTotalClients,
-        entry.pool.totalCount,
-      );
-      entry.stats.peakWaitingClients = Math.max(
-        entry.stats.peakWaitingClients,
-        entry.pool.waitingCount,
-      );
-      let poolClient;
+      const halfOpenProbe = breakerGate(entry);
+      let capacityObserved = false;
       try {
-        poolClient = await entry.pool.connect();
-      } catch (error) {
-        if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
+        if (entry.pool.waitingCount >= entry.maxAcquireQueueDepth) {
           entry.stats.saturationRejections += 1;
-          console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
-            maxClients: entry.maxClients,
-            totalClients: entry.pool.totalCount,
-            idleClients: entry.pool.idleCount,
-            waitingClients: entry.pool.waitingCount,
-          });
           throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
         }
-        if (isPgBouncerCapacityError(error)) {
-          recordCapacityRejection(entry, error);
-          throw new RuntimeSchedulerPgBouncerCapacityError(
-            error instanceof Error ? error.message : String(error),
-          );
-        }
-        recordBreakerFailure(entry, error);
-        entry.stats.connectionErrors += 1;
-        lastError = error;
-        if (attempt + 1 < maxAttempts) {
-          entry.stats.retries += 1;
-          await sleep(jitteredBackoffMs(attempt));
-          throwIfSchedulerAborted(signal);
-          continue;
-        }
-        throw error;
-      }
-      entry.stats.acquires += 1;
-      const acquireWaitMs = performance.now() - acquireStart;
-      entry.stats.peakTotalClients = Math.max(
-        entry.stats.peakTotalClients,
-        entry.pool.totalCount,
-      );
-      pushSample(entry.stats.acquireWait, acquireWaitMs);
-
-      // A checked-out pg client emits 'error' (not via the pool) when its TCP
-      // connection dies while idle — e.g. a worker holding a client across a
-      // long play execution. With no listener that is an uncaught exception
-      // that kills the process (observed: worker crash mid-150k-row run).
-      // Swallow it here; the next query on the dead client fails normally and
-      // flows through the connection-error retry/poison path below.
-      let checkedOutError: unknown = null;
-      const onCheckedOutError = (error: Error) => {
-        checkedOutError = error;
-        entry.stats.connectionErrors += 1;
-        console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
-          error: error.message,
-        });
-      };
-      poolClient.on('error', onCheckedOutError);
-      let clientReleased = false;
-      const releasePoolClient = (error?: Error) => {
-        if (clientReleased) return;
-        clientReleased = true;
-        poolClient.release(error);
-      };
-      const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
-      signal?.addEventListener('abort', onAbort, { once: true });
-
-      let queryCount = 0;
-      let queryMs = 0;
-      try {
-        throwIfSchedulerAborted(signal);
-        const result = await fn({
-          query: async (sql, params) => {
-            throwIfSchedulerAborted(signal);
-            const queryStartedAt = performance.now();
-            queryCount += 1;
-            try {
-              try {
-                const queryResult = await poolClient.query(sql, params);
-                throwIfSchedulerAborted(signal);
-                return { rows: queryResult.rows };
-              } catch (error) {
-                // node-postgres' query timeout and an AbortSignal reject the
-                // promise, but the server may still be processing the SQL.
-                // Never return that socket to the pool: doing so lets the
-                // next retry overlap the first statement on the same backend.
-                if (isRuntimeSchedulerQueryCancellationError(error)) {
-                  releasePoolClient(
-                    error instanceof Error ? error : new Error(String(error)),
-                  );
-                }
-                throw error;
-              }
-            } finally {
-              queryMs += performance.now() - queryStartedAt;
-            }
-          },
-        });
-        throwIfSchedulerAborted(signal);
-        recordBreakerSuccess(entry);
-        signal?.removeEventListener('abort', onAbort);
-        poolClient.removeListener('error', onCheckedOutError);
-        // The connection died mid-fn but fn happened to finish without another
-        // query — destroy the dead connection rather than pooling it.
-        releasePoolClient(
-          checkedOutError instanceof Error ? checkedOutError : undefined,
+        const acquireStart = performance.now();
+        entry.stats.peakTotalClients = Math.max(
+          entry.stats.peakTotalClients,
+          entry.pool.totalCount,
         );
-        if (trace) {
-          console.info(
-            '[runtime-scheduler-pool.operation] ' +
-              JSON.stringify({
-                runId: trace.runId,
-                phase: trace.phase,
-                outcome: 'success',
-                attempt: attempt + 1,
-                priority: trace.priority ?? 'ordinary',
-                admissionWaitMs,
-                acquireWaitMs,
-                queryCount,
-                queryMs,
-                totalMs: performance.now() - operationStartedAt,
-              }),
-          );
-        }
-        return result;
-      } catch (error) {
-        if (trace) {
-          console.info(
-            '[runtime-scheduler-pool.operation] ' +
-              JSON.stringify({
-                runId: trace.runId,
-                phase: trace.phase,
-                outcome: 'failed',
-                attempt: attempt + 1,
-                priority: trace.priority ?? 'ordinary',
-                admissionWaitMs,
-                acquireWaitMs,
-                queryCount,
-                queryMs,
-                totalMs: performance.now() - operationStartedAt,
-                errorName: error instanceof Error ? error.name : 'UnknownError',
-              }),
-          );
-        }
-        signal?.removeEventListener('abort', onAbort);
-        poolClient.removeListener('error', onCheckedOutError);
-        if (signal?.aborted) {
-          const cancellation = schedulerAbortReason(signal);
-          releasePoolClient(cancellation);
-          throw cancellation;
-        }
-        if (isPgBouncerCapacityError(error)) {
-          recordCapacityRejection(entry, error);
-          releasePoolClient();
-          throw new RuntimeSchedulerPgBouncerCapacityError(
-            error instanceof Error ? error.message : String(error),
-          );
-        }
-        if (isRuntimeSchedulerQueryCancellationError(error)) {
-          entry.stats.queryErrors += 1;
-          recordBreakerSuccess(entry);
-          releasePoolClient(
-            error instanceof Error ? error : new Error(String(error)),
-          );
-          throw error;
-        }
-        if (isConnectionCategoryError(error)) {
-          recordBreakerFailure(entry, error);
+        entry.stats.peakWaitingClients = Math.max(
+          entry.stats.peakWaitingClients,
+          entry.pool.waitingCount,
+        );
+        let poolClient;
+        try {
+          poolClient = await entry.pool.connect();
+        } catch (error) {
+          if (isRuntimeSchedulerLocalAcquireTimeout(error)) {
+            entry.stats.saturationRejections += 1;
+            console.warn('[runtime-scheduler-pool.local_acquire_saturated]', {
+              maxClients: entry.maxClients,
+              totalClients: entry.pool.totalCount,
+              idleClients: entry.pool.idleCount,
+              waitingClients: entry.pool.waitingCount,
+            });
+            throw new RuntimeSchedulerSaturatedError(entry.pool.waitingCount);
+          }
+          if (isPgBouncerCapacityError(error)) {
+            recordCapacityRejection(entry, error);
+            capacityObserved = true;
+            throw new RuntimeSchedulerPgBouncerCapacityError(
+              error instanceof Error ? error.message : String(error),
+            );
+          }
+          recordBreakerFailure(entry, error, halfOpenProbe);
           entry.stats.connectionErrors += 1;
           lastError = error;
-          // Destroy the broken connection instead of returning it to the pool.
-          releasePoolClient(
-            error instanceof Error ? error : new Error(String(error)),
-          );
           if (attempt + 1 < maxAttempts) {
             entry.stats.retries += 1;
             await sleep(jitteredBackoffMs(attempt));
@@ -990,30 +857,197 @@ export async function withRuntimeSchedulerClient<T>(
           }
           throw error;
         }
-        if (isRuntimeSchedulerRetryableTransactionError(error)) {
-          // PostgreSQL has aborted the transaction, so never return this
-          // client to the pool. Re-run the idempotent scheduler operation on a
-          // new session without opening the connection-failure breaker.
-          entry.stats.queryErrors += 1;
+        entry.stats.acquires += 1;
+        const acquireWaitMs = performance.now() - acquireStart;
+        entry.stats.peakTotalClients = Math.max(
+          entry.stats.peakTotalClients,
+          entry.pool.totalCount,
+        );
+        pushSample(entry.stats.acquireWait, acquireWaitMs);
+
+        // A checked-out pg client emits 'error' (not via the pool) when its TCP
+        // connection dies while idle — e.g. a worker holding a client across a
+        // long play execution. With no listener that is an uncaught exception
+        // that kills the process (observed: worker crash mid-150k-row run).
+        // Swallow it here; the next query on the dead client fails normally and
+        // flows through the connection-error retry/poison path below.
+        let checkedOutError: unknown = null;
+        const onCheckedOutError = (error: Error) => {
+          checkedOutError = error;
+          entry.stats.connectionErrors += 1;
+          console.warn('[runtime-scheduler-pool.checked_out_client_error]', {
+            error: error.message,
+          });
+        };
+        poolClient.on('error', onCheckedOutError);
+        let clientReleased = false;
+        const releasePoolClient = (error?: Error) => {
+          if (clientReleased) return;
+          clientReleased = true;
+          poolClient.release(error);
+        };
+        const onAbort = () => releasePoolClient(schedulerAbortReason(signal!));
+        signal?.addEventListener('abort', onAbort, { once: true });
+
+        let queryCount = 0;
+        let queryMs = 0;
+        try {
+          throwIfSchedulerAborted(signal);
+          const result = await fn({
+            query: async (sql, params) => {
+              throwIfSchedulerAborted(signal);
+              const queryStartedAt = performance.now();
+              queryCount += 1;
+              try {
+                try {
+                  const queryResult = await poolClient.query(sql, params);
+                  throwIfSchedulerAborted(signal);
+                  return { rows: queryResult.rows };
+                } catch (error) {
+                  // node-postgres' query timeout and an AbortSignal reject the
+                  // promise, but the server may still be processing the SQL.
+                  // Never return that socket to the pool: doing so lets the
+                  // next retry overlap the first statement on the same backend.
+                  if (isRuntimeSchedulerQueryCancellationError(error)) {
+                    releasePoolClient(
+                      error instanceof Error ? error : new Error(String(error)),
+                    );
+                  }
+                  throw error;
+                }
+              } finally {
+                queryMs += performance.now() - queryStartedAt;
+              }
+            },
+          });
+          throwIfSchedulerAborted(signal);
           recordBreakerSuccess(entry);
-          lastError = error;
+          signal?.removeEventListener('abort', onAbort);
+          poolClient.removeListener('error', onCheckedOutError);
+          // The connection died mid-fn but fn happened to finish without another
+          // query — destroy the dead connection rather than pooling it.
           releasePoolClient(
-            error instanceof Error ? error : new Error(String(error)),
+            checkedOutError instanceof Error ? checkedOutError : undefined,
           );
-          if (attempt + 1 < maxAttempts) {
-            entry.stats.retries += 1;
-            await sleep(jitteredBackoffMs(attempt));
-            throwIfSchedulerAborted(signal);
-            continue;
+          if (trace) {
+            console.info(
+              '[runtime-scheduler-pool.operation] ' +
+                JSON.stringify({
+                  runId: trace.runId,
+                  phase: trace.phase,
+                  outcome: 'success',
+                  attempt: attempt + 1,
+                  priority: trace.priority ?? 'ordinary',
+                  admissionWaitMs,
+                  acquireWaitMs,
+                  queryCount,
+                  queryMs,
+                  totalMs: performance.now() - operationStartedAt,
+                }),
+            );
           }
+          return result;
+        } catch (error) {
+          if (trace) {
+            console.info(
+              '[runtime-scheduler-pool.operation] ' +
+                JSON.stringify({
+                  runId: trace.runId,
+                  phase: trace.phase,
+                  outcome: 'failed',
+                  attempt: attempt + 1,
+                  priority: trace.priority ?? 'ordinary',
+                  admissionWaitMs,
+                  acquireWaitMs,
+                  queryCount,
+                  queryMs,
+                  totalMs: performance.now() - operationStartedAt,
+                  errorName:
+                    error instanceof Error ? error.name : 'UnknownError',
+                }),
+            );
+          }
+          signal?.removeEventListener('abort', onAbort);
+          poolClient.removeListener('error', onCheckedOutError);
+          if (signal?.aborted) {
+            const cancellation = schedulerAbortReason(signal);
+            releasePoolClient(cancellation);
+            throw cancellation;
+          }
+          if (isPgBouncerCapacityError(error)) {
+            recordCapacityRejection(entry, error);
+            capacityObserved = true;
+            releasePoolClient();
+            throw new RuntimeSchedulerPgBouncerCapacityError(
+              error instanceof Error ? error.message : String(error),
+            );
+          }
+          if (isRuntimeSchedulerQueryCancellationError(error)) {
+            entry.stats.queryErrors += 1;
+            recordBreakerSuccess(entry);
+            releasePoolClient(
+              error instanceof Error ? error : new Error(String(error)),
+            );
+            throw error;
+          }
+          if (isConnectionCategoryError(error)) {
+            recordBreakerFailure(entry, error, halfOpenProbe);
+            entry.stats.connectionErrors += 1;
+            lastError = error;
+            // Destroy the broken connection instead of returning it to the pool.
+            releasePoolClient(
+              error instanceof Error ? error : new Error(String(error)),
+            );
+            if (attempt + 1 < maxAttempts) {
+              entry.stats.retries += 1;
+              await sleep(jitteredBackoffMs(attempt));
+              throwIfSchedulerAborted(signal);
+              continue;
+            }
+            throw error;
+          }
+          if (isRuntimeSchedulerRetryableTransactionError(error)) {
+            // PostgreSQL has aborted the transaction, so never return this
+            // client to the pool. Re-run the idempotent scheduler operation on a
+            // new session without opening the connection-failure breaker.
+            entry.stats.queryErrors += 1;
+            recordBreakerSuccess(entry);
+            lastError = error;
+            releasePoolClient(
+              error instanceof Error ? error : new Error(String(error)),
+            );
+            if (attempt + 1 < maxAttempts) {
+              entry.stats.retries += 1;
+              await sleep(jitteredBackoffMs(attempt));
+              throwIfSchedulerAborted(signal);
+              continue;
+            }
+            throw error;
+          }
+          // Query-level failure on a healthy connection: count it, return the
+          // connection to the pool, surface the error without retry.
+          entry.stats.queryErrors += 1;
+          recordBreakerSuccess(entry);
+          releasePoolClient();
           throw error;
         }
-        // Query-level failure on a healthy connection: count it, return the
-        // connection to the pool, surface the error without retry.
-        entry.stats.queryErrors += 1;
-        recordBreakerSuccess(entry);
-        releasePoolClient();
-        throw error;
+      } finally {
+        // A canceled, locally saturated, or capacity-rejected trial did not
+        // call either breaker recorder. Never let it own the probe forever.
+        // Only the trial that claimed this token may release it: another
+        // operation can close and reopen the breaker while this one awaits.
+        if (
+          halfOpenProbe !== null &&
+          entry.breaker.halfOpenProbe === halfOpenProbe
+        ) {
+          entry.breaker.halfOpenProbe = null;
+          if (capacityObserved && entry.breaker.openedAt !== null) {
+            // Capacity is not a connection failure. Keep the existing open
+            // state and try once more after the ordinary cooldown instead of
+            // probing the still-full database on every request.
+            entry.breaker.openedAt = Date.now();
+          }
+        }
       }
     }
     throw lastError instanceof Error

package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts

@@ -239,7 +239,7 @@ function defaultThreshold(
 }
 
 function formatCredits(value: number): string {
-  return value.toLocaleString('en-US', { maximumFractionDigits: 2 });
+  return value.toLocaleString('en-US', { maximumFractionDigits: 3 });
 }
 
 export const DEEPLINE_DOCS_URL = 'https://deepline.com/docs';

package/dist/cli/index.js

@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.202",
+  version: "0.3.204",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1
@@ -29298,7 +29298,12 @@ async function handleRunStop(args) {
   }
   const client2 = new DeeplineClient();
   const result = await client2.runs.stop(runId, { reason });
-  if (result.cancelling === true) {
+  if (result.cancelling === true && result.hitlCancellationPending !== true) {
+    const lines2 = [
+      `Cancellation queued for ${result.runId || runId}; sandbox cleanup is pending`,
+      ...result.hitlCancelledCount > 0 ? [`cancelled HITL waits: ${result.hitlCancelledCount}`] : [],
+      `inspect: deepline runs get ${runId} --full --json`
+    ];
     printCommandEnvelope(
       {
         ...result,
@@ -29306,14 +29311,7 @@ async function handleRunStop(args) {
           sections: [
             {
               title: "run stop",
-              lines: [
-                `Cancellation queued for ${result.runId || runId}; sandbox cleanup is pending`,
-                ...result.hitlCancelledCount > 0 ? [`cancelled HITL waits: ${result.hitlCancelledCount}`] : [],
-                ...result.hitlCancellationPending === true ? [
-                  "warning: HITL cancellation or same-card Slack refresh is still pending"
-                ] : [],
-                `inspect: deepline runs get ${runId} --full --json`
-              ]
+              lines: lines2
             }
           ]
         }
@@ -50804,8 +50802,8 @@ function readPendingToolExecution(path) {
   }
   return saved.idempotencyKey;
 }
-async function completePendingToolExecution(path) {
-  if (!path || !(0, import_node_fs26.existsSync)(path)) return;
+async function completePendingToolExecution(path, idempotencyKey) {
+  if (!path || !idempotencyKey || !(0, import_node_fs26.existsSync)(path)) return;
   await new Promise((resolve22, reject) => {
     const onError = (error) => {
       process.stdout.off("error", onError);
@@ -50821,7 +50819,12 @@ async function completePendingToolExecution(path) {
       }
     });
   });
-  if ((0, import_node_fs26.existsSync)(path)) (0, import_node_fs26.unlinkSync)(path);
+  try {
+    if (readPendingToolExecution(path) !== idempotencyKey) return;
+    (0, import_node_fs26.unlinkSync)(path);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
 }
 function safeFileStem(value) {
   return value.trim().replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || "tool";
@@ -51169,7 +51172,7 @@ async function executeTool(args) {
   const executeExitCode = toolExecuteExitCode(baseEnvelope);
   if (!parsed.outPath && (parsed.outputFormat === "json" || parsed.outputFormat === "auto" && shouldEmitJson())) {
     printCommandEnvelope(baseEnvelope, { json: true });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (parsed.outputFormat === "json_file") {
@@ -51187,7 +51190,7 @@ async function executeTool(args) {
       },
       { json: true }
     );
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (!listConversion) {
@@ -51216,11 +51219,11 @@ async function executeTool(args) {
         },
         { json: parsed.outputFormat === "csv_file" || shouldEmitJson() }
       );
-      await completePendingToolExecution(pendingRecoveryPath);
+      await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
       return executeExitCode;
     }
     printCommandEnvelope(baseEnvelope, { json: false });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   const rowOutput = projectRowOutput(listConversion);
@@ -51243,7 +51246,7 @@ async function executeTool(args) {
   });
   if (parsed.outputFormat === "csv_file") {
     printCommandEnvelope(materializedEnvelope, { json: true });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (parsed.outPath) {
@@ -51252,7 +51255,7 @@ async function executeTool(args) {
       text: `Wrote ${csv.rowCount} row(s) to ${csv.path}
 `
     });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (parsed.noPreview) {
@@ -51267,11 +51270,11 @@ async function executeTool(args) {
       { json: shouldEmitJson(), text: `${csv.path}
 ` }
     );
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   printCommandEnvelope(materializedEnvelope, { json: false });
-  await completePendingToolExecution(pendingRecoveryPath);
+  await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
   return executeExitCode;
 }
 

package/dist/cli/index.mjs

@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.202",
+  version: "0.3.204",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1
@@ -29371,7 +29371,12 @@ async function handleRunStop(args) {
   }
   const client2 = new DeeplineClient();
   const result = await client2.runs.stop(runId, { reason });
-  if (result.cancelling === true) {
+  if (result.cancelling === true && result.hitlCancellationPending !== true) {
+    const lines2 = [
+      `Cancellation queued for ${result.runId || runId}; sandbox cleanup is pending`,
+      ...result.hitlCancelledCount > 0 ? [`cancelled HITL waits: ${result.hitlCancelledCount}`] : [],
+      `inspect: deepline runs get ${runId} --full --json`
+    ];
     printCommandEnvelope(
       {
         ...result,
@@ -29379,14 +29384,7 @@ async function handleRunStop(args) {
           sections: [
             {
               title: "run stop",
-              lines: [
-                `Cancellation queued for ${result.runId || runId}; sandbox cleanup is pending`,
-                ...result.hitlCancelledCount > 0 ? [`cancelled HITL waits: ${result.hitlCancelledCount}`] : [],
-                ...result.hitlCancellationPending === true ? [
-                  "warning: HITL cancellation or same-card Slack refresh is still pending"
-                ] : [],
-                `inspect: deepline runs get ${runId} --full --json`
-              ]
+              lines: lines2
             }
           ]
         }
@@ -50952,8 +50950,8 @@ function readPendingToolExecution(path) {
   }
   return saved.idempotencyKey;
 }
-async function completePendingToolExecution(path) {
-  if (!path || !existsSync19(path)) return;
+async function completePendingToolExecution(path, idempotencyKey) {
+  if (!path || !idempotencyKey || !existsSync19(path)) return;
   await new Promise((resolve22, reject) => {
     const onError = (error) => {
       process.stdout.off("error", onError);
@@ -50969,7 +50967,12 @@ async function completePendingToolExecution(path) {
       }
     });
   });
-  if (existsSync19(path)) unlinkSync4(path);
+  try {
+    if (readPendingToolExecution(path) !== idempotencyKey) return;
+    unlinkSync4(path);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
 }
 function safeFileStem(value) {
   return value.trim().replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || "tool";
@@ -51317,7 +51320,7 @@ async function executeTool(args) {
   const executeExitCode = toolExecuteExitCode(baseEnvelope);
   if (!parsed.outPath && (parsed.outputFormat === "json" || parsed.outputFormat === "auto" && shouldEmitJson())) {
     printCommandEnvelope(baseEnvelope, { json: true });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (parsed.outputFormat === "json_file") {
@@ -51335,7 +51338,7 @@ async function executeTool(args) {
       },
       { json: true }
     );
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (!listConversion) {
@@ -51364,11 +51367,11 @@ async function executeTool(args) {
         },
         { json: parsed.outputFormat === "csv_file" || shouldEmitJson() }
       );
-      await completePendingToolExecution(pendingRecoveryPath);
+      await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
       return executeExitCode;
     }
     printCommandEnvelope(baseEnvelope, { json: false });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   const rowOutput = projectRowOutput(listConversion);
@@ -51391,7 +51394,7 @@ async function executeTool(args) {
   });
   if (parsed.outputFormat === "csv_file") {
     printCommandEnvelope(materializedEnvelope, { json: true });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (parsed.outPath) {
@@ -51400,7 +51403,7 @@ async function executeTool(args) {
       text: `Wrote ${csv.rowCount} row(s) to ${csv.path}
 `
     });
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   if (parsed.noPreview) {
@@ -51415,11 +51418,11 @@ async function executeTool(args) {
       { json: shouldEmitJson(), text: `${csv.path}
 ` }
     );
-    await completePendingToolExecution(pendingRecoveryPath);
+    await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
     return executeExitCode;
   }
   printCommandEnvelope(materializedEnvelope, { json: false });
-  await completePendingToolExecution(pendingRecoveryPath);
+  await completePendingToolExecution(pendingRecoveryPath, idempotencyKey);
   return executeExitCode;
 }
 

package/dist/index.js

@@ -864,7 +864,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.202",
+  version: "0.3.204",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.mjs

@@ -768,7 +768,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.202",
+  version: "0.3.204",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.d.mts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.202";
+    readonly version: "0.3.204";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.d.ts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.202";
+    readonly version: "0.3.204";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.js

@@ -74,7 +74,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.202",
+  version: "0.3.204",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.mjs

@@ -48,7 +48,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.202",
+  version: "0.3.204",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.3.202",
+  "version": "0.3.204",
   "description": "GTM data CLI and TypeScript SDK for coding agents",
   "license": "MIT",
   "homepage": "https://code.deepline.com",