deepline 0.3.148 → 0.3.150

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -561,6 +561,24 @@
     "failureAlert": "vercel_route_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "vercel.public-tool-execution-blob-cleanup",
+    "name": "Public tool execution blob cleanup",
+    "scheduler": "vercel_cron",
+    "cadence": "*/15 * * * *",
+    "timezone": "UTC",
+    "source": "apps/deepline-api/vercel.json",
+    "locator": "/api/v2/cron/public-tool-execution-blob-cleanup",
+    "owner": "integrations",
+    "criticality": "maintenance",
+    "purpose": "Delete expired public execution response and launch blobs while retaining idempotency key and fingerprint tombstones.",
+    "successDefinition": "Every claimed expired receipt's recorded blobs are confirmed absent before their references are cleared; no eligible receipts is a valid no-op.",
+    "failureDefinition": "An R2 deletion/absence check fails or the fenced tombstone cleanup acknowledgement cannot complete.",
+    "startGraceMs": 300000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "vercel_route_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "vercel.scheduled-work-health-snapshot",
     "name": "Scheduled-work health snapshot",

package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts

@@ -144,6 +144,39 @@ export async function r2ObjectExists(
   }
 }
 
+/**
+ * Confirm an R2 object is absent without confusing a HEAD/auth/network failure
+ * with a successful deletion.
+ */
+export async function confirmR2ObjectAbsent(
+  key: string,
+  kind: R2BucketKind = 'default',
+): Promise<boolean> {
+  const config = getR2Config(kind);
+  const client = getR2Client(config);
+  try {
+    await client.send(
+      new HeadObjectCommand({ Bucket: config.bucket, Key: key }),
+    );
+    return false;
+  } catch (error) {
+    const details = error as {
+      name?: string;
+      Code?: string;
+      $metadata?: { httpStatusCode?: number };
+    };
+    if (
+      details.$metadata?.httpStatusCode === 404 ||
+      details.name === 'NotFound' ||
+      details.name === 'NoSuchKey' ||
+      details.Code === 'NotFound' ||
+      details.Code === 'NoSuchKey'
+    )
+      return true;
+    throw error;
+  }
+}
+
 export async function uploadBufferToR2(input: {
   key: string;
   body: Buffer | Uint8Array;

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -757,6 +757,9 @@ function isRetryableAppRuntimeResponse(input: {
   // than reducing every 4xx to permanent at the worker boundary.
   const explicitRetryable = appRuntimeExplicitRetryable(input.body);
   if (explicitRetryable !== null) return explicitRetryable;
+  if (isComputeBillingTooManyWritesResponse(input)) {
+    return true;
+  }
   if (
     input.action === 'append_run_events' &&
     input.status === 500 &&
@@ -809,6 +812,43 @@ function appRuntimeExplicitRetryable(body: string): boolean | null {
   }
 }
 
+function isComputeBillingTooManyWritesResponse(input: {
+  action: RuntimeApiRequest['action'];
+  status: number;
+  body: string;
+}): boolean {
+  if (
+    input.status !== 500 ||
+    (input.action !== 'compute_billing_upsert' &&
+      input.action !== 'compute_billing_record_item' &&
+      input.action !== 'compute_billing_finalize')
+  ) {
+    return false;
+  }
+  // The runtime route's unexpected-error envelope exposes the Convex error
+  // name and redacted message but has no retryable flag. A TooManyWrites
+  // conflict is transient; treating this 500 as permanent blocks a durable
+  // billing job. Only the replay-safe compute actions accept this signature.
+  try {
+    const parsed = JSON.parse(input.body) as unknown;
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      return false;
+    }
+    const error = parsed as Record<string, unknown>;
+    if (error.action != null && error.action !== input.action) {
+      return false;
+    }
+    return (
+      error.errorName === 'TooManyWrites' ||
+      error.code === 'TooManyWrites' ||
+      (typeof error.debug_error === 'string' &&
+        /\bTooManyWrites\b/.test(error.debug_error))
+    );
+  } catch {
+    return false;
+  }
+}
+
 function isRetryableAppRuntimeResponseStatus(status: number): boolean {
   return (
     status === 429 ||

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts

@@ -5,6 +5,7 @@ import {
   lowerBound,
   notApplicable,
   unavailable,
+  runtimeQueueItemRevision,
   type QueueDefinition,
   type QueueHealth,
   type QueueObservation,
@@ -236,7 +237,42 @@ function operatorRetirementCandidates(
         itemId = candidate.itemId;
       }
       const ageMs = numberValue(candidate.ageMs as string | number | undefined);
-      if (itemId !== undefined) candidates.push({ itemId, bucket, ageMs });
+      const revisionFields = candidate.revisionFields;
+      let sourceRevision: string | undefined;
+      if (
+        typeof revisionFields === 'object' &&
+        revisionFields !== null &&
+        !Array.isArray(revisionFields)
+      ) {
+        const fields = revisionFields as Record<string, unknown>;
+        const sourceDate = fields.updatedAt ?? fields.createdAt;
+        const requiresState = kind === 'absurd';
+        const requiresStatus =
+          kind === 'compute_billing' || kind === 'sandbox_cleanup';
+        if (
+          (typeof sourceDate === 'string' || typeof sourceDate === 'number') &&
+          Number.isFinite(new Date(sourceDate).getTime()) &&
+          (!requiresState || typeof fields.state === 'string') &&
+          (!requiresStatus || typeof fields.status === 'string')
+        ) {
+          sourceRevision = runtimeQueueItemRevision({
+            updatedAt: fields.updatedAt,
+            createdAt: fields.createdAt,
+            state: fields.state,
+            status: fields.status,
+            leaseExpiresAt: fields.leaseExpiresAt,
+            failedAt: fields.failedAt,
+          });
+        }
+      }
+      if (itemId !== undefined) {
+        candidates.push({
+          itemId,
+          bucket,
+          ageMs,
+          ...(sourceRevision ? { sourceRevision } : {}),
+        });
+      }
     }
   };
   append(row.blocked_retirement_candidates, 'blocked');
@@ -517,45 +553,53 @@ function absurdProjectionQuery(
   candidateSampleLimit: number,
 ): string {
   const table = `absurd.${quoteIdentifier(queueTable, 'queue table')}`;
+  const revisionFields = includeRetirementCandidates
+    ? `json_build_object(
+          'createdAt', created_at,
+          'state', state,
+          'leaseExpiresAt', claim_expires_at,
+          'failedAt', failed_at
+        )`
+    : 'NULL::json';
   const retirementColumns = includeRetirementCandidates
     ? `,
       COALESCE((
-        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
-        FROM (SELECT item_id, age_at FROM sampled WHERE bucket = 'blocked' ORDER BY age_at ASC, item_id ASC LIMIT $4::int) candidates
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}, 'revisionFields', revision_fields))
+        FROM (SELECT item_id, age_at, revision_fields FROM sampled WHERE bucket = 'blocked' ORDER BY age_at ASC, item_id ASC LIMIT $4::int) candidates
       ), '[]'::json) AS blocked_retirement_candidates,
       COALESCE((
-        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
-        FROM (SELECT item_id, age_at FROM sampled WHERE bucket = 'dead_lettered' ORDER BY age_at ASC, item_id ASC LIMIT $4::int) candidates
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}, 'revisionFields', revision_fields))
+        FROM (SELECT item_id, age_at, revision_fields FROM sampled WHERE bucket = 'dead_lettered' ORDER BY age_at ASC, item_id ASC LIMIT $4::int) candidates
       ), '[]'::json) AS dead_lettered_retirement_candidates`
     : '';
   return `
     WITH sampled AS (
-      (SELECT 'ready'::text AS bucket, run_id::text AS item_id, available_at AS age_at
+      (SELECT 'ready'::text AS bucket, run_id::text AS item_id, available_at AS age_at, NULL::json AS revision_fields
       FROM ${table}
       WHERE state = 'pending' AND available_at <= $1::timestamptz
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'delayed'::text, run_id::text, available_at
+      (SELECT 'delayed'::text, run_id::text, available_at, NULL::json
       FROM ${table}
       WHERE state = 'sleeping'
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'delayed'::text, run_id::text, available_at
+      (SELECT 'delayed'::text, run_id::text, available_at, NULL::json
       FROM ${table}
       WHERE state = 'pending' AND available_at > $1::timestamptz
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'in_flight'::text, run_id::text, coalesce(started_at, created_at)
+      (SELECT 'in_flight'::text, run_id::text, coalesce(started_at, created_at), NULL::json
       FROM ${table}
       WHERE state = 'running'
         AND (claim_expires_at IS NULL OR claim_expires_at > $1::timestamptz)
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'blocked'::text, run_id::text, coalesce(claim_expires_at, started_at, created_at)
+      (SELECT 'blocked'::text, run_id::text, coalesce(claim_expires_at, started_at, created_at), ${revisionFields}
       FROM ${table}
       WHERE state = 'running'
         AND claim_expires_at IS NOT NULL
@@ -563,7 +607,7 @@ function absurdProjectionQuery(
       ORDER BY claim_expires_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'dead_lettered'::text, run_id::text, coalesce(failed_at, created_at)
+      (SELECT 'dead_lettered'::text, run_id::text, coalesce(failed_at, created_at), ${revisionFields}
       FROM ${table}
       WHERE state = 'failed'
       ORDER BY available_at, run_id
@@ -602,15 +646,22 @@ function outboxProjectionQuery(
   includeRetirementCandidates: boolean,
 ): string {
   const table = `${schema}."outbox"`;
+  const revisionFields = includeRetirementCandidates
+    ? `json_build_object(
+          'createdAt', created_at,
+          'leaseExpiresAt', lease_expires_at,
+          'failedAt', failed_at
+        )`
+    : 'NULL::json';
   const retirementColumns = includeRetirementCandidates
     ? `,
       COALESCE((
-        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
-        FROM (SELECT item_id, age_at FROM blocked ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}, 'revisionFields', revision_fields))
+        FROM (SELECT item_id, age_at, revision_fields FROM blocked ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
       ), '[]'::json) AS blocked_retirement_candidates,
       COALESCE((
-        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
-        FROM (SELECT item_id, age_at FROM dead_lettered ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}, 'revisionFields', revision_fields))
+        FROM (SELECT item_id, age_at, revision_fields FROM dead_lettered ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
       ), '[]'::json) AS dead_lettered_retirement_candidates`
     : '';
   return `
@@ -642,7 +693,8 @@ function outboxProjectionQuery(
       ORDER BY lease_expires_at, created_at, event_id
       LIMIT $2::int
     ), blocked AS (
-      SELECT event_id::text AS item_id, coalesce(lease_expires_at, created_at) AS age_at
+      SELECT event_id::text AS item_id, coalesce(lease_expires_at, created_at) AS age_at,
+             ${revisionFields} AS revision_fields
       FROM ${table}
       WHERE delivered_at IS NULL
         AND failed_at IS NULL
@@ -651,7 +703,8 @@ function outboxProjectionQuery(
       ORDER BY coalesce(lease_expires_at, created_at), created_at, event_id
       LIMIT $2::int
     ), dead_lettered AS (
-      SELECT event_id::text AS item_id, coalesce(failed_at, created_at) AS age_at
+      SELECT event_id::text AS item_id, coalesce(failed_at, created_at) AS age_at,
+             ${revisionFields} AS revision_fields
       FROM ${table}
       WHERE delivered_at IS NULL AND failed_at IS NOT NULL
       ORDER BY failed_at, event_id
@@ -689,11 +742,19 @@ function cleanupProjectionQuery(
   includeRetirementCandidates: boolean,
 ): string {
   const table = `${schema}."sandbox_cleanup_jobs_v2"`;
+  const revisionFields = includeRetirementCandidates
+    ? `json_build_object(
+          'updatedAt', updated_at,
+          'createdAt', created_at,
+          'status', status,
+          'leaseExpiresAt', lease_expires_at
+        )`
+    : 'NULL::json';
   const retirementColumns = includeRetirementCandidates
     ? `,
       COALESCE((
-        SELECT json_agg(json_build_object('provider', provider, 'sandboxId', sandbox_id, 'routingDomainKey', routing_domain_key, 'ageMs', ${intervalExpr('age_at')}))
-        FROM (SELECT provider, sandbox_id, routing_domain_key, created_at, age_at FROM blocked ORDER BY age_at ASC, created_at ASC, provider ASC, sandbox_id ASC LIMIT $3::int) candidates
+        SELECT json_agg(json_build_object('provider', provider, 'sandboxId', sandbox_id, 'routingDomainKey', routing_domain_key, 'ageMs', ${intervalExpr('age_at')}, 'revisionFields', revision_fields))
+        FROM (SELECT provider, sandbox_id, routing_domain_key, created_at, age_at, revision_fields FROM blocked ORDER BY age_at ASC, created_at ASC, provider ASC, sandbox_id ASC LIMIT $3::int) candidates
       ), '[]'::json) AS blocked_retirement_candidates,
       '[]'::json AS dead_lettered_retirement_candidates`
     : '';
@@ -736,7 +797,8 @@ function cleanupProjectionQuery(
         CASE
           WHEN blocked_at IS NOT NULL THEN blocked_at
           ELSE coalesce(lease_expires_at, created_at)
-        END AS age_at
+        END AS age_at,
+        ${revisionFields} AS revision_fields
       FROM ${table}
       WHERE status <> 'deleted'
         AND (
@@ -779,11 +841,19 @@ function computeBillingProjectionQuery(
   includeRetirementCandidates: boolean,
 ): string {
   const table = `${schema}."compute_billing_jobs_v2"`;
+  const revisionFields = includeRetirementCandidates
+    ? `json_build_object(
+          'updatedAt', updated_at,
+          'createdAt', created_at,
+          'status', status,
+          'leaseExpiresAt', lease_expires_at
+        )`
+    : 'NULL::json';
   const retirementColumns = includeRetirementCandidates
     ? `,
       COALESCE((
-        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
-        FROM (SELECT item_id, age_at FROM blocked ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}, 'revisionFields', revision_fields))
+        FROM (SELECT item_id, age_at, revision_fields FROM blocked ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
       ), '[]'::json) AS blocked_retirement_candidates,
       '[]'::json AS dead_lettered_retirement_candidates`
     : '';
@@ -816,7 +886,8 @@ function computeBillingProjectionQuery(
       ORDER BY lease_expires_at, created_at, run_id
       LIMIT $2::int
     ), blocked AS (
-      SELECT run_id::text AS item_id, coalesce(blocked_at, lease_expires_at, created_at) AS age_at
+      SELECT run_id::text AS item_id, coalesce(blocked_at, lease_expires_at, created_at) AS age_at,
+             ${revisionFields} AS revision_fields
       FROM ${table}
       WHERE status <> 'settled'
         AND (