deepline 0.3.460 → 0.3.462

package/dist/bundling-sources/shared_libs/observability/dlq.ts

@@ -281,6 +281,26 @@ export type FieldDefinition = QueueItemFieldDefinition;
 /** A normalized item returned by an owner connector. `nativeState` is always
  * retained so an operator can distinguish a connector mapping from the
  * backend's actual state. */
+/**
+ * What one retained monitor callback is, without its content: enough for an
+ * operator to pair rows, choose what to redrive and see which monitor a row
+ * belongs to. Never carries the callback token, headers, body or raw error.
+ */
+export type QueueItemMonitorEvent = {
+  readonly version: 1;
+  readonly eventClass: 'finding' | 'notice' | 'failure' | 'unknown';
+  /** The provider's `update_type`, when it is a plain identifier. */
+  readonly updateType: string | null;
+  /** Hash of the provider's update id: equal across one finding's rows. */
+  readonly findingGroup: string | null;
+  /** Null on the shared bulk route, whose row is not bound to one monitor. */
+  readonly monitorKey: string | null;
+  readonly route: 'monitor' | 'shared' | 'unknown';
+  readonly bindingVersion: number | null;
+  /** What the inbound worker answered for this row's last redrive, if any. */
+  readonly lastRedriveStatus: string | null;
+};
+
 export type QueueItem = {
   readonly queueId: string;
   readonly id: string;
@@ -301,6 +321,8 @@ export type QueueItem = {
   readonly summary?: string;
   /** Bounded, owner-projected explanation for a queue item. */
   readonly diagnosis?: QueueItemDiagnosis;
+  /** Additive; only monitor inbound dead letters carry it. */
+  readonly monitorEvent?: QueueItemMonitorEvent;
   readonly retiredAt?: string;
   readonly retirementReason?: string;
 };
@@ -1530,3 +1552,13 @@ export function readOnlyDlqAdapter(input: {
     },
   };
 }
+
+/**
+ * Whether a retained failure is `ChargeAuthorizationNotOpenError`. The error
+ * names the charge's status, so match the sentence around it, not one status.
+ */
+export function isChargeAuthorizationNotOpenFailure(text: string): boolean {
+  return /Charge authorization is [a-z_]+; provider work requires an open charge\./.test(
+    text,
+  );
+}

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -219,6 +219,24 @@
     "failureAlert": "github_workflow_run",
     "domainReportRequired": true
   },
+  {
+    "id": "github.session-health-digest",
+    "name": "Session Health Digest",
+    "scheduler": "github_actions",
+    "cadence": "0 12 * * *",
+    "timezone": "UTC",
+    "source": ".github/workflows/session-health-digest.yml",
+    "locator": "schedule:",
+    "owner": "product",
+    "criticality": "reporting",
+    "purpose": "Score the last 24 hours of customer agent sessions from production Axiom telemetry and post the sessions predicted to be going poorly to the reporting Slack channel.",
+    "successDefinition": "All four Axiom reads return complete results, the scored report is retained as an artifact, and Slack accepts the digest; a day with no poor sessions still posts its counts.",
+    "failureDefinition": "Axiom returns an error, a partial result or a row-limit-sized result, or Slack rejects the digest.",
+    "startGraceMs": 600000,
+    "runTimeoutMs": 900000,
+    "failureAlert": "github_workflow_run",
+    "domainReportRequired": true
+  },
   {
     "id": "github.eval-agent-cli-matrix",
     "name": "Eval Agent CLI Matrix",
@@ -532,6 +550,24 @@
     "failureAlert": "vercel_route_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "vercel.finalized-usage-cycle-reconcile",
+    "name": "Finalized usage-cycle reconciliation",
+    "scheduler": "vercel_cron",
+    "cadence": "*/5 * * * *",
+    "timezone": "UTC",
+    "source": "apps/deepline-api/vercel.json",
+    "locator": "/api/v2/cron/finalized-usage-cycle-reconcile",
+    "owner": "billing",
+    "criticality": "critical",
+    "purpose": "Compare finalized Metronome usage invoices with local terminal and delivered usage receipts.",
+    "successDefinition": "Scanned invoices are durably reconciled and parity mismatches are recorded for review.",
+    "failureDefinition": "Provider, local usage, cursor, or persistence failures prevent the scan from advancing safely.",
+    "startGraceMs": 300000,
+    "runTimeoutMs": 300000,
+    "failureAlert": "vercel_route_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "vercel.metronome-usage-export",
     "name": "Metronome usage export",
@@ -1352,9 +1388,9 @@
     "locator": "deepline-play-execution-heartbeat",
     "owner": "runtime",
     "criticality": "critical",
-    "purpose": "Exercise the Play execution heartbeat that keeps active runs and queue leases alive.",
-    "successDefinition": "Active executions receive heartbeats and idle/terminal runs are reported; no active executions is a valid no-op.",
-    "failureDefinition": "Heartbeat dispatch, lease refresh, or terminal reporting fails.",
+    "purpose": "Run provider-free API, webhook and Modal-pinned canary Plays, then ping Better Stack: heartbeat 473888 for the public Play execution row and the internal heartbeat Deepline Play execution on Modal.",
+    "successDefinition": "The API and webhook canaries complete and heartbeat 473888 is pinged; the canary pinned to modal_only@1 completes and its Modal heartbeat is pinged.",
+    "failureDefinition": "Cron metadata is stale, a canary fails or times out, or a Better Stack call fails. A Modal-leg failure is reported in checks.modal and withholds only the Modal heartbeat.",
     "startGraceMs": 120000,
     "runTimeoutMs": 1800000,
     "failureAlert": "better_stack_heartbeat",
@@ -1593,5 +1629,23 @@
     "runTimeoutMs": 60000,
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
+  },
+  {
+    "id": "fly.runtime-detached-runner-liveness-sweep",
+    "name": "Runtime detached-runner liveness sweep",
+    "scheduler": "fly_controller",
+    "cadence": "runAfter each fixed runtime controller reconcile, about every 15 seconds",
+    "timezone": "UTC",
+    "source": "apps/absurd-release-scaler/index.ts",
+    "locator": "recoverConfiguredAbsurdRuntimeStaleDetachedRunners",
+    "owner": "runtime",
+    "criticality": "critical",
+    "purpose": "Read each parked detached runner whose heartbeat is stale, outside org-ranked claims, and report the ones whose sandbox the provider proves ended. It settles nothing until it moves to enforce.",
+    "successDefinition": "Each pass finishes inside its budget, logs every proven runner it reached as would_settle, names any provider it could not read, and leaves every sandbox parked.",
+    "failureDefinition": "Discovery or a terminal settlement fails. A provider outage is not a failure: its runs stay parked and the hard ceiling still bounds them.",
+    "startGraceMs": 60000,
+    "runTimeoutMs": 12000,
+    "failureAlert": "fly_scaler_lifecycle_metric",
+    "domainReportRequired": true
   }
 ]

package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts

@@ -6,6 +6,7 @@ export type ScheduledWorkScheduler =
   | 'convex_cron'
   | 'convex_scheduled_action'
   | 'convex_scheduled_mutation'
+  | 'fly_controller'
   | 'github_actions'
   | 'inngest'
   | 'scheduled_play'

package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts

@@ -4,7 +4,9 @@ import {
   GetObjectCommand,
   HeadObjectCommand,
   DeleteObjectCommand,
+  ListObjectsV2Command,
 } from '@aws-sdk/client-s3';
+import { createHash } from 'node:crypto';
 import { NodeHttpHandler } from '@smithy/node-http-handler';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Agent as HttpsAgent } from 'node:https';
@@ -56,9 +58,23 @@ function getRuntimeEnv(): keyof typeof PLAY_ARTIFACT_BUCKETS {
   return 'dev';
 }
 
+function resolvePlayArtifactStorage() {
+  const environment = getRuntimeEnv();
+  return { environment, bucket: PLAY_ARTIFACT_BUCKETS[environment] };
+}
+
+/** Nonsecret identity only: does not resolve credentials or access storage. */
+export function getPlayArtifactStorageIdentity() {
+  const { environment, bucket } = resolvePlayArtifactStorage();
+  return {
+    environment,
+    bucketSha256: createHash('sha256').update(bucket).digest('hex'),
+  };
+}
+
 function resolveBucket(kind: R2BucketKind): string {
   if (kind === 'playArtifacts') {
-    return PLAY_ARTIFACT_BUCKETS[getRuntimeEnv()];
+    return resolvePlayArtifactStorage().bucket;
   }
   return requireEnv('CLOUDFLARE_R2_BUCKET_NAME');
 }
@@ -73,9 +89,28 @@ export function getR2Config(kind: R2BucketKind = 'default') {
   };
 }
 
+export type R2TransportPolicy = {
+  connectionTimeoutMs: number;
+  socketIdleTimeoutMs: number;
+};
+
+const DEFAULT_R2_TRANSPORT_POLICY: Readonly<R2TransportPolicy> = {
+  connectionTimeoutMs: 10_000,
+  socketIdleTimeoutMs: 30_000,
+};
+
 export function createR2Client(
   config: ReturnType<typeof getR2Config>,
+  transportPolicy: Readonly<R2TransportPolicy> = DEFAULT_R2_TRANSPORT_POLICY,
 ): S3Client {
+  for (const timeout of [
+    transportPolicy.connectionTimeoutMs,
+    transportPolicy.socketIdleTimeoutMs,
+  ]) {
+    if (!Number.isSafeInteger(timeout) || timeout <= 0) {
+      throw new Error('R2 transport timeouts must be positive milliseconds');
+    }
+  }
   return new S3Client({
     region: 'auto',
     endpoint: config.endpoint,
@@ -84,6 +119,11 @@ export function createR2Client(
       secretAccessKey: config.secretAccessKey,
     },
     requestHandler: new NodeHttpHandler({
+      // Bound connection acquisition/connect and stalled I/O separately. A
+      // large upload that keeps moving must not hit a small total deadline.
+      // The SDK can retry an interrupted PUT with the same object key/body.
+      connectionTimeout: transportPolicy.connectionTimeoutMs,
+      socketTimeout: transportPolicy.socketIdleTimeoutMs,
       httpsAgent: new HttpsAgent({
         keepAlive: true,
         maxSockets: 64,
@@ -239,6 +279,31 @@ export async function getBufferFromR2(
   return Buffer.from(bytes);
 }
 
+/** Every key under a prefix, across as many listing pages as it takes. */
+export async function listR2Keys(
+  prefix: string,
+  kind: R2BucketKind = 'default',
+): Promise<string[]> {
+  const config = getR2Config(kind);
+  const client = getR2Client(config);
+  const keys: string[] = [];
+  let continuationToken: string | undefined;
+  do {
+    const response = await client.send(
+      new ListObjectsV2Command({
+        Bucket: config.bucket,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      }),
+    );
+    for (const object of response.Contents ?? []) {
+      if (object.Key) keys.push(object.Key);
+    }
+    continuationToken = response.NextContinuationToken;
+  } while (continuationToken);
+  return keys;
+}
+
 /**
  * Delete one immutable runtime object. Callers must retain their own durable
  * deletion intent and confirm absence with HEAD before they acknowledge this

package/dist/bundling-sources/shared_libs/play-data-plane/sheet-column-storage.ts

@@ -10,6 +10,15 @@ type QueryClient = {
 type SheetIdentity = { schema: string; table: string };
 type StorageType = 'boolean' | 'double precision' | 'text' | 'jsonb';
 type ColumnState = { name: string; type: string; marker: string | null };
+type WriteLockMode = 'shared' | 'exclusive';
+
+/** Restart the whole transaction exclusively before changing a type or marker. */
+export class RuntimeSheetColumnSchemaChangeRequired extends Error {
+  constructor() {
+    super('Runtime Sheet column preparation requires an exclusive transaction');
+    this.name = 'RuntimeSheetColumnSchemaChangeRequired';
+  }
+}
 export type RuntimeSheetColumnTypes = ReadonlyMap<string, string>;
 
 const MARKER = 'deepline:runtime-sheet-column:';
@@ -38,12 +47,19 @@ function valueType(value: unknown): StorageType | null {
   return 'jsonb';
 }
 
-async function lock(client: QueryClient, sheet: SheetIdentity): Promise<void> {
-  // The caller holds this transaction through DML. Both SQL writers use the
-  // same lock, so type inspection, widening and the affected write are atomic.
-  await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [
-    `runtime-sheet-columns:${relation(sheet)}`,
-  ]);
+async function lock(
+  client: QueryClient,
+  sheet: SheetIdentity,
+  mode: WriteLockMode = 'exclusive',
+): Promise<void> {
+  // Stable writers may share the type fence. Schema mutations and all existing
+  // callers keep the exclusive fence; either mode is held through row DML.
+  await client.query(
+    mode === 'shared'
+      ? 'SELECT pg_advisory_xact_lock_shared(hashtextextended($1, 0))'
+      : 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))',
+    [`runtime-sheet-columns:${relation(sheet)}`],
+  );
 }
 
 async function inspect(
@@ -160,9 +176,11 @@ export const RuntimeSheetColumns = {
         | readonly Record<string, unknown>[]
         | (() => Promise<readonly Record<string, unknown>[]>);
       columns: readonly PhysicalSheetColumnProjection[];
+      /** Opt in only when the caller can roll back and restart the whole transaction. */
+      lockMode?: WriteLockMode;
     },
   ): Promise<RuntimeSheetColumnTypes> {
-    await lock(client, input);
+    await lock(client, input, input.lockMode);
     const rows =
       typeof input.rows === 'function' ? await input.rows() : input.rows;
     const columns = new Map(
@@ -178,6 +196,17 @@ export const RuntimeSheetColumns = {
         projection.typeEvidence?.valueType === 'json' ? 'jsonb' : null;
       for (const row of rows)
         desired = combine(desired, valueType(row[projection.fieldName]));
+      if (
+        input.lockMode === 'shared' &&
+        desired !== null &&
+        (column.marker === UNDECIDED ||
+          (column.marker === NATIVE &&
+            combine(column.type as StorageType, desired) !== column.type))
+      ) {
+        // Even an unchanged SQL type can need a new enrollment marker. Never
+        // upgrade the shared lock or perform partial schema changes here.
+        throw new RuntimeSheetColumnSchemaChangeRequired();
+      }
       await changeType(client, input, column, desired);
     }
     return new Map([...columns].map(([name, column]) => [name, column.type]));