deepline 0.3.90 → 0.3.91

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -650,5 +650,41 @@
     "owner": "platform",
     "criticality": "maintenance",
     "failureAlert": "convex_action_wrapper"
+  },
+  {
+    "id": "github.modal-ci-runner-weekly-proof",
+    "name": "Modal CI runner",
+    "scheduler": "github_actions",
+    "cadence": "23 6 * * 1",
+    "timezone": "UTC",
+    "source": ".github/workflows/modal-ci-runner.yml",
+    "locator": "schedule:",
+    "owner": "platform",
+    "criticality": "maintenance",
+    "failureAlert": "github_workflow_run"
+  },
+  {
+    "id": "convex.async-usage-recovery-continuation",
+    "name": "Async usage recovery continuation",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter per attached async provider job, re-armed from provider truth",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "dispatchAsyncUsageRecoveryContinuation",
+    "owner": "billing",
+    "criticality": "critical",
+    "failureAlert": "convex_action_wrapper"
+  },
+  {
+    "id": "convex.charge-reconciliation-backlog-continuation",
+    "name": "Charge reconciliation backlog continuation",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter per bounded due-reservation page while a Convex cursor drain is active",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "dispatchChargeReconciliationBacklogContinuation",
+    "owner": "billing",
+    "criticality": "critical",
+    "failureAlert": "convex_action_wrapper"
   }
 ]

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -242,7 +242,7 @@ type RuntimeApiRequest =
       sheetContract?: PlaySheetContract | null;
       contractSnapshot?: unknown;
       runId: string;
-      userEmail: string;
+      userEmail?: string | null;
       updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
     }
   | {

package/dist/bundling-sources/shared_libs/play-runtime/db-session-lease-store.ts

@@ -0,0 +1,578 @@
+import { createHash } from 'node:crypto';
+import type { Redis } from '@upstash/redis';
+import { getRedisClient } from '@deepline/redis/client';
+import {
+  createDbSessionResponseSchema,
+  type CreateDbSessionResponse,
+  type DbSessionLimits,
+  type DbSessionOperation,
+} from './/db-session';
+
+const KEY_PREFIX = 'plays:db-session-lease:v1';
+const MIN_REUSABLE_LEASE_MS = 60_000;
+const TARGET_RESERVATION_TTL_SECONDS = 60;
+
+// Redis stores active scoped Postgres URLs, so every key must include org and
+// run identity. Never key this only by play/table: play names are not globally
+// unique, and table namespaces are caller-authored SDK strings.
+export type DbSessionLeaseLookup = {
+  orgId: string;
+  workflowId: string;
+  runId: string;
+  playName: string;
+  tableNamespace: string;
+  logicalTable: string;
+  operations: readonly DbSessionOperation[];
+  limits?: DbSessionLimits | undefined;
+  userEmail: string;
+};
+
+export type DbSessionLeaseStore = {
+  getActive(
+    input: DbSessionLeaseLookup,
+  ): Promise<CreateDbSessionResponse | null>;
+  reserveTarget(
+    input: DbSessionLeaseLookup,
+  ): Promise<
+    | { status: 'reserved'; reservationId: string }
+    | { status: 'active'; response: CreateDbSessionResponse }
+    | { status: 'conflict' }
+  >;
+  put(
+    input: DbSessionLeaseLookup,
+    response: CreateDbSessionResponse,
+    reservationId: string,
+  ): Promise<void>;
+  releaseTargetReservation(
+    input: DbSessionLeaseLookup,
+    reservationId: string,
+  ): Promise<void>;
+};
+
+const memoryLeaseEntries = new Map<string, LeaseEntry>();
+const memoryTargetEntries = new Map<string, TargetEntry>();
+
+type LeaseMetadata = {
+  orgId: string;
+  workflowId: string;
+  runId: string;
+  playName: string;
+  tableNamespace: string;
+  logicalTable: string;
+  operations: string[];
+  limits: DbSessionLimits;
+  userEmail: string;
+};
+type LeaseEntry = {
+  kind: 'lease';
+  metadata: LeaseMetadata;
+  response: CreateDbSessionResponse;
+};
+type TargetReservation = {
+  kind: 'reservation';
+  reservationId: string;
+  metadata: LeaseMetadata;
+};
+type TargetEntry = LeaseEntry | TargetReservation;
+
+function stableJson(value: unknown): string {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return JSON.stringify(value);
+  }
+  const record = value as Record<string, unknown>;
+  return JSON.stringify(
+    Object.keys(record)
+      .sort()
+      .reduce<Record<string, unknown>>((acc, key) => {
+        acc[key] = record[key];
+        return acc;
+      }, {}),
+  );
+}
+
+function leaseKey(input: DbSessionLeaseLookup): string {
+  // Exact-shape lease key. This is the idempotency path: the same org/run/table
+  // request with the same operations/limits/user gets the same active URL back
+  // instead of minting another short-lived Postgres role.
+  const fingerprint = createHash('sha256')
+    .update(
+      stableJson({
+        orgId: input.orgId,
+        workflowId: input.workflowId,
+        runId: input.runId,
+        playName: input.playName,
+        tableNamespace: input.tableNamespace,
+        logicalTable: input.logicalTable,
+        operations: [...input.operations].sort(),
+        limits: input.limits ?? {},
+        userEmail: input.userEmail,
+      }),
+    )
+    .digest('hex')
+    .slice(0, 32);
+  return `${KEY_PREFIX}:org:${input.orgId}:${fingerprint}`;
+}
+
+function targetLeaseKey(input: DbSessionLeaseLookup): string {
+  // Target key intentionally excludes operations/limits/user. It is a guardrail
+  // for "one active credential shape per org/run/play/table target"; callers can
+  // reuse the exact lease, but cannot widen privileges while the first URL lives.
+  const fingerprint = createHash('sha256')
+    .update(
+      stableJson({
+        orgId: input.orgId,
+        workflowId: input.workflowId,
+        runId: input.runId,
+        playName: input.playName,
+        tableNamespace: input.tableNamespace,
+        logicalTable: input.logicalTable,
+      }),
+    )
+    .digest('hex')
+    .slice(0, 32);
+  return `${KEY_PREFIX}:target:org:${input.orgId}:${fingerprint}`;
+}
+
+function reservationId(input: DbSessionLeaseLookup): string {
+  return createHash('sha256')
+    .update(`${leaseKey(input)}:${Date.now()}:${Math.random()}`)
+    .digest('hex')
+    .slice(0, 32);
+}
+
+function metadataFor(input: DbSessionLeaseLookup): LeaseMetadata {
+  return {
+    orgId: input.orgId,
+    workflowId: input.workflowId,
+    runId: input.runId,
+    playName: input.playName,
+    tableNamespace: input.tableNamespace,
+    logicalTable: input.logicalTable,
+    operations: [...input.operations].sort(),
+    limits: input.limits ?? {},
+    userEmail: input.userEmail,
+  };
+}
+
+function remainingLeaseMs(response: CreateDbSessionResponse): number {
+  const expiresAtMs = Date.parse(response.expiresAt);
+  if (!Number.isFinite(expiresAtMs)) {
+    return 0;
+  }
+  return expiresAtMs - Date.now();
+}
+
+function sameLimits(
+  left: DbSessionLimits | undefined,
+  right: DbSessionLimits | undefined,
+): boolean {
+  return stableJson(left ?? {}) === stableJson(right ?? {});
+}
+
+function sameOperations(
+  left: readonly DbSessionOperation[],
+  right: readonly DbSessionOperation[],
+): boolean {
+  return stableJson([...left].sort()) === stableJson([...right].sort());
+}
+
+function operationsSatisfyRequest(
+  existing: readonly DbSessionOperation[],
+  requested: readonly DbSessionOperation[],
+): boolean {
+  const existingSet = new Set(existing);
+  return requested.every((operation) => existingSet.has(operation));
+}
+
+function limitsSatisfyRequest(
+  existing: DbSessionLimits | undefined,
+  requested: DbSessionLimits | undefined,
+): boolean {
+  const existingLimits = existing ?? {};
+  const requestedLimits = requested ?? {};
+  for (const [key, requestedValue] of Object.entries(requestedLimits)) {
+    if (
+      typeof requestedValue !== 'number' ||
+      !Number.isFinite(requestedValue)
+    ) {
+      if (
+        stableJson((existingLimits as Record<string, unknown>)[key]) !==
+        stableJson(requestedValue)
+      ) {
+        return false;
+      }
+      continue;
+    }
+    const existingValue = (existingLimits as Record<string, unknown>)[key];
+    if (typeof existingValue !== 'number' || !Number.isFinite(existingValue)) {
+      return false;
+    }
+    if (existingValue < requestedValue) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function metadataMatchesLookup(
+  input: DbSessionLeaseLookup,
+  metadata: LeaseMetadata,
+  options: { includeExactShape: boolean },
+): boolean {
+  const targetMatches =
+    metadata.orgId === input.orgId &&
+    metadata.workflowId === input.workflowId &&
+    metadata.runId === input.runId &&
+    metadata.playName === input.playName &&
+    metadata.tableNamespace === input.tableNamespace &&
+    metadata.logicalTable === input.logicalTable;
+  if (!targetMatches) {
+    return false;
+  }
+  if (!options.includeExactShape) {
+    return true;
+  }
+  return (
+    sameOperations(
+      metadata.operations as DbSessionOperation[],
+      input.operations,
+    ) &&
+    sameLimits(metadata.limits, input.limits) &&
+    metadata.userEmail === input.userEmail
+  );
+}
+
+function leaseMatchesLookup(
+  input: DbSessionLeaseLookup,
+  entry: LeaseEntry,
+): boolean {
+  // Redis is not the authority. If a key is poisoned, stale, or computed by a
+  // future bug, the response payload still has to match the requested org/table
+  // scope before we hand a database URL back to the worker.
+  const response = entry.response;
+  return (
+    metadataMatchesLookup(input, entry.metadata, { includeExactShape: true }) &&
+    response.target.orgId === input.orgId &&
+    response.playName === input.playName &&
+    response.target.tableNamespace === input.tableNamespace &&
+    response.target.logicalTable === input.logicalTable &&
+    sameOperations(response.operations, input.operations) &&
+    sameLimits(response.limits, input.limits)
+  );
+}
+
+function activeTargetLeaseSatisfiesLookup(
+  input: DbSessionLeaseLookup,
+  entry: LeaseEntry,
+): boolean {
+  const response = entry.response;
+  return (
+    metadataMatchesLookup(input, entry.metadata, {
+      includeExactShape: false,
+    }) &&
+    entry.metadata.userEmail === input.userEmail &&
+    response.target.orgId === input.orgId &&
+    response.playName === input.playName &&
+    response.target.tableNamespace === input.tableNamespace &&
+    response.target.logicalTable === input.logicalTable &&
+    operationsSatisfyRequest(response.operations, input.operations) &&
+    limitsSatisfyRequest(response.limits, input.limits)
+  );
+}
+
+function parseLeaseEntry(raw: unknown): LeaseEntry | null {
+  if (!raw || typeof raw !== 'object') {
+    return null;
+  }
+  const record = raw as Record<string, unknown>;
+  if (record.kind !== 'lease' || !isLeaseMetadata(record.metadata)) {
+    return null;
+  }
+  const response = createDbSessionResponseSchema.safeParse(record.response);
+  if (!response.success) {
+    return null;
+  }
+  return {
+    kind: 'lease',
+    metadata: record.metadata,
+    response: response.data,
+  };
+}
+
+function parseTargetEntry(raw: unknown): TargetEntry | null {
+  const lease = parseLeaseEntry(raw);
+  if (lease) {
+    return lease;
+  }
+  if (!raw || typeof raw !== 'object') {
+    return null;
+  }
+  const record = raw as Record<string, unknown>;
+  if (
+    record.kind !== 'reservation' ||
+    typeof record.reservationId !== 'string' ||
+    record.reservationId.length === 0 ||
+    !isLeaseMetadata(record.metadata)
+  ) {
+    return null;
+  }
+  return {
+    kind: 'reservation',
+    reservationId: record.reservationId,
+    metadata: record.metadata,
+  };
+}
+
+function isLeaseMetadata(value: unknown): value is LeaseMetadata {
+  if (!value || typeof value !== 'object') {
+    return false;
+  }
+  const record = value as Record<string, unknown>;
+  return (
+    typeof record.orgId === 'string' &&
+    record.orgId.length > 0 &&
+    typeof record.workflowId === 'string' &&
+    record.workflowId.length > 0 &&
+    typeof record.runId === 'string' &&
+    record.runId.length > 0 &&
+    typeof record.playName === 'string' &&
+    record.playName.length > 0 &&
+    typeof record.tableNamespace === 'string' &&
+    record.tableNamespace.length > 0 &&
+    typeof record.logicalTable === 'string' &&
+    record.logicalTable.length > 0 &&
+    Array.isArray(record.operations) &&
+    record.operations.every((operation) => typeof operation === 'string') &&
+    (!record.limits ||
+      (typeof record.limits === 'object' && !Array.isArray(record.limits))) &&
+    typeof record.userEmail === 'string'
+  );
+}
+
+export function createRedisDbSessionLeaseStore(
+  redis: Redis | null = getRedisClient(),
+): DbSessionLeaseStore {
+  if (!redis && process.env.NODE_ENV !== 'production') {
+    return createInMemoryDbSessionLeaseStore();
+  }
+
+  async function readActiveEntry(key: string): Promise<LeaseEntry | null> {
+    if (!redis) {
+      return null;
+    }
+    const raw = await redis.get<unknown>(key);
+    if (!raw) {
+      return null;
+    }
+    const parsed = parseLeaseEntry(raw);
+    if (!parsed || remainingLeaseMs(parsed.response) <= MIN_REUSABLE_LEASE_MS) {
+      await redis.del(key);
+      return null;
+    }
+    return parsed;
+  }
+
+  return {
+    async getActive(input) {
+      if (!redis) {
+        return null;
+      }
+      const key = leaseKey(input);
+      const active = await readActiveEntry(key);
+      if (!active) {
+        return null;
+      }
+      if (!leaseMatchesLookup(input, active)) {
+        // Delete mismatched cache entries instead of falling back silently; a
+        // mismatch here means the security invariant around org/run scope broke.
+        await redis.del(key);
+        return null;
+      }
+      return active.response;
+    },
+    async reserveTarget(input) {
+      if (!redis) {
+        throw new Error(
+          'Redis is required to reserve scoped DB session leases.',
+        );
+      }
+      const key = targetLeaseKey(input);
+      const existing = await redis.get<unknown>(key);
+      if (existing) {
+        const parsed = parseTargetEntry(existing);
+        if (!parsed) {
+          await redis.del(key);
+        } else if (parsed.kind === 'lease') {
+          if (remainingLeaseMs(parsed.response) <= MIN_REUSABLE_LEASE_MS) {
+            await redis.del(key);
+          } else if (
+            !metadataMatchesLookup(input, parsed.metadata, {
+              includeExactShape: false,
+            }) ||
+            parsed.response.target.orgId !== input.orgId ||
+            parsed.response.playName !== input.playName ||
+            parsed.response.target.tableNamespace !== input.tableNamespace ||
+            parsed.response.target.logicalTable !== input.logicalTable
+          ) {
+            await redis.del(key);
+          } else {
+            return activeTargetLeaseSatisfiesLookup(input, parsed)
+              ? { status: 'active', response: parsed.response }
+              : { status: 'conflict' };
+          }
+        } else if (
+          metadataMatchesLookup(input, parsed.metadata, {
+            includeExactShape: false,
+          })
+        ) {
+          return { status: 'conflict' };
+        } else {
+          await redis.del(key);
+        }
+      }
+
+      const id = reservationId(input);
+      const reserved = await redis.set(
+        key,
+        {
+          kind: 'reservation',
+          reservationId: id,
+          metadata: metadataFor(input),
+        } satisfies TargetReservation,
+        { ex: TARGET_RESERVATION_TTL_SECONDS, nx: true } as never,
+      );
+      if (reserved !== 'OK') {
+        return { status: 'conflict' };
+      }
+      return { status: 'reserved', reservationId: id };
+    },
+    async put(input, response, reservationId) {
+      if (!redis) {
+        throw new Error('Redis is required to store scoped DB session leases.');
+      }
+      const key = targetLeaseKey(input);
+      const targetEntry = parseTargetEntry(await redis.get<unknown>(key));
+      if (
+        !targetEntry ||
+        targetEntry.kind !== 'reservation' ||
+        targetEntry.reservationId !== reservationId ||
+        !metadataMatchesLookup(input, targetEntry.metadata, {
+          includeExactShape: true,
+        })
+      ) {
+        throw new Error(
+          'Scoped DB session target reservation was not held by this request.',
+        );
+      }
+      const ttlSeconds = Math.floor(remainingLeaseMs(response) / 1000);
+      if (ttlSeconds <= 0) {
+        await redis.del(key);
+        return;
+      }
+      const entry = {
+        kind: 'lease',
+        metadata: metadataFor(input),
+        response,
+      } satisfies LeaseEntry;
+      await Promise.all([
+        redis.set(leaseKey(input), entry, { ex: ttlSeconds }),
+        redis.set(key, entry, { ex: ttlSeconds }),
+      ]);
+    },
+    async releaseTargetReservation(input, reservationId) {
+      if (!redis) {
+        return;
+      }
+      const key = targetLeaseKey(input);
+      const targetEntry = parseTargetEntry(await redis.get<unknown>(key));
+      if (
+        targetEntry?.kind === 'reservation' &&
+        targetEntry.reservationId === reservationId
+      ) {
+        // Same target-index rule: target entries are only hints for active lease
+        // exclusivity. Validate the payload before using it for any decision.
+        await redis.del(key);
+      }
+    },
+  };
+}
+
+function createInMemoryDbSessionLeaseStore(): DbSessionLeaseStore {
+  async function readActiveEntry(key: string): Promise<LeaseEntry | null> {
+    const entry = memoryLeaseEntries.get(key) ?? null;
+    if (!entry || remainingLeaseMs(entry.response) <= MIN_REUSABLE_LEASE_MS) {
+      memoryLeaseEntries.delete(key);
+      return null;
+    }
+    return entry;
+  }
+
+  return {
+    async getActive(input) {
+      const active = await readActiveEntry(leaseKey(input));
+      if (!active) {
+        return null;
+      }
+      if (!leaseMatchesLookup(input, active)) {
+        memoryLeaseEntries.delete(leaseKey(input));
+        return null;
+      }
+      return active.response;
+    },
+    async reserveTarget(input) {
+      const key = targetLeaseKey(input);
+      const existing = memoryTargetEntries.get(key);
+      if (existing) {
+        if (existing.kind === 'lease') {
+          if (remainingLeaseMs(existing.response) <= MIN_REUSABLE_LEASE_MS) {
+            memoryTargetEntries.delete(key);
+          } else if (activeTargetLeaseSatisfiesLookup(input, existing)) {
+            return { status: 'active', response: existing.response };
+          } else {
+            return { status: 'conflict' };
+          }
+        } else if (
+          metadataMatchesLookup(input, existing.metadata, {
+            includeExactShape: false,
+          })
+        ) {
+          return { status: 'conflict' };
+        }
+      }
+
+      const id = reservationId(input);
+      memoryTargetEntries.set(key, {
+        kind: 'reservation',
+        reservationId: id,
+        metadata: metadataFor(input),
+      });
+      return { status: 'reserved', reservationId: id };
+    },
+    async put(input, response, reservationId) {
+      const leaseEntry: LeaseEntry = {
+        kind: 'lease',
+        metadata: metadataFor(input),
+        response,
+      };
+      memoryLeaseEntries.set(leaseKey(input), leaseEntry);
+      const targetKey = targetLeaseKey(input);
+      const target = memoryTargetEntries.get(targetKey);
+      if (
+        target?.kind === 'reservation' &&
+        target.reservationId === reservationId
+      ) {
+        memoryTargetEntries.set(targetKey, leaseEntry);
+      }
+    },
+    async releaseTargetReservation(input, reservationId) {
+      const key = targetLeaseKey(input);
+      const existing = memoryTargetEntries.get(key);
+      if (
+        existing?.kind === 'reservation' &&
+        existing.reservationId === reservationId
+      ) {
+        memoryTargetEntries.delete(key);
+      }
+    },
+  };
+}