deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts

@@ -0,0 +1,35 @@
+/**
+ * The Convex cron roots that an authenticated platform operator may run on
+ * demand. Keep this list aligned with `packages/convex/crons.ts`; the catalog
+ * check makes a newly added Convex cron fail closed until it joins here.
+ *
+ * Other scheduler types deliberately are not listed. They need their own
+ * narrowly-scoped execution authority (rather than letting a Convex deploy
+ * key impersonate GitHub, Vercel, or Inngest).
+ */
+export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
+  'convex.fold-usage-rollups',
+  'convex.record-neon-managed-project-usage',
+  'convex.cleanup-expired-mcp-oauth-codes',
+  'convex.cleanup-expired-analytics-ip2company-cache',
+  'convex.cleanup-expired-domain-health-cache',
+  'convex.salesforce-oauth-refresh-maintenance',
+  'convex.monitor-fleet-convergence-tick',
+  'convex.provider-monitor-global-audit-daily',
+  'convex.recover-notification-delivery-work',
+  'convex.cleanup-expired-notification-audit-records',
+  'convex.fail-stale-play-deployments',
+] as const;
+
+export type ManuallyRunnableConvexCronJobId =
+  (typeof MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS)[number];
+
+const manuallyRunnableConvexCronJobIds = new Set<string>(
+  MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS,
+);
+
+export function isManuallyRunnableConvexCronJob(
+  jobId: string,
+): jobId is ManuallyRunnableConvexCronJobId {
+  return manuallyRunnableConvexCronJobIds.has(jobId);
+}

package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts

@@ -10,6 +10,7 @@ export type ScheduledWorkScheduler =
   | 'vercel_cron';
 
 export type ScheduledWorkOutcome = 'running' | 'succeeded' | 'failed';
+export type ScheduledWorkTrigger = 'scheduled' | 'manual';
 
 export type ScheduledWorkVolume = {
   unit: string;
@@ -34,6 +35,7 @@ export type ScheduledWorkObservation = {
   occurredAt: number;
   outcome: ScheduledWorkOutcome;
   expectedWithinMs: number;
+  trigger?: ScheduledWorkTrigger;
   durationMs?: number;
   errorSummary?: string;
   runUrl?: string;
@@ -53,6 +55,9 @@ export function scheduledWorkTelemetryFields(
     scheduledWorkOutcome: observation.outcome,
     scheduledWorkInvocationId: observation.invocationId,
     scheduledWorkExpectedWithinMs: observation.expectedWithinMs,
+    ...(observation.trigger === undefined
+      ? {}
+      : { scheduledWorkTrigger: observation.trigger }),
     ...(observation.durationMs === undefined
       ? {}
       : { scheduledWorkDurationMs: observation.durationMs }),

package/dist/bundling-sources/shared_libs/play-data-plane/neon-constants.ts

@@ -0,0 +1,9 @@
+/** Shared tenant data-plane topology and database role names. */
+export const TENANT_DATA_PLANE_TOPOLOGY = 'project_per_org';
+
+export const TENANT_OWNER_ROLE_NAME = 'dl_platform_owner';
+export const TENANT_RUNTIME_ROLE_NAME = 'dl_platform_runtime';
+export const TENANT_READ_ROLE_NAME = 'dl_tenant_read';
+export const TENANT_OVERRIDE_ROLE_NAME = 'dl_tenant_override';
+
+export const TENANT_BASELINE_SCHEMA_VERSION = 1;

package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts

@@ -0,0 +1,272 @@
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+} from '@aws-sdk/client-s3';
+import { NodeHttpHandler } from '@smithy/node-http-handler';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { Agent as HttpsAgent } from 'node:https';
+import { isBuildContext } from '../observability/build-context';
+
+type R2PresignerClient = Parameters<typeof getSignedUrl>[0];
+type R2PresignerCommand = Parameters<typeof getSignedUrl>[1];
+
+export type R2BucketKind = 'default' | 'playArtifacts';
+
+const PLAY_ARTIFACT_BUCKETS = {
+  dev: 'plays-artifacts-v1-dev',
+  prod: 'plays-artifacts-v1-prod',
+} as const;
+
+function requireEnv(name: string): string {
+  const value = process.env[name]?.trim();
+  if (!value) {
+    throw new Error(`Missing ${name}`);
+  }
+  return value;
+}
+
+function getRuntimeEnv(): keyof typeof PLAY_ARTIFACT_BUCKETS {
+  const candidates = [
+    process.env.DEEPLINE_ENV,
+    process.env.VERCEL_ENV,
+    process.env.NODE_ENV,
+  ]
+    .filter(Boolean)
+    .map((value) => String(value).trim().toLowerCase());
+
+  for (const value of candidates) {
+    if (value === 'production' || value === 'prod' || value === 'live') {
+      return 'prod';
+    }
+    if (
+      value === 'development' ||
+      value === 'dev' ||
+      value === 'preview' ||
+      value === 'staging' ||
+      value === 'stage' ||
+      value === 'test'
+    ) {
+      return 'dev';
+    }
+  }
+
+  return 'dev';
+}
+
+function resolveBucket(kind: R2BucketKind): string {
+  if (kind === 'playArtifacts') {
+    return PLAY_ARTIFACT_BUCKETS[getRuntimeEnv()];
+  }
+  return requireEnv('CLOUDFLARE_R2_BUCKET_NAME');
+}
+
+export function getR2Config(kind: R2BucketKind = 'default') {
+  return {
+    endpoint: requireEnv('CLOUDFLARE_R2_ENDPOINT'),
+    accessKeyId: requireEnv('CLOUDFLARE_R2_ACCESS_KEY_ID'),
+    secretAccessKey: requireEnv('CLOUDFLARE_R2_SECRET_ACCESS_KEY'),
+    bucket: resolveBucket(kind),
+    publicUrl: requireEnv('CLOUDFLARE_R2_PUBLIC_URL'),
+  };
+}
+
+export function createR2Client(
+  config: ReturnType<typeof getR2Config>,
+): S3Client {
+  return new S3Client({
+    region: 'auto',
+    endpoint: config.endpoint,
+    credentials: {
+      accessKeyId: config.accessKeyId,
+      secretAccessKey: config.secretAccessKey,
+    },
+    requestHandler: new NodeHttpHandler({
+      httpsAgent: new HttpsAgent({
+        keepAlive: true,
+        maxSockets: 64,
+      }),
+    }),
+  });
+}
+
+const r2ClientMemo = new Map<string, S3Client>();
+
+function getR2Client(config: ReturnType<typeof getR2Config>): S3Client {
+  const key = `${config.endpoint}|${config.bucket}|${config.accessKeyId}`;
+  const existing = r2ClientMemo.get(key);
+  if (existing) {
+    return existing;
+  }
+  const client = createR2Client(config);
+  r2ClientMemo.set(key, client);
+  return client;
+}
+
+function logR2Perf(
+  operation: string,
+  key: string,
+  startedAt: number,
+  extra: Record<string, unknown> = {},
+) {
+  // `[perf][r2]` fires on every object op; during the prebuilt-artifact build it
+  // floods the log. Keep it at runtime (Axiom) but silence it in the build.
+  if (isBuildContext()) return;
+  console.info('[perf][r2]', {
+    operation,
+    key,
+    ms: Date.now() - startedAt,
+    ...extra,
+  });
+}
+
+export async function r2ObjectExists(
+  key: string,
+  kind: R2BucketKind = 'default',
+): Promise<boolean> {
+  const startedAt = Date.now();
+  const config = getR2Config(kind);
+  const client = getR2Client(config);
+  try {
+    await client.send(
+      new HeadObjectCommand({
+        Bucket: config.bucket,
+        Key: key,
+      }),
+    );
+    logR2Perf('head', key, startedAt, { kind, exists: true });
+    return true;
+  } catch {
+    logR2Perf('head', key, startedAt, { kind, exists: false });
+    return false;
+  }
+}
+
+export async function uploadBufferToR2(input: {
+  key: string;
+  body: Buffer | Uint8Array;
+  contentType: string;
+  bucketKind?: R2BucketKind;
+}): Promise<string> {
+  const startedAt = Date.now();
+  const bucketKind = input.bucketKind ?? 'default';
+  const configStartedAt = Date.now();
+  const config = getR2Config(bucketKind);
+  logR2Perf('config', input.key, configStartedAt, { kind: bucketKind });
+  const clientStartedAt = Date.now();
+  const client = getR2Client(config);
+  logR2Perf('client', input.key, clientStartedAt, { kind: bucketKind });
+
+  const sendStartedAt = Date.now();
+  await client.send(
+    new PutObjectCommand({
+      Bucket: config.bucket,
+      Key: input.key,
+      Body:
+        input.body instanceof Buffer ? new Uint8Array(input.body) : input.body,
+      ContentType: input.contentType,
+    }),
+  );
+  logR2Perf('put-send', input.key, sendStartedAt, {
+    kind: bucketKind,
+    bytes: input.body.byteLength,
+  });
+  logR2Perf('put', input.key, startedAt, {
+    kind: bucketKind,
+    bytes: input.body.byteLength,
+  });
+
+  if (bucketKind === 'playArtifacts') {
+    return `r2://${config.bucket}/${input.key}`;
+  }
+  return `${config.publicUrl.replace(/\/+$/, '')}/${input.key}`;
+}
+
+export async function getBufferFromR2(
+  key: string,
+  kind: R2BucketKind = 'default',
+): Promise<Buffer> {
+  const startedAt = Date.now();
+  const config = getR2Config(kind);
+  const client = getR2Client(config);
+
+  const response = await client.send(
+    new GetObjectCommand({
+      Bucket: config.bucket,
+      Key: key,
+    }),
+  );
+
+  const bytes = await response.Body!.transformToByteArray();
+  logR2Perf('get', key, startedAt, { kind, bytes: bytes.byteLength });
+  return Buffer.from(bytes);
+}
+
+export async function createSignedR2ReadUrl(input: {
+  key: string;
+  kind?: R2BucketKind;
+  expiresInSeconds: number;
+}): Promise<{ url: string; expiresAt: string }> {
+  const startedAt = Date.now();
+  const kind = input.kind ?? 'default';
+  const config = getR2Config(kind);
+  const client = createR2Client(config);
+  const expiresInSeconds = Math.max(1, Math.floor(input.expiresInSeconds));
+  const command = new GetObjectCommand({
+    Bucket: config.bucket,
+    Key: input.key,
+  });
+  const url = await getSignedUrl(
+    client as R2PresignerClient,
+    command as R2PresignerCommand,
+    { expiresIn: expiresInSeconds },
+  );
+  logR2Perf('sign-get', input.key, startedAt, {
+    kind,
+    expiresInSeconds,
+  });
+  return {
+    url,
+    expiresAt: new Date(Date.now() + expiresInSeconds * 1000).toISOString(),
+  };
+}
+
+/**
+ * Mint a short-lived presigned PUT URL so a client can upload an object body
+ * straight to R2, bypassing the serverless request-body ceiling that a proxied
+ * upload through the Next.js route would hit.
+ *
+ * The URL signs `ContentType`, so the caller MUST send a matching
+ * `Content-Type` header on the PUT for the signature to validate.
+ */
+export async function createSignedR2WriteUrl(input: {
+  key: string;
+  kind?: R2BucketKind;
+  contentType: string;
+  expiresInSeconds: number;
+}): Promise<{ url: string; expiresAt: string }> {
+  const startedAt = Date.now();
+  const kind = input.kind ?? 'default';
+  const config = getR2Config(kind);
+  const client = createR2Client(config);
+  const expiresInSeconds = Math.max(1, Math.floor(input.expiresInSeconds));
+  const command = new PutObjectCommand({
+    Bucket: config.bucket,
+    Key: input.key,
+    ContentType: input.contentType,
+  });
+  const url = await getSignedUrl(
+    client as R2PresignerClient,
+    command as R2PresignerCommand,
+    { expiresIn: expiresInSeconds },
+  );
+  logR2Perf('sign-put', input.key, startedAt, {
+    kind,
+    expiresInSeconds,
+  });
+  return {
+    url,
+    expiresAt: new Date(Date.now() + expiresInSeconds * 1000).toISOString(),
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/activity-observation.ts

@@ -101,7 +101,11 @@ export type PlayActivityState =
     };
 
 export type PlayActivityObservation = {
-  schemaVersion: 1;
+  /**
+   * Retired marker. Producers stopped writing it; envelopes still in flight
+   * from an older producer carry `1`, so readers tolerate both shapes.
+   */
+  schemaVersion?: 1;
   /** Stable within one run, for example `tool:discover_ctos` or `dataset:leads`. */
   activityId: string;
   /** Optional owning play step. */
@@ -126,7 +130,7 @@ export function isPlayActivityObservation(
 ): value is PlayActivityObservation {
   if (
     !isRecord(value) ||
-    value.schemaVersion !== 1 ||
+    (value.schemaVersion !== undefined && value.schemaVersion !== 1) ||
     typeof value.activityId !== 'string' ||
     !value.activityId.trim() ||
     typeof value.observedAt !== 'number' ||
@@ -316,7 +320,6 @@ export function createPlayActivityReporter(input: {
         type: 'activity.observed',
         observation: {
           ...observation,
-          schemaVersion: 1,
           observedAt: observation.observedAt ?? input.now?.() ?? Date.now(),
         },
       });
@@ -402,7 +405,6 @@ export function projectPlayRunActivity(input: {
   if (input.waitKind === 'detached_runner') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'runner:managed',
         target: {
           kind: 'runner',
@@ -418,7 +420,6 @@ export function projectPlayRunActivity(input: {
   if (input.waitKind === 'sleep') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'run:sleep',
         target: {
           kind: 'step',
@@ -441,7 +442,6 @@ export function projectPlayRunActivity(input: {
   ) {
     return projection(
       {
-        schemaVersion: 1,
         activityId: `event:${input.activeNodeId ?? 'run'}`,
         stepId: input.activeNodeId ?? undefined,
         target: {
@@ -468,7 +468,6 @@ export function projectPlayRunActivity(input: {
   if (normalizedStatus === 'waiting') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'run:legacy-wait',
         target: {
           kind: 'step',
@@ -506,7 +505,6 @@ export function projectPlayRunActivity(input: {
     const datasetProgress = datasetStep?.progress ?? null;
     return projection(
       {
-        schemaVersion: 1,
         activityId: `dataset:${pendingDataset.datasetId ?? pendingDataset.tableNamespace}`,
         target: {
           kind: 'dataset',
@@ -548,7 +546,6 @@ export function projectPlayRunActivity(input: {
     const provider = toolId?.split(/[._]/)[0]?.trim() || null;
     return projection(
       {
-        schemaVersion: 1,
         activityId: `step:${activeStep.nodeId}`,
         stepId: activeStep.nodeId,
         target:
@@ -577,7 +574,6 @@ export function projectPlayRunActivity(input: {
   if (normalizedStatus === 'queued') {
     return projection(
       {
-        schemaVersion: 1,
         activityId: 'capacity:worker',
         target: { kind: 'capacity', pool: 'worker' },
         state: { kind: 'queued', reason: 'capacity' },
@@ -588,7 +584,6 @@ export function projectPlayRunActivity(input: {
   }
 
   const fallback: PlayActivityObservation = {
-    schemaVersion: 1,
     activityId: 'run:legacy',
     target: {
       kind: 'step',

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -4,6 +4,7 @@ import { stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import type { PlaySheetContract } from '../plays/static-pipeline';
+import type { AsyncPlayRunRef } from '../plays/authoring-contract';
 import type { PlayRowUpdate } from './/ctx-types';
 import type { PlayRunTimelineEntry } from './/live-events';
 import type { PlayDatasetBornFrom } from './/cell-provenance';
@@ -179,6 +180,58 @@ type RuntimeApiRequest =
       runId: string;
       firstClaimLatencyMs: number;
     }
+  | {
+      action: 'launch_play_async';
+      key: string;
+      invocationKey: string;
+      playRef: string;
+      input: Record<string, unknown>;
+      description: string;
+      schedulerSchema?: string | null;
+      ancestryPlayIds: string[];
+    }
+  | {
+      action: 'project_run_created';
+      idempotencyKey: string;
+      playName: string;
+      playReference?: string | null;
+      workflowId: string;
+      runId: string;
+      workflowFamilyKey: string;
+      /** Optional for root and older durable admission envelopes. */
+      parentRunId?: string | null;
+      rootRunId?: string | null;
+      asyncLaunchKey?: string | null;
+      asyncInvocationKey?: string | null;
+      asyncRequestFingerprint?: string | null;
+      asyncAncestryPlayIds?: string[] | null;
+      definitionScope?: 'org' | 'system';
+      revisionId?: string | null;
+      artifactStorageKey?: string | null;
+      artifactHash?: string | null;
+      graphHash?: string | null;
+      runtimeBackend?: string | null;
+      schedulerBackend?: string | null;
+      schedulerSchema?: string | null;
+      executionProfile?: string | null;
+      runtimeReleaseId?: string | null;
+      runtimeDeployVersion?: string | null;
+      runtimeCallbackBaseUrl?: string | null;
+      coordinatorUrl?: string | null;
+      coordinatorWorkerName?: string | null;
+      runtimeWorkflowName?: string | null;
+      runtimeHarnessWorkerName?: string | null;
+      maxCreditsPerRun?: number | null;
+      staticPipeline?: unknown;
+      source?: 'published' | 'ad_hoc' | 'draft';
+      triggerSource?: 'webhook' | 'cron' | 'sql_listener' | 'api';
+      inputFileId?: string;
+      inputBytes?: number;
+      inputSha256?: string;
+      replayedFromRunId?: string | null;
+      secretRefs?: unknown[];
+      inputSummary?: Record<string, unknown>;
+    }
   | ({
       action: 'save_results';
     } & RuntimeSaveResults)
@@ -546,6 +599,17 @@ function isRetryableAppRuntimeResponse(input: {
   ) {
     return true;
   }
+  // Convex can deploy the projector before the Vercel app knows this action.
+  // That old app returns this exact unstructured 403. The projection is
+  // idempotent and the outbox keeps its delivery receipt, so retrying bridges
+  // the release skew without classifying real executor authorization failures
+  // as transient.
+  if (
+    input.action === 'project_run_created' &&
+    isLegacyProjectRunCreatedUnsupportedActionResponse(input)
+  ) {
+    return true;
+  }
   // The app runtime may explicitly classify an otherwise-client-error status
   // as transient. Keep the structured delivery contract authoritative rather
   // than reducing every 4xx to permanent at the worker boundary.
@@ -564,6 +628,32 @@ function isRetryableAppRuntimeResponse(input: {
   return isRetryableAppRuntimeResponseStatus(input.status);
 }
 
+/**
+ * The pre-reader app returns this one precise response for an action it does
+ * not yet know. Do not use a substring match here: an authorization failure
+ * that merely repeats the phrase must remain terminal and reach an operator.
+ */
+function isLegacyProjectRunCreatedUnsupportedActionResponse(input: {
+  status: number;
+  body: string;
+}): boolean {
+  if (input.status !== 403) return false;
+  const expected = 'Unsupported runtime action capability scope.';
+  try {
+    const parsed = JSON.parse(input.body) as unknown;
+    return (
+      !!parsed &&
+      typeof parsed === 'object' &&
+      !Array.isArray(parsed) &&
+      (parsed as { error?: unknown }).error === expected
+    );
+  } catch {
+    // The oldest route can return the bare legacy message. Retain that bridge
+    // while keeping it exact rather than accepting an arbitrary 403 body.
+    return input.body.trim() === expected;
+  }
+}
+
 function appRuntimeExplicitRetryable(body: string): boolean | null {
   try {
     const parsed = JSON.parse(body) as unknown;
@@ -716,10 +806,12 @@ function isRetryableAppRuntimeAction(
     action === 'rate_state_release' ||
     action === 'rate_state_penalize' ||
     action === 'record_first_claim_latency' ||
+    action === 'project_run_created' ||
     action === 'release_runtime_step_receipt' ||
     action === 'save_results' ||
     action === 'skip_runtime_step_receipt' ||
-    action === 'start_run'
+    action === 'start_run' ||
+    action === 'launch_play_async'
   );
 }
 
@@ -1073,7 +1165,8 @@ export class AppRuntimeApiResponseError extends Error {
     this.requestId = input.requestId?.trim() || null;
     this.retryable = input.retryable;
     this.retryAfterMs =
-      typeof input.retryAfterMs === 'number' && Number.isFinite(input.retryAfterMs)
+      typeof input.retryAfterMs === 'number' &&
+      Number.isFinite(input.retryAfterMs)
         ? Math.max(0, Math.floor(input.retryAfterMs))
         : null;
     this.detail = input.detail;
@@ -1963,16 +2056,14 @@ export async function appendRunEventsViaAppRuntime(
   let terminalBatchIndex = -1;
   for (let index = batches.length - 1; index >= 0; index -= 1) {
     if (
-      batches[index]!.some(
-        (event) => {
-          const key = event.key ?? event.type;
-          return (
-            key === 'run.completed' ||
-            key === 'run.failed' ||
-            key === 'run.cancelled'
-          );
-        },
-      )
+      batches[index]!.some((event) => {
+        const key = event.key ?? event.type;
+        return (
+          key === 'run.completed' ||
+          key === 'run.failed' ||
+          key === 'run.cancelled'
+        );
+      })
     ) {
       terminalBatchIndex = index;
       break;
@@ -2021,6 +2112,19 @@ export async function startRunViaAppRuntime(
   });
 }
 
+export async function projectRunCreatedViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: Omit<
+    Extract<RuntimeApiRequest, { action: 'project_run_created' }>,
+    'action'
+  >,
+): Promise<void> {
+  await postAppRuntimeApi<{ ok: true; status: 'queued' }>(context, {
+    action: 'project_run_created',
+    ...input,
+  });
+}
+
 export async function recordFirstClaimLatencyViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: { runId: string; firstClaimLatencyMs: number },
@@ -2031,6 +2135,25 @@ export async function recordFirstClaimLatencyViaAppRuntime(
   });
 }
 
+export async function launchPlayAsyncViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: {
+    key: string;
+    invocationKey: string;
+    playRef: string;
+    input: Record<string, unknown>;
+    description: string;
+    schedulerSchema?: string | null;
+    ancestryPlayIds: readonly string[];
+  },
+): Promise<AsyncPlayRunRef> {
+  return await postAppRuntimeApi<AsyncPlayRunRef>(context, {
+    action: 'launch_play_async',
+    ...input,
+    ancestryPlayIds: [...input.ancestryPlayIds],
+  });
+}
+
 export async function saveResultsViaAppRuntime(
   context: WorkerRuntimeApiContext,
   input: RuntimeSaveResults,