deepline 0.3.473 → 0.3.475

package/dist/bundling-sources/sdk/src/client-core.ts

@@ -2555,7 +2555,10 @@ export type BillingNamespace = {
 
 export type BillingQuoteRequest = {
   quantity: number;
-  sandbox_runtime: { timeout?: string; size?: 'standard' };
+  sandbox_runtime: {
+    timeout?: string;
+    size?: 'standard' | 'large' | 'xlarge';
+  };
 };
 
 export type BillingQuoteResult = {

package/dist/bundling-sources/sdk/src/play.ts

@@ -184,8 +184,13 @@ export type PlayFetchResponse = PlayAuthoringFetchResponse;
  *   `deepline describe <id>` for a tool's streams and row columns).
  *   The changed row is delivered as the listener event's `after` in the play function's input.
  *
- * The default Play runtime is 30 minutes. For bounded long-running batches, add
- * `runtime: { timeout: '90m', size: 'standard' }`; duration values are whole minutes or hours, up to `4h`.
+ * The default Play runtime is 30 minutes with the `standard` 1 GiB / 1 CPU
+ * profile. Preview deployments also support `large` (4 GiB / 2 CPU) and
+ * `xlarge` (8 GiB / 4 CPU) through `runtime.size`. Production uses `standard`.
+ * Compute pricing follows the workspace plan; use `deepline billing quote`
+ * for the session ceiling.
+ * Set `runtime.timeout` for longer bounded runs; duration values are whole
+ * minutes or hours, up to `4h`.
  * It differs from `deepline.tools.execute({ timeoutMs })`, which limits one provider call.
  *
  * @example Webhook with HMAC verification. `secretEnv` is the Deepline Secret

package/dist/bundling-sources/sdk/src/release.ts

@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.473',
+  version: '0.3.475',
   updateSummary: {
     introducedIn: '0.3.90',
     summary:

package/dist/bundling-sources/shared_libs/integration-contracts/billing-entitlements.ts

@@ -74,7 +74,7 @@ export type BillingEntitlements = {
 export type IncludedCompute = {
   /** `unavailable`: the workspace is outside the November pricing rollout. */
   status: 'on' | 'off' | 'unavailable';
-  /** Credits per minute of sandbox runtime in force; null at provider cost. */
+  /** Credits per standard-equivalent sandbox minute; null at provider cost. */
   credits_per_minute: number | null;
   included_minutes: number;
   /** What turning included compute on gives this workspace's plan. */
@@ -83,6 +83,8 @@ export type IncludedCompute = {
   period_end_at: string;
   resets_on: 'calendar_month' | 'billing_cycle';
   included_minutes_used: number;
+  /** Exact Deepline credits consumed from included compute; older servers may omit it. */
+  included_credits_used?: number;
   minutes_used: number;
   days: Array<{ day: string; minutes: number }>;
 };

package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json

@@ -946,6 +946,24 @@
     "failureAlert": "convex_action_wrapper",
     "domainReportRequired": true
   },
+  {
+    "id": "convex.observe-large-usage-charges",
+    "name": "Observe unusually large usage charges",
+    "scheduler": "convex_scheduled_action",
+    "cadence": "runAfter(0) from each consumed positive usage-notification batch",
+    "timezone": "UTC",
+    "source": "packages/convex/scheduledJobsNode.ts",
+    "locator": "observeLargeUsageCharges",
+    "owner": "billing",
+    "criticality": "critical",
+    "purpose": "Observe committed individual charges against their tool thresholds, including cumulative async settlements.",
+    "successDefinition": "The bounded source batch and each billing-owner total are read independently, and any threshold breach is reported.",
+    "failureDefinition": "Committed usage or its billing-owner total cannot be observed; failure remains visible for operator recovery.",
+    "startGraceMs": 120000,
+    "runTimeoutMs": 1800000,
+    "failureAlert": "convex_action_wrapper",
+    "domainReportRequired": true
+  },
   {
     "id": "convex.credit-balance-auto-recharge-trigger",
     "name": "Credit-balance automatic recharge trigger",

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts

@@ -657,6 +657,20 @@ function assertMinSecretsVersion(value: unknown): asserts value is number {
   }
 }
 
+export type FlyMachinePhaseEvent = Readonly<{
+  phase:
+    | 'machine-create'
+    | 'machine-update'
+    | 'machine-start'
+    | 'readiness-read';
+  component: 'worker';
+  source: 'machines-api';
+  startedAtMs: number;
+  finishedAtMs: number;
+  elapsedMs: number;
+  outcome: 'success' | 'error';
+}>;
+
 export class FlyMachinesClient {
   constructor(
     private readonly app: string,
@@ -666,6 +680,11 @@ export class FlyMachinesClient {
       wait?: (ms: number) => Promise<void>;
       retryDelaysMs?: readonly number[];
       requestTimeoutMs?: number;
+      /** Preview CLI diagnostics only; no provider arguments or bodies. */
+      phaseTiming?: Readonly<{
+        now?: () => number;
+        report?: (event: FlyMachinePhaseEvent) => void;
+      }>;
     }> = {},
   ) {}
 
@@ -674,6 +693,67 @@ export class FlyMachinesClient {
     path: string,
     body?: unknown,
     leaseNonce?: string,
+  ): Promise<unknown> {
+    const timing = this.rateLimitOptions.phaseTiming;
+    const phase: FlyMachinePhaseEvent['phase'] | undefined =
+      method === 'GET' && /^\/machines(?:\/[^/]+)?$/.test(path)
+        ? 'readiness-read'
+        : method === 'POST' && path === '/machines'
+          ? 'machine-create'
+          : method === 'POST' && /^\/machines\/[^/]+$/.test(path)
+            ? 'machine-update'
+            : method === 'POST' && /^\/machines\/[^/]+\/start$/.test(path)
+              ? 'machine-start'
+              : undefined;
+    if (!timing || !phase)
+      return this.requestUntimed(method, path, body, leaseNonce);
+    const now = timing.now ?? Date.now;
+    let startedAtMs: number;
+    try {
+      startedAtMs = now();
+    } catch {
+      this.reportTimingUnavailable();
+      return this.requestUntimed(method, path, body, leaseNonce);
+    }
+    let outcome: FlyMachinePhaseEvent['outcome'] = 'error';
+    try {
+      const result = await this.requestUntimed(method, path, body, leaseNonce);
+      outcome = 'success';
+      return result;
+    } finally {
+      // Diagnostic failures cannot change a provider outcome or retry an effect.
+      try {
+        const finishedAtMs = now();
+        const event: FlyMachinePhaseEvent = {
+          phase,
+          component: 'worker',
+          source: 'machines-api',
+          startedAtMs,
+          finishedAtMs,
+          elapsedMs: finishedAtMs - startedAtMs,
+          outcome,
+        };
+        if (timing.report) timing.report(event);
+        else console.error(`[runtime-fly-phase] ${JSON.stringify(event)}`);
+      } catch {
+        this.reportTimingUnavailable();
+      }
+    }
+  }
+
+  private reportTimingUnavailable(): void {
+    try {
+      console.error('[runtime-fly-phase] timing_unavailable');
+    } catch {
+      // A closed diagnostic stream must preserve the original provider result.
+    }
+  }
+
+  private async requestUntimed(
+    method: string,
+    path: string,
+    body?: unknown,
+    leaseNonce?: string,
   ): Promise<unknown> {
     const retryDelaysMs =
       this.rateLimitOptions.retryDelaysMs ??
@@ -1458,6 +1538,23 @@ export async function replaceLane(input: {
   return replacements;
 }
 
+function previewMachinePhaseTiming(
+  flags: Record<string, string | boolean>,
+): Readonly<{ report: (event: FlyMachinePhaseEvent) => void }> | undefined {
+  if (flags.environment !== 'preview') return undefined;
+  const output =
+    flags['phase-timings-file'] === undefined
+      ? undefined
+      : requireStringFlag(flags, 'phase-timings-file');
+  return {
+    report(event) {
+      const record = JSON.stringify(event);
+      if (output) appendFileSync(output, `${record}\n`);
+      console.error(`[runtime-fly-phase] ${record}`);
+    },
+  };
+}
+
 async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
   const app = resolveApp(flags);
   const releaseId = requireReleaseId(flags);
@@ -1505,6 +1602,7 @@ async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
     app,
     requireFlyToken(),
     globalThis.fetch as FetchLike,
+    { phaseTiming: previewMachinePhaseTiming(flags) },
   );
   await bootLane({
     client,
@@ -1549,6 +1647,7 @@ async function cmdReplace(
     app,
     requireFlyToken(),
     globalThis.fetch as FetchLike,
+    { phaseTiming: previewMachinePhaseTiming(flags) },
   );
   await replaceLane({
     client,
@@ -2214,8 +2313,8 @@ const USAGE = `fly-absurd-lane <command> [flags]
 Commands:
   push-image --sha <sha> --release <id> [--app <app>]
   plan-image --release <id> [--image-ref <ref>] [--app <app>]
-  boot       --release <id> --sha <sha> --count <K> [--image-ref <ref>] [--reuse-existing] [--component-input-hash <sha256>] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
-  replace    --release <id> --sha <sha> --count <K> [--image-ref <ref>] --environment preview [--component-input-hash <sha256>] [--app <app>] [--region <r>]
+  boot       --release <id> --sha <sha> --count <K> [--image-ref <ref>] [--phase-timings-file <path>] [--reuse-existing] [--component-input-hash <sha256>] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
+  replace    --release <id> --sha <sha> --count <K> [--image-ref <ref>] [--phase-timings-file <path>] --environment preview [--component-input-hash <sha256>] [--app <app>] [--region <r>]
   list       [--release <id>] [--json] [--app <app>]
   drain          --release <id> [--if-present] [--app <app>]
   stop-claiming  --release <id> [--environment production|preview]

package/dist/bundling-sources/shared_libs/play-runtime/modal-runtime-config.ts

@@ -1,8 +1,12 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
 import type { ModalClient } from 'modal';
-import { isIsolatedRuntimeSchedulerSchema } from './runtime-scheduler-topology';
+import {
+  isIsolatedRuntimeSchedulerSchema,
+  normalizeRuntimeSchedulerSchema,
+} from './runtime-scheduler-topology';
 import {
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+  modalSandboxMemoryLimitGiBForPlay,
   validatePlaySandboxRuntimeLimits,
   type PlaySandboxRuntimeLimits,
 } from '../plays/sandbox-runtime-limits';
@@ -10,9 +14,47 @@ import {
 export const MODAL_RUNNER_APP_NAME = 'deepline-play-runner';
 export const MODAL_RUNNER_IMAGE = 'node:20-bookworm-slim';
 export const MODAL_RUNNER_WORKDIR = '/root/deepline';
-export { MODAL_SANDBOX_CPU_CORES } from './worker-api-types';
+export {
+  MODAL_SANDBOX_CPU_CORES,
+  modalSandboxResourcesForPlay,
+} from './worker-api-types';
 export const MODAL_SANDBOX_MEMORY_MIB = 1024;
 
+export function isModalPreviewEvaluation(input: {
+  runtimeSchedulerSchema: string | null | undefined;
+  env?: NodeJS.ProcessEnv;
+}): boolean {
+  const schema = normalizeRuntimeSchedulerSchema(input.runtimeSchedulerSchema);
+  return (
+    (input.env ?? process.env).DEEPLINE_RUNTIME_ENVIRONMENT === 'preview' &&
+    schema !== null &&
+    isIsolatedRuntimeSchedulerSchema(schema)
+  );
+}
+
+/**
+ * Reservation and burst ceiling are separate provider facts. Keep the launch's
+ * profile unchanged: it still selects guaranteed resources. Preview evaluates
+ * headroom before any Production rollout. Preview workers and a valid isolated
+ * scheduler schema are both required; absence keeps the Production policy.
+ */
+export function resolveModalSandboxMemoryPolicy(input: {
+  limits: PlaySandboxRuntimeLimits;
+  runtimeSchedulerSchema: string | null | undefined;
+  env?: NodeJS.ProcessEnv;
+}): { reservationMemoryMiB: number; memoryLimitMiB: number } {
+  const limits = validatePlaySandboxRuntimeLimits(input.limits);
+  const reservationMemoryMiB = limits.memoryGiB * 1024;
+  const enableBurst = isModalPreviewEvaluation(input);
+  // The retained buffered Node/Python transport completes in 4 GiB. Larger
+  // reservations can burst to the existing product ceiling, without increasing
+  // their CPU reservation or paying for unused memory headroom.
+  const memoryLimitMiB = enableBurst
+    ? modalSandboxMemoryLimitGiBForPlay(limits) * 1024
+    : reservationMemoryMiB;
+  return { reservationMemoryMiB, memoryLimitMiB };
+}
+
 export type ModalRequiredConfig = {
   client: ModalClient;
   appName: string;
@@ -105,24 +147,24 @@ export function withModalCallSignal<T>(
 
 // Modal runs custom middleware outside its own retry loop, so the signal also
 // ends a retry or throttle wait.
-const cancelModalCallsWithScopedSignal: ModalGrpcMiddleware =
-  async function* (call, options) {
-    const signal = modalCallSignal.getStore();
-    // Every caller shares one auth token refresh, so one read's deadline must
-    // not cancel it.
-    if (
-      !signal ||
-      call.method.path === '/modal.client.ModalClient/AuthTokenGet'
-    ) {
-      return yield* call.next(call.request, options);
-    }
-    return yield* call.next(call.request, {
-      ...options,
-      signal: options.signal
-        ? AbortSignal.any([options.signal, signal])
-        : signal,
-    });
-  };
+const cancelModalCallsWithScopedSignal: ModalGrpcMiddleware = async function* (
+  call,
+  options,
+) {
+  const signal = modalCallSignal.getStore();
+  // Every caller shares one auth token refresh, so one read's deadline must
+  // not cancel it.
+  if (
+    !signal ||
+    call.method.path === '/modal.client.ModalClient/AuthTokenGet'
+  ) {
+    return yield* call.next(call.request, options);
+  }
+  return yield* call.next(call.request, {
+    ...options,
+    signal: options.signal ? AbortSignal.any([options.signal, signal]) : signal,
+  });
+};
 
 let sharedModalClient: {
   tokenId: string;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts

@@ -12,12 +12,12 @@ import {
   DaytonaConflictError,
   DaytonaRateLimitError,
   DaytonaValidationError,
-  type CreateSandboxFromSnapshotParams,
 } from '@daytonaio/sdk';
 import axios, { type InternalAxiosRequestConfig } from 'axios';
 import { Readable } from 'node:stream';
 import { setTimeout as delay } from 'node:timers/promises';
 import type { DaytonaClientOptions } from '../../daytona-runtime-config';
+import type { CreateSandboxFromSnapshotParams } from '@daytonaio/sdk';
 import type { DaytonaDeleteRetryAuthorization } from './daytona-lifecycle';
 import {
   DAYTONA_CONTROL_TIMEOUT_MS,
@@ -429,7 +429,7 @@ function deadline(timeoutMs: number, signal?: AbortSignal) {
   };
 }
 
-/** Deepline's default-snapshot launch contract; image/build orchestration stays in the SDK. */
+/** Deepline's snapshot launch contract; image/build orchestration stays in the SDK. */
 export function createDaytonaLaunchClient(
   options: DaytonaClientOptions,
   cancellation?: AbortSignal,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -20,8 +20,10 @@ import { isIsolatedRuntimeSchedulerSchema } from '../../runtime-scheduler-topolo
 import { PLAY_RUNNER_TIMEOUT_SECONDS } from '../../runtime-constants';
 import {
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+  playSandboxSizeForLimits,
   validatePlaySandboxRuntimeLimits,
 } from '../../../plays/sandbox-runtime-limits';
+import type { PlaySandboxSize } from '../../../plays/sandbox-runtime-limits';
 import type {
   PlayRunnerDaytonaCleanupIntent,
   PlayRunnerRuntimeLifecycleEvent,
@@ -65,8 +67,15 @@ const DAYTONA_ABSENT_OPERATION_RECREATE_AFTER_MS = 3 * 60_000;
 // inactivity stop is a wider crash backstop measured from sandbox creation, so
 // setup time cannot consume the terminal-flush grace.
 const DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES = 15;
-// Daytona's Deepline-managed default snapshot is the only admitted prebuilt.
-// Resources are verified during acquisition, before customer code or billing.
+// Daytona snapshots carry fixed resource allocations. The API rejects resource
+// fields on snapshot creates, so select the provider snapshot that matches each
+// Deepline profile instead of trying to resize after creation.
+export const DAYTONA_SNAPSHOT_BY_PLAY_SANDBOX_SIZE = {
+  standard: 'daytona-small',
+  large: 'daytona-medium',
+  xlarge: 'daytona-large',
+} as const satisfies Record<PlaySandboxSize, string>;
+
 export const DAYTONA_SANDBOX_CPU = STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS.cpu;
 export const DAYTONA_SANDBOX_MEMORY_GIB =
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS.memoryGiB;
@@ -98,6 +107,7 @@ export type DaytonaClient = {
     timeoutMs: number,
   ): { sandbox: Sandbox; dispose(): void };
 };
+
 export type DaytonaSandbox = Awaited<ReturnType<DaytonaClient['create']>>;
 export type DaytonaExecutionContext = PlayRunnerExecutionConfig['context'];
 export type DaytonaStageEmitter = (
@@ -569,6 +579,12 @@ async function createOneShotDaytonaSandbox(input: {
       ...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
     },
   );
+  const sandboxSize = playSandboxSizeForLimits(limits);
+  if (!sandboxSize) {
+    throw new Error(
+      'Validated Daytona sandbox profile has no snapshot mapping.',
+    );
+  }
   const orgId = normalizeLabelValue(input.orgId);
   const workflowId = normalizeLabelValue(input.context.workflowId);
   const playId = normalizeLabelValue(input.context.playId);
@@ -596,6 +612,7 @@ async function createOneShotDaytonaSandbox(input: {
   const commonParams = {
     name: input.sandboxName,
     labels,
+    snapshot: DAYTONA_SNAPSHOT_BY_PLAY_SANDBOX_SIZE[sandboxSize],
     // This is also the provider-owned backstop for an HTTP create timeout
     // whose named sandbox never becomes lookup-addressable to this process:
     // Daytona deletes an ephemeral sandbox immediately when it stops.
@@ -1227,8 +1244,8 @@ async function acquireOneShotDaytonaSandbox(input: {
       cancellationSignal: input.cancellationSignal,
     });
   if (
-    granted.cpu !== limits.cpu ||
-    granted.memoryGiB !== limits.memoryGiB ||
+    granted.cpu > limits.cpu ||
+    granted.memoryGiB > limits.memoryGiB ||
     granted.diskGiB !== limits.diskGiB ||
     granted.gpu !== DAYTONA_SANDBOX_GPU
   ) {
@@ -1298,10 +1315,9 @@ async function acquireOneShotDaytonaSandbox(input: {
     elapsedMs: billingStartedAt - input.startedAt,
     admissionWaitMs,
     attemptElapsedMs: result.attemptElapsedMs,
-    // Daytona's default snapshot is intentionally provider-managed. Record the
-    // resolved immutable snapshot identifier so a slow acquisition can be
-    // distinguished from a bad warm-pool/default-snapshot selection without
-    // logging provider credentials or customer-code details.
+    // Record the resolved immutable snapshot identifier so slow acquisition
+    // can be distinguished from a bad profile snapshot without logging
+    // provider credentials or customer-code details.
     snapshot: sandbox.snapshot?.trim() || null,
     cpu: granted.cpu,
     memoryGiB: granted.memoryGiB,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -383,17 +383,21 @@ async function main() {
     runtimeLimitMarkerPath && fs.existsSync(runtimeLimitMarkerPath),
   );
   // 137 is SIGKILL, not proof of an OOM. A watchdog, platform eviction, or
-  // operator action can produce the same status. Only label this an OOM when
-  // the captured runtime output carries an explicit V8 allocation signature.
+  // operator action can produce the same status. OOM requires an explicit V8
+  // allocation signature or the sandbox's cgroup OOM-kill evidence.
   // The shell sees the watchdog's exit code, not necessarily the nested
   // runner child's signal. Read the watchdog's strict marker so a child
   // SIGKILL does not collapse into the wrapper's generic exit code 1.
   const sandboxKilled = exitCode === 137 || watchdogObservedChildSigkill();
   const memoryAbort = watchdogMemoryAbortDiagnostic();
+  const explicitOomSignature =
+    (sandboxKilled || exitCode === 134) && outputHasExplicitOomSignature();
   const sandboxOom =
     !memoryAbort &&
-    sandboxKilled &&
-    (outputHasExplicitOomSignature() || cgroupOomKillObserved());
+    ((sandboxKilled && (explicitOomSignature || cgroupOomKillObserved())) ||
+      // V8 aborts itself on heap exhaustion. SIGABRT alone is also authored
+      // program output, so 134 needs the explicit allocation signature.
+      (exitCode === 134 && explicitOomSignature));
   const synthesizedError = runtimeLimitExceeded
     ? 'RUNTIME_LIMIT_EXCEEDED: ' + ${JSON.stringify(runtimeLimitMessage)}
     : memoryAbort
@@ -580,6 +584,8 @@ export async function stageRunnerPayload(input: {
   sandbox: RemoteRunnerPayloadSandbox;
   bundlePromise: Promise<string>;
   config: PlayRunnerExecutionConfig;
+  /** Provider-selected burst ceiling; the stored profile remains a reservation. */
+  memoryLimitMiB?: number;
   workDir: string;
   startedAt: number;
   emitStage: DaytonaStageEmitter;
@@ -674,11 +680,12 @@ export async function stageRunnerPayload(input: {
     materializedFiles: remoteMaterializedFiles,
     context: remoteRuntimeContextForDaytona(input.config.context),
   };
-  const runtimeLimitSeconds = validatePlaySandboxRuntimeLimits(
+  const sandboxRuntimeLimits = validatePlaySandboxRuntimeLimits(
     input.config.context.sandboxRuntimeLimits ?? {
       ...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
     },
-  ).timeoutSeconds;
+  );
+  const runtimeLimitSeconds = sandboxRuntimeLimits.timeoutSeconds;
 
   const envelopeUpload = input.bundlePromise.then((bundle) =>
     timedRunnerPayloadUpload({
@@ -732,7 +739,12 @@ export async function stageRunnerPayload(input: {
   const watchedRunnerCommand = [
     'node',
     '-e',
-    shellQuote(buildDaytonaRuntimeWatchdogSource()),
+    shellQuote(
+      buildDaytonaRuntimeWatchdogSource(
+        sandboxRuntimeLimits,
+        input.memoryLimitMiB,
+      ),
+    ),
     shellQuote(runnerPath),
     shellQuote(configPath),
     shellQuote(runtimeStartedPath),