deepline 0.1.288 → 0.1.289

package/dist/bundling-sources/sdk/src/release.ts

@@ -155,7 +155,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.288',
+  version: '0.1.289',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -27,7 +27,8 @@ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
 
 export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
 
-export type DaytonaClient = Pick<Daytona, 'create'>;
+export type DaytonaClient = Pick<Daytona, 'create'> &
+  Partial<Pick<Daytona, 'get'>>;
 export type DaytonaSandbox = Awaited<ReturnType<DaytonaClient['create']>>;
 export type DaytonaExecutionContext = PlayRunnerExecutionConfig['context'];
 export type DaytonaStageEmitter = (
@@ -37,6 +38,7 @@ export type DaytonaStageEmitter = (
 
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
+  daytonaOrganizationId: string;
   billingStartedAt: number;
   billingEndedAt?: number;
 };
@@ -56,6 +58,22 @@ type DaytonaCreateResult = {
   attemptElapsedMs: number;
 };
 
+async function rejectAcquiredSandbox(
+  sandbox: DaytonaSandbox,
+  reason: string,
+): Promise<never> {
+  try {
+    await sandbox.delete(30);
+  } catch (error) {
+    const cleanupError = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `${reason} Defensive deletion of Daytona sandbox ${sandbox.id} also failed: ${cleanupError}`,
+      { cause: error },
+    );
+  }
+  throw new Error(reason);
+}
+
 function normalizeLabelValue(value: string | null | undefined): string | null {
   const trimmed = value?.trim();
   return trimmed ? trimmed.slice(0, 63) : null;
@@ -233,7 +251,15 @@ async function createRetriedOneShotDaytonaSandbox(input: {
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       errors.push(message);
+      input.emitStage('create:attempt_failed', {
+        attempt,
+        elapsedMs: Date.now() - input.startedAt,
+        attemptElapsedMs: Date.now() - attemptStartedAt,
+        error: message,
+      });
       console.warn('[play-runner.daytona.create_attempt_failed]', {
+        workflowId: input.context.workflowId ?? null,
+        runId: input.context.runId ?? null,
         attempt,
         error: message,
       });
@@ -265,11 +291,46 @@ async function acquireOneShotDaytonaSandbox(input: {
     granted.diskGiB !== DAYTONA_SANDBOX_DISK_GIB ||
     granted.gpu !== DAYTONA_SANDBOX_GPU
   ) {
-    await result.sandbox.delete(30).catch(() => undefined);
-    throw new Error(
+    await rejectAcquiredSandbox(
+      result.sandbox,
       `Daytona sandbox resource boundary mismatch: expected cpu=${DAYTONA_SANDBOX_CPU} memoryGiB=${DAYTONA_SANDBOX_MEMORY_GIB} diskGiB=${DAYTONA_SANDBOX_DISK_GIB} gpu=${DAYTONA_SANDBOX_GPU}, granted cpu=${granted.cpu} memoryGiB=${granted.memoryGiB} diskGiB=${granted.diskGiB} gpu=${granted.gpu}`,
     );
   }
+  const configuredOrganizationId =
+    process.env.DAYTONA_ORGANIZATION_ID?.trim() || null;
+  const observedOrganizationId = result.sandbox.organizationId?.trim() || null;
+  if (
+    configuredOrganizationId &&
+    observedOrganizationId &&
+    configuredOrganizationId !== observedOrganizationId
+  ) {
+    await rejectAcquiredSandbox(
+      result.sandbox,
+      'Daytona sandbox organization routing mismatch. Refusing to run customer code in a sandbox whose observed organization differs from the configured organization.',
+    );
+  }
+  let lookupOrganizationId: string | null = null;
+  if (
+    !observedOrganizationId &&
+    !configuredOrganizationId &&
+    input.daytona.get
+  ) {
+    try {
+      const lookedUpSandbox = await input.daytona.get(result.sandbox.id);
+      lookupOrganizationId = lookedUpSandbox.organizationId?.trim() || null;
+    } catch {
+      // The failure below is intentionally about the invariant, not the
+      // provider response. The newly created sandbox is still deleted.
+    }
+  }
+  const daytonaOrganizationId =
+    observedOrganizationId ?? configuredOrganizationId ?? lookupOrganizationId;
+  if (!daytonaOrganizationId) {
+    return await rejectAcquiredSandbox(
+      result.sandbox,
+      'Daytona sandbox organization routing identity is missing. Refusing to run customer code without a durable cleanup routing domain.',
+    );
+  }
   const billingStartedAt = Date.now();
   const sandbox = result.sandbox;
   input.emitStage('create:done', {
@@ -281,7 +342,7 @@ async function acquireOneShotDaytonaSandbox(input: {
     memoryGiB: granted.memoryGiB,
     diskGiB: granted.diskGiB,
   });
-  return { sandbox, billingStartedAt };
+  return { sandbox, daytonaOrganizationId, billingStartedAt };
 }
 
 export function createOneShotDaytonaSandboxLifecycle(input: {

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -61,6 +61,7 @@ export type StagedDaytonaPayload = {
   command: string;
   outputPath: string;
   exitCodePath: string;
+  runtimeCompletedPath: string;
   progressEventPath: string;
 };
 
@@ -557,6 +558,7 @@ export async function stageDaytonaRunnerPayload(input: {
     command,
     outputPath,
     exitCodePath,
+    runtimeCompletedPath,
     progressEventPath,
   };
 }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -289,27 +289,161 @@ export async function inspectDetachedDaytonaRunner(input: {
   }
 }
 
-export async function deleteDaytonaSandboxById(input: {
+/** Read the runner's exact customer-code completion fence before terminal GC. */
+export async function readDetachedDaytonaRuntimeCompletion(input: {
+  sandboxId: string;
+  runtimeCompletedPath: string;
+}): Promise<number | null> {
+  try {
+    const { clientOptions } = loadDaytonaRequiredConfig();
+    const sandbox = (await daytonaSdkClientFactory
+      .createFull(clientOptions)
+      .get(input.sandboxId)) as DaytonaSandbox;
+    const marker = JSON.parse(
+      (await sandbox.fs.downloadFile(input.runtimeCompletedPath, 5)).toString(
+        'utf-8',
+      ),
+    ) as { at?: unknown };
+    return typeof marker.at === 'number' && Number.isFinite(marker.at)
+      ? marker.at
+      : null;
+  } catch (error) {
+    console.warn(
+      '[play-runner.daytona.runtime_completion_marker_unavailable]',
+      {
+        sandboxId: input.sandboxId,
+        // Path is generated per attempt and contains no customer data.
+        runtimeCompletedPath: input.runtimeCompletedPath,
+        error: error instanceof Error ? error.message : String(error),
+      },
+    );
+    return null;
+  }
+}
+
+export type DaytonaSandboxDeleteOutcome =
+  | {
+      kind: 'deleted' | 'already_absent';
+      organizationId: string | null;
+    }
+  | {
+      kind: 'timed_out' | 'rate_limited' | 'failed';
+      organizationId: string | null;
+      code: string;
+      detail: string;
+    };
+
+export async function deleteDaytonaSandboxByIdWithOutcome(input: {
   sandboxId: string;
   timeoutSeconds?: number;
-}): Promise<boolean> {
+  expectedOrganizationId?: string | null;
+  allowUnscopedAlreadyAbsent?: boolean;
+}): Promise<DaytonaSandboxDeleteOutcome> {
   const sandboxId = input.sandboxId?.trim();
-  if (!sandboxId) return false;
+  const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
+  if (!sandboxId) {
+    return {
+      kind: 'failed',
+      organizationId: expectedOrganizationId,
+      code: 'invalid_sandbox_id',
+      detail: 'Sandbox ID is required.',
+    };
+  }
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
     const daytona = daytonaSdkClientFactory.createFull(clientOptions);
     const sandbox = await daytona.get(sandboxId);
+    const observedOrganizationId = sandbox.organizationId?.trim() || null;
+    if (
+      expectedOrganizationId &&
+      observedOrganizationId !== expectedOrganizationId
+    ) {
+      return {
+        kind: 'failed',
+        organizationId: observedOrganizationId,
+        code: 'wrong_routing_domain',
+        detail: 'Sandbox belongs to a different Daytona organization.',
+      };
+    }
     await daytona.delete(sandbox, input.timeoutSeconds ?? 30);
-    return true;
+    return {
+      kind: 'deleted',
+      organizationId: observedOrganizationId,
+    };
   } catch (error) {
+    const failure = describeDaytonaLookupFailure(error);
+    // Cleanup is an idempotent "ensure absent" operation. Daytona returning
+    // not-found means another cleanup owner already satisfied the obligation.
+    // The durable expected organization came from the sandbox returned by the
+    // same organization-scoped credential at creation time. When an explicit
+    // worker organization is configured it must still match that evidence;
+    // deployments which rely only on the provider-returned organization retain
+    // that durable creation-domain proof. Legacy eager cleanup retains its
+    // previous unscoped behavior through the explicit compatibility option.
+    if (failure.httpStatus === 404) {
+      const configuredOrganizationId =
+        process.env.DAYTONA_ORGANIZATION_ID?.trim() || null;
+      if (
+        !input.allowUnscopedAlreadyAbsent &&
+        (!expectedOrganizationId ||
+          (configuredOrganizationId &&
+            configuredOrganizationId !== expectedOrganizationId))
+      ) {
+        return {
+          kind: 'failed',
+          organizationId: configuredOrganizationId,
+          code: expectedOrganizationId
+            ? 'wrong_routing_domain'
+            : 'missing_routing_domain',
+          detail:
+            'Daytona returned not-found without an exact creation-domain match.',
+        };
+      }
+      console.info('[play-runner.daytona.reclaim_sandbox_already_absent]', {
+        sandboxId,
+      });
+      return {
+        kind: 'already_absent',
+        organizationId:
+          configuredOrganizationId ?? expectedOrganizationId ?? null,
+      };
+    }
     console.warn('[play-runner.daytona.reclaim_sandbox_delete_failed]', {
       sandboxId,
-      error: error instanceof Error ? error.message : String(error),
+      failure,
     });
-    return false;
+    const timedOut =
+      failure.httpStatus === 408 ||
+      /(?:timeout|timed out|ETIMEDOUT)/i.test(
+        `${failure.errorCode ?? ''} ${failure.detail}`,
+      );
+    return {
+      kind:
+        failure.httpStatus === 429
+          ? 'rate_limited'
+          : timedOut
+            ? 'timed_out'
+            : 'failed',
+      organizationId: expectedOrganizationId,
+      code:
+        failure.errorCode ??
+        (failure.httpStatus ? `http_${failure.httpStatus}` : 'delete_failed'),
+      detail: failure.detail,
+    };
   }
 }
 
+export async function deleteDaytonaSandboxById(input: {
+  sandboxId: string;
+  timeoutSeconds?: number;
+}): Promise<boolean> {
+  const outcome = await deleteDaytonaSandboxByIdWithOutcome({
+    ...input,
+    allowUnscopedAlreadyAbsent: true,
+  });
+  return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
+}
+
 function formatDaytonaExecutionError(error: unknown): string {
   const message = formatDaytonaError(error);
   return isConfiguredDaytonaRuntimeLimit(error)
@@ -630,6 +764,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     let cancellationCleanupStarted = false;
     let activeAcquiredResource: {
       sandbox: DaytonaSandbox;
+      daytonaOrganizationId: string;
       billingStartedAt: number;
       billingEndedAt?: number;
     } | null = null;
@@ -639,6 +774,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     const runtimeResourceReportErrors = new Set<unknown>();
     const reportRuntimeResource = async (acquired: {
       sandbox: DaytonaSandbox;
+      daytonaOrganizationId: string;
       billingStartedAt: number;
       billingEndedAt?: number;
     }) => {
@@ -658,6 +794,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
               ? 'preview'
               : 'production',
+          daytonaOrganizationId: acquired.daytonaOrganizationId,
           billingStartedAt: acquired.billingStartedAt,
           billingEndedAt,
           cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
@@ -673,6 +810,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     };
     const reportRetiringRuntimeResource = async (acquired: {
       sandbox: DaytonaSandbox;
+      daytonaOrganizationId: string;
       billingStartedAt: number;
       billingEndedAt?: number;
     }) => {
@@ -929,6 +1067,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               cmdId: start.cmdId,
               outputPath: stagedPayload.outputPath,
               exitCodePath: stagedPayload.exitCodePath,
+              runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
               startedAtMs: Date.now(),
               ceilingMs: DAYTONA_DETACHED_CEILING_SECONDS * 1_000,
             },
@@ -988,8 +1127,28 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
       // Resource persistence belongs to the scheduler control plane. Preserve
       // its typed capacity/fence errors so Absurd can defer or fence the
       // attempt; converting them into a runner failure would terminally fail a
-      // play that never began executing customer code.
-      if (runtimeResourceReportErrors.has(error)) throw error;
+      // play that never began executing customer code. The sandbox exists
+      // before that durable callback can succeed, so synchronously delete it
+      // before returning the control-plane error. The old fire-and-forget
+      // cleanup path was skipped by this rethrow and leaked the acquisition.
+      if (runtimeResourceReportErrors.has(error)) {
+        const sandbox = sandboxCleanup.currentSandbox();
+        if (sandbox) {
+          try {
+            await sandbox.delete(30);
+            console.info(
+              '[play-runner.daytona.resource_report_failure_cleanup_done]',
+              { sandboxId: sandbox.id },
+            );
+          } catch (cleanupError) {
+            throw new AggregateError(
+              [error, cleanupError],
+              `Failed to persist or delete acquired Daytona sandbox ${sandbox.id}.`,
+            );
+          }
+        }
+        throw error;
+      }
       emitDaytonaStage(callbacks, config.context, 'execute:error', {
         sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
         error: formatDaytonaError(error),

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -13,6 +13,8 @@ export type PlayRunnerRuntimeResource = {
   kind: 'daytona_sandbox';
   sandboxId: string;
   daytonaEnvironment?: 'preview' | 'production';
+  /** Stable, non-secret provider ownership domain returned by Daytona. */
+  daytonaOrganizationId?: string;
   billingStartedAt: number;
   billingEndedAt?: number | null;
   terminalReason?: RuntimeResourceTerminalReason | null;

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -39,6 +39,8 @@ export type PlayExecutionSuspension =
        * timeout-wake salvage verification. */
       outputPath: string;
       exitCodePath: string;
+      /** Exact customer-code completion fence inside the Daytona sandbox. */
+      runtimeCompletedPath?: string;
       startedAtMs: number;
       /** Overall run ceiling; the park timeout. */
       ceilingMs: number;

package/dist/cli/index.js

@@ -718,7 +718,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.288",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/cli/index.mjs

@@ -703,7 +703,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.288",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.js

@@ -438,7 +438,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.288",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.mjs

@@ -367,7 +367,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.288",
+  version: "0.1.289",
   contracts: {
     api: {
       name: "sdk-http-api",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.288",
+  "version": "0.1.289",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {