deepline 0.2.1 → 0.2.2

package/dist/bundling-sources/sdk/src/client.ts

@@ -744,7 +744,10 @@ export type MonitorUpdateChangeSummary = {
   };
   upstream: {
     resource_replaced: boolean;
-    strategy: 'unchanged' | 'create_then_delete_previous';
+    strategy:
+      | 'unchanged'
+      | 'create_then_delete_previous'
+      | 'deferred_until_reactivation';
   };
 };
 export type MonitorUpdateResult = {

package/dist/bundling-sources/sdk/src/release.ts

@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: '0.2.1',
+  version: '0.2.2',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/observability/scheduled-job-errors.ts

@@ -0,0 +1,95 @@
+import { redactTelemetryText } from './redaction';
+
+const MAX_ERROR_SUMMARY_LENGTH = 900;
+
+const SECRET_ASSIGNMENT =
+  /\b((?:[A-Za-z][A-Za-z0-9_-]*[_-])?(?:token|secret|password|api[_ -]?key|access[_ -]?key))\b(\s*[:=]\s*)([^\s,;]+)/gi;
+const AUTHORIZATION_VALUE =
+  /\bauthorization\b(\s*[:=]\s*)(?:Bearer\s+)?[^\s,;|]+/gi;
+const BEARER_TOKEN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi;
+const SLACK_WEBHOOK =
+  /https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]+/gi;
+const URL_SECRET = /([?&](?:token|secret|key|signature|sig|code)=)[^&\s]+/gi;
+const BARE_CREDENTIAL_PATTERNS = [
+  /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/gi,
+  /\bgithub_pat_[A-Za-z0-9_]{20,}\b/gi,
+  /\b(?:xaat|xain)-[A-Za-z0-9_-]{16,}\b/gi,
+  /\bsk-ant-(?:api|oat)[A-Za-z0-9_-]{8,}\b/gi,
+  /\bxox(?:a|b|p|r|s)-[A-Za-z0-9-]{10,}\b/gi,
+] as const;
+
+export type ScheduledJobFailure = {
+  jobId: string;
+  jobName: string;
+  scheduler: string;
+  error: unknown;
+  runUrl?: string | null;
+  stage?: string | null;
+  attempt?: number | null;
+  occurredAt?: string;
+  test?: boolean;
+};
+
+export function sanitizeScheduledJobError(
+  error: unknown,
+  maxLength = MAX_ERROR_SUMMARY_LENGTH,
+): string {
+  const raw =
+    error instanceof Error
+      ? `${error.name}: ${error.message}`
+      : typeof error === 'string'
+        ? error
+        : safeJson(error);
+  let compact = redactTelemetryText(raw)
+    .replace(/\u001b\[[0-9;]*m/g, '')
+    .replace(/\r/g, '')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join(' | ')
+    .replace(
+      AUTHORIZATION_VALUE,
+      (_match, separator) => `Authorization${separator}[REDACTED]`,
+    )
+    .replace(BEARER_TOKEN, 'Bearer [REDACTED]')
+    .replace(SLACK_WEBHOOK, '[REDACTED_SLACK_WEBHOOK]')
+    .replace(SECRET_ASSIGNMENT, (_match, label, separator) => {
+      return `${label}${separator}[REDACTED]`;
+    })
+    .replace(URL_SECRET, '$1[REDACTED]')
+    .replace(/\s+/g, ' ')
+    .trim();
+  for (const pattern of BARE_CREDENTIAL_PATTERNS) {
+    compact = compact.replace(pattern, '[REDACTED_CREDENTIAL]');
+  }
+  if (!compact) return 'No error detail was captured.';
+  return compact.length <= maxLength
+    ? compact
+    : `${compact.slice(0, Math.max(0, maxLength - 1))}…`;
+}
+
+export function formatScheduledJobFailure(input: ScheduledJobFailure): string {
+  const title = input.test
+    ? '[TEST] Scheduled job failure alert'
+    : 'Scheduled job failed';
+  return [
+    title,
+    `Job: ${input.jobName} (${input.jobId})`,
+    `Scheduler: ${input.scheduler}`,
+    input.stage ? `Stage: ${input.stage}` : null,
+    input.attempt ? `Attempt: ${input.attempt}` : null,
+    `Error: ${sanitizeScheduledJobError(input.error)}`,
+    `Occurred: ${input.occurredAt ?? new Date().toISOString()}`,
+    input.runUrl ? `Diagnostics: ${input.runUrl}` : null,
+  ]
+    .filter((line): line is string => Boolean(line))
+    .join('\n');
+}
+
+function safeJson(value: unknown): string {
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -9,6 +9,7 @@ import {
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
   validatePlaySandboxRuntimeLimits,
 } from '@shared_libs/play-runtime/sandbox-runtime-limits';
+import type { PlayRunnerRuntimeLifecycleEvent } from '../types';
 
 const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
 const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
@@ -41,6 +42,10 @@ export type DaytonaStageEmitter = (
   extra?: Record<string, unknown>,
 ) => void;
 
+type DaytonaCreateCallObserver = (
+  event: PlayRunnerRuntimeLifecycleEvent,
+) => Promise<void>;
+
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
   daytonaOrganizationId: string;
@@ -104,6 +109,16 @@ type DaytonaCreateResult = {
   attemptElapsedMs: number;
 };
 
+function daytonaCreateErrorClass(
+  error: unknown,
+): 'timeout' | 'capacity' | 'rate_limit' | 'other' {
+  const message = error instanceof Error ? error.message : String(error);
+  if (/timed out|timeout/i.test(message)) return 'timeout';
+  if (/total cpu limit|capacity/i.test(message)) return 'capacity';
+  if (/too many requests|rate limit|429/i.test(message)) return 'rate_limit';
+  return 'other';
+}
+
 async function rejectAcquiredSandbox(
   sandbox: DaytonaSandbox,
   reason: string,
@@ -307,11 +322,13 @@ async function createRetriedOneShotDaytonaSandbox(input: {
   orgId: string;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
+  observeCreateCall?: DaytonaCreateCallObserver;
+  nextProviderAttempt: () => number;
   startedAt: number;
 }): Promise<DaytonaCreateResult> {
   const errors: string[] = [];
   for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
-    const attempt = index + 1;
+    const attempt = input.nextProviderAttempt();
     const sandboxName = `dl-${crypto.randomUUID()}`;
     if (delayMs > 0) {
       input.emitStage('create:retry', {
@@ -323,19 +340,26 @@ async function createRetriedOneShotDaytonaSandbox(input: {
       await new Promise((resolve) => setTimeout(resolve, delayMs));
     }
     const attemptStartedAt = Date.now();
+    await input.observeCreateCall?.({
+      type: 'daytona_create_call_started',
+      occurredAtMs: attemptStartedAt,
+      providerAttempt: attempt,
+    });
+    let sandbox: DaytonaSandbox;
     try {
-      const sandbox = await createOneShotDaytonaSandbox({
+      sandbox = await createOneShotDaytonaSandbox({
         daytona: input.daytona,
         orgId: input.orgId,
         context: input.context,
         sandboxName,
       });
-      return {
-        sandbox,
-        attempt,
-        attemptElapsedMs: Date.now() - attemptStartedAt,
-      };
     } catch (error) {
+      await input.observeCreateCall?.({
+        type: 'daytona_create_call_failed',
+        occurredAtMs: Date.now(),
+        providerAttempt: attempt,
+        errorClass: daytonaCreateErrorClass(error),
+      });
       const message = error instanceof Error ? error.message : String(error);
       errors.push(message);
       input.emitStage('create:attempt_failed', {
@@ -366,7 +390,38 @@ async function createRetriedOneShotDaytonaSandbox(input: {
           { cause: error },
         );
       }
+      continue;
     }
+    // The outcome must be attributed immediately after Daytona acknowledges
+    // creation, before resource-policy validation can reject it. Otherwise a
+    // created-but-rejected sandbox is indistinguishable from an unknown
+    // create result. Do not turn a failed *post-create* journal write into a
+    // new provider create: the sandbox is already real. The missing durable
+    // outcome is deliberately loud in worker logs and makes the capture gate
+    // fail as incomplete telemetry, while the normal resource ledger still
+    // records the known sandbox for cleanup below.
+    const acquiredAt = Date.now();
+    try {
+      await input.observeCreateCall?.({
+        type: 'daytona_create_call_succeeded',
+        occurredAtMs: acquiredAt,
+        providerAttempt: attempt,
+        sandboxId: sandbox.id,
+      });
+    } catch {
+      console.error('[play-runner.daytona.create_lifecycle_event_unrecorded]', {
+        workflowId: input.context.workflowId ?? null,
+        runId: input.context.runId ?? null,
+        attempt,
+        sandboxId: sandbox.id,
+        eventType: 'daytona_create_call_succeeded',
+      });
+    }
+    return {
+      sandbox,
+      attempt,
+      attemptElapsedMs: acquiredAt - attemptStartedAt,
+    };
   }
   const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`;
   const fallbackReason =
@@ -385,6 +440,8 @@ async function acquireOneShotDaytonaSandbox(input: {
   orgId: string;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
+  observeCreateCall?: DaytonaCreateCallObserver;
+  nextProviderAttempt: () => number;
   startedAt: number;
 }): Promise<AcquiredDaytonaSandbox> {
   const limits = validatePlaySandboxRuntimeLimits(
@@ -464,11 +521,13 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
   daytona: DaytonaClient;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
+  observeCreateCall?: DaytonaCreateCallObserver;
   startedAt?: number;
 }): OneShotDaytonaSandboxLifecycle {
   const orgId = validateDaytonaExecutionContext(input.context);
   const startedAt = input.startedAt ?? Date.now();
   let disposed = false;
+  let providerAttempt = 0;
   const acquiredSandboxes = new Map<string, AcquiredDaytonaSandbox>();
   let latestAcquiredSandboxPromise: Promise<AcquiredDaytonaSandbox>;
   const createFreshSandbox = () => {
@@ -477,6 +536,11 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       orgId,
       context: input.context,
       emitStage: input.emitStage,
+      observeCreateCall: input.observeCreateCall,
+      nextProviderAttempt: () => {
+        providerAttempt += 1;
+        return providerAttempt;
+      },
       startedAt,
     }).then((acquired) => {
       acquiredSandboxes.set(acquired.sandbox.id, acquired);

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -670,6 +670,7 @@ function prepareDaytonaExecution(
     context: input.context,
     emitStage: (stage, extra) =>
       emitDaytonaStage(callbacks, input.context, stage, extra),
+    observeCreateCall: callbacks?.onRuntimeLifecycleEvent,
   });
   return {
     kind: 'daytona',

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -40,6 +40,22 @@ export type RuntimeResourceTerminalReason =
   | 'cancelled'
   | 'lease_expired';
 
+/**
+ * Safe, provider-edge lifecycle evidence. The scheduler records these before
+ * relying on a non-idempotent provider outcome; raw provider responses,
+ * commands, inputs, and credentials are intentionally not part of the type.
+ */
+export type PlayRunnerRuntimeLifecycleEvent = {
+  type:
+    | 'daytona_create_call_started'
+    | 'daytona_create_call_succeeded'
+    | 'daytona_create_call_failed';
+  occurredAtMs: number;
+  providerAttempt: number;
+  sandboxId?: string;
+  errorClass?: 'timeout' | 'capacity' | 'rate_limit' | 'other';
+};
+
 export class RuntimeResourceFenceLostError extends Error {
   constructor(message: string) {
     super(message);
@@ -55,6 +71,13 @@ export interface PlayRunnerCallbacks {
   onRuntimeResourceAcquired?: (
     resource: PlayRunnerRuntimeResource,
   ) => void | Promise<void>;
+  /**
+   * Durable scheduler-owned evidence for a provider create call. Backends
+   * await this callback at the create boundary; it is not best-effort logging.
+   */
+  onRuntimeLifecycleEvent?: (
+    event: PlayRunnerRuntimeLifecycleEvent,
+  ) => Promise<void>;
   /**
    * Scheduler-owned readiness read for a detached runner attempt. Daytona uses
    * this direct control-plane port before parking; it is never serialized into

package/dist/cli/index.js

@@ -1040,7 +1040,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.1",
+  version: "0.2.2",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/cli/index.mjs

@@ -1025,7 +1025,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.1",
+  version: "0.2.2",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.d.mts

@@ -1992,7 +1992,7 @@ type MonitorUpdateChangeSummary = {
     };
     upstream: {
         resource_replaced: boolean;
-        strategy: 'unchanged' | 'create_then_delete_previous';
+        strategy: 'unchanged' | 'create_then_delete_previous' | 'deferred_until_reactivation';
     };
 };
 type MonitorUpdateResult = {

package/dist/index.d.ts

@@ -1992,7 +1992,7 @@ type MonitorUpdateChangeSummary = {
     };
     upstream: {
         resource_replaced: boolean;
-        strategy: 'unchanged' | 'create_then_delete_previous';
+        strategy: 'unchanged' | 'create_then_delete_previous' | 'deferred_until_reactivation';
     };
 };
 type MonitorUpdateResult = {

package/dist/index.js

@@ -763,7 +763,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.1",
+  version: "0.2.2",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.mjs

@@ -689,7 +689,7 @@ var SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: "0.2.1",
+  version: "0.2.2",
   contracts: {
     api: {
       name: "sdk-http-api",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {