deepline 0.1.119 → 0.1.121

package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts

@@ -0,0 +1,10 @@
+import { createHash } from 'node:crypto';
+
+export function createRuntimeDatasetId(
+  playName: string,
+  tableNamespace: string,
+): string {
+  return createHash('sha1')
+    .update(`${playName}:${tableNamespace}`)
+    .digest('hex');
+}

package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts

@@ -0,0 +1,50 @@
+import type { Daytona } from '@daytonaio/sdk';
+
+export const DAYTONA_DEFAULT_WORKDIR = '/home/daytona';
+export const DAYTONA_BACKGROUND_CLEANUP_ENV = 'DAYTONA_BACKGROUND_CLEANUP';
+
+export type DaytonaClientOptions = NonNullable<
+  ConstructorParameters<typeof Daytona>[0]
+>;
+
+export type DaytonaRequiredConfig = {
+  apiKey: string;
+  clientOptions: DaytonaClientOptions;
+};
+
+export type DaytonaRunnerPathsConfig = {
+  workdir: string;
+};
+
+export function daytonaClientOptions(apiKey: string): DaytonaClientOptions {
+  return {
+    apiKey,
+    ...(process.env.DAYTONA_API_URL?.trim()
+      ? { apiUrl: process.env.DAYTONA_API_URL.trim() }
+      : {}),
+    ...(process.env.DAYTONA_TARGET?.trim()
+      ? { target: process.env.DAYTONA_TARGET.trim() }
+      : {}),
+  };
+}
+
+export function shouldCleanupSandboxInBackground(): boolean {
+  return process.env[DAYTONA_BACKGROUND_CLEANUP_ENV]?.trim() === '1';
+}
+
+export function loadDaytonaRequiredConfig(): DaytonaRequiredConfig {
+  const apiKey = process.env.DAYTONA_API_KEY?.trim();
+  if (!apiKey) {
+    throw new Error('Missing required Daytona configuration: DAYTONA_API_KEY.');
+  }
+  return {
+    apiKey,
+    clientOptions: daytonaClientOptions(apiKey),
+  };
+}
+
+export function loadDaytonaRunnerPathsConfig(): DaytonaRunnerPathsConfig {
+  return {
+    workdir: process.env.DAYTONA_WORKDIR?.trim() || DAYTONA_DEFAULT_WORKDIR,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts

@@ -0,0 +1,20 @@
+export type MapRowOutcomeStatus = 'completed' | 'failed';
+
+export type MapRowOutcome = {
+  key: string;
+  /**
+   * Original row position within this dataset invocation. The deterministic
+   * key is the primary identity; inputIndex is a same-run repair identity for
+   * completion writes after the start phase has already stamped `_run_id`.
+   */
+  inputIndex?: number | null;
+  data: Record<string, unknown>;
+  cellMetaPatch?: Record<string, unknown>;
+  /**
+   * Terminal row status for this write. Defaults to completed. Failed rows keep
+   * partial data, persist `_error`, and are returned as pending on the next run.
+   */
+  status?: MapRowOutcomeStatus;
+  /** Row-level error message persisted when `status` is failed. */
+  error?: string | null;
+};

package/dist/bundling-sources/shared_libs/play-runtime/event-wait-tools.ts

@@ -0,0 +1,9 @@
+export const EVENT_WAIT_TOOL_IDS: ReadonlySet<string> = new Set([
+  'slack_message_with_hitl',
+  'slack_send_message_with_hitl',
+  'test_wait_for_event',
+]);
+
+export function isIntegrationEventWaitToolId(toolId: string): boolean {
+  return EVENT_WAIT_TOOL_IDS.has(toolId.trim());
+}

package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts

@@ -0,0 +1,171 @@
+import type {
+  PacingPermit,
+  PacingRule,
+  RateStateBackend,
+} from './rate-state-backend';
+
+/**
+ * In-memory Rate State Backend for the single-process `cjs_node20` runner.
+ *
+ * In one Node process the local buckets ARE the global state, so this is the
+ * correct, zero-round-trip backend. The window/concurrency math is lifted from
+ * the previous `PlayRateLimitScheduler` (sliding window + per-rule concurrency +
+ * serialized acquire), plus a `penalize` cooldown the scheduler lacked so the
+ * in-process runner also honors a server-observed Retry-After.
+ */
+
+const MIN_CONCURRENCY_WAIT_MS = 10;
+
+interface RuleState {
+  windowStartedAt: number;
+  startedInWindow: number;
+  activeCount: number;
+}
+
+interface Options {
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+}
+
+export class InMemoryRateStateBackend implements RateStateBackend {
+  private readonly ruleStates = new Map<string, RuleState>();
+  private readonly cooldownUntilByBucket = new Map<string, number>();
+  private lock: Promise<void> = Promise.resolve();
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+
+  constructor(options: Options = {}) {
+    this.now = options.now ?? (() => Date.now());
+    this.sleep =
+      options.sleep ??
+      ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+  }
+
+  async acquire(input: {
+    bucketId: string;
+    rules: readonly PacingRule[];
+    signal?: AbortSignal;
+  }): Promise<PacingPermit> {
+    const { bucketId, rules, signal } = input;
+    if (rules.length === 0) {
+      return { release() {} };
+    }
+    // Stable order to avoid deadlock between concurrent acquirers on shared rules.
+    const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
+
+    while (true) {
+      if (signal?.aborted) {
+        throw signal.reason instanceof Error
+          ? signal.reason
+          : new Error('Rate-state acquire aborted.');
+      }
+      const decision = await this.withLock(() => {
+        const now = this.now();
+        let waitMs = 0;
+
+        const cooldownUntil = this.cooldownUntilByBucket.get(bucketId) ?? 0;
+        if (cooldownUntil > now) waitMs = Math.max(waitMs, cooldownUntil - now);
+
+        for (const rule of ordered) {
+          const state = this.getRuleState(bucketId, rule, now);
+          this.resetExpiredWindow(state, rule.windowMs, now);
+          if (state.startedInWindow >= rule.requestsPerWindow) {
+            waitMs = Math.max(waitMs, state.windowStartedAt + rule.windowMs - now);
+          }
+          if (rule.maxConcurrency != null && state.activeCount >= rule.maxConcurrency) {
+            waitMs = Math.max(waitMs, MIN_CONCURRENCY_WAIT_MS);
+          }
+        }
+
+        if (waitMs > 0) return { acquired: false, waitMs: Math.max(1, waitMs) } as const;
+
+        for (const rule of ordered) {
+          const state = this.getRuleState(bucketId, rule, now);
+          state.startedInWindow += 1;
+          if (rule.maxConcurrency != null) state.activeCount += 1;
+        }
+        return { acquired: true, waitMs: 0 } as const;
+      });
+
+      if (decision.acquired) {
+        let released = false;
+        return {
+          // The concurrency decrement is intentionally not awaitable (the permit
+          // signature is sync `release(): void`). It still runs serialized under
+          // the lock; do not "fix" this by trying to await a void.
+          release: () => {
+            if (released) return;
+            released = true;
+            void this.withLock(() => {
+              for (const rule of ordered) {
+                if (rule.maxConcurrency == null) continue;
+                const state = this.ruleStates.get(this.key(bucketId, rule.ruleId));
+                if (state && state.activeCount > 0) state.activeCount -= 1;
+              }
+            });
+          },
+        };
+      }
+      await this.sleep(decision.waitMs);
+    }
+  }
+
+  penalize(input: { bucketId: string; cooldownMs: number }): void {
+    if (input.cooldownMs <= 0) return;
+    const until = this.now() + input.cooldownMs;
+    const existing = this.cooldownUntilByBucket.get(input.bucketId) ?? 0;
+    this.cooldownUntilByBucket.set(input.bucketId, Math.max(existing, until));
+  }
+
+  private key(bucketId: string, ruleId: string): string {
+    return `${bucketId}::${ruleId}`;
+  }
+
+  private getRuleState(
+    bucketId: string,
+    rule: PacingRule,
+    now: number,
+  ): RuleState {
+    const key = this.key(bucketId, rule.ruleId);
+    const existing = this.ruleStates.get(key);
+    if (existing) return existing;
+    const created: RuleState = {
+      windowStartedAt: now,
+      startedInWindow: 0,
+      activeCount: 0,
+    };
+    this.ruleStates.set(key, created);
+    return created;
+  }
+
+  private resetExpiredWindow(
+    state: RuleState,
+    windowMs: number,
+    now: number,
+  ): void {
+    if (windowMs <= 0) {
+      state.windowStartedAt = now;
+      state.startedInWindow = 0;
+      return;
+    }
+    if (now - state.windowStartedAt < windowMs) return;
+    const elapsed = Math.floor((now - state.windowStartedAt) / windowMs);
+    state.windowStartedAt += elapsed * windowMs;
+    state.startedInWindow = 0;
+    if (now - state.windowStartedAt >= windowMs) state.windowStartedAt = now;
+  }
+
+  private async withLock<T>(fn: () => T | Promise<T>): Promise<T> {
+    const previous = this.lock;
+    let releaseLock!: () => void;
+    this.lock = new Promise<void>((resolve) => {
+      releaseLock = resolve;
+    });
+    await previous;
+    try {
+      return await fn();
+    } finally {
+      releaseLock();
+    }
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-diagnosis.ts

@@ -0,0 +1,321 @@
+import {
+  HATCHET_COLD_EXECUTION_PHASES,
+  HATCHET_COLD_EXECUTION_TARGET,
+  type HatchetColdExecutionPhaseId,
+  type HatchetColdExecutionPhaseSummary,
+} from './hatchet-cold-execution-target';
+
+export type HatchetColdExecutionPhaseSeverity =
+  | 'missing'
+  | 'ok'
+  | 'near_budget'
+  | 'over_budget';
+
+export type HatchetColdExecutionPhaseDiagnosis = {
+  id: HatchetColdExecutionPhaseId;
+  phase: string;
+  budgetMs: number;
+  p95: number | null;
+  overBudgetMs: number | null;
+  severity: HatchetColdExecutionPhaseSeverity;
+  ownerModule: string;
+  ownerFiles: readonly string[];
+  interpretation: string;
+  nextAction: string;
+};
+
+export type HatchetColdExecutionDiagnosis = {
+  ok: boolean;
+  targetName: string;
+  thresholdMs: number;
+  overallP95: number | null;
+  overallOverThresholdMs: number | null;
+  bottleneck: HatchetColdExecutionPhaseDiagnosis | null;
+  phases: HatchetColdExecutionPhaseDiagnosis[];
+  observations: string[];
+  nextActions: string[];
+  summary: string;
+};
+
+type DiagnoseInput = {
+  overallP95: number | null;
+  thresholdMs?: number;
+  phaseSummaries: readonly HatchetColdExecutionPhaseSummary[];
+};
+
+type PhaseOwnership = {
+  ownerModule: string;
+  ownerFiles: readonly string[];
+  interpretation: string;
+  nextAction: string;
+};
+
+const PHASE_OWNERSHIP: Record<HatchetColdExecutionPhaseId, PhaseOwnership> = {
+  submit: {
+    ownerModule: 'Play Run Launch Plan',
+    ownerFiles: [
+      'src/lib/plays/start-run.ts',
+      'src/lib/plays/launch/',
+      'scripts/check-hatchet-cold-execution-latency.ts',
+    ],
+    interpretation:
+      'Submit time is being spent before the worker can claim the run: app validation, authority minting, start-row persistence, scheduler-row creation, or Hatchet dispatch.',
+    nextAction:
+      'Inspect routeTimings plus schema_ready / scheduler_row_submitted / run_no_wait_done deltas, then move non-launch work out of the hot path or overlap insert and dispatch.',
+  },
+  hatchet_dispatch_claim: {
+    ownerModule: 'Play Scheduler Adapter',
+    ownerFiles: [
+      'src/lib/plays/scheduler-backends/hatchet-dispatch.ts',
+      'src/lib/plays/scheduler-backends/hatchet-runtime-worker.ts',
+      'deploy/fly/hatchet/',
+    ],
+    interpretation:
+      'Hatchet assignment or worker claim is slower than budget, so the run is waiting in orchestration before Daytona work begins.',
+    nextAction:
+      'Check Hatchet engine placement, worker replica health, worker slots, and dispatch mode before changing Daytona or payload transport.',
+  },
+  daytona_create: {
+    ownerModule: 'One-Shot Sandbox Runner',
+    ownerFiles: [
+      'apps/temporal-worker/src/runner-backends/backends/daytona.ts',
+      'apps/temporal-worker/src/runner-backends/backends/daytona-lifecycle.ts',
+    ],
+    interpretation:
+      'Fresh Daytona sandbox creation is consuming the cold-execution budget before runner files upload.',
+    nextAction:
+      'Compare default image versus configured snapshot startup, then verify create timing in Daytona lifecycle events before touching artifact upload.',
+  },
+  daytona_upload: {
+    ownerModule: 'Play Artifact Target',
+    ownerFiles: [
+      'apps/temporal-worker/src/runner-backends/backends/daytona-payload-transport.ts',
+      'shared_libs/plays/artifact-transport.ts',
+      'apps/play-runner/src/entry.ts',
+    ],
+    interpretation:
+      'Runner bundle, play artifact, config, or materialized-file staging is now the dominant transfer cost into the one-shot sandbox.',
+    nextAction:
+      'Measure uploaded bytes by label, then reduce runner/artifact/config bytes or move immutable runner material into the snapshot without changing the one-shot sandbox Interface.',
+  },
+  daytona_execute: {
+    ownerModule: 'One-Shot Sandbox Runner',
+    ownerFiles: [
+      'apps/temporal-worker/src/runner-backends/backends/daytona.ts',
+      'apps/play-runner/src/entry.ts',
+      'shared_libs/play-runtime/protocol.ts',
+    ],
+    interpretation:
+      'The sandbox command is slow after files are present, usually runner boot, artifact load/decode, module startup, or the hello-world execution path.',
+    nextAction:
+      'Inspect the captured runner output and runtimeTiming; shrink startup imports or shift decode/load work into upload/snapshot prep instead of adding pools.',
+  },
+  terminal_observation: {
+    ownerModule: 'Run Snapshot',
+    ownerFiles: [
+      'src/lib/plays/scheduler-backends/hatchet-completion-publisher.ts',
+      'src/lib/plays/scheduler-backends/postgres-progress.ts',
+      'convex/model/runLogStream.ts',
+    ],
+    interpretation:
+      'The worker has reached terminal state, but the customer-visible observer is seeing it late.',
+    nextAction:
+      'Check terminal write ordering, NOTIFY/direct observer connection, and polling fallback before changing execution or scheduler code.',
+  },
+};
+
+export function diagnoseHatchetColdExecution(
+  input: DiagnoseInput,
+): HatchetColdExecutionDiagnosis {
+  const thresholdMs =
+    input.thresholdMs ?? HATCHET_COLD_EXECUTION_TARGET.thresholdMs;
+  const summariesById = new Map(
+    input.phaseSummaries.map((summary) => [summary.id, summary]),
+  );
+  const phases = HATCHET_COLD_EXECUTION_PHASES.map((definition) => {
+    const summary = summariesById.get(definition.id);
+    const p95 = summary?.p95 ?? null;
+    const overBudgetMs =
+      p95 === null ? null : Math.max(0, p95 - definition.budgetMs);
+    const severity = phaseSeverity(p95, definition.budgetMs);
+    return {
+      id: definition.id,
+      phase: definition.phase,
+      budgetMs: definition.budgetMs,
+      p95,
+      overBudgetMs,
+      severity,
+      ...PHASE_OWNERSHIP[definition.id],
+    };
+  });
+
+  const overallOverThresholdMs =
+    input.overallP95 === null
+      ? null
+      : Math.max(0, input.overallP95 - thresholdMs);
+  const ok = input.overallP95 !== null && input.overallP95 <= thresholdMs;
+  const bottleneck = chooseBottleneck(phases);
+  const observations = buildObservations({
+    ok,
+    thresholdMs,
+    overallP95: input.overallP95,
+    overallOverThresholdMs,
+    phases,
+    bottleneck,
+  });
+  const nextActions = buildNextActions(bottleneck, phases);
+
+  return {
+    ok,
+    targetName: HATCHET_COLD_EXECUTION_TARGET.name,
+    thresholdMs,
+    overallP95: input.overallP95,
+    overallOverThresholdMs,
+    bottleneck,
+    phases,
+    observations,
+    nextActions,
+    summary: buildSummary({ ok, input, thresholdMs, bottleneck }),
+  };
+}
+
+export function formatHatchetColdExecutionDiagnosis(
+  diagnosis: HatchetColdExecutionDiagnosis,
+): string[] {
+  const lines = [
+    `[hatchet-cold-diagnosis] ${diagnosis.summary}`,
+    ...diagnosis.observations.map(
+      (observation) => `[hatchet-cold-diagnosis] ${observation}`,
+    ),
+  ];
+  if (diagnosis.bottleneck) {
+    lines.push(
+      `[hatchet-cold-diagnosis] owner=${diagnosis.bottleneck.ownerModule}`,
+      `[hatchet-cold-diagnosis] next=${diagnosis.bottleneck.nextAction}`,
+    );
+  }
+  return lines;
+}
+
+function phaseSeverity(
+  p95: number | null,
+  budgetMs: number,
+): HatchetColdExecutionPhaseSeverity {
+  if (p95 === null) return 'missing';
+  if (p95 > budgetMs) return 'over_budget';
+  if (p95 >= Math.floor(budgetMs * 0.85)) return 'near_budget';
+  return 'ok';
+}
+
+function chooseBottleneck(
+  phases: readonly HatchetColdExecutionPhaseDiagnosis[],
+): HatchetColdExecutionPhaseDiagnosis | null {
+  const overBudget = phases
+    .filter((phase) => phase.severity === 'over_budget')
+    .sort(
+      (left, right) =>
+        (right.overBudgetMs ?? 0) - (left.overBudgetMs ?? 0) ||
+        (right.p95 ?? 0) - (left.p95 ?? 0),
+    );
+  if (overBudget[0]) return overBudget[0];
+
+  const nearBudget = phases
+    .filter((phase) => phase.severity === 'near_budget')
+    .sort((left, right) => (right.p95 ?? 0) - (left.p95 ?? 0));
+  if (nearBudget[0]) return nearBudget[0];
+
+  const missing = phases.find((phase) => phase.severity === 'missing');
+  return missing ?? null;
+}
+
+function buildObservations(input: {
+  ok: boolean;
+  thresholdMs: number;
+  overallP95: number | null;
+  overallOverThresholdMs: number | null;
+  phases: readonly HatchetColdExecutionPhaseDiagnosis[];
+  bottleneck: HatchetColdExecutionPhaseDiagnosis | null;
+}): string[] {
+  const observations: string[] = [];
+  if (input.overallP95 === null) {
+    observations.push(
+      'No overall p95 was observed; latency target cannot be evaluated.',
+    );
+  } else if (input.ok) {
+    observations.push(
+      `Overall p95=${input.overallP95}ms is within threshold=${input.thresholdMs}ms.`,
+    );
+  } else {
+    observations.push(
+      `Overall p95=${input.overallP95}ms exceeds threshold=${input.thresholdMs}ms by ${input.overallOverThresholdMs}ms.`,
+    );
+  }
+
+  const overBudget = input.phases.filter(
+    (phase) => phase.severity === 'over_budget',
+  );
+  if (overBudget.length > 0) {
+    observations.push(
+      `Over-budget phases: ${overBudget
+        .map(
+          (phase) =>
+            `${phase.id} p95=${phase.p95}ms budget=${phase.budgetMs}ms (+${phase.overBudgetMs}ms)`,
+        )
+        .join(', ')}.`,
+    );
+  }
+
+  const missing = input.phases.filter((phase) => phase.severity === 'missing');
+  if (missing.length > 0) {
+    observations.push(
+      `Missing phase samples: ${missing.map((phase) => phase.id).join(', ')}.`,
+    );
+  }
+
+  if (input.bottleneck) {
+    observations.push(
+      `Primary bottleneck: ${input.bottleneck.id} owned by ${input.bottleneck.ownerModule}.`,
+    );
+  }
+  return observations;
+}
+
+function buildNextActions(
+  bottleneck: HatchetColdExecutionPhaseDiagnosis | null,
+  phases: readonly HatchetColdExecutionPhaseDiagnosis[],
+): string[] {
+  const actions: string[] = [];
+  if (bottleneck) {
+    actions.push(bottleneck.nextAction);
+  }
+  const missing = phases.find((phase) => phase.severity === 'missing');
+  if (missing) {
+    actions.push(
+      'Fix missing runtimeTiming/phase sample emission before trusting the latency diagnosis.',
+    );
+  }
+  if (actions.length === 0) {
+    actions.push(
+      'Keep the current phase budgets and rerun the same gate after the next runtime change.',
+    );
+  }
+  return [...new Set(actions)];
+}
+
+function buildSummary(input: {
+  ok: boolean;
+  input: DiagnoseInput;
+  thresholdMs: number;
+  bottleneck: HatchetColdExecutionPhaseDiagnosis | null;
+}): string {
+  if (input.input.overallP95 === null) {
+    return `${HATCHET_COLD_EXECUTION_TARGET.name}: inconclusive; no overall p95 sample.`;
+  }
+  if (input.ok) {
+    return `${HATCHET_COLD_EXECUTION_TARGET.name}: pass at p95=${input.input.overallP95}ms.`;
+  }
+  const owner = input.bottleneck
+    ? `; primary owner ${input.bottleneck.ownerModule}`
+    : '';
+  return `${HATCHET_COLD_EXECUTION_TARGET.name}: fail at p95=${input.input.overallP95}ms over threshold=${input.thresholdMs}ms${owner}.`;
+}