deepline 0.0.1 → 0.1.1

package/dist/repo/shared_libs/play-runtime/runtime-constraints.ts

@@ -0,0 +1,2 @@
+export const DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE =
+  'No run_javascript tool in plays. Just write regular code.';

package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts

@@ -0,0 +1,201 @@
+/**
+ * `@neondatabase/serverless` Pool factory. Installed at boot by the
+ * workers_edge dynamic Worker harness. The serverless driver speaks the
+ * same Postgres wire protocol as `pg` but uses Neon's WebSocket proxy
+ * (or HTTP endpoint) so it works inside the workerd V8 isolate where raw
+ * TCP is unavailable.
+ *
+ * ## Lazy loading (the actual implementation)
+ *
+ * `@neondatabase/serverless` is ~143 KB minified — a non-trivial fraction
+ * of every per-graphHash play Worker bundle. The harness's
+ * `installNeonServerlessRuntimePoolDriver()` fires unconditionally at
+ * isolate startup, but the heavy module is loaded ON FIRST `pool.connect()`
+ * via dynamic `import('@neondatabase/serverless')`. Plays that never open
+ * a Postgres pool (csv-only, computation-only, tool-only plays) skip the
+ * import entirely, even though the driver registration ran at boot.
+ *
+ * Concretely:
+ *   - Bundle still contains the dynamic-import path → `@neondatabase/serverless`
+ *     is still in the per-play bundle (esbuild emits it as a deferred chunk
+ *     in `format: 'esm'`, but workerd inlines all chunks for a Worker —
+ *     so the bytes are still in the worker's compiled code).
+ *   - V8 PARSE / COMPILE for those bytes is also deferred until the first
+ *     `import()` call resolves; modern V8 does parse-on-demand for ESM
+ *     dynamic imports. So plays that never connect skip the parse cost.
+ *   - Plays that DO connect pay the import cost once per isolate, then
+ *     reuse the cached module + pool for the rest of the run.
+ *
+ * If you change this file, double-check by:
+ *   1. Bundling a play that doesn't use `ctx.dataset` / postgres ops
+ *      (e.g. tests/v2-plays/plays/03-tool-basic.play.ts).
+ *   2. Confirm the resulting bundle still references
+ *      `@neondatabase/serverless` (the dynamic import path is preserved
+ *      as a string), but no top-level `Pool` symbol is constructed.
+ *   3. Confirm that creating an actual pool via `createRuntimePool` in
+ *      a test still resolves and returns a working client.
+ *
+ * Why we don't just `external: ['@neondatabase/serverless']` in
+ * bundle-play-file.ts:
+ *   The `@neondatabase/serverless` package isn't provided by the
+ *   workerd runtime — marking it external would crash the Worker on
+ *   first import lookup. The lazy dynamic import is the right pattern:
+ *   esbuild bundles the module, but its CODE STARTUP COST (parse +
+ *   compile) is paid only when the import resolves.
+ */
+
+import {
+  registerRuntimePoolFactory,
+  type RuntimePool,
+  type RuntimePoolClient,
+} from './runtime-pg-driver';
+
+/**
+ * Minimal local types for the Neon module's runtime surface.
+ *
+ * We INTENTIONALLY don't `import type { Pool, PoolClient } from
+ * '@neondatabase/serverless'` here — that would force tsc to read the
+ * package's full type tree on every typecheck, slowing the build, and
+ * the package's overloaded `connect()` resolves through types in a way
+ * that surfaces as `void` in some TS configurations.
+ *
+ * The minimal shape is just:
+ *   - Pool ctor takes a config bag
+ *   - Pool instance has connect() / end()
+ *   - PoolClient instance has query() / release()
+ *
+ * If Neon ever changes this surface, the runtime cast will fail loudly
+ * the first time a play tries to use Postgres — which is exactly when
+ * we want to find out.
+ */
+interface NeonPoolClient {
+  query<R = Record<string, unknown>>(
+    text: string,
+    params?: unknown[],
+  ): Promise<{ rows: R[] }>;
+  release(): void;
+}
+
+interface NeonPoolInstance {
+  connect(): Promise<NeonPoolClient>;
+  end(): Promise<void>;
+}
+
+interface NeonPoolConfig {
+  connectionString: string;
+  max?: number;
+  idleTimeoutMillis?: number;
+  connectionTimeoutMillis?: number;
+}
+
+interface NeonModule {
+  Pool: new (config: NeonPoolConfig) => NeonPoolInstance;
+}
+
+/**
+ * Cached promise of the loaded Neon module. Once the first connect()
+ * resolves, every subsequent connect on the same isolate hits this
+ * cache instead of re-importing.
+ *
+ * We cache the PROMISE rather than the resolved module so concurrent
+ * first-connect calls coalesce onto a single import — V8's module
+ * loader is already idempotent for the same specifier, but caching at
+ * this layer makes the contract obvious to readers.
+ */
+let neonModulePromise: Promise<NeonModule> | null = null;
+function loadNeonModule(): Promise<NeonModule> {
+  if (!neonModulePromise) {
+    neonModulePromise = import('@neondatabase/serverless') as unknown as Promise<NeonModule>;
+  }
+  return neonModulePromise;
+}
+
+function wrapNeonClient(client: NeonPoolClient): RuntimePoolClient {
+  return {
+    query: <R extends Record<string, unknown> = Record<string, unknown>>(
+      text: string,
+      params?: unknown[],
+    ) =>
+      (client.query as unknown as (
+        t: string,
+        p?: unknown[],
+      ) => Promise<{ rows: R[] }>)(text, params),
+    release: () => {
+      client.release();
+    },
+  };
+}
+
+/**
+ * Wrap a Neon `Pool` instance as our internal `RuntimePool` shape.
+ * `connect()` and `end()` are async on both sides, so this is a thin
+ * adapter — no logic, just type marshaling.
+ */
+function wrapNeonPool(pool: NeonPoolInstance): RuntimePool {
+  return {
+    connect: async () =>
+      wrapNeonClient((await pool.connect()) as unknown as NeonPoolClient),
+    end: async () => {
+      await pool.end();
+    },
+  };
+}
+
+/**
+ * Install the Neon-backed pool factory. Called once at Worker isolate
+ * boot — `registerRuntimePoolFactory` is idempotent and throws if a
+ * different factory was already registered (build-config bug).
+ *
+ * The factory itself is SYNCHRONOUS (returns a `RuntimePool`) but the
+ * underlying Neon Pool is created LAZILY: we capture the connection
+ * config in a closure and instantiate the real `Pool` on first
+ * `connect()`. Until then, no `@neondatabase/serverless` runtime code
+ * runs in the isolate — V8 doesn't parse the dynamic-imported chunk
+ * until the import() resolves.
+ */
+export function installNeonServerlessRuntimePoolDriver(): void {
+  registerRuntimePoolFactory((input) => {
+    // Captured lazily so a Pool is only constructed if the play actually
+    // connects. Most plays that don't touch Postgres directly never even
+    // reach this branch — the factory is registered but never invoked.
+    let lazyPool: NeonPoolInstance | null = null;
+
+    /**
+     * Resolve the lazy Pool, creating it on first call.
+     *
+     * Race safety: workerd executes within a single JS event loop, so
+     * two concurrent `connect()` calls on the SAME wrapper run their
+     * "lazyPool == null" checks in turn — the first creates the Pool,
+     * the second sees it. No locking needed.
+     *
+     * If pool creation fails (e.g. DNS, bad connection string), the
+     * error propagates out of connect() to the caller. We don't cache
+     * a failed pool so retries can succeed on transient failures.
+     */
+    async function ensurePool(): Promise<NeonPoolInstance> {
+      if (lazyPool) return lazyPool;
+      const neon = await loadNeonModule();
+      const pool = new neon.Pool({
+        connectionString: input.connectionString,
+        max: input.maxConnections ?? 4,
+        idleTimeoutMillis: input.idleTimeoutMs ?? 15_000,
+        connectionTimeoutMillis: input.connectTimeoutMs ?? 10_000,
+      });
+      lazyPool = pool;
+      return pool;
+    }
+
+    return {
+      connect: async () => {
+        const pool = await ensurePool();
+        return wrapNeonClient(await pool.connect());
+      },
+      end: async () => {
+        if (lazyPool) {
+          await lazyPool.end();
+          lazyPool = null;
+        }
+      },
+    };
+  });
+}

package/dist/repo/shared_libs/play-runtime/runtime-pg-driver-pg.ts

@@ -0,0 +1,48 @@
+/**
+ * pg-backed Pool factory. Installed at boot by the Node-runtime
+ * play-runner (Daytona/local). NOT used by the workers_edge harness — pg's
+ * raw TCP socket usage is banned in workerd.
+ */
+import { Pool, type PoolClient } from 'pg';
+import {
+  registerRuntimePoolFactory,
+  type RuntimePool,
+  type RuntimePoolClient,
+} from './runtime-pg-driver';
+
+function wrapPgClient(client: PoolClient): RuntimePoolClient {
+  return {
+    query: <R extends Record<string, unknown> = Record<string, unknown>>(
+      text: string,
+      params?: unknown[],
+    ) =>
+      (client.query as unknown as (
+        t: string,
+        p?: unknown[],
+      ) => Promise<{ rows: R[] }>)(text, params).then((result) => ({
+        rows: result.rows,
+      })),
+    release: () => {
+      client.release();
+    },
+  };
+}
+
+function wrapPgPool(pool: Pool): RuntimePool {
+  return {
+    connect: async () => wrapPgClient(await pool.connect()),
+    end: () => pool.end(),
+  };
+}
+
+export function installPgRuntimePoolDriver(): void {
+  registerRuntimePoolFactory((input) => {
+    const pool = new Pool({
+      connectionString: input.connectionString,
+      max: input.maxConnections ?? 4,
+      idleTimeoutMillis: input.idleTimeoutMs ?? 15_000,
+      connectionTimeoutMillis: input.connectTimeoutMs ?? 10_000,
+    });
+    return wrapPgPool(pool);
+  });
+}

package/dist/repo/shared_libs/play-runtime/runtime-pg-driver.ts

@@ -0,0 +1,84 @@
+/**
+ * Driver-agnostic Postgres pool/client surface used by the runtime data plane.
+ *
+ * The runtime data plane (`runtime-api.ts` direct-Neon writes) needs to run in
+ * two very different environments:
+ *
+ *   - Node (Daytona/local play-runner) — uses the `pg` library, raw TCP to Neon.
+ *   - Cloudflare Workers (workers_edge harness) — `pg` cannot bundle (raw TCP
+ *     is banned at workerd boundary), so we use `@neondatabase/serverless`
+ *     which speaks the same Postgres wire protocol over a WebSocket proxy.
+ *
+ * Rather than `if-platform` branches inside runtime-api.ts, each runtime
+ * registers its driver once at boot and the rest of the code uses the
+ * abstract `RuntimePool` / `RuntimePoolClient` types. Both `pg` and
+ * `@neondatabase/serverless` already expose the relevant subset of the
+ * `pg` API surface (`pool.connect()` → `client.query() / .release()`), so
+ * the wrappers are thin.
+ */
+
+export interface RuntimePoolClient {
+  query<R extends Record<string, unknown> = Record<string, unknown>>(
+    text: string,
+    params?: unknown[],
+  ): Promise<{ rows: R[] }>;
+  release(): void;
+}
+
+export interface RuntimePool {
+  connect(): Promise<RuntimePoolClient>;
+  end(): Promise<void>;
+}
+
+export type RuntimePoolFactory = (input: {
+  connectionString: string;
+  /** Soft cap on simultaneous connections; defaults to 4. */
+  maxConnections?: number;
+  /** Idle timeout in ms before pool tears down a connection; defaults to 15000. */
+  idleTimeoutMs?: number;
+  /** Connect timeout in ms; defaults to 10000. */
+  connectTimeoutMs?: number;
+}) => RuntimePool;
+
+let registeredFactory: RuntimePoolFactory | null = null;
+
+/**
+ * Install a Pool factory at process / Worker boot. Idempotent — calling twice
+ * with different factories will throw to prevent silent driver swaps that
+ * would otherwise drop in-flight pooled connections.
+ */
+export function registerRuntimePoolFactory(factory: RuntimePoolFactory): void {
+  if (registeredFactory && registeredFactory !== factory) {
+    throw new Error(
+      'A different runtime Postgres pool factory is already registered. ' +
+        'This is a build-config bug — only one driver should be installed per process.',
+    );
+  }
+  registeredFactory = factory;
+}
+
+export function isRuntimePoolFactoryRegistered(): boolean {
+  return registeredFactory !== null;
+}
+
+export function createRuntimePool(input: {
+  connectionString: string;
+  maxConnections?: number;
+  idleTimeoutMs?: number;
+  connectTimeoutMs?: number;
+}): RuntimePool {
+  if (!registeredFactory) {
+    throw new Error(
+      'No runtime Postgres pool factory registered. Call ' +
+        'installPgRuntimePoolDriver() (Node) or ' +
+        'installNeonServerlessRuntimePoolDriver() (Workers) at boot before ' +
+        'using runtime-api Postgres helpers.',
+    );
+  }
+  return registeredFactory(input);
+}
+
+/** Test seam — clears the registered factory so a test can swap drivers. */
+export function __resetRuntimePoolFactoryForTests(): void {
+  registeredFactory = null;
+}

package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts

@@ -0,0 +1,174 @@
+/**
+ * Scheduler-backend interface — owns play workflow lifecycle.
+ *
+ * One of three pluggable axes (alongside runner-backends and dedup-backends).
+ * Selected per-run via PlayExecutionProfile.
+ *
+ * Temporal is the existing production scheduler. Cloudflare Workflows is the
+ * edge scheduler used by the workers_edge profile.
+ *
+ * Customer plays are unaffected — this is purely the orchestration layer.
+ */
+
+import type {
+  PlayCheckpoint,
+  PlayExecutionEvent,
+  PlayRowUpdate,
+} from './ctx-types';
+import type { ExecutionPlan } from './execution-plan';
+import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
+
+export const PLAY_SCHEDULER_BACKENDS = {
+  temporal: 'temporal',
+  cfWorkflows: 'cf-workflows',
+  inProcess: 'in-process',
+} as const;
+
+export type PlaySchedulerBackendId =
+  (typeof PLAY_SCHEDULER_BACKENDS)[keyof typeof PLAY_SCHEDULER_BACKENDS];
+
+export type PlayCallGovernanceSnapshot = {
+  rootRunId: string;
+  parentRunId: string;
+  parentPlayName: string;
+  key: string;
+  ancestryPlayIds: string[];
+  callDepth: number;
+};
+
+export type PlaySchedulerSubmitInput = {
+  runId: string;
+  playId: string;
+  playName: string;
+  artifactStorageKey: string;
+  artifactHash: string;
+  graphHash: string;
+  input: Record<string, unknown>;
+  inputFile?: {
+    name?: string;
+    path?: string;
+    r2Key?: string;
+    storageKey?: string;
+    fileName?: string;
+    logicalPath?: string;
+  } | null;
+  inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
+  /**
+   * Tiny input files (<= WORKERS_EDGE_INLINE_INPUT_MAX_BYTES) carried as raw
+   * base64 bytes through the scheduler submit so the cf-workflows backend can
+   * build inlineCsv without an R2 PUT+GET round trip. Mirrors the
+   * Temporal-side `inlineInputFile` fast path.
+   */
+  inlineInputFile?: {
+    logicalPath: string;
+    fileName: string;
+    contentBase64: string;
+    contentType: string;
+    bytes: number;
+  } | null;
+  packagedFiles?: Array<{
+    playPath?: string;
+    logicalPath?: string;
+    fileName?: string;
+    storageKey: string;
+    contentType?: string;
+    bytes?: number;
+    inlineText?: string;
+  }> | null;
+  contractSnapshot?: unknown;
+  executionPlan?: ExecutionPlan | null;
+  childPlayManifests?: PlayRuntimeManifestMap | null;
+  playCallGovernance?: PlayCallGovernanceSnapshot | null;
+  /** Optional immutable Worker module source for local Dynamic Worker loading. */
+  dynamicWorkerCode?: string | null;
+  executorToken: string;
+  baseUrl: string;
+  orgId: string;
+  userEmail: string;
+  userId?: string | null;
+  /** runner backend to use for executing attempts */
+  runtimeBackend: string;
+  /** dedup backend for cross-attempt cross-process idempotency */
+  dedupBackend: string;
+  /** If known at submit time, total input rows (for partition decisions). */
+  totalRows?: number;
+  /** Internal scheduler/coordinator URL for Worker-side capabilities. */
+  coordinatorUrl?: string | null;
+};
+
+export type PlaySchedulerProgressEvent =
+  | { type: 'status'; status: string; logs?: string[]; ts: number }
+  | { type: 'log'; line: string; ts: number }
+  | { type: 'row'; update: PlayRowUpdate; ts: number }
+  | { type: 'execution_event'; event: PlayExecutionEvent; ts: number }
+  | { type: 'completed'; result: unknown; ts: number }
+  | { type: 'failed'; error: string; ts: number }
+  | { type: 'suspended'; reason: string; ts: number };
+
+export type PlaySchedulerRunHandle = {
+  runId: string;
+  /**
+   * Optional snapshot of the run's state captured during submit() so callers
+   * can skip an immediate follow-up status read. Populated by backends that
+   * piggyback the read on the create round-trip (currently cf-workflows).
+   */
+  initialState?: Record<string, unknown> | null;
+  /**
+   * Stream live progress events. Implementations may use SSE, polling, etc.
+   * The contract: yields events in order until terminal status.
+   */
+  observe(): AsyncIterable<PlaySchedulerProgressEvent>;
+  /** Cooperatively cancel the run. */
+  cancel(): Promise<void>;
+  /** Inject an external event (HITL, webhook). */
+  signal(payload: PlaySchedulerSignalPayload): Promise<void>;
+  /** Block until terminal state and return final envelope. */
+  result(): Promise<PlaySchedulerResultEnvelope>;
+};
+
+export type PlaySchedulerSignalPayload = {
+  kind: 'integration_event' | 'cancel' | 'custom';
+  eventKey?: string;
+  data?: unknown;
+};
+
+export type PlaySchedulerResultEnvelope = {
+  runId: string;
+  status: 'completed' | 'failed' | 'cancelled';
+  output?: unknown;
+  error?: string;
+  finalCheckpoint?: PlayCheckpoint;
+  totalRows?: number;
+  durationMs?: number;
+};
+
+export interface PlaySchedulerBackend {
+  readonly id: PlaySchedulerBackendId;
+
+  /** Submit a play run; returns a handle to observe / cancel / signal it. */
+  submit(input: PlaySchedulerSubmitInput): Promise<PlaySchedulerRunHandle>;
+
+  /** Open a handle to an already-submitted run (e.g. for tail-reconnect). */
+  attach(
+    runId: string,
+    options?: { coordinatorUrl?: string | null },
+  ): Promise<PlaySchedulerRunHandle>;
+}
+
+export function normalizePlaySchedulerBackend(
+  value?: string | null,
+): PlaySchedulerBackendId {
+  const normalized = value?.trim().toLowerCase();
+  if (!normalized || normalized === 'temporal') {
+    return PLAY_SCHEDULER_BACKENDS.temporal;
+  }
+  if (normalized === 'cf-workflows' || normalized === 'cf_workflows') {
+    return PLAY_SCHEDULER_BACKENDS.cfWorkflows;
+  }
+  if (normalized === 'in-process' || normalized === 'in_process') {
+    return PLAY_SCHEDULER_BACKENDS.inProcess;
+  }
+  throw new Error(
+    `Unsupported scheduler backend "${normalized}". Expected one of: ${Object.values(PLAY_SCHEDULER_BACKENDS).join(', ')}.`,
+  );
+}

package/dist/repo/shared_libs/play-runtime/static-pipeline-types.ts

@@ -0,0 +1,147 @@
+export interface PlayStaticPipeline {
+  tableNamespace?: string;
+  inputFields?: string[];
+  csvArg?: string;
+  hasInlineData?: boolean;
+  csvDescription?: string;
+  mapDescription?: string;
+  fields: string[];
+  stages?: PlayStaticSubstep[];
+  substeps: PlayStaticSubstep[];
+  sheetContract?: PlaySheetContract | null;
+  sheetContractErrors?: string[];
+}
+
+export type PlaySheetColumnSource =
+  | 'input'
+  | 'mapField'
+  | 'waterfallStep'
+  | 'childPlayColumn';
+
+export interface PlaySheetColumnContract {
+  id: string;
+  sqlName: string;
+  source: PlaySheetColumnSource;
+  field?: string;
+  parentField?: string;
+  playId?: string;
+  waterfallId?: string;
+  outputField?: string;
+  outputSqlName?: string;
+  stepId?: string;
+  toolId?: string;
+}
+
+export interface PlaySheetContract {
+  tableNamespace: string;
+  columns: PlaySheetColumnContract[];
+}
+
+export interface PlayStaticSourceRange {
+  sourcePath?: string;
+  startLine: number;
+  endLine: number;
+  startColumn: number;
+  endColumn: number;
+}
+
+type PlayStaticSubstepMetadata = {
+  conditional?: boolean;
+};
+
+export type PlayStaticSubstep = PlayStaticSubstepMetadata &
+  (
+    | {
+        type: 'csv';
+        field: string;
+        path?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'map';
+        field: string;
+        name?: string;
+        tableNamespace?: string;
+        inputFields?: string[];
+        outputFields?: string[];
+        waterfallIds?: string[];
+        sheetContract?: PlaySheetContract | null;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'tool';
+        toolId: string;
+        field: string;
+        description?: string;
+        inLoop?: boolean;
+        isEventWait?: boolean;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'waterfall';
+        tool?: string;
+        field: string;
+        inLoop?: boolean;
+        id?: string;
+        output?: string;
+        minResults?: number;
+        sourceText?: string;
+        steps?: Array<{
+          id: string;
+          kind?: 'tool' | 'code';
+          toolId?: string;
+          paramsSource?: string;
+        }>;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'step_suite';
+        field: string;
+        steps: PlayStaticSubstep[];
+        returnSource?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'play_call';
+        playId: string;
+        field: string;
+        inLoop?: boolean;
+        pipeline?: PlayStaticPipeline | null;
+        cycleDetected?: boolean;
+        resolutionError?: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'run_javascript';
+        alias: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+    | {
+        type: 'code';
+        field: string;
+        description?: string;
+        sourceRange?: PlayStaticSourceRange;
+        callDepth?: number;
+        callPath?: string[];
+      }
+  );