deepline 0.1.119 → 0.1.121

package/dist/bundling-sources/shared_libs/play-runtime/runtime-constraints.ts

@@ -0,0 +1,2 @@
+export const DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE =
+  'No run_javascript tool in plays. Just write regular code.';

package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts

@@ -0,0 +1,238 @@
+/**
+ * `@neondatabase/serverless` Pool factory. Installed at boot by the
+ * workers_edge dynamic Worker harness. The serverless driver speaks the
+ * same Postgres wire protocol as `pg` but uses Neon's WebSocket proxy
+ * (or HTTP endpoint) so it works inside the workerd V8 isolate where raw
+ * TCP is unavailable.
+ *
+ * ## Lazy loading (the actual implementation)
+ *
+ * `@neondatabase/serverless` is ~143 KB minified — a non-trivial fraction
+ * of every per-graphHash play Worker bundle. The harness's
+ * `installNeonServerlessRuntimePoolDriver()` fires unconditionally at
+ * isolate startup, but the heavy module is loaded ON FIRST `pool.connect()`
+ * via dynamic `import('@neondatabase/serverless')`. Plays that never open
+ * a Postgres pool (csv-only, computation-only, tool-only plays) skip the
+ * import entirely, even though the driver registration ran at boot.
+ *
+ * Concretely:
+ *   - Bundle still contains the dynamic-import path → `@neondatabase/serverless`
+ *     is still in the per-play bundle (esbuild emits it as a deferred chunk
+ *     in `format: 'esm'`, but workerd inlines all chunks for a Worker —
+ *     so the bytes are still in the worker's compiled code).
+ *   - V8 PARSE / COMPILE for those bytes is also deferred until the first
+ *     `import()` call resolves; modern V8 does parse-on-demand for ESM
+ *     dynamic imports. So plays that never connect skip the parse cost.
+ *   - Plays that DO connect pay the import cost once per isolate, then
+ *     reuse the cached module + pool for the rest of the run.
+ *
+ * If you change this file, double-check by:
+ *   1. Bundling a play that doesn't use `ctx.dataset` / postgres ops
+ *      (e.g. tests/v2-plays/plays/03-tool-basic.play.ts).
+ *   2. Confirm the resulting bundle still references
+ *      `@neondatabase/serverless` (the dynamic import path is preserved
+ *      as a string), but no top-level `Pool` symbol is constructed.
+ *   3. Confirm that creating an actual pool via `createRuntimePool` in
+ *      a test still resolves and returns a working client.
+ *
+ * Why we don't just `external: ['@neondatabase/serverless']` in
+ * bundle-play-file.ts:
+ *   The `@neondatabase/serverless` package isn't provided by the
+ *   workerd runtime — marking it external would crash the Worker on
+ *   first import lookup. The lazy dynamic import is the right pattern:
+ *   esbuild bundles the module, but its CODE STARTUP COST (parse +
+ *   compile) is paid only when the import resolves.
+ */
+
+import {
+  registerRuntimePoolFactory,
+  registerRuntimeOneShotQueryFactory,
+  type RuntimePool,
+  type RuntimePoolClient,
+} from './runtime-pg-driver';
+
+/**
+ * Minimal local types for the Neon module's runtime surface.
+ *
+ * We INTENTIONALLY don't `import type { Pool, PoolClient } from
+ * '@neondatabase/serverless'` here — that would force tsc to read the
+ * package's full type tree on every typecheck, slowing the build, and
+ * the package's overloaded `connect()` resolves through types in a way
+ * that surfaces as `void` in some TS configurations.
+ *
+ * The minimal shape is just:
+ *   - Pool ctor takes a config bag
+ *   - Pool instance has connect() / end()
+ *   - PoolClient instance has query() / release()
+ *
+ * If Neon ever changes this surface, the runtime cast will fail loudly
+ * the first time a play tries to use Postgres — which is exactly when
+ * we want to find out.
+ */
+interface NeonPoolClient {
+  query<R = Record<string, unknown>>(
+    text: string,
+    params?: unknown[],
+  ): Promise<{ rows: R[] }>;
+  release(): void;
+}
+
+interface NeonPoolInstance {
+  connect(): Promise<NeonPoolClient>;
+  end(): Promise<void>;
+}
+
+interface NeonPoolConfig {
+  connectionString: string;
+  max?: number;
+  idleTimeoutMillis?: number;
+  connectionTimeoutMillis?: number;
+}
+
+interface NeonModule {
+  Pool: new (config: NeonPoolConfig) => NeonPoolInstance;
+  neon: (
+    connectionString: string,
+    options?: { fullResults?: boolean },
+  ) => {
+    query<R = Record<string, unknown>>(
+      text: string,
+      params?: unknown[],
+      options?: { fullResults?: boolean },
+    ): Promise<{ rows: R[] }>;
+  };
+}
+
+/**
+ * Cached promise of the loaded Neon module. Once the first connect()
+ * resolves, every subsequent connect on the same isolate hits this
+ * cache instead of re-importing.
+ *
+ * We cache the PROMISE rather than the resolved module so concurrent
+ * first-connect calls coalesce onto a single import — V8's module
+ * loader is already idempotent for the same specifier, but caching at
+ * this layer makes the contract obvious to readers.
+ */
+let neonModulePromise: Promise<NeonModule> | null = null;
+function loadNeonModule(): Promise<NeonModule> {
+  if (!neonModulePromise) {
+    neonModulePromise =
+      import('@neondatabase/serverless') as unknown as Promise<NeonModule>;
+  }
+  return neonModulePromise;
+}
+
+function wrapNeonClient(client: NeonPoolClient): RuntimePoolClient {
+  return {
+    query: <R extends Record<string, unknown> = Record<string, unknown>>(
+      text: string,
+      params?: unknown[],
+    ) =>
+      (
+        client.query as unknown as (
+          t: string,
+          p?: unknown[],
+        ) => Promise<{ rows: R[] }>
+      )(text, params),
+    release: () => {
+      client.release();
+    },
+  };
+}
+
+/**
+ * Wrap a Neon `Pool` instance as our internal `RuntimePool` shape.
+ * `connect()` and `end()` are async on both sides, so this is a thin
+ * adapter — no logic, just type marshaling.
+ */
+function wrapNeonPool(pool: NeonPoolInstance): RuntimePool {
+  return {
+    connect: async () =>
+      wrapNeonClient((await pool.connect()) as unknown as NeonPoolClient),
+    end: async () => {
+      await pool.end();
+    },
+  };
+}
+
+/**
+ * Install the Neon-backed pool factory. Called once at Worker isolate
+ * boot — `registerRuntimePoolFactory` is idempotent and throws if a
+ * different factory was already registered (build-config bug).
+ *
+ * The factory itself is SYNCHRONOUS (returns a `RuntimePool`) but the
+ * underlying Neon Pool is created LAZILY: we capture the connection
+ * config in a closure and instantiate the real `Pool` on first
+ * `connect()`. Until then, no `@neondatabase/serverless` runtime code
+ * runs in the isolate — V8 doesn't parse the dynamic-imported chunk
+ * until the import() resolves.
+ */
+export function installNeonServerlessRuntimePoolDriver(): void {
+  registerRuntimeOneShotQueryFactory((input) => {
+    let sqlPromise: Promise<ReturnType<NeonModule['neon']>> | null = null;
+    async function getSql(): Promise<ReturnType<NeonModule['neon']>> {
+      if (!sqlPromise) {
+        sqlPromise = loadNeonModule().then((neon) =>
+          neon.neon(input.connectionString, { fullResults: true }),
+        );
+      }
+      return sqlPromise;
+    }
+    return {
+      query: async <
+        R extends Record<string, unknown> = Record<string, unknown>,
+      >(
+        text: string,
+        params?: unknown[],
+      ) => {
+        const sql = await getSql();
+        return await sql.query<R>(text, params, { fullResults: true });
+      },
+    };
+  });
+
+  registerRuntimePoolFactory((input) => {
+    // Captured lazily so a Pool is only constructed if the play actually
+    // connects. Most plays that don't touch Postgres directly never even
+    // reach this branch — the factory is registered but never invoked.
+    let lazyPool: NeonPoolInstance | null = null;
+
+    /**
+     * Resolve the lazy Pool, creating it on first call.
+     *
+     * Race safety: workerd executes within a single JS event loop, so
+     * two concurrent `connect()` calls on the SAME wrapper run their
+     * "lazyPool == null" checks in turn — the first creates the Pool,
+     * the second sees it. No locking needed.
+     *
+     * If pool creation fails (e.g. DNS, bad connection string), the
+     * error propagates out of connect() to the caller. We don't cache
+     * a failed pool so retries can succeed on transient failures.
+     */
+    async function ensurePool(): Promise<NeonPoolInstance> {
+      if (lazyPool) return lazyPool;
+      const neon = await loadNeonModule();
+      const pool = new neon.Pool({
+        connectionString: input.connectionString,
+        max: input.maxConnections ?? 4,
+        idleTimeoutMillis: input.idleTimeoutMs ?? 15_000,
+        connectionTimeoutMillis: input.connectTimeoutMs ?? 10_000,
+      });
+      lazyPool = pool;
+      return pool;
+    }
+
+    return {
+      connect: async () => {
+        const pool = await ensurePool();
+        return wrapNeonClient(await pool.connect());
+      },
+      end: async () => {
+        if (lazyPool) {
+          await lazyPool.end();
+          lazyPool = null;
+        }
+      },
+    };
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts

@@ -0,0 +1,53 @@
+/**
+ * pg-backed Pool factory. Installed at boot by the Node-runtime
+ * play-runner (Daytona/local). NOT used by the workers_edge harness — pg's
+ * raw TCP socket usage is banned in workerd.
+ */
+import { Pool, type PoolClient } from 'pg';
+import {
+  registerRuntimePoolFactory,
+  type RuntimePool,
+  type RuntimePoolClient,
+  type RuntimePoolFactory,
+} from './runtime-pg-driver';
+
+function wrapPgClient(client: PoolClient): RuntimePoolClient {
+  return {
+    query: <R extends Record<string, unknown> = Record<string, unknown>>(
+      text: string,
+      params?: unknown[],
+    ) =>
+      (
+        client.query as unknown as (
+          t: string,
+          p?: unknown[],
+        ) => Promise<{ rows: R[] }>
+      )(text, params).then((result) => ({
+        rows: result.rows,
+      })),
+    release: () => {
+      client.release();
+    },
+  };
+}
+
+function wrapPgPool(pool: Pool): RuntimePool {
+  return {
+    connect: async () => wrapPgClient(await pool.connect()),
+    end: () => pool.end(),
+  };
+}
+
+const pgRuntimePoolFactory: RuntimePoolFactory = (input): RuntimePool => {
+  const pool = new Pool({
+    connectionString: input.connectionString,
+    max: input.maxConnections ?? 2,
+    idleTimeoutMillis: input.idleTimeoutMs ?? 15_000,
+    connectionTimeoutMillis: input.connectTimeoutMs ?? 10_000,
+  });
+  return wrapPgPool(pool);
+};
+
+export function installPgRuntimePoolDriver(): void {
+  registerRuntimePoolFactory(pgRuntimePoolFactory);
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts

@@ -0,0 +1,149 @@
+/**
+ * Driver-agnostic Postgres pool/client surface used by the runtime data plane.
+ *
+ * The runtime data plane (`runtime-api.ts` direct-Neon writes) needs to run in
+ * two very different environments:
+ *
+ *   - Node (Daytona/local play-runner) — uses the `pg` library, raw TCP to Neon.
+ *   - Cloudflare Workers (workers_edge harness) — `pg` cannot bundle (raw TCP
+ *     is banned at workerd boundary), so we use `@neondatabase/serverless`
+ *     which speaks the same Postgres wire protocol over a WebSocket proxy.
+ *
+ * Rather than `if-platform` branches inside runtime-api.ts, each runtime
+ * registers its driver once at boot and the rest of the code uses the
+ * abstract `RuntimePool` / `RuntimePoolClient` types. Both `pg` and
+ * `@neondatabase/serverless` already expose the relevant subset of the
+ * `pg` API surface (`pool.connect()` → `client.query() / .release()`), so
+ * the wrappers are thin.
+ */
+
+export interface RuntimePoolClient {
+  query<R extends Record<string, unknown> = Record<string, unknown>>(
+    text: string,
+    params?: unknown[],
+  ): Promise<{ rows: R[] }>;
+  release(): void;
+}
+
+export interface RuntimePool {
+  connect(): Promise<RuntimePoolClient>;
+  end(): Promise<void>;
+}
+
+export interface RuntimeOneShotQueryClient {
+  query<R extends Record<string, unknown> = Record<string, unknown>>(
+    text: string,
+    params?: unknown[],
+  ): Promise<{ rows: R[] }>;
+}
+
+export type RuntimePoolFactory = (input: {
+  connectionString: string;
+  /** Soft cap on simultaneous connections; defaults to 4. */
+  maxConnections?: number;
+  /** Idle timeout in ms before pool tears down a connection; defaults to 15000. */
+  idleTimeoutMs?: number;
+  /** Connect timeout in ms; defaults to 10000. */
+  connectTimeoutMs?: number;
+}) => RuntimePool;
+
+export type RuntimeOneShotQueryFactory = (input: {
+  connectionString: string;
+}) => RuntimeOneShotQueryClient;
+
+const runtimePgDriverRegistryKey = Symbol.for(
+  'deepline.play-runtime.runtime-pg-driver.registry',
+);
+
+type RuntimePgDriverRegistry = {
+  poolFactory: RuntimePoolFactory | null;
+  oneShotQueryFactory: RuntimeOneShotQueryFactory | null;
+};
+
+function getRuntimePgDriverRegistry(): RuntimePgDriverRegistry {
+  const globalWithRegistry = globalThis as typeof globalThis & {
+    [runtimePgDriverRegistryKey]?: RuntimePgDriverRegistry;
+  };
+  globalWithRegistry[runtimePgDriverRegistryKey] ??= {
+    poolFactory: null,
+    oneShotQueryFactory: null,
+  };
+  return globalWithRegistry[runtimePgDriverRegistryKey];
+}
+
+/**
+ * Install a Pool factory at process / Worker boot. Idempotent — calling twice
+ * with different factories will throw to prevent silent driver swaps that
+ * would otherwise drop in-flight pooled connections.
+ */
+export function registerRuntimePoolFactory(factory: RuntimePoolFactory): void {
+  const registry = getRuntimePgDriverRegistry();
+  if (registry.poolFactory && registry.poolFactory !== factory) {
+    throw new Error(
+      'A different runtime Postgres pool factory is already registered. ' +
+        'This is a build-config bug — only one driver should be installed per process.',
+    );
+  }
+  registry.poolFactory = factory;
+}
+
+export function isRuntimePoolFactoryRegistered(): boolean {
+  return getRuntimePgDriverRegistry().poolFactory !== null;
+}
+
+export function registerRuntimeOneShotQueryFactory(
+  factory: RuntimeOneShotQueryFactory,
+): void {
+  const registry = getRuntimePgDriverRegistry();
+  if (
+    registry.oneShotQueryFactory &&
+    registry.oneShotQueryFactory !== factory
+  ) {
+    throw new Error(
+      'A different runtime one-shot Postgres query factory is already registered. ' +
+        'This is a build-config bug — only one driver should be installed per process.',
+    );
+  }
+  registry.oneShotQueryFactory = factory;
+}
+
+export function isRuntimeOneShotQueryFactoryRegistered(): boolean {
+  return getRuntimePgDriverRegistry().oneShotQueryFactory !== null;
+}
+
+export function createRuntimeOneShotQueryClient(input: {
+  connectionString: string;
+}): RuntimeOneShotQueryClient {
+  const { oneShotQueryFactory } = getRuntimePgDriverRegistry();
+  if (!oneShotQueryFactory) {
+    throw new Error(
+      'No runtime one-shot Postgres query factory registered. Call the runtime Postgres driver installer at boot before using one-shot query helpers.',
+    );
+  }
+  return oneShotQueryFactory(input);
+}
+
+export function createRuntimePool(input: {
+  connectionString: string;
+  maxConnections?: number;
+  idleTimeoutMs?: number;
+  connectTimeoutMs?: number;
+}): RuntimePool {
+  const { poolFactory } = getRuntimePgDriverRegistry();
+  if (!poolFactory) {
+    throw new Error(
+      'No runtime Postgres pool factory registered. Call ' +
+        'installPgRuntimePoolDriver() (Node) or ' +
+        'installNeonServerlessRuntimePoolDriver() (Workers) at boot before ' +
+        'using runtime-api Postgres helpers.',
+    );
+  }
+  return poolFactory(input);
+}
+
+/** Test seam — clears the registered factory so a test can swap drivers. */
+export function __resetRuntimePoolFactoryForTests(): void {
+  const registry = getRuntimePgDriverRegistry();
+  registry.poolFactory = null;
+  registry.oneShotQueryFactory = null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -0,0 +1,68 @@
+export type PlayExecutionSuspension =
+  | {
+      kind: 'sleep';
+      boundaryId: string;
+      delayMs: number;
+    }
+  | {
+      kind: 'integration_event';
+      boundaryId: string;
+      eventKey: string;
+      timeoutMs: number;
+    }
+  | {
+      kind: 'integration_event_batch';
+      boundaries: Array<{
+        boundaryId: string;
+        eventKey: string;
+        timeoutMs: number;
+      }>;
+    };
+
+export class PlayExecutionSuspendedError extends Error {
+  readonly suspension: PlayExecutionSuspension;
+
+  constructor(suspension: PlayExecutionSuspension) {
+    super(
+      suspension.kind === 'sleep'
+        ? `Play execution suspended for durable sleep (${suspension.delayMs}ms).`
+        : suspension.kind === 'integration_event_batch'
+          ? `Play execution suspended waiting for ${suspension.boundaries.length} integration events.`
+          : `Play execution suspended waiting for integration event ${JSON.stringify(suspension.eventKey)}.`,
+    );
+    this.name = 'PlayExecutionSuspendedError';
+    this.suspension = suspension;
+  }
+}
+
+export class PlayRowExecutionSuspendedError extends Error {
+  readonly boundary: {
+    boundaryId: string;
+    eventKey: string;
+    timeoutMs: number;
+  };
+
+  constructor(boundary: {
+    boundaryId: string;
+    eventKey: string;
+    timeoutMs: number;
+  }) {
+    super(
+      `Play row execution suspended waiting for integration event ${JSON.stringify(boundary.eventKey)}.`,
+    );
+    this.name = 'PlayRowExecutionSuspendedError';
+    this.boundary = boundary;
+  }
+}
+
+export function isPlayRowExecutionSuspendedError(
+  value: unknown,
+): value is PlayRowExecutionSuspendedError {
+  return value instanceof PlayRowExecutionSuspendedError;
+}
+
+export function isPlayExecutionSuspendedError(
+  value: unknown,
+): value is PlayExecutionSuspendedError {
+  return value instanceof PlayExecutionSuspendedError;
+}

package/dist/bundling-sources/shared_libs/play-runtime/tool-batch-executor.ts

@@ -0,0 +1,149 @@
+export type ToolBatchItem = {
+  itemKey: string;
+  payload: Record<string, unknown>;
+  inputHash?: string | null;
+};
+
+export type ToolBatchRequest = {
+  runId: string;
+  orgId: string;
+  toolId: string;
+  operation: string;
+  provider: string;
+  items: ToolBatchItem[];
+  waterfallId?: string | null;
+  stageId?: string | null;
+  fieldName?: string | null;
+  mapName?: string | null;
+  chunkIndex?: number | null;
+  userProvidedRateLimitKey?: string | null;
+  providerBatchSize: number;
+};
+
+export type ToolBatchItemResult = {
+  itemKey: string;
+  result: unknown;
+  cached?: boolean;
+};
+
+export type ToolBatchResult = {
+  runId: string;
+  toolId: string;
+  operation: string;
+  provider: string;
+  batchCount: number;
+  itemCount: number;
+  results: ToolBatchItemResult[];
+};
+
+export type ToolBatchExecutorTransport = {
+  executeProviderBatch(input: {
+    request: ToolBatchRequest;
+    batchIndex: number;
+    idempotencyKeys: string[];
+    rateLimitKey: string;
+    items: ToolBatchItem[];
+  }): Promise<ToolBatchItemResult[]>;
+};
+
+export type ToolBatchExecutor = {
+  executeToolBatch(request: ToolBatchRequest): Promise<ToolBatchResult>;
+};
+
+export function createToolBatchExecutor(
+  transport: ToolBatchExecutorTransport,
+): ToolBatchExecutor {
+  return {
+    async executeToolBatch(request) {
+      const providerBatchSize = Math.max(
+        1,
+        Math.floor(request.providerBatchSize),
+      );
+      const batches = chunkToolBatchItems(request.items, providerBatchSize);
+      const results: ToolBatchItemResult[] = [];
+      for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
+        const items = batches[batchIndex]!;
+        results.push(
+          ...(await transport.executeProviderBatch({
+            request,
+            batchIndex,
+            items,
+            rateLimitKey: buildToolBatchRateLimitKey(request),
+            idempotencyKeys: items.map((item) =>
+              buildToolBatchIdempotencyKey(request, item),
+            ),
+          })),
+        );
+      }
+      return {
+        runId: request.runId,
+        toolId: request.toolId,
+        operation: request.operation,
+        provider: request.provider,
+        batchCount: batches.length,
+        itemCount: request.items.length,
+        results,
+      };
+    },
+  };
+}
+
+export function buildToolBatchIdempotencyKey(
+  request: ToolBatchRequest,
+  item: ToolBatchItem,
+): string {
+  return [
+    request.runId,
+    request.mapName ?? '',
+    request.chunkIndex ?? '',
+    item.itemKey,
+    request.fieldName ?? '',
+    request.waterfallId ?? '',
+    request.stageId ?? '',
+    item.inputHash ?? stableToolBatchHash(item.payload),
+  ].join(':');
+}
+
+export function buildToolBatchRateLimitKey(request: ToolBatchRequest): string {
+  return [
+    request.orgId,
+    request.provider,
+    request.operation,
+    request.userProvidedRateLimitKey ?? '',
+  ].join(':');
+}
+
+function chunkToolBatchItems(
+  items: readonly ToolBatchItem[],
+  size: number,
+): ToolBatchItem[][] {
+  const chunks: ToolBatchItem[][] = [];
+  for (let index = 0; index < items.length; index += size) {
+    chunks.push(items.slice(index, index + size));
+  }
+  return chunks;
+}
+
+function stableToolBatchHash(value: unknown): string {
+  const text = stableStringify(value);
+  let hash = 2166136261;
+  for (let index = 0; index < text.length; index += 1) {
+    hash ^= text.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0).toString(36);
+}
+
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== 'object') {
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map(stableStringify).join(',')}]`;
+  }
+  const record = value as Record<string, unknown>;
+  return `{${Object.keys(record)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
+    .join(',')}}`;
+}