deepline 0.1.78 → 0.1.80

package/dist/repo/apps/play-runner-workers/src/workflow-retry-state.ts

@@ -0,0 +1,209 @@
+import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
+import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
+import type {
+  PlayRuntimeManifest,
+  PlayRuntimeManifestMap,
+} from '../../../shared_libs/plays/compiler-manifest';
+
+import {
+  PLAY_SUBMIT_INPUT_INLINE_MAX_BYTES,
+  PLAY_SUBMIT_INPUT_MAX_BYTES,
+} from '../../../shared_libs/play-runtime/submit-limits';
+
+export const WORKFLOW_RETRY_STATE_TARGET_BYTES =
+  PLAY_SUBMIT_INPUT_INLINE_MAX_BYTES;
+export const WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES =
+  WORKFLOW_RETRY_STATE_TARGET_BYTES;
+export const WORKFLOW_RETRY_PARAMS_MAX_BYTES = PLAY_SUBMIT_INPUT_MAX_BYTES;
+
+export type WorkflowRetryParamsRef = {
+  storageKind: 'r2';
+  storageKey: string;
+  bytes: number;
+  hash: string;
+  expiresAt: number;
+};
+
+export type WorkflowRetryPlayParams = {
+  runId: string;
+  playId: string;
+  playName: string;
+  artifactStorageKey: string;
+  artifactHash: string;
+  graphHash: string;
+  input: Record<string, unknown>;
+  inputFile?: {
+    name?: string;
+    r2Key?: string;
+    storageKey?: string;
+    path?: string;
+    fileName?: string;
+    logicalPath?: string;
+    contentType?: string;
+    bytes?: number;
+  } | null;
+  inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
+  packagedFiles?: Array<{
+    playPath: string;
+    storageKey: string;
+    contentType?: string;
+    bytes?: number;
+    inlineText?: string;
+  }> | null;
+  contractSnapshot?: unknown;
+  executionPlan?: ExecutionPlan | null;
+  childPlayManifests?: PlayRuntimeManifestMap | null;
+  playCallGovernance?: PlayCallGovernanceSnapshot | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
+  preloadedDbSessionRef?: {
+    runId: string;
+    sessionCount: number;
+    expiresAt: number;
+  } | null;
+  dynamicWorkerCode?: string | null;
+  executorToken: string;
+  baseUrl: string;
+  orgId: string;
+  userEmail: string;
+  userId?: string | null;
+  runtimeBackend: string;
+  dedupBackend: string;
+  totalRows?: number;
+  coordinatorUrl?: string | null;
+  coordinatorInternalToken?: string | null;
+};
+
+export type WorkflowRetryStatePayload<TParams = WorkflowRetryPlayParams> =
+  | {
+      params: TParams;
+      paramsRef?: null;
+      paramsBytes: number;
+    }
+  | {
+      params: null;
+      paramsRef: WorkflowRetryParamsRef;
+      paramsBytes: number;
+    };
+
+export function buildWorkflowRetryParams<
+  TParams extends WorkflowRetryPlayParams,
+>(params: TParams): TParams {
+  const retryParams = {
+    ...params,
+    dynamicWorkerCode: null,
+    contractSnapshot: stripRetrySourceSnapshot(params.contractSnapshot),
+    childPlayManifests: stripRetryChildManifestCode(params.childPlayManifests),
+    packagedFiles: stripRetryPackagedFiles(params.packagedFiles),
+  } satisfies WorkflowRetryPlayParams as TParams;
+  if (jsonByteLength(retryParams) <= WORKFLOW_RETRY_STATE_TARGET_BYTES) {
+    return retryParams;
+  }
+  return {
+    ...retryParams,
+    contractSnapshot: stripRetryContractSnapshotToArtifact(
+      retryParams.contractSnapshot,
+      params,
+    ),
+    childPlayManifests: stripRetryChildManifestToArtifact(
+      retryParams.childPlayManifests,
+    ),
+  } satisfies WorkflowRetryPlayParams as TParams;
+}
+
+export function jsonByteLength(value: unknown): number {
+  return new TextEncoder().encode(JSON.stringify(value)).length;
+}
+
+export function workflowRetryParamsStorageKey(input: {
+  runId: string;
+  hash: string;
+}): string {
+  const safeRunId = input.runId
+    .toLowerCase()
+    .replace(/[^a-z0-9_-]+/g, '-')
+    .slice(0, 140);
+  return `plays/workflow-retry-params/${safeRunId || 'run'}/${input.hash}.json`;
+}
+
+function stripRetryPackagedFiles(
+  files: WorkflowRetryPlayParams['packagedFiles'],
+): WorkflowRetryPlayParams['packagedFiles'] {
+  return (
+    files?.map((file) => ({
+      playPath: file.playPath,
+      storageKey: file.storageKey,
+      contentType: file.contentType,
+      bytes: file.bytes,
+    })) ?? null
+  );
+}
+
+function stripRetrySourceSnapshot(snapshot: unknown): unknown {
+  if (!isRecord(snapshot)) return snapshot;
+  const rest = { ...snapshot };
+  delete rest.sourceCode;
+  delete rest.sourceFiles;
+  delete rest.bundledCode;
+  return rest;
+}
+
+function stripRetryContractSnapshotToArtifact(
+  snapshot: unknown,
+  params: WorkflowRetryPlayParams,
+): unknown {
+  if (!isRecord(snapshot)) return snapshot;
+  return {
+    source: snapshot.source ?? 'artifact',
+    revisionVersion: snapshot.revisionVersion ?? null,
+    staticPipeline: snapshot.staticPipeline ?? null,
+    billingLimit: snapshot.billingLimit ?? null,
+    artifactMetadata: {
+      storageKey: params.artifactStorageKey,
+      artifactHash: params.artifactHash,
+      graphHash: params.graphHash,
+    },
+    codeFormat: snapshot.codeFormat ?? null,
+    compatibility: snapshot.compatibility ?? null,
+  };
+}
+
+function stripRetryChildManifestCode(
+  manifests: PlayRuntimeManifestMap | null | undefined,
+): PlayRuntimeManifestMap | null {
+  if (!manifests) return null;
+  const stripped: PlayRuntimeManifestMap = {};
+  for (const [key, manifest] of Object.entries(manifests)) {
+    const rest = { ...manifest };
+    delete rest.bundledCode;
+    delete rest.sourceCode;
+    delete (rest as Record<string, unknown>).sourceMap;
+    stripped[key] = rest;
+  }
+  return stripped;
+}
+
+function stripRetryChildManifestToArtifact(
+  manifests: PlayRuntimeManifestMap | null | undefined,
+): PlayRuntimeManifestMap | null {
+  if (!manifests) return null;
+  const stripped: PlayRuntimeManifestMap = {};
+  for (const [key, manifest] of Object.entries(manifests)) {
+    stripped[key] = {
+      playName: manifest.playName,
+      graphHash: manifest.graphHash,
+      artifactStorageKey: manifest.artifactStorageKey,
+      artifactHash: manifest.artifactHash,
+      staticPipelineHash: manifest.staticPipelineHash,
+      staticPipeline: manifest.staticPipeline,
+      compiledAt: manifest.compiledAt,
+      compilerVersion: manifest.compilerVersion,
+      maxCreditsPerRun: manifest.maxCreditsPerRun,
+    };
+  }
+  return stripped;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}

package/dist/repo/sdk/src/client.ts

@@ -642,6 +642,7 @@ export class DeeplineClient {
     categories?: string;
     grep?: string;
     grepMode?: 'all' | 'any' | 'phrase';
+    compact?: boolean;
   }): Promise<ToolDefinition[]> {
     const params = new URLSearchParams();
     if (options?.categories?.trim()) {
@@ -651,6 +652,7 @@ export class DeeplineClient {
       params.set('grep', options.grep.trim());
       params.set('grep_mode', options.grepMode ?? 'all');
     }
+    params.set('compact', options?.compact === true ? 'true' : 'false');
     const suffix = params.toString() ? `?${params.toString()}` : '';
     const res = await this.http.get<{ tools: ToolDefinition[] }>(
       `/api/v2/tools${suffix}`,
@@ -1433,6 +1435,7 @@ export class DeeplineClient {
     if (status) {
       params.set('status', status);
     }
+    params.set('compact', 'true');
     const response = await this.http.get<{ runs: PlayRunListItem[] }>(
       `/api/v2/runs?${params.toString()}`,
     );
@@ -1636,10 +1639,14 @@ export class DeeplineClient {
    * @param name - Play name
    * @returns Version list (newest first)
    */
-  async listPlayVersions(name: string): Promise<PlayRevisionSummary[]> {
+  async listPlayVersions(
+    name: string,
+    options?: { full?: boolean },
+  ): Promise<PlayRevisionSummary[]> {
     const encodedName = encodeURIComponent(name);
+    const suffix = options?.full ? '?full=true' : '';
     const response = await this.http.get<{ versions: PlayRevisionSummary[] }>(
-      `/api/v2/plays/${encodedName}/versions`,
+      `/api/v2/plays/${encodedName}/versions${suffix}`,
     );
     return response.versions ?? [];
   }

package/dist/repo/sdk/src/release.ts

@@ -50,10 +50,10 @@ export type SdkRelease = {
 };
 
 export const SDK_RELEASE = {
-  version: '0.1.78',
+  version: '0.1.80',
   apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
   supportPolicy: {
-    latest: '0.1.78',
+    latest: '0.1.80',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
   },

package/dist/repo/sdk/src/types.ts

@@ -131,6 +131,10 @@ export interface ToolDefinition {
   operationId?: string;
   /** Alternative names that resolve to this tool. */
   operationAliases?: string[];
+  /** Whether detailed input schema is available from `tools describe`. */
+  hasInputSchema?: boolean;
+  /** Whether detailed output schema is available from `tools describe`. */
+  hasOutputSchema?: boolean;
   /** JSON Schema describing the tool's input parameters. */
   inputSchema?: Record<string, unknown>;
   /** JSON Schema describing the tool's output shape. */
@@ -661,6 +665,7 @@ export interface PlayListItem {
   currentPublishedVersion?: number | null;
   tableNamespace?: string | null;
   isDraftDirty?: boolean;
+  hasInputSchema?: boolean;
   inputSchema?: Record<string, unknown> | null;
   outputSchema?: Record<string, unknown> | null;
   staticPipeline?: unknown;

package/dist/repo/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts

@@ -0,0 +1,231 @@
+import {
+  noopPacingPermit,
+  type PacingPermit,
+  type PacingRule,
+  type RateStateBackend,
+} from './rate-state-backend';
+
+/**
+ * Distributed Rate State Backend for the `esm_workers` substrate.
+ *
+ * On Cloudflare a single play run fans child plays and map rows across many
+ * V8 isolates, so the per-`(org, provider)` request window cannot be
+ * process-local — each isolate would otherwise pace against its own private
+ * counter and the org could blow past a provider's real limit by the number of
+ * isolates. This backend makes the window GLOBAL by RPCing the coordinator
+ * Durable Object addressed per bucket (`idFromName('rate:<orgId>:<provider>')`).
+ * The DO is single-threaded, so it runs the same sliding-window algorithm as
+ * `InMemoryRateStateBackend` correctly for all isolates at once.
+ *
+ * Latency: a full DO round-trip on every outbound tool call would tax the
+ * hello-world latency baseline. Instead the backend LEASES SMALL PERMIT BLOCKS:
+ * one `/rate-acquire` round-trip debits up to {@link LEASE_BLOCK_SIZE} permits
+ * from the global window, and subsequent acquires draw from the local block
+ * until it is exhausted or its short TTL expires. This bounds round-trips to
+ * roughly `calls / LEASE_BLOCK_SIZE` while keeping over-issuance bounded by one
+ * block per isolate per window.
+ *
+ * Fail-open: if the coordinator is unreachable the backend logs once and
+ * PROCEEDS (grants the permit) rather than stalling the run, matching the
+ * semantics of `src/lib/redis/customer-rate-limiter.ts` — a degraded limiter
+ * must never become an availability outage. The Governor's global
+ * tool-concurrency semaphore remains the unconditional backstop.
+ */
+
+/** Permits leased per round-trip. Tuned to amortize the DO hop, not to batch. */
+const LEASE_BLOCK_SIZE = 16;
+/**
+ * Max age of a leased block. A leased permit debited the global window already,
+ * but if it is held past roughly one window it could let an isolate run ahead
+ * of a rolled-over window. Discarding stale blocks bounds that to sub-window.
+ */
+const LEASE_BLOCK_TTL_MS = 250;
+/** Cap on how long the backend will park waiting on a saturated window. */
+const MAX_ACQUIRE_WAIT_MS = 5_000;
+
+export interface CoordinatorRatePort {
+  /**
+   * Lease up to `requested` request-window permits for `bucketId` under all
+   * `rules` from the coordinator DO. Returns how many were `granted` (0 when the
+   * window is saturated) and a `waitMs` hint before retrying.
+   */
+  rateAcquire(input: {
+    bucketId: string;
+    rules: PacingRule[];
+    requested: number;
+  }): Promise<{ granted: number; waitMs: number }>;
+  /** Feed a Retry-After cooldown back into the global bucket. */
+  ratePenalize(input: { bucketId: string; cooldownMs: number }): Promise<void>;
+}
+
+interface LeasedBlock {
+  remaining: number;
+  expiresAt: number;
+  /** Stable signature of the rules this block was leased under. */
+  rulesKey: string;
+}
+
+interface Options {
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  onDegraded?: (info: { bucketId: string; error: string }) => void;
+}
+
+function rulesSignature(rules: readonly PacingRule[]): string {
+  return [...rules]
+    .map(
+      (rule) =>
+        `${rule.ruleId}:${rule.requestsPerWindow}:${rule.windowMs}:${rule.maxConcurrency ?? ''}`,
+    )
+    .sort()
+    .join('|');
+}
+
+export class CoordinatorRateStateBackend implements RateStateBackend {
+  private readonly port: CoordinatorRatePort;
+  private readonly now: () => number;
+  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly onDegraded: (info: {
+    bucketId: string;
+    error: string;
+  }) => void;
+  private readonly blocks = new Map<string, LeasedBlock>();
+  private degradedLogged = false;
+
+  constructor(port: CoordinatorRatePort, options: Options = {}) {
+    this.port = port;
+    this.now = options.now ?? (() => Date.now());
+    this.sleep =
+      options.sleep ??
+      ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.onDegraded =
+      options.onDegraded ??
+      ((info) => {
+        if (this.degradedLogged) return;
+        this.degradedLogged = true;
+        console.warn('[coordinator-rate-state] acquire failed open', info);
+      });
+  }
+
+  async acquire(input: {
+    bucketId: string;
+    rules: readonly PacingRule[];
+    signal?: AbortSignal;
+  }): Promise<PacingPermit> {
+    const { bucketId, rules, signal } = input;
+    if (rules.length === 0) {
+      return noopPacingPermit();
+    }
+    const rulesKey = rulesSignature(rules);
+
+    // Draw from a still-valid local block first — no round-trip.
+    if (this.drawFromBlock(bucketId, rulesKey)) {
+      return noopPacingPermit();
+    }
+
+    const waitStartedAt = this.now();
+    while (true) {
+      if (signal?.aborted) {
+        throw signal.reason instanceof Error
+          ? signal.reason
+          : new Error('Rate-state acquire aborted.');
+      }
+      let response: { granted: number; waitMs: number };
+      try {
+        response = await this.port.rateAcquire({
+          bucketId,
+          rules: [...rules],
+          requested: LEASE_BLOCK_SIZE,
+        });
+      } catch (error) {
+        // Fail open: a degraded coordinator must not stall the run.
+        this.onDegraded({
+          bucketId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        return noopPacingPermit();
+      }
+      if (response.granted > 0) {
+        // Consume one for this call; cache the rest as a short-lived block.
+        const remaining = response.granted - 1;
+        if (remaining > 0) {
+          this.mergeBlock(bucketId, remaining, rulesKey);
+        }
+        return noopPacingPermit();
+      }
+      // Window saturated. Park for the hint, then re-acquire. Cap total wait so
+      // a stuck bucket surfaces through the Governor's wall-clock guard instead
+      // of hanging forever.
+      if (this.now() - waitStartedAt >= MAX_ACQUIRE_WAIT_MS) {
+        return noopPacingPermit();
+      }
+      const waitMs = Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_WAIT_MS));
+      await this.sleep(waitMs);
+    }
+  }
+
+  penalize(input: { bucketId: string; cooldownMs: number }): void {
+    if (input.cooldownMs <= 0) return;
+    // Drop any cached block for this bucket so the cooldown takes effect on the
+    // very next acquire instead of being masked by already-leased permits.
+    this.blocks.delete(input.bucketId);
+    void this.port
+      .ratePenalize({
+        bucketId: input.bucketId,
+        cooldownMs: input.cooldownMs,
+      })
+      .catch((error) => {
+        this.onDegraded({
+          bucketId: input.bucketId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      });
+  }
+
+  /**
+   * Add freshly-leased permits to the bucket's block instead of overwriting it.
+   * Two concurrent acquires can both miss the local block and both round-trip;
+   * each debited the global window, so the DO already issued both blocks'
+   * permits. Overwriting would drop one set — under-issuance that wastes window
+   * capacity and over-throttles. Merging preserves every debited permit:
+   *  - same rulesKey + still valid → sum remaining, keep the earlier expiry so
+   *    the merged block never outlives the older lease's sub-window bound.
+   *  - missing / stale / different rules → start fresh from this lease.
+   */
+  private mergeBlock(
+    bucketId: string,
+    remaining: number,
+    rulesKey: string,
+  ): void {
+    const freshExpiresAt = this.now() + LEASE_BLOCK_TTL_MS;
+    const existing = this.blocks.get(bucketId);
+    if (
+      existing &&
+      existing.rulesKey === rulesKey &&
+      existing.expiresAt > this.now()
+    ) {
+      existing.remaining += remaining;
+      existing.expiresAt = Math.min(existing.expiresAt, freshExpiresAt);
+      return;
+    }
+    this.blocks.set(bucketId, {
+      remaining,
+      expiresAt: freshExpiresAt,
+      rulesKey,
+    });
+  }
+
+  private drawFromBlock(bucketId: string, rulesKey: string): boolean {
+    const block = this.blocks.get(bucketId);
+    if (!block) return false;
+    if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
+      this.blocks.delete(bucketId);
+      return false;
+    }
+    block.remaining -= 1;
+    if (block.remaining <= 0) {
+      this.blocks.delete(bucketId);
+    }
+    return true;
+  }
+}