deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts

@@ -0,0 +1,78 @@
+import {
+  resolvePlayRunTerminalAuthority,
+  type PlayRunTerminalInput,
+  type PlayRunTerminalJob,
+  type PlayRunTerminalOutcome,
+} from '../../play-runs/terminal/contracts';
+import type { PlayRunTerminalPorts } from '../../play-runs/terminal/ports';
+
+/**
+ * Play Run Terminal Job.
+ *
+ * The job owns the terminal transition law:
+ *
+ *   atomic direct transition → terminal outcome
+ *   otherwise durable compatibility receipt → wake
+ *
+ * Infrastructure only supplies the three capabilities. In particular, no
+ * caller can accidentally publish a wake before the compatibility receipt is
+ * durable, or publish after a stale fence rejected its write.
+ */
+export function createPlayRunTerminalJob(
+  ports: PlayRunTerminalPorts,
+): PlayRunTerminalJob {
+  return {
+    async accept(input: PlayRunTerminalInput): Promise<PlayRunTerminalOutcome> {
+      // The public compatibility facade historically accepted a manually
+      // constructed authority as well as the parsed ingress value. Normalize
+      // it again here so whitespace cannot turn an invalid lease into a direct
+      // terminal write when a caller bypasses the parser.
+      const authority = resolvePlayRunTerminalAuthority(input.authority ?? {});
+      const direct = authority
+        ? await ports.directCommitter.commit({
+            runId: input.runId,
+            attempt: input.attempt,
+            authority,
+            result: input.result,
+            queueName: input.queueName,
+            // An authority exists only for the one terminal contract this
+            // build recognizes, and that contract is what proves the durable
+            // compute-billing controller. An absent or unrecognized contract
+            // never reaches here: it resolves to null above and wakes instead.
+            allowCappedCompletion: true,
+          })
+        : 'wake_required';
+
+      if (direct === 'not_found') {
+        return { kind: 'not_found_or_fence_rejected' };
+      }
+      if (direct === 'completed') {
+        return { kind: 'direct_completed' };
+      }
+      if (direct === 'stale_cleaned') {
+        return { kind: 'stale_attempt_cleaned' };
+      }
+      if (direct === 'already_terminal') {
+        // Mixed-version recovery is first-write-wins. Re-emitting can retire a
+        // historical parked invocation whose prior transition predates atomic
+        // retirement-event recording.
+        await ports.wakePublisher.publish(input);
+        return { kind: 'already_terminal' };
+      }
+
+      const recorded = await ports.compatibilityRecorder.record({
+        runId: input.runId,
+        attempt: input.attempt,
+        result: input.result,
+      });
+      if (!recorded) {
+        return { kind: 'not_found_or_fence_rejected' };
+      }
+
+      // The event carries no terminal payload. A resumed attempt reads truth
+      // from its durable store rather than trusting a message body.
+      await ports.wakePublisher.publish(input);
+      return { kind: 'compatibility_wake_queued' };
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts

@@ -0,0 +1,54 @@
+import type {
+  PlayRunTerminalAuthority,
+  PlayRunTerminalInput,
+} from '../../play-runs/terminal/contracts';
+
+/** The result of the atomic terminal-transition capability. */
+export type DirectTerminalCommitOutcome =
+  | 'completed'
+  | 'stale_cleaned'
+  | 'already_terminal'
+  | 'not_found'
+  | 'wake_required';
+
+/**
+ * Capability for the modern atomic terminal transition. An implementation
+ * commits run truth, cleanup/billing obligations, terminal log outbox, and
+ * retirement event as one transaction when it returns `completed`.
+ */
+export type DirectTerminalCommitter = {
+  commit(input: {
+    runId: string;
+    attempt: number;
+    authority: PlayRunTerminalAuthority;
+    result: unknown;
+    queueName: string;
+    allowCappedCompletion: boolean;
+  }): Promise<DirectTerminalCommitOutcome>;
+};
+
+/**
+ * Compatibility-only durable write. It persists a runner terminal before the
+ * wake is published; false means a stale/missing attempt won and no event may
+ * be emitted.
+ */
+export type CompatibilityTerminalRecorder = {
+  record(input: {
+    runId: string;
+    attempt: number;
+    result: unknown;
+  }): Promise<boolean>;
+};
+
+/** Publishes the wake only after terminal truth has become durable. */
+export type RunnerTerminalWakePublisher = {
+  publish(
+    input: Pick<PlayRunTerminalInput, 'queueName' | 'runId' | 'attempt'>,
+  ): Promise<void>;
+};
+
+export type PlayRunTerminalPorts = {
+  directCommitter: DirectTerminalCommitter;
+  compatibilityRecorder: CompatibilityTerminalRecorder;
+  wakePublisher: RunnerTerminalWakePublisher;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts

@@ -0,0 +1,71 @@
+import { ABSURD_WAKE_CHANNEL } from '@deepline/absurd/sdk';
+
+import { absurdRunnerTerminalEventName } from '../../scheduler-backends/absurd-shared';
+import {
+  directCompletePostgresSchedulerRunnerTerminal,
+  recordPostgresSchedulerRunnerTerminal,
+  type PostgresSchedulerQueryClient,
+} from '../../scheduler-backends/postgres';
+import type { PlayRunTerminalPorts } from '../../play-runs/terminal/ports';
+
+/**
+ * Postgres implementation of the Play Run Terminal capabilities.
+ *
+ * This adapter is intentionally the only place this module knows scheduler
+ * SQL, the Absurd event name, or PostgreSQL notification transport.
+ */
+export function createPostgresPlayRunTerminalPorts(input: {
+  client: PostgresSchedulerQueryClient;
+  schema?: string;
+}): PlayRunTerminalPorts {
+  const options = input.schema ? { schema: input.schema } : undefined;
+  return {
+    directCommitter: {
+      async commit(terminal) {
+        return await directCompletePostgresSchedulerRunnerTerminal(
+          input.client,
+          {
+            runId: terminal.runId,
+            attempt: terminal.attempt,
+            leaseToken: terminal.authority.leaseToken,
+            result: terminal.result,
+            terminalLogOutbox: 'ctx_log_tail',
+            allowCappedCompletion: terminal.allowCappedCompletion,
+            engineWake: {
+              queueName: terminal.queueName,
+              eventName: absurdRunnerTerminalEventName(
+                terminal.runId,
+                terminal.attempt,
+              ),
+              wakeChannel: ABSURD_WAKE_CHANNEL,
+            },
+          },
+          options,
+        );
+      },
+    },
+    compatibilityRecorder: {
+      async record(terminal) {
+        return await recordPostgresSchedulerRunnerTerminal(
+          input.client,
+          terminal,
+          options,
+        );
+      },
+    },
+    wakePublisher: {
+      async publish(terminal) {
+        await input.client.query(
+          `WITH emitted AS (
+             SELECT absurd.emit_event($1, $2, 'null'::jsonb)
+           )
+           SELECT pg_notify('${ABSURD_WAKE_CHANNEL}', $1) FROM emitted`,
+          [
+            terminal.queueName,
+            absurdRunnerTerminalEventName(terminal.runId, terminal.attempt),
+          ],
+        );
+      },
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts

@@ -1,8 +1,10 @@
+import { bouncebanBatchStrategies } from './/bounceban-batching';
 import type { AnyBatchOperationStrategy } from './batching-types';
 import { bettercontactBatchStrategies } from './bettercontact-batching';
 import { DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES } from './default-batch-strategies';
 import { fullenrichBatchStrategies } from './fullenrich-batching';
 import { opensosdataBatchStrategies } from './opensosdata-batching';
+import { zerobounceBatchStrategies } from './zerobounce-batching';
 import { testAsyncBatchStrategies } from './test-async-batching';
 
 export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
@@ -11,9 +13,11 @@ export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
 > = {
   ...DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES,
   ...bettercontactBatchStrategies,
+  ...bouncebanBatchStrategies,
   ...fullenrichBatchStrategies,
   ...opensosdataBatchStrategies,
   ...testAsyncBatchStrategies,
+  ...zerobounceBatchStrategies,
 };
 
 export function getPlayRuntimeBatchStrategy(

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -152,6 +152,8 @@ export interface PlayRunnerContextConfig {
   workflowId?: string;
   playId?: string;
   runId?: string;
+  /** Original run for an explicit user-requested replay. */
+  replayedFromRunId?: string | null;
   /** Shared governance/billing root. Differs from runId for async children. */
   rootRunId?: string;
   runAttempt?: number | null;

package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts

@@ -1,13 +1,10 @@
 import type { WorkReceiptStatus } from './work-receipts';
+import { WORK_RECEIPT_STATUS_CODE } from './work-receipt-storage-contract';
 
-export const RECEIPT_STATUS_CODE = {
-  queued: 0,
-  pending: 0,
-  running: 1,
-  completed: 2,
-  failed: 3,
-  skipped: 4,
-} as const satisfies Record<WorkReceiptStatus, number>;
+export const RECEIPT_STATUS_CODE = WORK_RECEIPT_STATUS_CODE satisfies Record<
+  WorkReceiptStatus,
+  number
+>;
 
 export type ReceiptStatusCode =
   (typeof RECEIPT_STATUS_CODE)[keyof typeof RECEIPT_STATUS_CODE];

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -63,10 +63,10 @@ export const WORKSPACE_STORAGE_NOT_READY_CODE = 'WORKSPACE_STORAGE_NOT_READY';
 // postgres-URI passwords). The raw pg error goes in the Error `cause` for server
 // logs only, never into this message or the normalized public failure.
 export const WORKSPACE_STORAGE_NOT_READY_MESSAGE =
-  'Workspace storage for this organization is unavailable, so this operation could not start. ' +
+  'Deepline could not prepare your workspace storage in time, so this operation did not start. ' +
+  'Try again in a few moments. You do not need to run a database command. ' +
   'This blocked operation did not use Deepline credits. ' +
   'Earlier completed operations in the same run may already have used credits. ' +
-  'Re-run the command. If it fails again, ask an organization admin to run `deepline db repair` (POST /api/v2/ingestion/repair), then re-run. ' +
   'If this keeps happening, contact Deepline support with the run ID.';
 
 export class WorkspaceStorageNotReadyError extends Error {
@@ -177,9 +177,7 @@ export class ProviderExhaustedError extends Error {
  * `ToolExecutionError` shape. Treat those two representations as the same
  * typed exhaustion outcome; do not fall back based on a message substring.
  */
-export function isProviderExhaustedFailure(
-  error: unknown,
-): error is
+export function isProviderExhaustedFailure(error: unknown): error is
   | ProviderExhaustedError
   | (ToolExecutionError & {
       readonly origin: 'provider';

package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts

@@ -0,0 +1,168 @@
+import { sha256Hex, stableStringify } from '../plays/row-identity';
+function sanitizePlayNameForWorkflowSegment(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9-]/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 64);
+}
+
+export function buildPlayWorkflowFamilyKey(playName: string): string {
+  const normalized = playName
+    .split('/')
+    .map(sanitizePlayNameForWorkflowSegment)
+    .filter(Boolean)
+    .join('/');
+  return normalized || 'play';
+}
+
+export function buildPlayWorkflowRunSuffix(now: Date): string {
+  const pad = (value: number, size = 2) => String(value).padStart(size, '0');
+  return [
+    now.getUTCFullYear(),
+    pad(now.getUTCMonth() + 1),
+    pad(now.getUTCDate()),
+    't',
+    pad(now.getUTCHours()),
+    pad(now.getUTCMinutes()),
+    pad(now.getUTCSeconds()),
+    '-',
+    pad(now.getUTCMilliseconds(), 3),
+  ].join('');
+}
+
+const CANONICAL_PLAY_RUN_TIMESTAMP_RE =
+  /\/run\/(\d{4})(\d{2})(\d{2})t(\d{2})(\d{2})(\d{2})-(\d{3})(?:-|$)/;
+
+/** Read the UTC creation time embedded in a canonical root play-run id. */
+export function canonicalPlayRunStartedAtMs(runId: string): number | null {
+  const match = CANONICAL_PLAY_RUN_TIMESTAMP_RE.exec(runId);
+  if (!match) return null;
+  const [, year, month, day, hour, minute, second, millisecond] = match;
+  const value = Date.UTC(
+    Number(year),
+    Number(month) - 1,
+    Number(day),
+    Number(hour),
+    Number(minute),
+    Number(second),
+    Number(millisecond),
+  );
+  if (!Number.isFinite(value)) return null;
+  const roundTrip = new Date(value);
+  return roundTrip.getUTCFullYear() === Number(year) &&
+    roundTrip.getUTCMonth() === Number(month) - 1 &&
+    roundTrip.getUTCDate() === Number(day) &&
+    roundTrip.getUTCHours() === Number(hour) &&
+    roundTrip.getUTCMinutes() === Number(minute) &&
+    roundTrip.getUTCSeconds() === Number(second) &&
+    roundTrip.getUTCMilliseconds() === Number(millisecond)
+    ? value
+    : null;
+}
+
+type BuildPlayWorkflowIdOptions = {
+  /**
+   * Add entropy for new run creation. Keep false for tests and explicit
+   * externally-owned ids that need deterministic reconstruction.
+   */
+  unique?: boolean;
+};
+
+function buildRunEntropySuffix(): string {
+  const randomUuid = globalThis.crypto?.randomUUID?.();
+  if (randomUuid) {
+    return randomUuid.replace(/-/g, '').slice(0, 8);
+  }
+  return Math.random().toString(36).slice(2, 10).padEnd(8, '0');
+}
+
+export function buildPlayWorkflowId(
+  playName: string,
+  now?: Date,
+  options: BuildPlayWorkflowIdOptions = {},
+): string {
+  const familyKey = buildPlayWorkflowFamilyKey(playName);
+  const timestamp = now ?? new Date();
+  const unique = options.unique ?? now === undefined;
+  const entropy = unique ? `-${buildRunEntropySuffix()}` : '';
+  return `play/${familyKey}/run/${buildPlayWorkflowRunSuffix(timestamp)}${entropy}`;
+}
+
+export type PlayRunIdentity = {
+  workflowId: string;
+  runId: string;
+  workflowFamilyKey: string;
+};
+
+/**
+ * Convex and runtime schedulers share one run identity contract.
+ *
+ * - `workflowId` is the unique execution id.
+ * - `runId` is the same execution id surfaced in dashboard/API state.
+ * - `workflowFamilyKey` is the stable play-name-derived family/group key.
+ *
+ * Keeping these three values explicit removes the old ambiguity where one field
+ * pretended to mean both "this specific run" and "this play family".
+ */
+export function buildPlayRunIdentity(
+  playName: string,
+  now?: Date,
+  options: BuildPlayWorkflowIdOptions = {},
+): PlayRunIdentity {
+  const workflowId = buildPlayWorkflowId(playName, now, {
+    unique: options.unique ?? now === undefined,
+  });
+  return {
+    workflowId,
+    runId: workflowId,
+    workflowFamilyKey: buildPlayWorkflowFamilyKey(playName),
+  };
+}
+
+/**
+ * Build the replay-stable identity for one ctx.runPlayAsync invocation.
+ *
+ * The parent's embedded timestamp keeps ordinary run ordering useful while
+ * the digest makes retries converge on exactly one independently scheduled
+ * child. Input is deliberately excluded: a changed request must conflict with
+ * the first immutable launch rather than silently minting another Run.
+ */
+export function buildAsyncPlayRunIdentity(input: {
+  playName: string;
+  parentRunId: string;
+  invocationKey: string;
+}): PlayRunIdentity {
+  const digest = sha256Hex(
+    `${input.parentRunId}\0${input.invocationKey}`,
+  ).slice(0, 20);
+  const parentStartedAt = canonicalPlayRunStartedAtMs(input.parentRunId) ?? 0;
+  // The run id is the durable identity of the authored invocation, so it must
+  // be reconstructible before resolving a mutable live Play reference. The
+  // workflow family remains the resolved child Play for grouping and display.
+  const base = buildPlayWorkflowId('async-child', new Date(parentStartedAt), {
+    unique: false,
+  });
+  const workflowId = `${base}-async-${digest}`;
+  return {
+    workflowId,
+    runId: workflowId,
+    workflowFamilyKey: buildPlayWorkflowFamilyKey(input.playName),
+  };
+}
+
+/** Immutable authored request identity, separate from the resolved revision. */
+export function buildAsyncPlayRequestFingerprint(input: {
+  key: string;
+  playRef: string;
+  childInput: Record<string, unknown>;
+}): string {
+  return sha256Hex(
+    stableStringify({
+      key: input.key,
+      playRef: input.playRef,
+      input: input.childInput,
+    }),
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -20,6 +20,7 @@ import {
   normalizeDatasetBornFrom,
   type PlayDatasetBornFrom,
 } from './cell-provenance';
+import { isLedgerTerminalResultRef } from './output-size-limits';
 
 export type PlayRunLedgerStatus =
   | 'queued'
@@ -1019,6 +1020,32 @@ function conflictingTerminalSnapshot(
   );
 }
 
+/**
+ * A legacy out-of-line terminal reference did not prove that its Postgres
+ * value was written in full. A later ref bearing that proof is an enrichment
+ * of the same terminal result, not a competing outcome or payload.
+ */
+function canPromoteTerminalResultReference(
+  current: unknown,
+  next: unknown,
+): boolean {
+  return (
+    isLedgerTerminalResultRef(current) &&
+    current.content !== 'full' &&
+    isLedgerTerminalResultRef(next) &&
+    next.content === 'full'
+  );
+}
+
+function mergeTerminalResult(input: {
+  current: unknown;
+  next: unknown;
+}): unknown {
+  return canPromoteTerminalResultReference(input.current, input.next)
+    ? input.next
+    : (input.next ?? input.current);
+}
+
 function retryablePlatformDeployFailureSnapshot(
   base: PlayRunLedgerSnapshot,
   eventError?: string | null,
@@ -1249,6 +1276,7 @@ export function reducePlayRunLedgerEvent(
         conflictingTerminalSnapshot(base, event.type, occurredAt, null, {
           allowStaleSameStatusPayloadMerge:
             (base.result === undefined && event.result !== undefined) ||
+            canPromoteTerminalResultReference(base.result, event.result) ||
             (base.resultSummary === undefined &&
               event.resultSummary !== undefined),
         }) ??
@@ -1261,7 +1289,10 @@ export function reducePlayRunLedgerEvent(
           activeStepId: null,
           activeActivityId: null,
           activeArtifactTableNamespace: null,
-          result: event.result ?? base.result,
+          result: mergeTerminalResult({
+            current: base.result,
+            next: event.result,
+          }),
           resultSummary: event.resultSummary ?? base.resultSummary,
           terminalSource: nextTerminalSource(base, event.source),
         })
@@ -1272,7 +1303,8 @@ export function reducePlayRunLedgerEvent(
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
           allowStaleSameStatusPayloadMerge:
             (base.error == null && event.error != null) ||
-            (base.result === undefined && event.result !== undefined),
+            (base.result === undefined && event.result !== undefined) ||
+            canPromoteTerminalResultReference(base.result, event.result),
         }) ??
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
@@ -1286,7 +1318,10 @@ export function reducePlayRunLedgerEvent(
           activeStepId: null,
           activeActivityId: null,
           activeArtifactTableNamespace: null,
-          result: event.result ?? base.result,
+          result: mergeTerminalResult({
+            current: base.result,
+            next: event.result,
+          }),
           terminalSource: nextTerminalSource(base, event.source),
         })
       );
@@ -1295,7 +1330,8 @@ export function reducePlayRunLedgerEvent(
         conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
           allowStaleSameStatusPayloadMerge:
             (base.error == null && event.error != null) ||
-            (base.result === undefined && event.result !== undefined),
+            (base.result === undefined && event.result !== undefined) ||
+            canPromoteTerminalResultReference(base.result, event.result),
         }) ??
         withTiming({
           ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
@@ -1309,7 +1345,10 @@ export function reducePlayRunLedgerEvent(
           activeStepId: null,
           activeActivityId: null,
           activeArtifactTableNamespace: null,
-          result: event.result ?? base.result,
+          result: mergeTerminalResult({
+            current: base.result,
+            next: event.result,
+          }),
           terminalSource: nextTerminalSource(base, event.source),
         })
       );

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/child-play-resolver.ts

@@ -0,0 +1,26 @@
+import { createSingleFlight } from '../single-flight';
+
+/**
+ * Cache authorized child metadata for one Node runner execution. The caller
+ * owns this resolver beside the root PlayContext, so all map rows share it and
+ * the cache becomes unreachable as soon as the run settles.
+ */
+export function createRunScopedChildPlayResolver<T>(input: {
+  resolvePlay: (playRef: string) => Promise<T | null>;
+}): (playRef: string) => Promise<T | null> {
+  const resolved = new Map<string, T>();
+  const flights = createSingleFlight<string, T | null>();
+
+  return async (playRef) => {
+    const cached = resolved.get(playRef);
+    if (cached) return cached;
+
+    return flights.run(playRef, async () => {
+      const cachedAfterAdmission = resolved.get(playRef);
+      if (cachedAfterAdmission) return cachedAfterAdmission;
+      const play = await input.resolvePlay(playRef);
+      if (play) resolved.set(playRef, play);
+      return play;
+    });
+  };
+}