deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts

@@ -0,0 +1,442 @@
+export type DurableInvocationResponse = {
+  status: number;
+  headers: Array<[string, string]>;
+  bodyBase64: string;
+};
+
+export type DurableInvocationRecord = {
+  invocationKey: string;
+  requestFingerprint: string;
+  ownerId: string;
+  parentRunId: string;
+  state: 'running' | 'completed' | 'compacted' | 'indeterminate';
+  response: DurableInvocationResponse | null;
+  updatedAt: string;
+};
+
+export type DurableInvocationStore = {
+  get(input: {
+    invocationKey: string;
+  }): Promise<DurableInvocationRecord | null>;
+  begin(input: {
+    invocationKey: string;
+    requestFingerprint: string;
+    acceptedRequestFingerprints?: readonly string[];
+    ownerId: string;
+    parentRunId: string;
+  }): Promise<DurableInvocationRecord>;
+  complete(input: {
+    invocationKey: string;
+    requestFingerprint: string;
+    ownerId: string;
+    response: DurableInvocationResponse;
+  }): Promise<DurableInvocationRecord | null>;
+  markIndeterminate(input: {
+    invocationKey: string;
+    requestFingerprint: string;
+    ownerId: string;
+  }): Promise<DurableInvocationRecord | null>;
+  abandon(input: {
+    invocationKey: string;
+    requestFingerprint: string;
+    ownerId: string;
+  }): Promise<boolean>;
+};
+
+export type DurableInvocationDiagnosticPhase =
+  | 'existing_indeterminate'
+  | 'existing_compacted'
+  | 'stale_running_indeterminate'
+  | 'foreign_owner_running'
+  | 'dispatch_failed'
+  | 'completion_failed'
+  | 'completion_missing'
+  | 'completion_recovered'
+  | 'fixture_abandon_succeeded'
+  | 'fixture_abandon_rejected'
+  | 'fixture_abandon_failed'
+  | 'wait_exhausted';
+
+export type DurableInvocationDiagnostic = {
+  phase: DurableInvocationDiagnosticPhase;
+  error?: unknown;
+};
+
+export class DurableInvocationConflictError extends Error {
+  readonly code = 'DURABLE_INVOCATION_REQUEST_CONFLICT';
+
+  constructor() {
+    super('The durable invocation key was reused for a different request.');
+    this.name = 'DurableInvocationConflictError';
+  }
+}
+
+export class DurableInvocationOutcomeUnknownError extends Error {
+  readonly code = 'DURABLE_INVOCATION_OUTCOME_UNKNOWN';
+
+  constructor() {
+    super(
+      'The durable invocation started but no response was committed. Deepline will not dispatch it again because its outcome may already have taken effect.',
+    );
+    this.name = 'DurableInvocationOutcomeUnknownError';
+  }
+}
+
+export class DurableInvocationInProgressError extends Error {
+  readonly code = 'GATEWAY_INVOCATION_IN_PROGRESS';
+
+  constructor() {
+    super(
+      'The same durable invocation is still executing. Retry this transport delivery with the same invocation attempt.',
+    );
+    this.name = 'DurableInvocationInProgressError';
+  }
+}
+
+export class DurableInvocationCommitError extends Error {
+  readonly code = 'DURABLE_INVOCATION_COMMIT_FAILED';
+
+  constructor() {
+    super(
+      'The integration response could not be committed before delivery. Its outcome is unknown and Deepline will not dispatch it again automatically.',
+    );
+    this.name = 'DurableInvocationCommitError';
+  }
+}
+
+function assertMatchingRequest(
+  record: DurableInvocationRecord,
+  requestFingerprint: string,
+  acceptedRequestFingerprints: readonly string[] = [],
+): void {
+  if (
+    record.requestFingerprint !== requestFingerprint &&
+    !acceptedRequestFingerprints.includes(record.requestFingerprint)
+  ) {
+    throw new DurableInvocationConflictError();
+  }
+}
+
+function completedResponse(
+  record: DurableInvocationRecord,
+): DurableInvocationResponse | null {
+  if (record.state !== 'completed') return null;
+  if (!record.response) throw new DurableInvocationCommitError();
+  return record.response;
+}
+
+function decodedBodyBytes(response: DurableInvocationResponse): number {
+  const padding = response.bodyBase64.endsWith('==')
+    ? 2
+    : response.bodyBase64.endsWith('=')
+      ? 1
+      : 0;
+  return Math.max(
+    0,
+    Math.floor((response.bodyBase64.length * 3) / 4) - padding,
+  );
+}
+
+/**
+ * Deep transport-redelivery Module for integration execution.
+ *
+ * A live owner is never stolen. An indeterminate outcome stays blocked inside
+ * its owning run, but a later explicit Play Run may replace it. That rerun is
+ * fresh user intent and may repeat a side effect whose first outcome was
+ * unknown; automatic transport and same-run retries remain fenced.
+ */
+export async function executeWithDurableInvocationFence(input: {
+  invocationKey: string;
+  requestFingerprint: string;
+  /** Exact historical fingerprint aliases accepted during a rolling upgrade. */
+  acceptedRequestFingerprints?: readonly string[];
+  ownerId: string;
+  parentRunId: string;
+  store: DurableInvocationStore;
+  execute: () => Promise<DurableInvocationResponse>;
+  /**
+   * Fixture execution has no provider-side effect. It may discard an
+   * ambiguous transport owner and redeliver the exact request. Live provider
+   * execution must leave this false and retain the fail-closed tombstone.
+   */
+  replaySafeAfterDispatch?: boolean;
+  onDispatchStart?: () => void;
+  beforeReplay?: () => Promise<void>;
+  wait?: (attempt: number) => Promise<void>;
+  maxWaitAttempts?: number;
+  runningStaleAfterMs?: number;
+  now?: () => number;
+  onOutcome?: (
+    outcome: 'live' | 'replay' | 'in_progress' | 'unknown',
+    responseBytes?: number,
+  ) => void;
+  /**
+   * Exceptional transition telemetry. This is intentionally silent on the
+   * normal dispatch/commit path so large maps do not produce per-row logs.
+   */
+  onDiagnostic?: (diagnostic: DurableInvocationDiagnostic) => void;
+}): Promise<DurableInvocationResponse> {
+  const diagnose = (
+    phase: DurableInvocationDiagnosticPhase,
+    error?: unknown,
+  ): void => {
+    try {
+      input.onDiagnostic?.({
+        phase,
+        ...(error === undefined ? {} : { error }),
+      });
+    } catch {
+      // Observability must never change invocation fencing semantics.
+    }
+  };
+  const retryReplaySafeInvocation = async (
+    record: Pick<
+      DurableInvocationRecord,
+      'invocationKey' | 'requestFingerprint' | 'ownerId'
+    >,
+  ): Promise<never> => {
+    try {
+      const abandoned = await input.store.abandon(record);
+      diagnose(
+        abandoned ? 'fixture_abandon_succeeded' : 'fixture_abandon_rejected',
+      );
+    } catch (error) {
+      diagnose('fixture_abandon_failed', error);
+      // The store may be the failed boundary. The next delivery retries this
+      // conditional cleanup after storage recovers; fixture execution remains
+      // safe to redeliver either way.
+    }
+    input.onOutcome?.('in_progress');
+    throw new DurableInvocationInProgressError();
+  };
+
+  const markStaleRunningUnknown = async (
+    record: DurableInvocationRecord,
+  ): Promise<DurableInvocationRecord> => {
+    if (record.state !== 'running') return record;
+    const updatedAt = Date.parse(record.updatedAt);
+    const staleAfterMs = input.runningStaleAfterMs ?? 30 * 60_000;
+    if (
+      Number.isFinite(updatedAt) &&
+      (input.now ?? Date.now)() - updatedAt >= staleAfterMs
+    ) {
+      const marked = await input.store.markIndeterminate({
+        invocationKey: record.invocationKey,
+        requestFingerprint: record.requestFingerprint,
+        ownerId: record.ownerId,
+      });
+      if (marked?.state === 'indeterminate') {
+        diagnose('stale_running_indeterminate');
+        if (marked.parentRunId !== input.parentRunId) {
+          return await input.store.begin({
+            invocationKey: input.invocationKey,
+            requestFingerprint: input.requestFingerprint,
+            acceptedRequestFingerprints: input.acceptedRequestFingerprints,
+            ownerId: input.ownerId,
+            parentRunId: input.parentRunId,
+          });
+        }
+        if (input.replaySafeAfterDispatch) {
+          return await retryReplaySafeInvocation(marked);
+        }
+        throw new DurableInvocationOutcomeUnknownError();
+      }
+      // The owner may have committed between this follower's stale read and
+      // the conditional transition. Read the winner before declaring the
+      // provider outcome unknown.
+      const winner =
+        marked ??
+        (await input.store.get({ invocationKey: record.invocationKey }));
+      if (!winner) {
+        diagnose('stale_running_indeterminate');
+        throw new DurableInvocationOutcomeUnknownError();
+      }
+      assertMatchingRequest(
+        winner,
+        input.requestFingerprint,
+        input.acceptedRequestFingerprints,
+      );
+      if (winner.state === 'indeterminate' || winner.state === 'compacted') {
+        diagnose(
+          winner.state === 'indeterminate'
+            ? 'stale_running_indeterminate'
+            : 'existing_compacted',
+        );
+        if (input.replaySafeAfterDispatch) {
+          return await retryReplaySafeInvocation(winner);
+        }
+        throw new DurableInvocationOutcomeUnknownError();
+      }
+      return winner;
+    }
+    return record;
+  };
+  let begun = await input.store.begin({
+    invocationKey: input.invocationKey,
+    requestFingerprint: input.requestFingerprint,
+    acceptedRequestFingerprints: input.acceptedRequestFingerprints,
+    ownerId: input.ownerId,
+    parentRunId: input.parentRunId,
+  });
+  assertMatchingRequest(
+    begun,
+    input.requestFingerprint,
+    input.acceptedRequestFingerprints,
+  );
+  if (begun.state === 'indeterminate' || begun.state === 'compacted') {
+    diagnose(
+      begun.state === 'indeterminate'
+        ? 'existing_indeterminate'
+        : 'existing_compacted',
+    );
+    if (input.replaySafeAfterDispatch) {
+      return await retryReplaySafeInvocation(begun);
+    }
+    input.onOutcome?.('unknown');
+    throw new DurableInvocationOutcomeUnknownError();
+  }
+  begun = await markStaleRunningUnknown(begun);
+  const replay = completedResponse(begun);
+  if (replay) {
+    await input.beforeReplay?.();
+    input.onOutcome?.('replay', decodedBodyBytes(replay));
+    return replay;
+  }
+
+  if (begun.ownerId === input.ownerId) {
+    let response: DurableInvocationResponse;
+    try {
+      // Store/control-plane failures before this boundary are safely
+      // retryable: no integration request has started. Unclassified failures
+      // after it must fail closed because the provider outcome may be unknown.
+      input.onDispatchStart?.();
+      response = await input.execute();
+    } catch (error) {
+      diagnose('dispatch_failed', error);
+      if (input.replaySafeAfterDispatch) {
+        return await retryReplaySafeInvocation(begun);
+      }
+      await input.store.markIndeterminate({
+        invocationKey: input.invocationKey,
+        requestFingerprint: input.requestFingerprint,
+        ownerId: input.ownerId,
+      });
+      throw error;
+    }
+    let completed: DurableInvocationRecord | null;
+    try {
+      completed = await input.store.complete({
+        invocationKey: input.invocationKey,
+        requestFingerprint: input.requestFingerprint,
+        ownerId: input.ownerId,
+        response,
+      });
+    } catch (error) {
+      diagnose('completion_failed', error);
+      if (!input.replaySafeAfterDispatch) throw error;
+      try {
+        const winner = await input.store.get({
+          invocationKey: input.invocationKey,
+        });
+        if (winner) {
+          assertMatchingRequest(
+            winner,
+            input.requestFingerprint,
+            input.acceptedRequestFingerprints,
+          );
+          const committed = completedResponse(winner);
+          if (committed) {
+            diagnose('completion_recovered');
+            input.onOutcome?.('replay', decodedBodyBytes(committed));
+            return committed;
+          }
+        }
+      } catch {
+        // Fall through to the fixture-safe conditional cleanup below.
+      }
+      return await retryReplaySafeInvocation(begun);
+    }
+    if (!completed) {
+      diagnose('completion_missing');
+      const winner = await input.store.get({
+        invocationKey: input.invocationKey,
+      });
+      if (winner) {
+        assertMatchingRequest(
+          winner,
+          input.requestFingerprint,
+          input.acceptedRequestFingerprints,
+        );
+        const committed = completedResponse(winner);
+        if (committed) {
+          diagnose('completion_recovered');
+          input.onOutcome?.('replay', decodedBodyBytes(committed));
+          return committed;
+        }
+      }
+      if (input.replaySafeAfterDispatch) {
+        return await retryReplaySafeInvocation(begun);
+      }
+      await input.store.markIndeterminate({
+        invocationKey: input.invocationKey,
+        requestFingerprint: input.requestFingerprint,
+        ownerId: input.ownerId,
+      });
+      input.onOutcome?.('unknown');
+      throw new DurableInvocationCommitError();
+    }
+    assertMatchingRequest(
+      completed,
+      input.requestFingerprint,
+      input.acceptedRequestFingerprints,
+    );
+    const committed = completedResponse(completed) ?? response;
+    input.onOutcome?.('live', decodedBodyBytes(committed));
+    return committed;
+  }
+
+  if (input.replaySafeAfterDispatch) {
+    diagnose('foreign_owner_running');
+    return await retryReplaySafeInvocation(begun);
+  }
+
+  const maxWaitAttempts = input.maxWaitAttempts ?? 20;
+  for (let attempt = 1; attempt <= maxWaitAttempts; attempt += 1) {
+    await (input.wait?.(attempt) ??
+      new Promise<void>((resolve) => setTimeout(resolve, 250)));
+    let latest = await input.store.get({
+      invocationKey: input.invocationKey,
+    });
+    if (!latest) continue;
+    assertMatchingRequest(
+      latest,
+      input.requestFingerprint,
+      input.acceptedRequestFingerprints,
+    );
+    if (
+      latest.state === 'indeterminate' &&
+      latest.parentRunId !== input.parentRunId
+    ) {
+      return await executeWithDurableInvocationFence(input);
+    }
+    if (latest.state === 'indeterminate' || latest.state === 'compacted') {
+      diagnose(
+        latest.state === 'indeterminate'
+          ? 'existing_indeterminate'
+          : 'existing_compacted',
+      );
+      input.onOutcome?.('unknown');
+      throw new DurableInvocationOutcomeUnknownError();
+    }
+    latest = await markStaleRunningUnknown(latest);
+    const response = completedResponse(latest);
+    if (response) {
+      await input.beforeReplay?.();
+      input.onOutcome?.('replay', decodedBodyBytes(response));
+      return response;
+    }
+  }
+  diagnose('wait_exhausted');
+  input.onOutcome?.('in_progress');
+  throw new DurableInvocationInProgressError();
+}

package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts

@@ -14,9 +14,9 @@ export const PLAY_RUNTIME_PROTOCOL = {
   /**
    * Current scheduler/runner protocol. This number STAYS AT 1: the additive
    * compatibility rule (`docs/runtime-compatibility-contract.md`) covers new
-   * fields and new named actions without a bump. A bump is a fence release —
-   * two PRs, expand worker support first, then enable — and needs a written
-   * reason the additive rule cannot serve.
+   * fields and new named actions without a bump. The required compatibility
+   * audit freezes these historical tags; ship a new action's reader before
+   * its writer instead of changing a global number.
    */
   current: 1,
 } as const;

package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts

@@ -51,6 +51,10 @@ const FIXTURE_MODE_BYPASS_PROVIDERS = new Set([
   'test',
   'test_high',
   'test_low',
+  // Dedicated stateful failure fixtures must reach their local custom handlers
+  // even when the surrounding Play runs in fixture mode.
+  'test_receipt',
+  'test_retry',
   'test_unhinted',
 ]);
 
@@ -443,6 +447,20 @@ export function shouldRouteFixtureToolId(toolId: string): boolean {
   );
 }
 
+/** Only canned fixture responses are safe to redispatch after response loss. */
+export function isReplaySafeFixtureToolId(toolId: string): boolean {
+  const normalized = toolId.trim().toLowerCase();
+  // Managed inference can use the fake Gateway while exercising real wallet
+  // authorization and settlement. Its invocation fence must survive retries.
+  // Keep all inference variants fenced, including aliases and local emulators.
+  return (
+    shouldRouteFixtureToolId(normalized) &&
+    normalized !== 'ai_inference' &&
+    normalized !== 'deeplineagent' &&
+    !normalized.startsWith('deeplineagent_')
+  );
+}
+
 function stableFixtureRequestHash(value: string): number {
   let hash = 0x811c9dc5;
   for (let index = 0; index < value.length; index += 1) {

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -483,6 +483,15 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
           );
           return;
         }
+        // Zero grants without a finite positive wait cannot make progress.
+        // Older gateways return this for an empty/invalid normalized rule set;
+        // fail exhausted instead of spinning once per millisecond forever.
+        if (!Number.isFinite(response.waitMs) || response.waitMs <= 0) {
+          throw this.providerExhaustedFromResponse(bucketId, {
+            ...response,
+            waitMs: 0,
+          });
+        }
         // Live path: the server told us to wait past the threshold. Fail fast
         // with a typed PROVIDER_EXHAUSTED instead of sleeping toward it. The
         // rejection propagates to every waiter parked on this single-flight

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -16,7 +16,7 @@
  *    is no `null`, no "off", no implicit infinity. Unaccounted resource use
  *    must be impossible.
  *
- * The docs catalog (`src/lib/plays/limits-catalog.ts`) and its generated public
+ * The docs catalog (`apps/deepline-api/src/lib/plays/limits-catalog.ts`) and its generated public
  * + internal pages source their numbers from this table.
  */
 

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -24,7 +24,7 @@ export interface PacingRule {
 
 /**
  * Per-tool queue-hint metadata produced by the runtime tool catalog
- * (`src/lib/plays/runtime-tool-metadata.ts`) and surfaced to the runtime via
+ * (`packages/integrations/runtime-tool-metadata.ts`) and surfaced to the runtime via
  * `ContextOptions.getToolQueueHints`. It is the raw provider rate-limit metadata
  * that the runtime maps into one {@link PacingRule} per hint before handing it
  * to the Governor's pacing resolver. The `provider` field identifies the pacing

package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts

@@ -0,0 +1,66 @@
+/**
+ * The storage format a gateway may write for one Runtime Postgres schema.
+ *
+ * `canonical_v1` is deliberately a reader capability, not a deploy toggle:
+ * every gateway which can encounter the schema must be able to materialize
+ * canonical operation outcomes before this mode is recorded.
+ */
+export type IntegrationOperationStorageMode =
+  | 'legacy_byte_reader'
+  | 'canonical_v1';
+
+export const INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY =
+  'integration-operation-canonical-v1' as const;
+
+export type RuntimeGatewayRole = 'receipt' | 'execution';
+
+/**
+ * Operator-verified evidence that both durable gateway reader fleets can
+ * understand canonical Integration Operation rows. The control-plane verifier
+ * collects this evidence; this module records and validates its shape.
+ */
+export type CanonicalReaderProof = {
+  capability: typeof INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY;
+  sourceSha: string;
+  machineIds: Readonly<Record<RuntimeGatewayRole, readonly string[]>>;
+};
+
+/**
+ * The local Runtime Postgres mirror of the control-plane reader fence.
+ * `schema` is the scheduler schema/namespace, never an application or lane.
+ */
+export type RuntimeGatewayReaderFence = {
+  schema: string;
+  /** Strictly positive, monotonically increasing within a schema. */
+  epoch: number;
+  mode: IntegrationOperationStorageMode;
+  canonicalReaderProof: CanonicalReaderProof | null;
+};
+
+export type RuntimeGatewayReaderFenceCandidate = RuntimeGatewayReaderFence;
+
+export type ReaderFenceWriterMode = {
+  mode: IntegrationOperationStorageMode;
+  /** `absent` and unavailable storage are intentionally legacy-only. */
+  source: 'fence' | 'absent' | 'fail_closed';
+  epoch: number | null;
+  failure?: 'store_unavailable' | 'invalid_record';
+};
+
+export type ReaderFenceApplyOutcome =
+  | { kind: 'advanced'; fence: RuntimeGatewayReaderFence }
+  | { kind: 'already_applied'; fence: RuntimeGatewayReaderFence }
+  | {
+      kind: 'rejected';
+      reason: 'stale_epoch' | 'epoch_conflict' | 'unsafe_mode_downgrade';
+      current: RuntimeGatewayReaderFence;
+    };
+
+export type RuntimeGatewayReaderFenceJob = {
+  /** Resolve the only format this gateway may write. Never returns canonical on uncertainty. */
+  writerMode(input: { schema: string }): Promise<ReaderFenceWriterMode>;
+  /** Advance the mirror only with a strictly newer epoch, or retry the exact same fence. */
+  apply(
+    candidate: RuntimeGatewayReaderFenceCandidate,
+  ): Promise<ReaderFenceApplyOutcome>;
+};