deepline 0.1.214 → 0.1.216

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +108 -0
  2. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +85 -10
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-runtime-proxy.ts +34 -0
  4. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +50 -36
  5. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +12 -0
  6. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +4 -0
  7. package/dist/bundling-sources/sdk/src/client.ts +1 -2
  8. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  9. package/dist/bundling-sources/sdk/src/types.ts +2 -3
  10. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +21 -3
  11. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +95 -144
  12. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +14 -3
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +37 -2
  14. package/dist/bundling-sources/shared_libs/play-runtime/profiles.ts +22 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +32 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +13 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -2
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +66 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +18 -55
  20. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +59 -27
  21. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +17 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -2
  23. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +11 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +10 -0
  25. package/dist/cli/index.js +72 -21
  26. package/dist/cli/index.mjs +72 -21
  27. package/dist/index.d.mts +2 -3
  28. package/dist/index.d.ts +2 -3
  29. package/dist/index.js +7 -6
  30. package/dist/index.mjs +7 -6
  31. package/package.json +1 -1
@@ -27,6 +27,22 @@ export type PlayRuntimeProviderDescriptor = {
27
27
  label: string;
28
28
  };
29
29
 
30
+ export const PLAY_RUNTIME_PROVIDER_DISABLED_CODE =
31
+ 'PLAY_RUNTIME_PROVIDER_DISABLED' as const;
32
+
33
+ export class PlayRuntimeProviderDisabledError extends Error {
34
+ readonly code = PLAY_RUNTIME_PROVIDER_DISABLED_CODE;
35
+ readonly profile: PlayRuntimeProviderId;
36
+
37
+ constructor(profile: PlayRuntimeProviderId) {
38
+ super(
39
+ `Play runtime profile "${profile}" is disabled. Deepline runs now execute on the "absurd" profile.`,
40
+ );
41
+ this.name = 'PlayRuntimeProviderDisabledError';
42
+ this.profile = profile;
43
+ }
44
+ }
45
+
30
46
  export const PLAY_RUNTIME_PROVIDERS: Record<
31
47
  PlayRuntimeProviderId,
32
48
  PlayRuntimeProviderDescriptor
@@ -57,6 +73,14 @@ export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
57
73
  return PLAY_RUNTIME_PROVIDERS.absurd;
58
74
  }
59
75
 
76
+ export function assertPlayRuntimeProviderEnabled(
77
+ provider: PlayRuntimeProviderDescriptor,
78
+ ): void {
79
+ if (provider.id === PLAY_RUNTIME_PROVIDER_IDS.workersEdge) {
80
+ throw new PlayRuntimeProviderDisabledError(provider.id);
81
+ }
82
+ }
83
+
60
84
  export function resolvePlayRuntimeProvider(
61
85
  override?: string | null,
62
86
  ): PlayRuntimeProviderDescriptor {
@@ -73,3 +97,11 @@ export function resolvePlayRuntimeProvider(
73
97
  }
74
98
  return defaultPlayRuntimeProvider();
75
99
  }
100
+
101
+ export function resolveEnabledPlayRuntimeProvider(
102
+ override?: string | null,
103
+ ): PlayRuntimeProviderDescriptor {
104
+ const provider = resolvePlayRuntimeProvider(override);
105
+ assertPlayRuntimeProviderEnabled(provider);
106
+ return provider;
107
+ }
@@ -2,6 +2,8 @@ const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
2
2
  /Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
3
3
  const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
4
4
  /Too many subrequests by single Worker invocation/i;
5
+ const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE =
6
+ /The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i;
5
7
  const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
6
8
  /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
7
9
 
@@ -9,6 +11,8 @@ export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
9
11
  'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
10
12
  export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
11
13
  'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
14
+ export const PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE =
15
+ 'Cloudflare workflow control was temporarily unavailable. Deepline retries automatically; if still visible, re-run.';
12
16
 
13
17
  export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
14
18
  'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
@@ -171,6 +175,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
171
175
  cause,
172
176
  };
173
177
  }
178
+ if (CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE.test(cause)) {
179
+ return {
180
+ code: 'PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED',
181
+ phase: 'runtime',
182
+ message: PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE,
183
+ retryable: true,
184
+ cause,
185
+ };
186
+ }
174
187
  const playDepthBudgetMatch = cause.match(
175
188
  /Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
176
189
  );
@@ -27,6 +27,7 @@ type DaytonaPayloadEnvelope = {
27
27
  export type StagedDaytonaPayload = {
28
28
  workDir: string;
29
29
  command: string;
30
+ readyPath: string;
30
31
  outputPath: string;
31
32
  exitCodePath: string;
32
33
  progressEventPath: string;
@@ -343,6 +344,7 @@ export async function stageDaytonaRunnerPayload(input: {
343
344
 
344
345
  const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
345
346
  const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
347
+ const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
346
348
  const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
347
349
  const runnerTraceEnv =
348
350
  process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
@@ -354,17 +356,18 @@ export async function stageDaytonaRunnerPayload(input: {
354
356
  configPath,
355
357
  artifactCodePath,
356
358
  crashPusherPath,
357
- })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
359
+ })} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
358
360
  // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
359
361
  // pushes the parsed (or synthesized) terminal to the gateway so the parked
360
362
  // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
361
363
  // SIGKILL, or a crash before the in-process push. Idempotent against a
362
364
  // successful in-runner push (terminals are first-write-wins).
363
- const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
365
+ const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
364
366
 
365
367
  return {
366
368
  workDir: input.workDir,
367
369
  command,
370
+ readyPath,
368
371
  outputPath,
369
372
  exitCodePath,
370
373
  progressEventPath,
@@ -112,6 +112,16 @@ export type DetachedDaytonaCommandStart = {
112
112
  cmdId: string;
113
113
  };
114
114
 
115
+ const DAYTONA_RUNNER_READY_TIMEOUT_MS = 3_000;
116
+ const DAYTONA_RUNNER_READY_POLL_MS = 250;
117
+
118
+ export class DaytonaRunnerInitializationError extends Error {
119
+ constructor(message: string) {
120
+ super(message);
121
+ this.name = 'DaytonaRunnerInitializationError';
122
+ }
123
+ }
124
+
115
125
  /**
116
126
  * Start `command` detached in a fresh session. Returns the session + command id
117
127
  * immediately (no await on completion).
@@ -137,3 +147,59 @@ export async function startDetachedDaytonaCommand(input: {
137
147
  }
138
148
  return { sessionId: input.sessionId, cmdId };
139
149
  }
150
+
151
+ /**
152
+ * A detached command id only proves that Daytona accepted the request. Before
153
+ * the worker parks, require a marker written by the initialized runner process.
154
+ * This closes the unobservable handoff where a sandbox or command disappears
155
+ * after executeSessionCommand but before the first gateway heartbeat.
156
+ */
157
+ export async function confirmDetachedDaytonaRunnerReady(input: {
158
+ sandbox: DaytonaSandbox;
159
+ sessionId: string;
160
+ cmdId: string;
161
+ readyPath: string;
162
+ exitCodePath: string;
163
+ timeoutMs?: number;
164
+ pollMs?: number;
165
+ now?: () => number;
166
+ sleep?: (ms: number) => Promise<void>;
167
+ }): Promise<void> {
168
+ const now = input.now ?? Date.now;
169
+ const sleep =
170
+ input.sleep ??
171
+ ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
172
+ const timeoutMs = Math.max(
173
+ 1,
174
+ input.timeoutMs ?? DAYTONA_RUNNER_READY_TIMEOUT_MS,
175
+ );
176
+ const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
177
+ const deadline = now() + timeoutMs;
178
+ let markerDiagnosis = 'ready_marker_unavailable';
179
+
180
+ while (now() < deadline) {
181
+ try {
182
+ const marker = await input.sandbox.fs.downloadFile(input.readyPath, 2);
183
+ const parsed = JSON.parse(marker.toString('utf-8')) as {
184
+ ready?: unknown;
185
+ };
186
+ if (parsed.ready === true) return;
187
+ markerDiagnosis = 'ready_marker_invalid';
188
+ } catch {
189
+ markerDiagnosis = 'ready_marker_unavailable';
190
+ }
191
+ await sleep(Math.min(pollMs, Math.max(1, deadline - now())));
192
+ }
193
+
194
+ const diagnostic = await inspectDetachedDaytonaStartup({
195
+ sandbox: input.sandbox,
196
+ sessionId: input.sessionId,
197
+ cmdId: input.cmdId,
198
+ exitCodePath: input.exitCodePath,
199
+ });
200
+ const oomIndicated =
201
+ diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
202
+ throw new DaytonaRunnerInitializationError(
203
+ `RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not initialize within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The play was stopped and was not retried.`,
204
+ );
205
+ }
@@ -33,7 +33,8 @@ import {
33
33
  } from './daytona-lifecycle';
34
34
  import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
35
35
  import {
36
- inspectDetachedDaytonaStartup,
36
+ confirmDetachedDaytonaRunnerReady,
37
+ DaytonaRunnerInitializationError,
37
38
  startDetachedDaytonaCommand,
38
39
  } from './daytona-session-execution';
39
40
 
@@ -45,8 +46,6 @@ const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
45
46
  const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
46
47
  const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
47
48
  const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
48
- const DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS = 15_000;
49
- const DAYTONA_STARTUP_TRACE_ENV = 'DEEPLINE_DAYTONA_STARTUP_TRACE';
50
49
 
51
50
  const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
52
51
  /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
@@ -397,6 +396,7 @@ export function retryableRuntimeInitializationFailureReason(
397
396
  }
398
397
 
399
398
  function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
399
+ if (error instanceof DaytonaRunnerInitializationError) return false;
400
400
  const message = error instanceof Error ? error.message : String(error);
401
401
  if (!message || /cancelled|aborted/i.test(message)) {
402
402
  return false;
@@ -715,12 +715,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
715
715
  });
716
716
  // Push execution (B2-final, park-and-wake): start the runner command
717
717
  // DETACHED via a Daytona session and return a `detached_runner`
718
- // suspension immediately the worker parks on it (releasing its
719
- // claim, freeing the slot) and wakes on the runner's pushed terminal
720
- // or the ceiling timeout. Nothing here awaits completion: no held
721
- // socket, no poll loop, no per-second sandbox file reads. The sandbox
722
- // MUST outlive this return (the runner is executing inside it); the
723
- // wake leg owns finalize + sandbox GC.
718
+ // suspension after the runner writes its one-time ready marker. The
719
+ // marker is the handoff proof: a Daytona cmdId alone does not prove
720
+ // the sandbox process initialized. After that bounded startup check,
721
+ // the worker parks (releasing its claim and slot) and wakes only on
722
+ // the runner's pushed terminal or the ceiling timeout.
724
723
  const sessionId = `deepline-play-${randomUUID()}`;
725
724
  let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
726
725
  try {
@@ -744,6 +743,16 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
744
743
  // classification below decide fresh-sandbox retry vs loud failure.
745
744
  throw error;
746
745
  }
746
+ await Promise.race([
747
+ confirmDetachedDaytonaRunnerReady({
748
+ sandbox,
749
+ sessionId: start.sessionId,
750
+ cmdId: start.cmdId,
751
+ readyPath: stagedPayload.readyPath,
752
+ exitCodePath: stagedPayload.exitCodePath,
753
+ }),
754
+ cancellationPromise,
755
+ ]);
747
756
  const runnerAttempt = Math.max(
748
757
  0,
749
758
  Math.floor(config.context.runAttempt ?? 0),
@@ -758,52 +767,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
758
767
  exitCodePath: stagedPayload.exitCodePath,
759
768
  elapsedMs: Date.now() - startedAt,
760
769
  });
761
- // Daytona acknowledges an async command with only a cmdId. It does
762
- // not expose a queued/running state, and the backend parks before the
763
- // runner contacts the runtime sheet. During startup investigations,
764
- // retain a bounded snapshot while the sandbox is still alive; the
765
- // normal cancellation path otherwise deletes the only evidence.
766
- // This is opt-in because it adds Daytona control-plane calls to every
767
- // run. The diagnostic excludes command lines and environment values.
768
- if (process.env[DAYTONA_STARTUP_TRACE_ENV] === '1') {
769
- const diagnosticTimer = setTimeout(() => {
770
- void inspectDetachedDaytonaStartup({
771
- sandbox,
772
- sessionId: start.sessionId,
773
- cmdId: start.cmdId,
774
- exitCodePath: stagedPayload.exitCodePath,
775
- }).then(
776
- (diagnostic) =>
777
- emitDaytonaStage(
778
- callbacks,
779
- config.context,
780
- 'execute:startup_diagnostic',
781
- {
782
- sandboxId: sandbox.id,
783
- sessionId: start.sessionId,
784
- cmdId: start.cmdId,
785
- diagnostic,
786
- elapsedMs: Date.now() - startedAt,
787
- },
788
- ),
789
- (error: unknown) =>
790
- emitDaytonaStage(
791
- callbacks,
792
- config.context,
793
- 'execute:startup_diagnostic_failed',
794
- {
795
- sandboxId: sandbox.id,
796
- sessionId: start.sessionId,
797
- cmdId: start.cmdId,
798
- errorType:
799
- error instanceof Error ? error.name : 'unknown',
800
- elapsedMs: Date.now() - startedAt,
801
- },
802
- ),
803
- );
804
- }, DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS);
805
- diagnosticTimer.unref?.();
806
- }
807
770
  return {
808
771
  status: 'suspended',
809
772
  suspension: {
@@ -95,7 +95,7 @@ import {
95
95
  PLAY_RUNTIME_CONTRACT,
96
96
  PLAY_RUNTIME_CONTRACT_HEADER,
97
97
  } from './runtime-contract';
98
- import { PLAY_RUNTIME_API_COMPAT_PATH } from './runtime-api-paths';
98
+ import { PLAY_RUNTIME_API_CURRENT_PATH } from './runtime-api-paths';
99
99
  import {
100
100
  PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
101
101
  PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
@@ -298,6 +298,7 @@ const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
298
298
  'lease_expires_at',
299
299
  ] as const;
300
300
  const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
301
+ const RUNTIME_WORK_RECEIPT_KEY_INDEX = 'idx_deepline_step_receipts_k_unique';
301
302
  const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
302
303
  ...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
303
304
  RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
@@ -504,7 +505,7 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
504
505
  if (!baseUrl) {
505
506
  throw new Error('Runner runtime API requires a baseUrl.');
506
507
  }
507
- return `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
508
+ return `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_CURRENT_PATH}`;
508
509
  }
509
510
 
510
511
  function resolveRuntimeApiHeaders(
@@ -586,11 +587,13 @@ async function postRuntimeApi<TResponse>(
586
587
  ? parsed.retry_after_ms
587
588
  : RUNTIME_API_DEFAULT_RETRY_AFTER_MS;
588
589
  const details =
589
- typeof parsed?.details === 'string'
590
- ? parsed.details
591
- : typeof parsed?.debug_error === 'string'
592
- ? parsed.debug_error
593
- : null;
590
+ typeof parsed?.detail === 'string'
591
+ ? parsed.detail
592
+ : typeof parsed?.details === 'string'
593
+ ? parsed.details
594
+ : typeof parsed?.debug_error === 'string'
595
+ ? parsed.debug_error
596
+ : null;
594
597
  const shouldRetryRuntimeResponse =
595
598
  (response.status === 503 &&
596
599
  parsed?.code === 'ingestion_plane_not_ready') ||
@@ -2040,6 +2043,22 @@ function isMissingRuntimeWorkReceiptSelfHealColumnError(
2040
2043
  );
2041
2044
  }
2042
2045
 
2046
+ function isMissingRuntimeWorkReceiptKeyConstraintError(
2047
+ error: unknown,
2048
+ ): boolean {
2049
+ if (!error || typeof error !== 'object') {
2050
+ return false;
2051
+ }
2052
+ const code = 'code' in error ? String(error.code) : '';
2053
+ const message = 'message' in error ? String(error.message) : '';
2054
+ return (
2055
+ code === '42P10' ||
2056
+ /no unique or exclusion constraint matching the ON CONFLICT specification/i.test(
2057
+ message,
2058
+ )
2059
+ );
2060
+ }
2061
+
2043
2062
  function runtimeWorkReceiptEnsureCacheKey(
2044
2063
  session: RuntimePostgresSession,
2045
2064
  ): string {
@@ -2119,22 +2138,29 @@ async function ensureRuntimeWorkReceiptTable(
2119
2138
  session,
2120
2139
  client,
2121
2140
  );
2122
- if (missingColumns.length === 0) return;
2141
+ if (missingColumns.length > 0) {
2142
+ await client.query(`
2143
+ ALTER TABLE ${workReceiptTable(session)}
2144
+ ${missingColumns
2145
+ .map((column) => {
2146
+ const type =
2147
+ column === 'lease_expires_at'
2148
+ ? 'timestamptz'
2149
+ : column === 'lease_owner_attempt'
2150
+ ? 'integer'
2151
+ : column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
2152
+ ? 'smallint NOT NULL DEFAULT 0'
2153
+ : 'text';
2154
+ return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
2155
+ })
2156
+ .join(',\n ')}
2157
+ `);
2158
+ }
2123
2159
  await client.query(`
2124
- ALTER TABLE ${workReceiptTable(session)}
2125
- ${missingColumns
2126
- .map((column) => {
2127
- const type =
2128
- column === 'lease_expires_at'
2129
- ? 'timestamptz'
2130
- : column === 'lease_owner_attempt'
2131
- ? 'integer'
2132
- : column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
2133
- ? 'smallint NOT NULL DEFAULT 0'
2134
- : 'text';
2135
- return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
2136
- })
2137
- .join(',\n ')}
2160
+ CREATE UNIQUE INDEX IF NOT EXISTS ${quoteIdentifier(
2161
+ RUNTIME_WORK_RECEIPT_KEY_INDEX,
2162
+ )}
2163
+ ON ${workReceiptTable(session)} (k)
2138
2164
  `);
2139
2165
  })
2140
2166
  .then(() => undefined);
@@ -2238,7 +2264,8 @@ async function withRuntimeWorkReceiptClient<T>(
2238
2264
  } catch (error) {
2239
2265
  if (
2240
2266
  isMissingRelationError(error) ||
2241
- isMissingRuntimeWorkReceiptSelfHealColumnError(error)
2267
+ isMissingRuntimeWorkReceiptSelfHealColumnError(error) ||
2268
+ isMissingRuntimeWorkReceiptKeyConstraintError(error)
2242
2269
  ) {
2243
2270
  runtimeWorkReceiptEnsureCache.delete(
2244
2271
  runtimeWorkReceiptEnsureCacheKey(session),
@@ -4636,6 +4663,7 @@ export async function completeRuntimeWorkReceipt(
4636
4663
  runAttempt?: number | null;
4637
4664
  leaseId?: string | null;
4638
4665
  output: unknown;
4666
+ returnOutput?: boolean;
4639
4667
  },
4640
4668
  ): Promise<WorkReceipt | null> {
4641
4669
  if (
@@ -4741,7 +4769,7 @@ export async function completeRuntimeWorkReceipt(
4741
4769
  })}
4742
4770
  RETURNING convert_from(k, 'UTF8') AS k,
4743
4771
  status,
4744
- output,
4772
+ CASE WHEN $7::boolean THEN output ELSE NULL::jsonb END AS output,
4745
4773
  error,
4746
4774
  failure_kind,
4747
4775
  run_id,
@@ -4754,7 +4782,7 @@ export async function completeRuntimeWorkReceipt(
4754
4782
  replay AS (
4755
4783
  SELECT convert_from(k, 'UTF8') AS k,
4756
4784
  status,
4757
- output,
4785
+ CASE WHEN $7::boolean THEN output ELSE NULL::jsonb END AS output,
4758
4786
  error,
4759
4787
  failure_kind,
4760
4788
  run_id,
@@ -4783,6 +4811,7 @@ export async function completeRuntimeWorkReceipt(
4783
4811
  input.runId,
4784
4812
  leaseId,
4785
4813
  runAttempt,
4814
+ input.returnOutput !== false,
4786
4815
  ],
4787
4816
  );
4788
4817
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -4801,6 +4830,7 @@ export async function completeRuntimeWorkReceipts(
4801
4830
  leaseId?: string | null;
4802
4831
  runAttempt?: number | null;
4803
4832
  }>;
4833
+ returnOutput?: boolean;
4804
4834
  },
4805
4835
  ): Promise<Array<WorkReceipt | null>> {
4806
4836
  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
@@ -4819,6 +4849,7 @@ export async function completeRuntimeWorkReceipts(
4819
4849
  await completeRuntimeWorkReceipts(context, {
4820
4850
  playName: input.playName,
4821
4851
  receipts: siblingReceipts,
4852
+ returnOutput: input.returnOutput,
4822
4853
  });
4823
4854
  } catch (error) {
4824
4855
  siblingError = error;
@@ -4938,7 +4969,7 @@ export async function completeRuntimeWorkReceipts(
4938
4969
  })}
4939
4970
  RETURNING target.k,
4940
4971
  target.status,
4941
- target.output,
4972
+ CASE WHEN $7::boolean THEN target.output ELSE NULL::jsonb END AS output,
4942
4973
  target.error,
4943
4974
  target.failure_kind,
4944
4975
  target.run_id,
@@ -4952,7 +4983,7 @@ export async function completeRuntimeWorkReceipts(
4952
4983
  replay AS (
4953
4984
  SELECT target.k,
4954
4985
  target.status,
4955
- target.output,
4986
+ CASE WHEN $7::boolean THEN target.output ELSE NULL::jsonb END AS output,
4956
4987
  target.error,
4957
4988
  target.failure_kind,
4958
4989
  target.run_id,
@@ -5020,6 +5051,7 @@ export async function completeRuntimeWorkReceipts(
5020
5051
  ),
5021
5052
  ),
5022
5053
  RECEIPT_STATUS_COMPLETED,
5054
+ input.returnOutput !== false,
5023
5055
  ],
5024
5056
  );
5025
5057
  return rows.map((row) =>
@@ -1,8 +1,15 @@
1
- export const PLAY_RUNTIME_CONTRACT = 2;
1
+ export const PLAY_RUNTIME_CONTRACT = 3;
2
2
  export const PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED = 1;
3
3
  export const PLAY_RUNTIME_CONTRACT_HEADER = 'x-deepline-runtime-contract';
4
4
 
5
5
  export const RUNTIME_CONTRACT_CHANGELOG = [
6
+ {
7
+ contract: 3,
8
+ changed:
9
+ 'Runtime receipt completion responses are compact acknowledgements; receipt reads remain the payload replay path.',
10
+ compatOwner: 'runtime',
11
+ compatRemoval: 'contract 2 support window closure',
12
+ },
6
13
  {
7
14
  contract: 2,
8
15
  changed:
@@ -12,6 +19,15 @@ export const RUNTIME_CONTRACT_CHANGELOG = [
12
19
  },
13
20
  ] as const;
14
21
 
22
+ export function runtimeReceiptCompletionResponse<
23
+ T extends { output?: unknown },
24
+ >(runtimeContract: number | null | undefined, receipt: T | null): T | null {
25
+ if (!receipt || (runtimeContract ?? 1) < 3) return receipt;
26
+ const { output, ...ack } = receipt;
27
+ void output;
28
+ return ack as T;
29
+ }
30
+
15
31
  export type RuntimeContractResolution =
16
32
  | { ok: true; contract: number }
17
33
  | {
@@ -4,8 +4,9 @@
4
4
  * One of three pluggable axes (alongside runner-backends and dedup-backends).
5
5
  * Selected per-run via PlayExecutionProfile.
6
6
  *
7
- * Absurd is the default production scheduler through the absurd profile.
8
- * Cloudflare Workflows remains available through the workers_edge profile.
7
+ * Absurd is the enabled production scheduler. The Cloudflare Workflows
8
+ * adapter is retained for historical run interpretation while workers_edge
9
+ * launch admission is disabled.
9
10
  *
10
11
  * Customer plays are unaffected — this is purely the orchestration layer.
11
12
  */
@@ -5,10 +5,18 @@ const JWT_RE =
5
5
  /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g;
6
6
  const PRIVATE_KEY_RE =
7
7
  /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
8
+ // Only redact self-identifying credential prefixes here. Generic words such as
9
+ // `key`, `token`, and `secret` are common in harmless play/run slugs (for
10
+ // example `absurd-key-rotation-check`) and need an assignment context before
11
+ // they are sensitive; HIGH_ENTROPY_ASSIGNMENT_RE handles that case below.
8
12
  const COMMON_SECRET_RE =
9
- /\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs]|key|token|secret|api[_-]?key)[A-Za-z0-9_./+=:-]{12,}\b/gi;
13
+ /\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs])[A-Za-z0-9_./+=:-]{12,}\b/gi;
14
+ const GENERIC_PREFIXED_SECRET_RE =
15
+ /\b(?:key|token|secret|api[_-]?key)_[A-Za-z0-9_./+=:-]{12,}\b/gi;
10
16
  const URL_SECRET_VALUE_RE =
11
17
  /\b(?:sk|pk|rk|pat|ghp|github_pat|xox)[A-Za-z0-9_./+=:-]{12,}\b/i;
18
+ const GENERIC_SECRET_ASSIGNMENT_RE =
19
+ /\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*["']?[^"',\s]{12,}["']?/gi;
12
20
  const HIGH_ENTROPY_ASSIGNMENT_RE =
13
21
  /\b(?:api[_-]?key|token|secret|password|authorization|access[_-]?token|refresh[_-]?token)\b\s*[:=]\s*["']?[^"',\s]{16,}["']?/gi;
14
22
  const SENSITIVE_URL_PARAM_RE =
@@ -39,6 +47,8 @@ export function redactSecretLikeString(value: string): string {
39
47
  .replace(BEARER_TOKEN_RE, `Bearer ${SECRET_REDACTION_PLACEHOLDER}`)
40
48
  .replace(JWT_RE, SECRET_REDACTION_PLACEHOLDER)
41
49
  .replace(COMMON_SECRET_RE, SECRET_REDACTION_PLACEHOLDER)
50
+ .replace(GENERIC_PREFIXED_SECRET_RE, SECRET_REDACTION_PLACEHOLDER)
51
+ .replace(GENERIC_SECRET_ASSIGNMENT_RE, SECRET_REDACTION_PLACEHOLDER)
42
52
  .replace(HIGH_ENTROPY_ASSIGNMENT_RE, (match) => {
43
53
  const separator = match.includes('=') ? '=' : ':';
44
54
  const [key] = match.split(separator, 1);
@@ -0,0 +1,10 @@
1
+ export function transientServiceUnavailableDetail(
2
+ error: unknown,
3
+ ): string | null {
4
+ const detail = error instanceof Error ? error.message : String(error);
5
+ return /ServiceUnavailable|Service temporarily unavailable|temporarily unavailable \(Code:\s*\d+\)/i.test(
6
+ detail,
7
+ )
8
+ ? detail
9
+ : null;
10
+ }