deepline 0.3.124 → 0.3.126

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +10 -0
  2. package/dist/bundling-sources/sdk/src/compat.ts +17 -2
  3. package/dist/bundling-sources/sdk/src/play.ts +3 -1
  4. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  5. package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +61 -10
  6. package/dist/bundling-sources/shared_libs/observability/redaction.ts +3 -0
  7. package/dist/bundling-sources/shared_libs/observability/runtime-console.ts +174 -0
  8. package/dist/bundling-sources/shared_libs/observability/telemetry.ts +26 -2
  9. package/dist/bundling-sources/shared_libs/play-runtime/play-admission-outcome.ts +3 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +20 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +1 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +28 -11
  13. package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +121 -1
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +91 -2
  15. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +87 -41
  16. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +4 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +5 -0
  18. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -0
  19. package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +56 -1
  20. package/dist/cli/index.js +528 -466
  21. package/dist/cli/index.mjs +528 -466
  22. package/dist/index.d.mts +13 -1
  23. package/dist/index.d.ts +13 -1
  24. package/dist/index.js +1 -1
  25. package/dist/index.mjs +1 -1
  26. package/dist/install-integrity.json +1 -0
  27. package/dist/release.d.mts +1 -1
  28. package/dist/release.d.ts +1 -1
  29. package/dist/release.js +1 -1
  30. package/dist/release.mjs +1 -1
  31. package/package.json +1 -1
@@ -1,4 +1,7 @@
1
- import { createTelemetry } from '../observability/telemetry';
1
+ import {
2
+ createTelemetry,
3
+ TELEMETRY_TRIAGE_SCHEMA_VERSION,
4
+ } from '../observability/telemetry';
2
5
 
3
6
  type PlayRuntimeTelemetryInput = {
4
7
  phase: string;
@@ -19,6 +22,123 @@ const playRuntimeTelemetry = createTelemetry({
19
22
 
20
23
  type PlayTelemetryLevel = 'debug' | 'info' | 'warn' | 'error';
21
24
 
25
+ export type PlayRunLaunchConflictTriggerSource =
26
+ | 'api'
27
+ | 'webhook'
28
+ | 'cron'
29
+ | 'sql_listener'
30
+ | 'async_child'
31
+ | null
32
+ | undefined;
33
+
34
+ export type PlayRunLaunchConflictTelemetryInput = {
35
+ orgId?: string | null;
36
+ runId: string;
37
+ playName?: string | null;
38
+ triggerSource?: PlayRunLaunchConflictTriggerSource;
39
+ workflowFamilyKey?: string | null;
40
+ revisionId?: string | null;
41
+ artifactHash?: string | null;
42
+ graphHash?: string | null;
43
+ inputSha256?: string | null;
44
+ error?: unknown;
45
+ };
46
+
47
+ export type PlayRunLaunchConflictTelemetryDetails = {
48
+ level: 'warn' | 'error';
49
+ conflictKind: 'caller' | 'internal' | 'unknown';
50
+ triggerSource: string;
51
+ launchFingerprint: string;
52
+ fields: Record<string, unknown>;
53
+ };
54
+
55
+ function launchConflictKindForTriggerSource(
56
+ triggerSource: PlayRunLaunchConflictTriggerSource,
57
+ ): PlayRunLaunchConflictTelemetryDetails['conflictKind'] {
58
+ if (triggerSource === 'api') return 'caller';
59
+ if (triggerSource) return 'internal';
60
+ return 'unknown';
61
+ }
62
+
63
+ function launchFingerprint(input: PlayRunLaunchConflictTelemetryInput): string {
64
+ // Fingerprint only durable, non-secret launch metadata. In particular, do
65
+ // not serialize the input payload or executor/coordinator credentials. Keep
66
+ // this a small pure-JS hash because this module is also bundled into the
67
+ // Worker runtime, where importing node:crypto is not portable.
68
+ const durableFields = [
69
+ input.orgId ?? '',
70
+ input.runId,
71
+ input.playName ?? '',
72
+ input.triggerSource ?? '',
73
+ input.workflowFamilyKey ?? '',
74
+ input.revisionId ?? '',
75
+ input.artifactHash ?? '',
76
+ input.graphHash ?? '',
77
+ input.inputSha256 ?? '',
78
+ ].join('\u001f');
79
+ let first = 0x811c9dc5;
80
+ let second = 0x9e3779b9;
81
+ for (let index = 0; index < durableFields.length; index += 1) {
82
+ const code = durableFields.charCodeAt(index);
83
+ first = Math.imul(first ^ code, 0x01000193);
84
+ second = Math.imul(second ^ code, 0x85ebca6b);
85
+ }
86
+ return `${(first >>> 0).toString(16).padStart(8, '0')}${(second >>> 0)
87
+ .toString(16)
88
+ .padStart(8, '0')}`;
89
+ }
90
+
91
+ export function buildPlayRunLaunchConflictTelemetry(
92
+ input: PlayRunLaunchConflictTelemetryInput,
93
+ ): PlayRunLaunchConflictTelemetryDetails {
94
+ const conflictKind = launchConflictKindForTriggerSource(input.triggerSource);
95
+ const level = conflictKind === 'caller' ? 'warn' : 'error';
96
+ const triggerSource = input.triggerSource ?? 'unknown';
97
+ return {
98
+ level,
99
+ conflictKind,
100
+ triggerSource,
101
+ launchFingerprint: launchFingerprint(input),
102
+ fields: {
103
+ orgId: input.orgId ?? null,
104
+ runId: input.runId,
105
+ playName: input.playName ?? null,
106
+ triggerSource,
107
+ conflictKind,
108
+ code: 'PLAY_RUN_LAUNCH_CONFLICT',
109
+ httpStatus: 409,
110
+ retryable: false,
111
+ launchFingerprint: launchFingerprint(input),
112
+ workflowFamilyKey: input.workflowFamilyKey ?? null,
113
+ revisionId: input.revisionId ?? null,
114
+ artifactHash: input.artifactHash ?? null,
115
+ graphHash: input.graphHash ?? null,
116
+ triage:
117
+ level === 'error'
118
+ ? {
119
+ schema_version: TELEMETRY_TRIAGE_SCHEMA_VERSION,
120
+ reportable: true,
121
+ class: 'play_run_launch_conflict',
122
+ }
123
+ : undefined,
124
+ },
125
+ };
126
+ }
127
+
128
+ /** Emit the single structured event for a durable run-id/launch conflict. */
129
+ export function emitPlayRunLaunchConflictTelemetry(
130
+ input: PlayRunLaunchConflictTelemetryInput,
131
+ ): void {
132
+ const details = buildPlayRunLaunchConflictTelemetry(input);
133
+ emitPlayTelemetry({
134
+ level: details.level,
135
+ event: 'play.run.admission_conflict',
136
+ tag: '[play.run.admission_conflict]',
137
+ ...(details.level === 'error' && input.error ? { error: input.error } : {}),
138
+ fields: details.fields,
139
+ });
140
+ }
141
+
22
142
  function stringField(
23
143
  fields: Record<string, unknown>,
24
144
  ...keys: readonly string[]
@@ -1,11 +1,15 @@
1
- import type { RetryStrategy } from '@deepline/absurd/sdk';
1
+ import { ABSURD_WAKE_CHANNEL, type RetryStrategy } from '@deepline/absurd/sdk';
2
2
  import {
3
3
  createAbsurdRuntimeClient,
4
4
  resolveAbsurdPoolOptions,
5
5
  resolveAbsurdQueueSuffix,
6
6
  } from './absurd-shared';
7
7
  import { ensureAbsurdSchema } from './postgres';
8
- import { withRuntimeSchedulerClient } from './postgres-pool';
8
+ import {
9
+ acquirePgNotifyHub,
10
+ withRuntimeSchedulerClient,
11
+ type PgNotifyHubLease,
12
+ } from './postgres-pool';
9
13
 
10
14
  /**
11
15
  * GitHub deploy-on-push build tasks ride their own fixed Absurd queue,
@@ -48,6 +52,91 @@ export const PLAY_DEPLOY_RETRY_STRATEGY: RetryStrategy = {
48
52
  export const PLAY_DEPLOY_CLEANUP_TTL = '6 hours';
49
53
  export const PLAY_DEPLOY_CLEANUP_LIMIT = 1_000;
50
54
 
55
+ /**
56
+ * Durable idle poll for the deploy worker. `spawn_task` emits a transactional
57
+ * NOTIFY on `absurd_wake` for this queue, so a healthy listener claims a new
58
+ * deploy within one round-trip; this interval only bounds how late a missed
59
+ * notification or a retry with a future `available_at` is discovered.
60
+ *
61
+ * The SDK default (250ms) is meant for a busy queue. Against the production
62
+ * runtime Postgres it made one idle worker the single largest
63
+ * `absurd.claim_task` caller: four claims per second, each running the
64
+ * cancellation sweep, the expired-lease sweep, and the fair-claim ranking
65
+ * over an empty queue.
66
+ */
67
+ export const PLAY_DEPLOY_IDLE_POLL_INTERVAL_SECONDS = 30;
68
+ /** Durable poll while no LISTEN session is available (connecting/down). */
69
+ export const PLAY_DEPLOY_WAKE_UNAVAILABLE_POLL_INTERVAL_MS = 5_000;
70
+
71
+ export type PlayDeployWakeHook = ((wake: () => void) => (() => void) | void) & {
72
+ isListening: () => boolean;
73
+ };
74
+
75
+ /**
76
+ * Wire the deploy worker's poll loop to the engine's `absurd_wake` channel.
77
+ * There is at most one deploy worker machine (two during a rolling deploy),
78
+ * so no leader election is needed: each process holds one direct LISTEN
79
+ * session. A missing hub degrades to the durable poll, never to a spin.
80
+ */
81
+ export function createPlayDeployWakeHook(input: {
82
+ url: string;
83
+ queue: string;
84
+ acquireNotifyHub?: (input: {
85
+ url: string;
86
+ channel: string;
87
+ applicationName?: string;
88
+ }) => PgNotifyHubLease | null;
89
+ }): PlayDeployWakeHook {
90
+ let listening = false;
91
+ const hook: PlayDeployWakeHook = (wake) => {
92
+ const acquireNotifyHub =
93
+ input.acquireNotifyHub ??
94
+ ((notifyInput: {
95
+ url: string;
96
+ channel: string;
97
+ applicationName?: string;
98
+ }) => acquirePgNotifyHub(notifyInput, { idleMs: 0 }));
99
+ const lease = acquireNotifyHub({
100
+ url: input.url,
101
+ channel: ABSURD_WAKE_CHANNEL,
102
+ applicationName: 'deepline-deploy-worker-wake',
103
+ });
104
+ if (!lease) return;
105
+ const unsubscribeNotify = lease.hub.subscribe((payload) => {
106
+ if (payload === input.queue) wake();
107
+ });
108
+ // A freshly connected listener rechecks durable state so a task spawned
109
+ // between the last poll and LISTEN taking effect is not missed.
110
+ const unsubscribeState = lease.hub.subscribeState((state) => {
111
+ const wasListening = listening;
112
+ listening = state === 'listening';
113
+ if (listening || wasListening) wake();
114
+ });
115
+ return () => {
116
+ unsubscribeNotify();
117
+ unsubscribeState();
118
+ listening = false;
119
+ lease.release();
120
+ };
121
+ };
122
+ hook.isListening = () => listening;
123
+ return hook;
124
+ }
125
+
126
+ /**
127
+ * Idle poll policy for the deploy worker. Unlike the runs lanes, the deploy
128
+ * queue keeps a durable poll even while listening: retries are scheduled with
129
+ * a future `available_at` and no later NOTIFY, and the worker reads no
130
+ * next-due hint (that read is itself queue-wide work).
131
+ */
132
+ export function resolvePlayDeployIdlePollFallbackMs(
133
+ hook: Pick<PlayDeployWakeHook, 'isListening'> | undefined,
134
+ ): number {
135
+ return hook?.isListening() === true
136
+ ? PLAY_DEPLOY_IDLE_POLL_INTERVAL_SECONDS * 1000
137
+ : PLAY_DEPLOY_WAKE_UNAVAILABLE_POLL_INTERVAL_MS;
138
+ }
139
+
51
140
  /**
52
141
  * Task payload stays tiny (32 KiB control-plane cap): the deploy token plus
53
142
  * addressing. The worker fetches everything else — repo, commit, project
@@ -23,6 +23,7 @@
23
23
  import { hostname } from 'node:os';
24
24
  import { randomUUID } from 'node:crypto';
25
25
  import { setTimeout as sleep } from 'node:timers/promises';
26
+ import { createRateLimitedRuntimeLogger } from '../../observability/runtime-console';
26
27
  import {
27
28
  ABSURD_WAKE_CHANNEL,
28
29
  CancelledTask,
@@ -32,6 +33,7 @@ import {
32
33
  TimeoutError,
33
34
  type Absurd,
34
35
  type TaskContext,
36
+ type WorkerOptions,
35
37
  } from '@deepline/absurd/sdk';
36
38
  import {
37
39
  AppRuntimeApiResponseError,
@@ -333,6 +335,21 @@ export const ABSURD_IDLE_POLL_INTERVAL_SECONDS = 30;
333
335
  * may add latency but must not invent a terminal result outside that policy.
334
336
  */
335
337
  export const ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS = 1_000;
338
+ /**
339
+ * An adopted (retired) lane is served by durable polling only. Its parked
340
+ * sleeps and event waits already tolerate the adoption rotation (minutes), so
341
+ * a 30-second discovery bound costs nothing customer-visible, while a direct
342
+ * LISTEN session per adopted queue did: with `ABSURD_LANE_ADOPTION_MAX_QUEUES`
343
+ * queues rotating every discovery pass on every Machine, each attach opened a
344
+ * fresh direct Postgres session (plus its advisory-lock election) and each
345
+ * detach closed one, and any adopted lane that lost the election or the
346
+ * process-wide hub budget fell back to `ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS`
347
+ * — one `claim_task` plus one next-due read per second per lane. On a
348
+ * 25-connection primary that churn is where the "remaining connection slots
349
+ * are reserved" rejections came from.
350
+ */
351
+ export const ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS =
352
+ ABSURD_IDLE_POLL_INTERVAL_SECONDS * 1000;
336
353
  /** Heartbeat throttle: keep the lease well ahead of expiry (leaseTimeout / 3). */
337
354
  const HEARTBEAT_THROTTLE_MS = (HEARTBEAT_CLAIM_TIMEOUT_SECONDS * 1000) / 3;
338
355
  const HEARTBEAT_TIMEOUT_RETRY_DELAY_MS = 1_000;
@@ -5324,6 +5341,41 @@ export function selectAbsurdAdoptionQueues(input: {
5324
5341
  };
5325
5342
  }
5326
5343
 
5344
+ /**
5345
+ * Poll-loop options for one adopted (retired) lane. Exported so the durable
5346
+ * polling contract is asserted without a Fly Machine or a live engine.
5347
+ *
5348
+ * - No `onWake`: an adopted queue never opens a direct LISTEN session (see
5349
+ * `ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS`).
5350
+ * - `idlePollFallbackMs` is the 30-second durable bound, never the one-second
5351
+ * "listener unavailable" cadence meant for the active lane's standby.
5352
+ * - `nextPollDelayMs` still shortens the wait to the lane's next indexed due
5353
+ * transition, so an already-due parked wake is claimed on the next pass.
5354
+ */
5355
+ export function absurdAdoptedLaneWorkerOptions(input: {
5356
+ workerBase: string;
5357
+ queue: string;
5358
+ nextPollDelayMs: () => Promise<number | null>;
5359
+ onError: (error: Error) => void;
5360
+ }): WorkerOptions {
5361
+ return {
5362
+ // The `:continuations` suffix is the engine-side half of the adoption
5363
+ // rule: `claim_task` admits this worker only for runs still sleeping
5364
+ // or carrying an event-wake payload.
5365
+ workerId: absurdContinuationWorkerId({
5366
+ workerBase: input.workerBase,
5367
+ queue: input.queue,
5368
+ }),
5369
+ concurrency: ABSURD_LANE_ADOPTION_SLOTS,
5370
+ claimTimeout: HEARTBEAT_CLAIM_TIMEOUT_SECONDS,
5371
+ pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
5372
+ nextPollDelayMs: input.nextPollDelayMs,
5373
+ idlePollFallbackMs: () => ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS,
5374
+ fatalOnLeaseTimeout: false,
5375
+ onError: input.onError,
5376
+ };
5377
+ }
5378
+
5327
5379
  /**
5328
5380
  * Continuously attach the queues of retired lanes that still hold adoptable
5329
5381
  * parked waits, and detach a queue as soon as it stops qualifying (its lane was
@@ -5661,6 +5713,9 @@ export async function runPostgresSchedulerComputeBillingController(input: {
5661
5713
  );
5662
5714
  const wait = input.sleep ?? ((ms: number) => sleep(ms));
5663
5715
  const active = new Set<Promise<void>>();
5716
+ const reportPollFailure = createRateLimitedRuntimeLogger({
5717
+ log: (...args) => console.warn(...args),
5718
+ });
5664
5719
  const startJob = (job: PostgresSchedulerComputeBillingJob): void => {
5665
5720
  const task = processPostgresSchedulerComputeBillingJob({
5666
5721
  job,
@@ -5725,7 +5780,7 @@ export async function runPostgresSchedulerComputeBillingController(input: {
5725
5780
  continue;
5726
5781
  }
5727
5782
  } catch (error) {
5728
- console.warn(
5783
+ reportPollFailure(
5729
5784
  '[absurd-runtime.worker] durable_compute_billing_poll_failed',
5730
5785
  {
5731
5786
  error: error instanceof Error ? error.message : String(error),
@@ -6333,6 +6388,12 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
6333
6388
  );
6334
6389
  const timeoutMs = ABSURD_SANDBOX_GC_TIMEOUT_MS;
6335
6390
  const active = new Set<Promise<void>>();
6391
+ const reportPollFailure = createRateLimitedRuntimeLogger({
6392
+ log: (...args) => console.warn(...args),
6393
+ });
6394
+ const reportReconciliationPollFailure = createRateLimitedRuntimeLogger({
6395
+ log: (...args) => console.warn(...args),
6396
+ });
6336
6397
 
6337
6398
  const startJob = (job: PostgresSchedulerSandboxCleanupJob): void => {
6338
6399
  const task = processPostgresSchedulerSandboxCleanupJob({
@@ -6394,7 +6455,7 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
6394
6455
  );
6395
6456
  }
6396
6457
  } catch (error) {
6397
- console.warn(
6458
+ reportReconciliationPollFailure(
6398
6459
  '[absurd-runtime.worker] modal_create_reconciliation_poll_failed',
6399
6460
  {
6400
6461
  error: error instanceof Error ? error.message : String(error),
@@ -6427,7 +6488,7 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
6427
6488
  continue;
6428
6489
  }
6429
6490
  } catch (error) {
6430
- console.warn(
6491
+ reportPollFailure(
6431
6492
  '[absurd-runtime.worker] durable_sandbox_cleanup_poll_failed',
6432
6493
  {
6433
6494
  error: error instanceof Error ? error.message : String(error),
@@ -6747,6 +6808,9 @@ export async function startRuntimeWorkerHost(
6747
6808
  );
6748
6809
  await reconcileDeadlineCancellations();
6749
6810
  };
6811
+ const reportQueueError = createRateLimitedRuntimeLogger({
6812
+ log: (...args) => console.error(...args),
6813
+ });
6750
6814
  const worker = await absurdClient.startWorker({
6751
6815
  workerId: `${workerBase}:${queue}`,
6752
6816
  concurrency: slots,
@@ -6773,7 +6837,7 @@ export async function startRuntimeWorkerHost(
6773
6837
  onWake,
6774
6838
  fatalOnLeaseTimeout: false,
6775
6839
  onError: (error) =>
6776
- console.error('[absurd-runtime.worker] queue_error', { queue, error }),
6840
+ reportQueueError('[absurd-runtime.worker] queue_error', { queue, error }),
6777
6841
  });
6778
6842
  // Compute settlement and cleanup each have their own controller and
6779
6843
  // database lease. Neither consumes an ordinary play slot or competes with
@@ -6872,47 +6936,29 @@ export async function startRuntimeWorkerHost(
6872
6936
  adoptedQueue,
6873
6937
  );
6874
6938
  createAbsurdRuntimeTasks(adoptedClient, workerDeps, adoptedQueue);
6875
- const adoptedWake = createAbsurdWorkerWakeHook({
6876
- poolOptions,
6877
- queue: adoptedQueue,
6878
- // The adopted lane has no other claimant, and the wake election is
6879
- // keyed per queue, so listening here never competes with the active
6880
- // lane's own session. A parked HITL resume keeps its low latency.
6881
- draining: false,
6882
- useDirectNotify: shouldUseRuntimeSchedulerDirectNotify(
6883
- schedulerOptions.schema,
6884
- ),
6939
+ const reportAdoptedQueueError = createRateLimitedRuntimeLogger({
6940
+ log: (...args) => console.error(...args),
6885
6941
  });
6886
- const adoptedWorker = await adoptedClient.startWorker({
6887
- // The `:continuations` suffix is the engine-side half of the adoption
6888
- // rule: `claim_task` admits this worker only for runs still sleeping
6889
- // or carrying an event-wake payload.
6890
- workerId: absurdContinuationWorkerId({
6942
+ const adoptedWorker = await adoptedClient.startWorker(
6943
+ absurdAdoptedLaneWorkerOptions({
6891
6944
  workerBase,
6892
6945
  queue: adoptedQueue,
6946
+ nextPollDelayMs: () =>
6947
+ withRuntimeSchedulerClient(poolOptions, async (client) => {
6948
+ const nextDue = await readAbsurdQueueNextDue(
6949
+ client,
6950
+ adoptedQueue,
6951
+ );
6952
+ if (!nextDue.dueAt) return null;
6953
+ return nextDue.dueAt.getTime() - Date.now();
6954
+ }),
6955
+ onError: (error) =>
6956
+ reportAdoptedQueueError(
6957
+ '[absurd-runtime.worker] adopted_queue_error',
6958
+ { queue: adoptedQueue, error },
6959
+ ),
6893
6960
  }),
6894
- concurrency: ABSURD_LANE_ADOPTION_SLOTS,
6895
- claimTimeout: HEARTBEAT_CLAIM_TIMEOUT_SECONDS,
6896
- pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
6897
- nextPollDelayMs: () =>
6898
- withRuntimeSchedulerClient(poolOptions, async (client) => {
6899
- const nextDue = await readAbsurdQueueNextDue(
6900
- client,
6901
- adoptedQueue,
6902
- );
6903
- if (!nextDue.dueAt) return null;
6904
- return nextDue.dueAt.getTime() - Date.now();
6905
- }),
6906
- idlePollFallbackMs: () =>
6907
- resolveAbsurdIdlePollFallbackMs(adoptedWake),
6908
- onWake: adoptedWake,
6909
- fatalOnLeaseTimeout: false,
6910
- onError: (error) =>
6911
- console.error('[absurd-runtime.worker] adopted_queue_error', {
6912
- queue: adoptedQueue,
6913
- error,
6914
- }),
6915
- });
6961
+ );
6916
6962
  return {
6917
6963
  queue: adoptedQueue,
6918
6964
  close: () => adoptedWorker.close(),
@@ -105,6 +105,8 @@ export function isKnownPooledRuntimePostgresUrl(url: string): boolean {
105
105
  }
106
106
 
107
107
  export class RuntimeSchedulerDirectDatabaseUrlError extends Error {
108
+ readonly code = 'RUNTIME_SCHEDULER_DIRECT_DATABASE_URL';
109
+
108
110
  constructor() {
109
111
  super(
110
112
  'WORKFLOW_RUNTIME_POSTGRES points at PlanetScale direct Postgres (port 5432). ' +
@@ -194,6 +196,8 @@ export type RuntimeSchedulerQueryClient = {
194
196
  };
195
197
 
196
198
  export class RuntimeSchedulerCircuitOpenError extends Error {
199
+ readonly code = 'RUNTIME_SCHEDULER_UNAVAILABLE';
200
+
197
201
  constructor(cause: string) {
198
202
  super(
199
203
  `Runtime scheduler DB circuit breaker is open after repeated connection failures: ${cause}`,
@@ -1,4 +1,5 @@
1
1
  export class RuntimeSchedulerSaturatedError extends Error {
2
+ readonly code = 'RUNTIME_SCHEDULER_SATURATED';
2
3
  readonly queueDepth: number;
3
4
 
4
5
  constructor(queueDepth: number) {
@@ -13,6 +14,7 @@ export class RuntimeSchedulerSaturatedError extends Error {
13
14
 
14
15
  /** Scheduler admission is temporarily full, not broken. */
15
16
  export class RuntimeSchedulerCapacityError extends Error {
17
+ readonly code: string = 'RUNTIME_SCHEDULER_CAPACITY';
16
18
  readonly retryAfterMs: number;
17
19
 
18
20
  constructor(message: string, retryAfterMs: number) {
@@ -23,6 +25,8 @@ export class RuntimeSchedulerCapacityError extends Error {
23
25
  }
24
26
 
25
27
  export class RuntimeSchedulerPgBouncerCapacityError extends RuntimeSchedulerCapacityError {
28
+ override readonly code = 'RUNTIME_SCHEDULER_PGBOUNCER_CAPACITY';
29
+
26
30
  constructor(cause: string) {
27
31
  super(
28
32
  `Runtime scheduler PgBouncer is at connection capacity: ${cause}`,
@@ -39,6 +43,7 @@ export class RuntimeSchedulerPgBouncerCapacityError extends RuntimeSchedulerCapa
39
43
  * atomic launch transaction commits.
40
44
  */
41
45
  export class RuntimeSchedulerSubmissionAdmissionError extends RuntimeSchedulerCapacityError {
46
+ override readonly code = 'RUNTIME_SCHEDULER_SUBMISSION_ADMISSION';
42
47
  readonly reason: 'full' | 'timed_out';
43
48
 
44
49
  constructor(reason: 'full' | 'timed_out', queueDepth: number) {
@@ -140,6 +140,7 @@ export const PLAY_AUTHORING_CONTRACT_ISSUE_CODES = [
140
140
  'play_authoring_fetch_http_error_branch',
141
141
  'play_authoring_tool_deprecated',
142
142
  'play_authoring_replay_unsafe_nondeterminism',
143
+ 'play_authoring_row_sleep_unsupported',
143
144
  'play_authoring_binding_invalid',
144
145
  'play_authoring_input_schema_unresolved',
145
146
  'tool_response_raw_access',
@@ -2,7 +2,7 @@ const REDACTED_VALUE = '[REDACTED]';
2
2
 
3
3
  type SecretRedactionOptions = {
4
4
  redactHighEntropy?: boolean;
5
- /** Preserve only known pagination cursor fields in authenticated result data. */
5
+ /** Preserve pagination cursors and signed transfer URLs in authenticated results. */
6
6
  keyPolicy?: 'strict' | 'public-result';
7
7
  /** Replace sample cursor values with a non-capability-bearing placeholder. */
8
8
  placeholderContinuationValues?: boolean;
@@ -121,6 +121,59 @@ const SAFE_IDENTIFIER_KEYS = new Set([
121
121
  'snowflake_query_ids',
122
122
  ]);
123
123
 
124
+ // Transfer URLs are bearer capabilities. Only return them in authenticated
125
+ // results; keep logs, errors, and generated samples redacted.
126
+ const PUBLIC_RESULT_TRANSFER_URL_KEYS = new Set([
127
+ 'uploadUrl',
128
+ 'upload_url',
129
+ 'downloadUrl',
130
+ 'download_url',
131
+ ]);
132
+ const TRANSFER_SIGNING_PARAM =
133
+ /^(?:sig|signature|x-amz-(?:algorithm|credential|date|expires|signedheaders|signature|security-token)|x-goog-(?:algorithm|credential|date|expires|signedheaders|signature))$/i;
134
+
135
+ function isPublicResultTransferUrl(
136
+ value: string,
137
+ key: string | undefined,
138
+ options: SecretRedactionOptions,
139
+ ): boolean {
140
+ if (
141
+ !key ||
142
+ !PUBLIC_RESULT_TRANSFER_URL_KEYS.has(key) ||
143
+ options.keyPolicy !== 'public-result' ||
144
+ options.placeholderContinuationValues
145
+ )
146
+ return false;
147
+ let url: URL;
148
+ try {
149
+ url = new URL(value);
150
+ } catch {
151
+ return false;
152
+ }
153
+ if (
154
+ url.protocol !== 'https:' ||
155
+ url.username ||
156
+ url.password ||
157
+ url.hash ||
158
+ /\s/.test(value)
159
+ )
160
+ return false;
161
+ let signed = false;
162
+ for (const [name, paramValue] of url.searchParams) {
163
+ if (
164
+ SENSITIVE_URL_PARAM_KEY_PATTERN.test(name) &&
165
+ !TRANSFER_SIGNING_PARAM.test(name)
166
+ )
167
+ return false;
168
+ if (
169
+ /^(?:sig|signature|x-amz-signature|x-goog-signature)$/i.test(name) &&
170
+ paramValue
171
+ )
172
+ signed = true;
173
+ }
174
+ return signed;
175
+ }
176
+
124
177
  const EXPLICIT_SECRET_VALUE_PATTERNS = [
125
178
  /\bnapi_[a-z0-9]{20,}\b/i,
126
179
  /\bgh[opsu]_[A-Za-z0-9_]{20,}\b/,
@@ -254,6 +307,7 @@ export function redactSecretsInValue<T>(
254
307
  if (EXPLICIT_SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(value))) {
255
308
  return REDACTED_VALUE as T;
256
309
  }
310
+ if (isPublicResultTransferUrl(value, parentKey, options)) return value as T;
257
311
  if (parentKey && SAFE_IDENTIFIER_KEYS.has(parentKey)) {
258
312
  return value as T;
259
313
  }
@@ -297,6 +351,7 @@ export function redactSecretsInValue<T>(
297
351
  if (EXPLICIT_SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(item))) {
298
352
  return REDACTED_VALUE;
299
353
  }
354
+ if (isPublicResultTransferUrl(item, itemParentKey, options)) return item;
300
355
  if (itemParentKey && SAFE_IDENTIFIER_KEYS.has(itemParentKey)) return item;
301
356
  if (itemParentKey && shouldRedactKey(itemParentKey)) {
302
357
  return REDACTED_VALUE;