deepline 0.3.124 → 0.3.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +10 -0
- package/dist/bundling-sources/sdk/src/compat.ts +17 -2
- package/dist/bundling-sources/sdk/src/play.ts +3 -1
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-batch-operation.ts +61 -10
- package/dist/bundling-sources/shared_libs/observability/redaction.ts +3 -0
- package/dist/bundling-sources/shared_libs/observability/runtime-console.ts +174 -0
- package/dist/bundling-sources/shared_libs/observability/telemetry.ts +26 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-admission-outcome.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/contracts.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/admission/postgres-adapter.ts +28 -11
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-telemetry.ts +121 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +91 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +87 -41
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts +5 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -0
- package/dist/bundling-sources/shared_libs/security/secret-redaction.ts +56 -1
- package/dist/cli/index.js +528 -466
- package/dist/cli/index.mjs +528 -466
- package/dist/index.d.mts +13 -1
- package/dist/index.d.ts +13 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/install-integrity.json +1 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
createTelemetry,
|
|
3
|
+
TELEMETRY_TRIAGE_SCHEMA_VERSION,
|
|
4
|
+
} from '../observability/telemetry';
|
|
2
5
|
|
|
3
6
|
type PlayRuntimeTelemetryInput = {
|
|
4
7
|
phase: string;
|
|
@@ -19,6 +22,123 @@ const playRuntimeTelemetry = createTelemetry({
|
|
|
19
22
|
|
|
20
23
|
type PlayTelemetryLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
21
24
|
|
|
25
|
+
export type PlayRunLaunchConflictTriggerSource =
|
|
26
|
+
| 'api'
|
|
27
|
+
| 'webhook'
|
|
28
|
+
| 'cron'
|
|
29
|
+
| 'sql_listener'
|
|
30
|
+
| 'async_child'
|
|
31
|
+
| null
|
|
32
|
+
| undefined;
|
|
33
|
+
|
|
34
|
+
export type PlayRunLaunchConflictTelemetryInput = {
|
|
35
|
+
orgId?: string | null;
|
|
36
|
+
runId: string;
|
|
37
|
+
playName?: string | null;
|
|
38
|
+
triggerSource?: PlayRunLaunchConflictTriggerSource;
|
|
39
|
+
workflowFamilyKey?: string | null;
|
|
40
|
+
revisionId?: string | null;
|
|
41
|
+
artifactHash?: string | null;
|
|
42
|
+
graphHash?: string | null;
|
|
43
|
+
inputSha256?: string | null;
|
|
44
|
+
error?: unknown;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type PlayRunLaunchConflictTelemetryDetails = {
|
|
48
|
+
level: 'warn' | 'error';
|
|
49
|
+
conflictKind: 'caller' | 'internal' | 'unknown';
|
|
50
|
+
triggerSource: string;
|
|
51
|
+
launchFingerprint: string;
|
|
52
|
+
fields: Record<string, unknown>;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
function launchConflictKindForTriggerSource(
|
|
56
|
+
triggerSource: PlayRunLaunchConflictTriggerSource,
|
|
57
|
+
): PlayRunLaunchConflictTelemetryDetails['conflictKind'] {
|
|
58
|
+
if (triggerSource === 'api') return 'caller';
|
|
59
|
+
if (triggerSource) return 'internal';
|
|
60
|
+
return 'unknown';
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function launchFingerprint(input: PlayRunLaunchConflictTelemetryInput): string {
|
|
64
|
+
// Fingerprint only durable, non-secret launch metadata. In particular, do
|
|
65
|
+
// not serialize the input payload or executor/coordinator credentials. Keep
|
|
66
|
+
// this a small pure-JS hash because this module is also bundled into the
|
|
67
|
+
// Worker runtime, where importing node:crypto is not portable.
|
|
68
|
+
const durableFields = [
|
|
69
|
+
input.orgId ?? '',
|
|
70
|
+
input.runId,
|
|
71
|
+
input.playName ?? '',
|
|
72
|
+
input.triggerSource ?? '',
|
|
73
|
+
input.workflowFamilyKey ?? '',
|
|
74
|
+
input.revisionId ?? '',
|
|
75
|
+
input.artifactHash ?? '',
|
|
76
|
+
input.graphHash ?? '',
|
|
77
|
+
input.inputSha256 ?? '',
|
|
78
|
+
].join('\u001f');
|
|
79
|
+
let first = 0x811c9dc5;
|
|
80
|
+
let second = 0x9e3779b9;
|
|
81
|
+
for (let index = 0; index < durableFields.length; index += 1) {
|
|
82
|
+
const code = durableFields.charCodeAt(index);
|
|
83
|
+
first = Math.imul(first ^ code, 0x01000193);
|
|
84
|
+
second = Math.imul(second ^ code, 0x85ebca6b);
|
|
85
|
+
}
|
|
86
|
+
return `${(first >>> 0).toString(16).padStart(8, '0')}${(second >>> 0)
|
|
87
|
+
.toString(16)
|
|
88
|
+
.padStart(8, '0')}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function buildPlayRunLaunchConflictTelemetry(
|
|
92
|
+
input: PlayRunLaunchConflictTelemetryInput,
|
|
93
|
+
): PlayRunLaunchConflictTelemetryDetails {
|
|
94
|
+
const conflictKind = launchConflictKindForTriggerSource(input.triggerSource);
|
|
95
|
+
const level = conflictKind === 'caller' ? 'warn' : 'error';
|
|
96
|
+
const triggerSource = input.triggerSource ?? 'unknown';
|
|
97
|
+
return {
|
|
98
|
+
level,
|
|
99
|
+
conflictKind,
|
|
100
|
+
triggerSource,
|
|
101
|
+
launchFingerprint: launchFingerprint(input),
|
|
102
|
+
fields: {
|
|
103
|
+
orgId: input.orgId ?? null,
|
|
104
|
+
runId: input.runId,
|
|
105
|
+
playName: input.playName ?? null,
|
|
106
|
+
triggerSource,
|
|
107
|
+
conflictKind,
|
|
108
|
+
code: 'PLAY_RUN_LAUNCH_CONFLICT',
|
|
109
|
+
httpStatus: 409,
|
|
110
|
+
retryable: false,
|
|
111
|
+
launchFingerprint: launchFingerprint(input),
|
|
112
|
+
workflowFamilyKey: input.workflowFamilyKey ?? null,
|
|
113
|
+
revisionId: input.revisionId ?? null,
|
|
114
|
+
artifactHash: input.artifactHash ?? null,
|
|
115
|
+
graphHash: input.graphHash ?? null,
|
|
116
|
+
triage:
|
|
117
|
+
level === 'error'
|
|
118
|
+
? {
|
|
119
|
+
schema_version: TELEMETRY_TRIAGE_SCHEMA_VERSION,
|
|
120
|
+
reportable: true,
|
|
121
|
+
class: 'play_run_launch_conflict',
|
|
122
|
+
}
|
|
123
|
+
: undefined,
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Emit the single structured event for a durable run-id/launch conflict. */
|
|
129
|
+
export function emitPlayRunLaunchConflictTelemetry(
|
|
130
|
+
input: PlayRunLaunchConflictTelemetryInput,
|
|
131
|
+
): void {
|
|
132
|
+
const details = buildPlayRunLaunchConflictTelemetry(input);
|
|
133
|
+
emitPlayTelemetry({
|
|
134
|
+
level: details.level,
|
|
135
|
+
event: 'play.run.admission_conflict',
|
|
136
|
+
tag: '[play.run.admission_conflict]',
|
|
137
|
+
...(details.level === 'error' && input.error ? { error: input.error } : {}),
|
|
138
|
+
fields: details.fields,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
22
142
|
function stringField(
|
|
23
143
|
fields: Record<string, unknown>,
|
|
24
144
|
...keys: readonly string[]
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { ABSURD_WAKE_CHANNEL, type RetryStrategy } from '@deepline/absurd/sdk';
|
|
2
2
|
import {
|
|
3
3
|
createAbsurdRuntimeClient,
|
|
4
4
|
resolveAbsurdPoolOptions,
|
|
5
5
|
resolveAbsurdQueueSuffix,
|
|
6
6
|
} from './absurd-shared';
|
|
7
7
|
import { ensureAbsurdSchema } from './postgres';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
acquirePgNotifyHub,
|
|
10
|
+
withRuntimeSchedulerClient,
|
|
11
|
+
type PgNotifyHubLease,
|
|
12
|
+
} from './postgres-pool';
|
|
9
13
|
|
|
10
14
|
/**
|
|
11
15
|
* GitHub deploy-on-push build tasks ride their own fixed Absurd queue,
|
|
@@ -48,6 +52,91 @@ export const PLAY_DEPLOY_RETRY_STRATEGY: RetryStrategy = {
|
|
|
48
52
|
export const PLAY_DEPLOY_CLEANUP_TTL = '6 hours';
|
|
49
53
|
export const PLAY_DEPLOY_CLEANUP_LIMIT = 1_000;
|
|
50
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Durable idle poll for the deploy worker. `spawn_task` emits a transactional
|
|
57
|
+
* NOTIFY on `absurd_wake` for this queue, so a healthy listener claims a new
|
|
58
|
+
* deploy within one round-trip; this interval only bounds how late a missed
|
|
59
|
+
* notification or a retry with a future `available_at` is discovered.
|
|
60
|
+
*
|
|
61
|
+
* The SDK default (250ms) is meant for a busy queue. Against the production
|
|
62
|
+
* runtime Postgres it made one idle worker the single largest
|
|
63
|
+
* `absurd.claim_task` caller: four claims per second, each running the
|
|
64
|
+
* cancellation sweep, the expired-lease sweep, and the fair-claim ranking
|
|
65
|
+
* over an empty queue.
|
|
66
|
+
*/
|
|
67
|
+
export const PLAY_DEPLOY_IDLE_POLL_INTERVAL_SECONDS = 30;
|
|
68
|
+
/** Durable poll while no LISTEN session is available (connecting/down). */
|
|
69
|
+
export const PLAY_DEPLOY_WAKE_UNAVAILABLE_POLL_INTERVAL_MS = 5_000;
|
|
70
|
+
|
|
71
|
+
export type PlayDeployWakeHook = ((wake: () => void) => (() => void) | void) & {
|
|
72
|
+
isListening: () => boolean;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Wire the deploy worker's poll loop to the engine's `absurd_wake` channel.
|
|
77
|
+
* There is at most one deploy worker machine (two during a rolling deploy),
|
|
78
|
+
* so no leader election is needed: each process holds one direct LISTEN
|
|
79
|
+
* session. A missing hub degrades to the durable poll, never to a spin.
|
|
80
|
+
*/
|
|
81
|
+
export function createPlayDeployWakeHook(input: {
|
|
82
|
+
url: string;
|
|
83
|
+
queue: string;
|
|
84
|
+
acquireNotifyHub?: (input: {
|
|
85
|
+
url: string;
|
|
86
|
+
channel: string;
|
|
87
|
+
applicationName?: string;
|
|
88
|
+
}) => PgNotifyHubLease | null;
|
|
89
|
+
}): PlayDeployWakeHook {
|
|
90
|
+
let listening = false;
|
|
91
|
+
const hook: PlayDeployWakeHook = (wake) => {
|
|
92
|
+
const acquireNotifyHub =
|
|
93
|
+
input.acquireNotifyHub ??
|
|
94
|
+
((notifyInput: {
|
|
95
|
+
url: string;
|
|
96
|
+
channel: string;
|
|
97
|
+
applicationName?: string;
|
|
98
|
+
}) => acquirePgNotifyHub(notifyInput, { idleMs: 0 }));
|
|
99
|
+
const lease = acquireNotifyHub({
|
|
100
|
+
url: input.url,
|
|
101
|
+
channel: ABSURD_WAKE_CHANNEL,
|
|
102
|
+
applicationName: 'deepline-deploy-worker-wake',
|
|
103
|
+
});
|
|
104
|
+
if (!lease) return;
|
|
105
|
+
const unsubscribeNotify = lease.hub.subscribe((payload) => {
|
|
106
|
+
if (payload === input.queue) wake();
|
|
107
|
+
});
|
|
108
|
+
// A freshly connected listener rechecks durable state so a task spawned
|
|
109
|
+
// between the last poll and LISTEN taking effect is not missed.
|
|
110
|
+
const unsubscribeState = lease.hub.subscribeState((state) => {
|
|
111
|
+
const wasListening = listening;
|
|
112
|
+
listening = state === 'listening';
|
|
113
|
+
if (listening || wasListening) wake();
|
|
114
|
+
});
|
|
115
|
+
return () => {
|
|
116
|
+
unsubscribeNotify();
|
|
117
|
+
unsubscribeState();
|
|
118
|
+
listening = false;
|
|
119
|
+
lease.release();
|
|
120
|
+
};
|
|
121
|
+
};
|
|
122
|
+
hook.isListening = () => listening;
|
|
123
|
+
return hook;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Idle poll policy for the deploy worker. Unlike the runs lanes, the deploy
|
|
128
|
+
* queue keeps a durable poll even while listening: retries are scheduled with
|
|
129
|
+
* a future `available_at` and no later NOTIFY, and the worker reads no
|
|
130
|
+
* next-due hint (that read is itself queue-wide work).
|
|
131
|
+
*/
|
|
132
|
+
export function resolvePlayDeployIdlePollFallbackMs(
|
|
133
|
+
hook: Pick<PlayDeployWakeHook, 'isListening'> | undefined,
|
|
134
|
+
): number {
|
|
135
|
+
return hook?.isListening() === true
|
|
136
|
+
? PLAY_DEPLOY_IDLE_POLL_INTERVAL_SECONDS * 1000
|
|
137
|
+
: PLAY_DEPLOY_WAKE_UNAVAILABLE_POLL_INTERVAL_MS;
|
|
138
|
+
}
|
|
139
|
+
|
|
51
140
|
/**
|
|
52
141
|
* Task payload stays tiny (32 KiB control-plane cap): the deploy token plus
|
|
53
142
|
* addressing. The worker fetches everything else — repo, commit, project
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
import { hostname } from 'node:os';
|
|
24
24
|
import { randomUUID } from 'node:crypto';
|
|
25
25
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
26
|
+
import { createRateLimitedRuntimeLogger } from '../../observability/runtime-console';
|
|
26
27
|
import {
|
|
27
28
|
ABSURD_WAKE_CHANNEL,
|
|
28
29
|
CancelledTask,
|
|
@@ -32,6 +33,7 @@ import {
|
|
|
32
33
|
TimeoutError,
|
|
33
34
|
type Absurd,
|
|
34
35
|
type TaskContext,
|
|
36
|
+
type WorkerOptions,
|
|
35
37
|
} from '@deepline/absurd/sdk';
|
|
36
38
|
import {
|
|
37
39
|
AppRuntimeApiResponseError,
|
|
@@ -333,6 +335,21 @@ export const ABSURD_IDLE_POLL_INTERVAL_SECONDS = 30;
|
|
|
333
335
|
* may add latency but must not invent a terminal result outside that policy.
|
|
334
336
|
*/
|
|
335
337
|
export const ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS = 1_000;
|
|
338
|
+
/**
|
|
339
|
+
* An adopted (retired) lane is served by durable polling only. Its parked
|
|
340
|
+
* sleeps and event waits already tolerate the adoption rotation (minutes), so
|
|
341
|
+
* a 30-second discovery bound costs nothing customer-visible, while a direct
|
|
342
|
+
* LISTEN session per adopted queue did: with `ABSURD_LANE_ADOPTION_MAX_QUEUES`
|
|
343
|
+
* queues rotating every discovery pass on every Machine, each attach opened a
|
|
344
|
+
* fresh direct Postgres session (plus its advisory-lock election) and each
|
|
345
|
+
* detach closed one, and any adopted lane that lost the election or the
|
|
346
|
+
* process-wide hub budget fell back to `ABSURD_WAKE_UNAVAILABLE_POLL_INTERVAL_MS`
|
|
347
|
+
* — one `claim_task` plus one next-due read per second per lane. On a
|
|
348
|
+
* 25-connection primary that churn is where the "remaining connection slots
|
|
349
|
+
* are reserved" rejections came from.
|
|
350
|
+
*/
|
|
351
|
+
export const ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS =
|
|
352
|
+
ABSURD_IDLE_POLL_INTERVAL_SECONDS * 1000;
|
|
336
353
|
/** Heartbeat throttle: keep the lease well ahead of expiry (leaseTimeout / 3). */
|
|
337
354
|
const HEARTBEAT_THROTTLE_MS = (HEARTBEAT_CLAIM_TIMEOUT_SECONDS * 1000) / 3;
|
|
338
355
|
const HEARTBEAT_TIMEOUT_RETRY_DELAY_MS = 1_000;
|
|
@@ -5324,6 +5341,41 @@ export function selectAbsurdAdoptionQueues(input: {
|
|
|
5324
5341
|
};
|
|
5325
5342
|
}
|
|
5326
5343
|
|
|
5344
|
+
/**
|
|
5345
|
+
* Poll-loop options for one adopted (retired) lane. Exported so the durable
|
|
5346
|
+
* polling contract is asserted without a Fly Machine or a live engine.
|
|
5347
|
+
*
|
|
5348
|
+
* - No `onWake`: an adopted queue never opens a direct LISTEN session (see
|
|
5349
|
+
* `ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS`).
|
|
5350
|
+
* - `idlePollFallbackMs` is the 30-second durable bound, never the one-second
|
|
5351
|
+
* "listener unavailable" cadence meant for the active lane's standby.
|
|
5352
|
+
* - `nextPollDelayMs` still shortens the wait to the lane's next indexed due
|
|
5353
|
+
* transition, so an already-due parked wake is claimed on the next pass.
|
|
5354
|
+
*/
|
|
5355
|
+
export function absurdAdoptedLaneWorkerOptions(input: {
|
|
5356
|
+
workerBase: string;
|
|
5357
|
+
queue: string;
|
|
5358
|
+
nextPollDelayMs: () => Promise<number | null>;
|
|
5359
|
+
onError: (error: Error) => void;
|
|
5360
|
+
}): WorkerOptions {
|
|
5361
|
+
return {
|
|
5362
|
+
// The `:continuations` suffix is the engine-side half of the adoption
|
|
5363
|
+
// rule: `claim_task` admits this worker only for runs still sleeping
|
|
5364
|
+
// or carrying an event-wake payload.
|
|
5365
|
+
workerId: absurdContinuationWorkerId({
|
|
5366
|
+
workerBase: input.workerBase,
|
|
5367
|
+
queue: input.queue,
|
|
5368
|
+
}),
|
|
5369
|
+
concurrency: ABSURD_LANE_ADOPTION_SLOTS,
|
|
5370
|
+
claimTimeout: HEARTBEAT_CLAIM_TIMEOUT_SECONDS,
|
|
5371
|
+
pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
|
|
5372
|
+
nextPollDelayMs: input.nextPollDelayMs,
|
|
5373
|
+
idlePollFallbackMs: () => ABSURD_ADOPTED_LANE_POLL_INTERVAL_MS,
|
|
5374
|
+
fatalOnLeaseTimeout: false,
|
|
5375
|
+
onError: input.onError,
|
|
5376
|
+
};
|
|
5377
|
+
}
|
|
5378
|
+
|
|
5327
5379
|
/**
|
|
5328
5380
|
* Continuously attach the queues of retired lanes that still hold adoptable
|
|
5329
5381
|
* parked waits, and detach a queue as soon as it stops qualifying (its lane was
|
|
@@ -5661,6 +5713,9 @@ export async function runPostgresSchedulerComputeBillingController(input: {
|
|
|
5661
5713
|
);
|
|
5662
5714
|
const wait = input.sleep ?? ((ms: number) => sleep(ms));
|
|
5663
5715
|
const active = new Set<Promise<void>>();
|
|
5716
|
+
const reportPollFailure = createRateLimitedRuntimeLogger({
|
|
5717
|
+
log: (...args) => console.warn(...args),
|
|
5718
|
+
});
|
|
5664
5719
|
const startJob = (job: PostgresSchedulerComputeBillingJob): void => {
|
|
5665
5720
|
const task = processPostgresSchedulerComputeBillingJob({
|
|
5666
5721
|
job,
|
|
@@ -5725,7 +5780,7 @@ export async function runPostgresSchedulerComputeBillingController(input: {
|
|
|
5725
5780
|
continue;
|
|
5726
5781
|
}
|
|
5727
5782
|
} catch (error) {
|
|
5728
|
-
|
|
5783
|
+
reportPollFailure(
|
|
5729
5784
|
'[absurd-runtime.worker] durable_compute_billing_poll_failed',
|
|
5730
5785
|
{
|
|
5731
5786
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -6333,6 +6388,12 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
|
|
|
6333
6388
|
);
|
|
6334
6389
|
const timeoutMs = ABSURD_SANDBOX_GC_TIMEOUT_MS;
|
|
6335
6390
|
const active = new Set<Promise<void>>();
|
|
6391
|
+
const reportPollFailure = createRateLimitedRuntimeLogger({
|
|
6392
|
+
log: (...args) => console.warn(...args),
|
|
6393
|
+
});
|
|
6394
|
+
const reportReconciliationPollFailure = createRateLimitedRuntimeLogger({
|
|
6395
|
+
log: (...args) => console.warn(...args),
|
|
6396
|
+
});
|
|
6336
6397
|
|
|
6337
6398
|
const startJob = (job: PostgresSchedulerSandboxCleanupJob): void => {
|
|
6338
6399
|
const task = processPostgresSchedulerSandboxCleanupJob({
|
|
@@ -6394,7 +6455,7 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
|
|
|
6394
6455
|
);
|
|
6395
6456
|
}
|
|
6396
6457
|
} catch (error) {
|
|
6397
|
-
|
|
6458
|
+
reportReconciliationPollFailure(
|
|
6398
6459
|
'[absurd-runtime.worker] modal_create_reconciliation_poll_failed',
|
|
6399
6460
|
{
|
|
6400
6461
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -6427,7 +6488,7 @@ export async function runPostgresSchedulerSandboxCleanupController(input: {
|
|
|
6427
6488
|
continue;
|
|
6428
6489
|
}
|
|
6429
6490
|
} catch (error) {
|
|
6430
|
-
|
|
6491
|
+
reportPollFailure(
|
|
6431
6492
|
'[absurd-runtime.worker] durable_sandbox_cleanup_poll_failed',
|
|
6432
6493
|
{
|
|
6433
6494
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -6747,6 +6808,9 @@ export async function startRuntimeWorkerHost(
|
|
|
6747
6808
|
);
|
|
6748
6809
|
await reconcileDeadlineCancellations();
|
|
6749
6810
|
};
|
|
6811
|
+
const reportQueueError = createRateLimitedRuntimeLogger({
|
|
6812
|
+
log: (...args) => console.error(...args),
|
|
6813
|
+
});
|
|
6750
6814
|
const worker = await absurdClient.startWorker({
|
|
6751
6815
|
workerId: `${workerBase}:${queue}`,
|
|
6752
6816
|
concurrency: slots,
|
|
@@ -6773,7 +6837,7 @@ export async function startRuntimeWorkerHost(
|
|
|
6773
6837
|
onWake,
|
|
6774
6838
|
fatalOnLeaseTimeout: false,
|
|
6775
6839
|
onError: (error) =>
|
|
6776
|
-
|
|
6840
|
+
reportQueueError('[absurd-runtime.worker] queue_error', { queue, error }),
|
|
6777
6841
|
});
|
|
6778
6842
|
// Compute settlement and cleanup each have their own controller and
|
|
6779
6843
|
// database lease. Neither consumes an ordinary play slot or competes with
|
|
@@ -6872,47 +6936,29 @@ export async function startRuntimeWorkerHost(
|
|
|
6872
6936
|
adoptedQueue,
|
|
6873
6937
|
);
|
|
6874
6938
|
createAbsurdRuntimeTasks(adoptedClient, workerDeps, adoptedQueue);
|
|
6875
|
-
const
|
|
6876
|
-
|
|
6877
|
-
queue: adoptedQueue,
|
|
6878
|
-
// The adopted lane has no other claimant, and the wake election is
|
|
6879
|
-
// keyed per queue, so listening here never competes with the active
|
|
6880
|
-
// lane's own session. A parked HITL resume keeps its low latency.
|
|
6881
|
-
draining: false,
|
|
6882
|
-
useDirectNotify: shouldUseRuntimeSchedulerDirectNotify(
|
|
6883
|
-
schedulerOptions.schema,
|
|
6884
|
-
),
|
|
6939
|
+
const reportAdoptedQueueError = createRateLimitedRuntimeLogger({
|
|
6940
|
+
log: (...args) => console.error(...args),
|
|
6885
6941
|
});
|
|
6886
|
-
const adoptedWorker = await adoptedClient.startWorker(
|
|
6887
|
-
|
|
6888
|
-
// rule: `claim_task` admits this worker only for runs still sleeping
|
|
6889
|
-
// or carrying an event-wake payload.
|
|
6890
|
-
workerId: absurdContinuationWorkerId({
|
|
6942
|
+
const adoptedWorker = await adoptedClient.startWorker(
|
|
6943
|
+
absurdAdoptedLaneWorkerOptions({
|
|
6891
6944
|
workerBase,
|
|
6892
6945
|
queue: adoptedQueue,
|
|
6946
|
+
nextPollDelayMs: () =>
|
|
6947
|
+
withRuntimeSchedulerClient(poolOptions, async (client) => {
|
|
6948
|
+
const nextDue = await readAbsurdQueueNextDue(
|
|
6949
|
+
client,
|
|
6950
|
+
adoptedQueue,
|
|
6951
|
+
);
|
|
6952
|
+
if (!nextDue.dueAt) return null;
|
|
6953
|
+
return nextDue.dueAt.getTime() - Date.now();
|
|
6954
|
+
}),
|
|
6955
|
+
onError: (error) =>
|
|
6956
|
+
reportAdoptedQueueError(
|
|
6957
|
+
'[absurd-runtime.worker] adopted_queue_error',
|
|
6958
|
+
{ queue: adoptedQueue, error },
|
|
6959
|
+
),
|
|
6893
6960
|
}),
|
|
6894
|
-
|
|
6895
|
-
claimTimeout: HEARTBEAT_CLAIM_TIMEOUT_SECONDS,
|
|
6896
|
-
pollInterval: ABSURD_IDLE_POLL_INTERVAL_SECONDS,
|
|
6897
|
-
nextPollDelayMs: () =>
|
|
6898
|
-
withRuntimeSchedulerClient(poolOptions, async (client) => {
|
|
6899
|
-
const nextDue = await readAbsurdQueueNextDue(
|
|
6900
|
-
client,
|
|
6901
|
-
adoptedQueue,
|
|
6902
|
-
);
|
|
6903
|
-
if (!nextDue.dueAt) return null;
|
|
6904
|
-
return nextDue.dueAt.getTime() - Date.now();
|
|
6905
|
-
}),
|
|
6906
|
-
idlePollFallbackMs: () =>
|
|
6907
|
-
resolveAbsurdIdlePollFallbackMs(adoptedWake),
|
|
6908
|
-
onWake: adoptedWake,
|
|
6909
|
-
fatalOnLeaseTimeout: false,
|
|
6910
|
-
onError: (error) =>
|
|
6911
|
-
console.error('[absurd-runtime.worker] adopted_queue_error', {
|
|
6912
|
-
queue: adoptedQueue,
|
|
6913
|
-
error,
|
|
6914
|
-
}),
|
|
6915
|
-
});
|
|
6961
|
+
);
|
|
6916
6962
|
return {
|
|
6917
6963
|
queue: adoptedQueue,
|
|
6918
6964
|
close: () => adoptedWorker.close(),
|
|
@@ -105,6 +105,8 @@ export function isKnownPooledRuntimePostgresUrl(url: string): boolean {
|
|
|
105
105
|
}
|
|
106
106
|
|
|
107
107
|
export class RuntimeSchedulerDirectDatabaseUrlError extends Error {
|
|
108
|
+
readonly code = 'RUNTIME_SCHEDULER_DIRECT_DATABASE_URL';
|
|
109
|
+
|
|
108
110
|
constructor() {
|
|
109
111
|
super(
|
|
110
112
|
'WORKFLOW_RUNTIME_POSTGRES points at PlanetScale direct Postgres (port 5432). ' +
|
|
@@ -194,6 +196,8 @@ export type RuntimeSchedulerQueryClient = {
|
|
|
194
196
|
};
|
|
195
197
|
|
|
196
198
|
export class RuntimeSchedulerCircuitOpenError extends Error {
|
|
199
|
+
readonly code = 'RUNTIME_SCHEDULER_UNAVAILABLE';
|
|
200
|
+
|
|
197
201
|
constructor(cause: string) {
|
|
198
202
|
super(
|
|
199
203
|
`Runtime scheduler DB circuit breaker is open after repeated connection failures: ${cause}`,
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/scheduler-capacity.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export class RuntimeSchedulerSaturatedError extends Error {
|
|
2
|
+
readonly code = 'RUNTIME_SCHEDULER_SATURATED';
|
|
2
3
|
readonly queueDepth: number;
|
|
3
4
|
|
|
4
5
|
constructor(queueDepth: number) {
|
|
@@ -13,6 +14,7 @@ export class RuntimeSchedulerSaturatedError extends Error {
|
|
|
13
14
|
|
|
14
15
|
/** Scheduler admission is temporarily full, not broken. */
|
|
15
16
|
export class RuntimeSchedulerCapacityError extends Error {
|
|
17
|
+
readonly code: string = 'RUNTIME_SCHEDULER_CAPACITY';
|
|
16
18
|
readonly retryAfterMs: number;
|
|
17
19
|
|
|
18
20
|
constructor(message: string, retryAfterMs: number) {
|
|
@@ -23,6 +25,8 @@ export class RuntimeSchedulerCapacityError extends Error {
|
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export class RuntimeSchedulerPgBouncerCapacityError extends RuntimeSchedulerCapacityError {
|
|
28
|
+
override readonly code = 'RUNTIME_SCHEDULER_PGBOUNCER_CAPACITY';
|
|
29
|
+
|
|
26
30
|
constructor(cause: string) {
|
|
27
31
|
super(
|
|
28
32
|
`Runtime scheduler PgBouncer is at connection capacity: ${cause}`,
|
|
@@ -39,6 +43,7 @@ export class RuntimeSchedulerPgBouncerCapacityError extends RuntimeSchedulerCapa
|
|
|
39
43
|
* atomic launch transaction commits.
|
|
40
44
|
*/
|
|
41
45
|
export class RuntimeSchedulerSubmissionAdmissionError extends RuntimeSchedulerCapacityError {
|
|
46
|
+
override readonly code = 'RUNTIME_SCHEDULER_SUBMISSION_ADMISSION';
|
|
42
47
|
readonly reason: 'full' | 'timed_out';
|
|
43
48
|
|
|
44
49
|
constructor(reason: 'full' | 'timed_out', queueDepth: number) {
|
|
@@ -140,6 +140,7 @@ export const PLAY_AUTHORING_CONTRACT_ISSUE_CODES = [
|
|
|
140
140
|
'play_authoring_fetch_http_error_branch',
|
|
141
141
|
'play_authoring_tool_deprecated',
|
|
142
142
|
'play_authoring_replay_unsafe_nondeterminism',
|
|
143
|
+
'play_authoring_row_sleep_unsupported',
|
|
143
144
|
'play_authoring_binding_invalid',
|
|
144
145
|
'play_authoring_input_schema_unresolved',
|
|
145
146
|
'tool_response_raw_access',
|
|
@@ -2,7 +2,7 @@ const REDACTED_VALUE = '[REDACTED]';
|
|
|
2
2
|
|
|
3
3
|
type SecretRedactionOptions = {
|
|
4
4
|
redactHighEntropy?: boolean;
|
|
5
|
-
/** Preserve
|
|
5
|
+
/** Preserve pagination cursors and signed transfer URLs in authenticated results. */
|
|
6
6
|
keyPolicy?: 'strict' | 'public-result';
|
|
7
7
|
/** Replace sample cursor values with a non-capability-bearing placeholder. */
|
|
8
8
|
placeholderContinuationValues?: boolean;
|
|
@@ -121,6 +121,59 @@ const SAFE_IDENTIFIER_KEYS = new Set([
|
|
|
121
121
|
'snowflake_query_ids',
|
|
122
122
|
]);
|
|
123
123
|
|
|
124
|
+
// Transfer URLs are bearer capabilities. Only return them in authenticated
|
|
125
|
+
// results; keep logs, errors, and generated samples redacted.
|
|
126
|
+
const PUBLIC_RESULT_TRANSFER_URL_KEYS = new Set([
|
|
127
|
+
'uploadUrl',
|
|
128
|
+
'upload_url',
|
|
129
|
+
'downloadUrl',
|
|
130
|
+
'download_url',
|
|
131
|
+
]);
|
|
132
|
+
const TRANSFER_SIGNING_PARAM =
|
|
133
|
+
/^(?:sig|signature|x-amz-(?:algorithm|credential|date|expires|signedheaders|signature|security-token)|x-goog-(?:algorithm|credential|date|expires|signedheaders|signature))$/i;
|
|
134
|
+
|
|
135
|
+
function isPublicResultTransferUrl(
|
|
136
|
+
value: string,
|
|
137
|
+
key: string | undefined,
|
|
138
|
+
options: SecretRedactionOptions,
|
|
139
|
+
): boolean {
|
|
140
|
+
if (
|
|
141
|
+
!key ||
|
|
142
|
+
!PUBLIC_RESULT_TRANSFER_URL_KEYS.has(key) ||
|
|
143
|
+
options.keyPolicy !== 'public-result' ||
|
|
144
|
+
options.placeholderContinuationValues
|
|
145
|
+
)
|
|
146
|
+
return false;
|
|
147
|
+
let url: URL;
|
|
148
|
+
try {
|
|
149
|
+
url = new URL(value);
|
|
150
|
+
} catch {
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
if (
|
|
154
|
+
url.protocol !== 'https:' ||
|
|
155
|
+
url.username ||
|
|
156
|
+
url.password ||
|
|
157
|
+
url.hash ||
|
|
158
|
+
/\s/.test(value)
|
|
159
|
+
)
|
|
160
|
+
return false;
|
|
161
|
+
let signed = false;
|
|
162
|
+
for (const [name, paramValue] of url.searchParams) {
|
|
163
|
+
if (
|
|
164
|
+
SENSITIVE_URL_PARAM_KEY_PATTERN.test(name) &&
|
|
165
|
+
!TRANSFER_SIGNING_PARAM.test(name)
|
|
166
|
+
)
|
|
167
|
+
return false;
|
|
168
|
+
if (
|
|
169
|
+
/^(?:sig|signature|x-amz-signature|x-goog-signature)$/i.test(name) &&
|
|
170
|
+
paramValue
|
|
171
|
+
)
|
|
172
|
+
signed = true;
|
|
173
|
+
}
|
|
174
|
+
return signed;
|
|
175
|
+
}
|
|
176
|
+
|
|
124
177
|
const EXPLICIT_SECRET_VALUE_PATTERNS = [
|
|
125
178
|
/\bnapi_[a-z0-9]{20,}\b/i,
|
|
126
179
|
/\bgh[opsu]_[A-Za-z0-9_]{20,}\b/,
|
|
@@ -254,6 +307,7 @@ export function redactSecretsInValue<T>(
|
|
|
254
307
|
if (EXPLICIT_SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(value))) {
|
|
255
308
|
return REDACTED_VALUE as T;
|
|
256
309
|
}
|
|
310
|
+
if (isPublicResultTransferUrl(value, parentKey, options)) return value as T;
|
|
257
311
|
if (parentKey && SAFE_IDENTIFIER_KEYS.has(parentKey)) {
|
|
258
312
|
return value as T;
|
|
259
313
|
}
|
|
@@ -297,6 +351,7 @@ export function redactSecretsInValue<T>(
|
|
|
297
351
|
if (EXPLICIT_SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(item))) {
|
|
298
352
|
return REDACTED_VALUE;
|
|
299
353
|
}
|
|
354
|
+
if (isPublicResultTransferUrl(item, itemParentKey, options)) return item;
|
|
300
355
|
if (itemParentKey && SAFE_IDENTIFIER_KEYS.has(itemParentKey)) return item;
|
|
301
356
|
if (itemParentKey && shouldRedactKey(itemParentKey)) {
|
|
302
357
|
return REDACTED_VALUE;
|