deepline 0.3.244 → 0.3.246
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +38 -49
- package/dist/bundling-sources/sdk/src/http.ts +10 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/runs/read-errors.ts +19 -0
- package/dist/bundling-sources/sdk/src/runs/watch-polling.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +15 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +20 -11
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +74 -0
- package/dist/bundling-sources/shared_libs/plays/execution-pipeline.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/retained-pipeline-graph.ts +12 -6
- package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts +2 -7
- package/dist/cli/index.js +36 -66
- package/dist/cli/index.mjs +36 -66
- package/dist/index.js +25 -32
- package/dist/index.mjs +25 -32
- package/dist/install-integrity.json +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/bundling-sources/sdk/src/stream-reconnect.ts +0 -47
|
@@ -113,7 +113,10 @@ import type { MonitorFleetDefinition } from './monitor-fleets.js';
|
|
|
113
113
|
import type { PlayStagedFileRef } from './plays/local-file-discovery.js';
|
|
114
114
|
import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest.js';
|
|
115
115
|
import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
|
|
116
|
-
import {
|
|
116
|
+
import {
|
|
117
|
+
RUNTIME_ENVIRONMENT_TOKEN_HEADER,
|
|
118
|
+
RETAINED_PIPELINE_RECEIVERS_HEADER,
|
|
119
|
+
} from '../../shared_libs/play-runtime/coordinator-headers.js';
|
|
117
120
|
import {
|
|
118
121
|
normalizePlayRuntimeEnvironment,
|
|
119
122
|
normalizePlayRuntimeNamespace,
|
|
@@ -123,6 +126,7 @@ import {
|
|
|
123
126
|
} from '../../shared_libs/play-runtime/runtime-environment.js';
|
|
124
127
|
import {
|
|
125
128
|
decodePlayRunPublicStatus,
|
|
129
|
+
isTerminalPlayRunLifecycleStatus,
|
|
126
130
|
type PlayRunPublicStatus,
|
|
127
131
|
} from '../../shared_libs/play-runtime/run-lifecycle-policy.js';
|
|
128
132
|
export type SlackNotificationTarget =
|
|
@@ -209,7 +213,6 @@ function notificationObjectKey(kind: 'group' | 'destination') {
|
|
|
209
213
|
return `${kind}.${globalThis.crypto.randomUUID()}`;
|
|
210
214
|
}
|
|
211
215
|
|
|
212
|
-
const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
|
|
213
216
|
const MONITOR_NON_RETRYABLE_MUTATION_OPTIONS = {
|
|
214
217
|
maxRetries: 0,
|
|
215
218
|
exactUrlOnly: true,
|
|
@@ -403,7 +406,12 @@ function runtimeSelectionHeaders(
|
|
|
403
406
|
'RUNTIME_ENVIRONMENT_TOKEN_REQUIRED',
|
|
404
407
|
);
|
|
405
408
|
}
|
|
406
|
-
|
|
409
|
+
const receivers =
|
|
410
|
+
process.env.DEEPLINE_RUNTIME_RETAINED_PIPELINE_RECEIVERS?.trim();
|
|
411
|
+
return {
|
|
412
|
+
[RUNTIME_ENVIRONMENT_TOKEN_HEADER]: token,
|
|
413
|
+
...(receivers ? { [RETAINED_PIPELINE_RECEIVERS_HEADER]: receivers } : {}),
|
|
414
|
+
};
|
|
407
415
|
}
|
|
408
416
|
|
|
409
417
|
function normalizeTestPolicyOverrides(
|
|
@@ -2306,7 +2314,9 @@ function normalizePlayRunStart(raw: Record<string, unknown>): PlayRunStart {
|
|
|
2306
2314
|
...(runPackage.run.dashboardUrl
|
|
2307
2315
|
? { dashboardUrl: runPackage.run.dashboardUrl }
|
|
2308
2316
|
: {}),
|
|
2309
|
-
...(
|
|
2317
|
+
...(isTerminalPlayRunLifecycleStatus(status)
|
|
2318
|
+
? { finalStatus: runPackage }
|
|
2319
|
+
: {}),
|
|
2310
2320
|
package: runPackage,
|
|
2311
2321
|
};
|
|
2312
2322
|
}
|
|
@@ -2469,18 +2479,22 @@ function getPlayLiveEventPayload(
|
|
|
2469
2479
|
: {};
|
|
2470
2480
|
}
|
|
2471
2481
|
|
|
2472
|
-
|
|
2482
|
+
/** Run lifecycle events are the only events that can finish a watch. */
|
|
2483
|
+
function playLiveRunStatus(event: PlayLiveEvent): PlayStatus['status'] | null {
|
|
2473
2484
|
if (
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
value === 'completed' ||
|
|
2478
|
-
value === 'failed' ||
|
|
2479
|
-
value === 'cancelled'
|
|
2485
|
+
event.type !== 'play.run.snapshot' &&
|
|
2486
|
+
event.type !== 'play.run.status' &&
|
|
2487
|
+
event.type !== 'play.run.final_status'
|
|
2480
2488
|
) {
|
|
2481
|
-
return
|
|
2489
|
+
return null;
|
|
2482
2490
|
}
|
|
2483
|
-
|
|
2491
|
+
const payload = getPlayLiveEventPayload(event);
|
|
2492
|
+
return (
|
|
2493
|
+
decodePlayRunPublicStatus(payload.status) ??
|
|
2494
|
+
(isPlayRunPackage(payload)
|
|
2495
|
+
? decodePlayRunPublicStatus(payload.run.status)
|
|
2496
|
+
: null)
|
|
2497
|
+
);
|
|
2484
2498
|
}
|
|
2485
2499
|
|
|
2486
2500
|
function appendPlayLiveLogLines(
|
|
@@ -2545,12 +2559,7 @@ function updatePlayLiveStatusState(
|
|
|
2545
2559
|
: isPlayRunPackage(payload)
|
|
2546
2560
|
? payload.run.id
|
|
2547
2561
|
: state.runId;
|
|
2548
|
-
const status =
|
|
2549
|
-
normalizeLiveStatus(payload.status) ??
|
|
2550
|
-
(isPlayRunPackage(payload)
|
|
2551
|
-
? normalizeLiveStatus(payload.run.status)
|
|
2552
|
-
: null) ??
|
|
2553
|
-
state.status;
|
|
2562
|
+
const status = playLiveRunStatus(event) ?? state.status;
|
|
2554
2563
|
const progressPayload = isRecord(payload.progress) ? payload.progress : {};
|
|
2555
2564
|
// Snapshots no longer REPLACE accumulated logs (ADR-0009): the snapshot
|
|
2556
2565
|
// only retains a bounded tail, so replacing would clobber the seq-keyed
|
|
@@ -4221,13 +4230,8 @@ export class DeeplineClient {
|
|
|
4221
4230
|
runId: string,
|
|
4222
4231
|
options?: RunsWatchOptions,
|
|
4223
4232
|
): Promise<PlayStatus> {
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
status: 'running',
|
|
4227
|
-
logs: [],
|
|
4228
|
-
lastLogSeq: 0,
|
|
4229
|
-
latest: null,
|
|
4230
|
-
};
|
|
4233
|
+
// Forward events without retaining their log history. Only lifecycle
|
|
4234
|
+
// evidence is needed here; the final package comes from the durable read.
|
|
4231
4235
|
let terminalObserved = false;
|
|
4232
4236
|
for await (const event of this.watchRunEvents(runId, {
|
|
4233
4237
|
signal: options?.signal,
|
|
@@ -4235,19 +4239,20 @@ export class DeeplineClient {
|
|
|
4235
4239
|
onRetry: options?.onRetry ?? options?.onReconnect,
|
|
4236
4240
|
})) {
|
|
4237
4241
|
options?.onEvent?.(event);
|
|
4238
|
-
|
|
4239
|
-
if (status && TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
4242
|
+
if (isTerminalPlayRunLifecycleStatus(playLiveRunStatus(event))) {
|
|
4240
4243
|
terminalObserved = true;
|
|
4241
4244
|
}
|
|
4242
4245
|
}
|
|
4243
4246
|
if (options?.signal?.aborted) {
|
|
4244
4247
|
throw new DeeplineError('Run watch aborted.', undefined, 'ABORTED');
|
|
4245
4248
|
}
|
|
4246
|
-
if (terminalObserved) {
|
|
4247
|
-
return await this.getRunStatus(runId);
|
|
4248
|
-
}
|
|
4249
4249
|
const refreshed = await this.getRunStatus(runId);
|
|
4250
|
-
if (
|
|
4250
|
+
if (
|
|
4251
|
+
terminalObserved ||
|
|
4252
|
+
isTerminalPlayRunLifecycleStatus(refreshed.status)
|
|
4253
|
+
) {
|
|
4254
|
+
return refreshed;
|
|
4255
|
+
}
|
|
4251
4256
|
throw new DeeplineError(
|
|
4252
4257
|
`Run watch for ${runId} ended before a terminal status.`,
|
|
4253
4258
|
undefined,
|
|
@@ -4474,23 +4479,7 @@ export class DeeplineClient {
|
|
|
4474
4479
|
`/api/v2/runs/${encodeURIComponent(runId)}${query}`,
|
|
4475
4480
|
{ headers: { 'x-deepline-run-status': '1' } },
|
|
4476
4481
|
);
|
|
4477
|
-
const responseRunStatus = isPlayRunPackage(response)
|
|
4478
|
-
? response.runStatus
|
|
4479
|
-
: isPlayRunPackage(response.package)
|
|
4480
|
-
? response.package.runStatus
|
|
4481
|
-
: response.runStatus;
|
|
4482
|
-
const responseOverview = isPlayRunPackage(response)
|
|
4483
|
-
? response.overview
|
|
4484
|
-
: isPlayRunPackage(response.package)
|
|
4485
|
-
? response.package.overview
|
|
4486
|
-
: response.overview;
|
|
4487
4482
|
const status = normalizePlayStatus(response);
|
|
4488
|
-
if (isRecord(responseRunStatus)) {
|
|
4489
|
-
status.runStatus = responseRunStatus as unknown as RunGetStatus;
|
|
4490
|
-
}
|
|
4491
|
-
if (isRecord(responseOverview)) {
|
|
4492
|
-
status.overview = responseOverview as unknown as RunLifecycleStatus;
|
|
4493
|
-
}
|
|
4494
4483
|
if (options?.failedLogs !== true || status.status !== 'failed') {
|
|
4495
4484
|
return status;
|
|
4496
4485
|
}
|
|
@@ -5796,7 +5785,7 @@ export class DeeplineClient {
|
|
|
5796
5785
|
options?.onProgress?.(status);
|
|
5797
5786
|
}
|
|
5798
5787
|
|
|
5799
|
-
if (
|
|
5788
|
+
if (isTerminalPlayRunLifecycleStatus(status.status)) {
|
|
5800
5789
|
terminalObserved = status;
|
|
5801
5790
|
}
|
|
5802
5791
|
}
|
|
@@ -38,6 +38,7 @@ import { readSdkSkillsLocalVersion } from './skills-version.js';
|
|
|
38
38
|
import { deploymentProtectionHeaders } from './deployment-protection.js';
|
|
39
39
|
import {
|
|
40
40
|
ABSURD_RELEASE_OVERRIDE_HEADER,
|
|
41
|
+
RETAINED_PIPELINE_RECEIVERS_HEADER,
|
|
41
42
|
COORDINATOR_INTERNAL_TOKEN_HEADER,
|
|
42
43
|
COORDINATOR_URL_OVERRIDE_HEADER,
|
|
43
44
|
PREFERRED_RECEIPT_GATEWAY_MACHINE_ID_HEADER,
|
|
@@ -345,6 +346,15 @@ export class HttpClient {
|
|
|
345
346
|
if (runtimeTestFault?.trim()) {
|
|
346
347
|
headers[PLAY_RUNTIME_TEST_FAULT_HEADER] = runtimeTestFault.trim();
|
|
347
348
|
}
|
|
349
|
+
const retainedReceivers =
|
|
350
|
+
typeof process !== 'undefined'
|
|
351
|
+
? process.env?.DEEPLINE_RUNTIME_RETAINED_PIPELINE_RECEIVERS?.trim()
|
|
352
|
+
: undefined;
|
|
353
|
+
if (retainedReceivers && coordinatorInternalToken?.trim()) {
|
|
354
|
+
headers[RETAINED_PIPELINE_RECEIVERS_HEADER] = retainedReceivers;
|
|
355
|
+
headers[COORDINATOR_INTERNAL_TOKEN_HEADER] =
|
|
356
|
+
coordinatorInternalToken.trim();
|
|
357
|
+
}
|
|
348
358
|
if (absurdReleaseOverride?.trim() && coordinatorInternalToken?.trim()) {
|
|
349
359
|
headers[ABSURD_RELEASE_OVERRIDE_HEADER] = absurdReleaseOverride.trim();
|
|
350
360
|
}
|
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.246',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { DeeplineError } from '../errors.js';
|
|
2
|
+
|
|
3
|
+
export function isTransientPlayRunReadError(error: unknown): boolean {
|
|
4
|
+
if (
|
|
5
|
+
error instanceof DeeplineError &&
|
|
6
|
+
error.code === 'PLAY_STREAM_NETWORK_ABORTED'
|
|
7
|
+
) {
|
|
8
|
+
return true;
|
|
9
|
+
}
|
|
10
|
+
if (error instanceof DeeplineError && typeof error.statusCode === 'number') {
|
|
11
|
+
// 5xx responses and network-level failures may recover. Client errors,
|
|
12
|
+
// especially 404, stay visible instead of disappearing into a retry loop.
|
|
13
|
+
return error.statusCode >= 500 && error.statusCode < 600;
|
|
14
|
+
}
|
|
15
|
+
const text = error instanceof Error ? error.message : String(error);
|
|
16
|
+
return /auth validation backend timed out|coordinator \/submit(?:\?[^ ]*)? 5\d\d|Worker threw exception|Internal Server Error|Service Unavailable|fetch failed|eaddrnotavail|econnreset|etimedout|eai_again|socket hang up/i.test(
|
|
17
|
+
text,
|
|
18
|
+
);
|
|
19
|
+
}
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
type PlayRunStreamDiffState,
|
|
8
8
|
type PlayRunStreamEvent,
|
|
9
9
|
} from '../../../shared_libs/play-runtime/run-snapshot-stream.js';
|
|
10
|
-
import { isTransientPlayRunReadError } from '
|
|
10
|
+
import { isTransientPlayRunReadError } from './read-errors.js';
|
|
11
11
|
|
|
12
12
|
/** The watch endpoint is a small authenticated REST read, polled every two seconds. */
|
|
13
13
|
export const RUN_WATCH_POLL_INTERVAL_MS = 2_000;
|
|
@@ -22,6 +22,16 @@ export const RUNTIME_SCHEDULER_SCHEMA_OVERRIDE_HEADER =
|
|
|
22
22
|
*/
|
|
23
23
|
export const RUNTIME_ENVIRONMENT_TOKEN_HEADER =
|
|
24
24
|
'x-deepline-runtime-environment-token';
|
|
25
|
+
/** Authenticated target-owner assertion, issued only after deployment reproof. */
|
|
26
|
+
export const RETAINED_PIPELINE_RECEIVERS_HEADER =
|
|
27
|
+
'x-deepline-retained-pipeline-receivers';
|
|
28
|
+
export type RetainedPipelineReceivers = {
|
|
29
|
+
namespace: string;
|
|
30
|
+
schedulerSchema?: string;
|
|
31
|
+
backend: string;
|
|
32
|
+
callbackOrigin: string;
|
|
33
|
+
runtimeApiOrigin: string;
|
|
34
|
+
};
|
|
25
35
|
/**
|
|
26
36
|
* Internal top-level `profile=absurd` launches, including deploy canaries, use
|
|
27
37
|
* this header to pin an exact candidate release before activation. Public API
|
|
@@ -10,9 +10,7 @@ import {
|
|
|
10
10
|
jsonByteLength,
|
|
11
11
|
jsonByteLengthUpTo,
|
|
12
12
|
} from '../output-size-limits';
|
|
13
|
-
import {
|
|
14
|
-
PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER,
|
|
15
|
-
} from '../runtime-contract';
|
|
13
|
+
import { PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER } from '../runtime-contract';
|
|
16
14
|
import {
|
|
17
15
|
createRunnerGatewayTransportAttemptId,
|
|
18
16
|
runnerGatewayTransportErrorCode,
|
|
@@ -595,8 +593,14 @@ export async function pushRunnerTerminal(
|
|
|
595
593
|
: 'redirect',
|
|
596
594
|
{ status: response.status },
|
|
597
595
|
);
|
|
598
|
-
//
|
|
599
|
-
|
|
596
|
+
// Gateway queue admission can shed this first-write-wins terminal with
|
|
597
|
+
// 429 before acquiring a database client. Retry the same terminal fact
|
|
598
|
+
// within the existing attempt budget; scope/fence rejections stay final.
|
|
599
|
+
if (
|
|
600
|
+
response.status >= 400 &&
|
|
601
|
+
response.status < 500 &&
|
|
602
|
+
response.status !== 429
|
|
603
|
+
) {
|
|
600
604
|
warn('terminal_push_rejected', {
|
|
601
605
|
runId: options.runId,
|
|
602
606
|
status: response.status,
|
|
@@ -606,9 +610,12 @@ export async function pushRunnerTerminal(
|
|
|
606
610
|
}
|
|
607
611
|
} catch (error) {
|
|
608
612
|
lastError = error instanceof Error ? error.message : String(error);
|
|
609
|
-
reportTransport(
|
|
610
|
-
|
|
611
|
-
|
|
613
|
+
reportTransport(
|
|
614
|
+
requestController.signal.aborted ? 'timeout' : 'network_error',
|
|
615
|
+
{
|
|
616
|
+
errorCode: runnerGatewayTransportErrorCode(error),
|
|
617
|
+
},
|
|
618
|
+
);
|
|
612
619
|
} finally {
|
|
613
620
|
clearTimeout(requestTimeout);
|
|
614
621
|
}
|
|
@@ -488,10 +488,15 @@ async function main() {
|
|
|
488
488
|
checkpointBytes: 0,
|
|
489
489
|
tableNamespace: null,
|
|
490
490
|
});
|
|
491
|
+
const requestBody = JSON.stringify({
|
|
492
|
+
action: 'runner_terminal',
|
|
493
|
+
runId: push.runId,
|
|
494
|
+
result,
|
|
495
|
+
});
|
|
491
496
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
497
|
+
const controller = new AbortController();
|
|
498
|
+
const timer = setTimeout(() => controller.abort(), 10000);
|
|
492
499
|
try {
|
|
493
|
-
const controller = new AbortController();
|
|
494
|
-
const timer = setTimeout(() => controller.abort(), 10000);
|
|
495
500
|
const response = await fetch(gateway + '/api/v2/plays/runtime', {
|
|
496
501
|
method: 'POST',
|
|
497
502
|
headers: {
|
|
@@ -501,28 +506,32 @@ async function main() {
|
|
|
501
506
|
? { 'fly-force-instance-id': preferredGatewayMachineId }
|
|
502
507
|
: {}),
|
|
503
508
|
},
|
|
504
|
-
body:
|
|
505
|
-
action: 'runner_terminal',
|
|
506
|
-
runId: push.runId,
|
|
507
|
-
result,
|
|
508
|
-
}),
|
|
509
|
+
body: requestBody,
|
|
509
510
|
signal: controller.signal,
|
|
510
511
|
redirect: 'manual',
|
|
511
512
|
});
|
|
512
|
-
|
|
513
|
+
// Keep the HTTP deadline through body consumption so retries do not
|
|
514
|
+
// leave an unread socket behind. The status already acknowledges the
|
|
515
|
+
// durable terminal even if its diagnostic response body is interrupted.
|
|
516
|
+
await response.text().catch(() => '');
|
|
513
517
|
if (response.ok) {
|
|
514
518
|
writeDiagnostic('[crash-terminal] pushed status=' + result.status);
|
|
515
519
|
return;
|
|
516
520
|
}
|
|
517
521
|
writeDiagnostic('[crash-terminal] rejected http=' + response.status);
|
|
518
|
-
//
|
|
519
|
-
|
|
522
|
+
// A 429 is gateway admission pressure, not a scope/fence rejection.
|
|
523
|
+
// Replaying this terminal fact never replays customer execution.
|
|
524
|
+
if (response.status >= 400 && response.status < 500 && response.status !== 429) return;
|
|
520
525
|
} catch (error) {
|
|
521
526
|
writeDiagnostic(
|
|
522
527
|
'[crash-terminal] transport ' + (error && error.message ? error.message : String(error)),
|
|
523
528
|
);
|
|
529
|
+
} finally {
|
|
530
|
+
clearTimeout(timer);
|
|
531
|
+
}
|
|
532
|
+
if (attempt < 3) {
|
|
533
|
+
await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1000));
|
|
524
534
|
}
|
|
525
|
-
await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1000));
|
|
526
535
|
}
|
|
527
536
|
writeDiagnostic('[crash-terminal] exhausted retries');
|
|
528
537
|
}
|
|
@@ -937,6 +937,7 @@ export async function withRuntimeSchedulerClient<T>(
|
|
|
937
937
|
throw error;
|
|
938
938
|
}
|
|
939
939
|
const admissionWaitMs = performance.now() - admissionStartedAt;
|
|
940
|
+
let unsampledAdmissionWaitMs = admissionWaitMs;
|
|
940
941
|
let lastError: unknown = null;
|
|
941
942
|
try {
|
|
942
943
|
throwIfSchedulerAborted(signal);
|
|
@@ -996,7 +997,14 @@ export async function withRuntimeSchedulerClient<T>(
|
|
|
996
997
|
entry.stats.peakTotalClients,
|
|
997
998
|
entry.pool.totalCount,
|
|
998
999
|
);
|
|
999
|
-
|
|
1000
|
+
// The admission gate owns the local queue before pg.Pool sees this
|
|
1001
|
+
// checkout. Include that wait once, on the operation's first acquired
|
|
1002
|
+
// client; retries retain the permit and only add their own connect wait.
|
|
1003
|
+
pushSample(
|
|
1004
|
+
entry.stats.acquireWait,
|
|
1005
|
+
unsampledAdmissionWaitMs + acquireWaitMs,
|
|
1006
|
+
);
|
|
1007
|
+
unsampledAdmissionWaitMs = 0;
|
|
1000
1008
|
|
|
1001
1009
|
// A checked-out pg client emits 'error' (not via the pool) when its TCP
|
|
1002
1010
|
// connection dies while idle — e.g. a worker holding a client across a
|
|
@@ -1255,6 +1263,7 @@ export type RuntimeSchedulerPoolStats = {
|
|
|
1255
1263
|
idleCount: number;
|
|
1256
1264
|
waitingCount: number;
|
|
1257
1265
|
acquires: number;
|
|
1266
|
+
/** Successful checkout latency, including local admission once per operation. */
|
|
1258
1267
|
acquireWaitP50Ms: number | null;
|
|
1259
1268
|
acquireWaitP95Ms: number | null;
|
|
1260
1269
|
saturationRejections: number;
|
|
@@ -1388,6 +1397,8 @@ export async function probeRuntimeSchedulerDirectSession(input: {
|
|
|
1388
1397
|
connectionString: deriveDirectDatabaseUrl(input.url),
|
|
1389
1398
|
application_name:
|
|
1390
1399
|
input.applicationName ?? 'deepline-runtime-direct-session-preflight',
|
|
1400
|
+
connectionTimeoutMillis: RUNTIME_SCHEDULER_POOL_DEFAULTS.connectTimeoutMs,
|
|
1401
|
+
query_timeout: RUNTIME_SCHEDULER_POOL_DEFAULTS.queryTimeoutMs,
|
|
1391
1402
|
});
|
|
1392
1403
|
try {
|
|
1393
1404
|
await client.connect();
|
|
@@ -1648,6 +1659,8 @@ export function createPgNotifyHub(input: {
|
|
|
1648
1659
|
const next = (input.createClient ?? ((config) => new Client(config)))({
|
|
1649
1660
|
connectionString: directUrl,
|
|
1650
1661
|
application_name: input.applicationName ?? 'deepline-runtime-notify-hub',
|
|
1662
|
+
connectionTimeoutMillis: RUNTIME_SCHEDULER_POOL_DEFAULTS.connectTimeoutMs,
|
|
1663
|
+
query_timeout: RUNTIME_SCHEDULER_POOL_DEFAULTS.queryTimeoutMs,
|
|
1651
1664
|
});
|
|
1652
1665
|
try {
|
|
1653
1666
|
await next.connect();
|
|
@@ -8453,6 +8453,80 @@ export async function readPostgresSchedulerLaunchForAttemptLease(
|
|
|
8453
8453
|
return row ? parseLaunchJson(row.launch_json) : null;
|
|
8454
8454
|
}
|
|
8455
8455
|
|
|
8456
|
+
/** Identity, signing authority, and origins needed to issue an attempt token. */
|
|
8457
|
+
const POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS = [
|
|
8458
|
+
'runId',
|
|
8459
|
+
'orgId',
|
|
8460
|
+
'playName',
|
|
8461
|
+
'userId',
|
|
8462
|
+
'userEmail',
|
|
8463
|
+
'baseUrl',
|
|
8464
|
+
'runtimeApiBaseUrl',
|
|
8465
|
+
'runtimeTokenOrigin',
|
|
8466
|
+
'runtimeBackend',
|
|
8467
|
+
'runtimeAuthority',
|
|
8468
|
+
] as const;
|
|
8469
|
+
|
|
8470
|
+
export type PostgresSchedulerAttemptAuthority = Pick<
|
|
8471
|
+
PlaySchedulerSubmitInput,
|
|
8472
|
+
(typeof POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS)[number]
|
|
8473
|
+
>;
|
|
8474
|
+
|
|
8475
|
+
/**
|
|
8476
|
+
* Prove the same attempt lease as the full execution reader, returning only
|
|
8477
|
+
* the token issuer's authority. Input, graph, and artifact bytes never cross
|
|
8478
|
+
* this SQL seam or occupy a Vercel checkout during JSON decoding. The authority
|
|
8479
|
+
* descriptor is retained intact for the existing validator and deploy skew;
|
|
8480
|
+
* its admitted secrets and fixture configuration are not execution payloads.
|
|
8481
|
+
*/
|
|
8482
|
+
export async function readPostgresSchedulerAttemptAuthorityForLease(
|
|
8483
|
+
client: PostgresSchedulerQueryClient,
|
|
8484
|
+
input: { runId: string; attempt: number; leaseToken: string },
|
|
8485
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
8486
|
+
): Promise<PostgresSchedulerAttemptAuthority | null> {
|
|
8487
|
+
const fields = POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS;
|
|
8488
|
+
// Keep JSON types intact. A text record column would coerce malformed launch
|
|
8489
|
+
// scalars into strings, bypassing the full reader's required-field checks.
|
|
8490
|
+
const columns = [...fields, 'playId', 'executorToken']
|
|
8491
|
+
.map((field) => `${quoteIdent(field)} jsonb`)
|
|
8492
|
+
.join(', ');
|
|
8493
|
+
const result = await client.query<
|
|
8494
|
+
PostgresSchedulerAttemptAuthority & {
|
|
8495
|
+
launch_required_fields_valid: boolean;
|
|
8496
|
+
}
|
|
8497
|
+
>(
|
|
8498
|
+
`
|
|
8499
|
+
SELECT ${fields.map((field) => `launch.${quoteIdent(field)}`).join(', ')},
|
|
8500
|
+
${['runId', 'playId', 'playName', 'executorToken', 'baseUrl']
|
|
8501
|
+
.map(
|
|
8502
|
+
(field) =>
|
|
8503
|
+
`jsonb_typeof(launch.${quoteIdent(field)}) = 'string'`,
|
|
8504
|
+
)
|
|
8505
|
+
.join(' AND ')} AS launch_required_fields_valid
|
|
8506
|
+
FROM ${tableName('runs', options)} AS runs
|
|
8507
|
+
JOIN ${tableName('attempts', options)} AS attempts
|
|
8508
|
+
ON attempts.run_id = runs.run_id
|
|
8509
|
+
CROSS JOIN LATERAL jsonb_to_record(runs.launch_json) AS launch(${columns})
|
|
8510
|
+
WHERE runs.run_id = $1
|
|
8511
|
+
AND runs.status IN ('queued', 'running', 'waiting')
|
|
8512
|
+
AND attempts.attempt = $2
|
|
8513
|
+
AND attempts.lease_token = $3
|
|
8514
|
+
AND attempts.status IN ('claimed', 'running', 'waiting')
|
|
8515
|
+
LIMIT 1
|
|
8516
|
+
`,
|
|
8517
|
+
[input.runId, input.attempt, input.leaseToken],
|
|
8518
|
+
);
|
|
8519
|
+
const row = result.rows[0];
|
|
8520
|
+
if (!row) return null;
|
|
8521
|
+
const { launch_required_fields_valid, ...authority } = row;
|
|
8522
|
+
if (launch_required_fields_valid !== true) {
|
|
8523
|
+
throw new Error(
|
|
8524
|
+
'Postgres scheduler launch_json is missing required fields.',
|
|
8525
|
+
);
|
|
8526
|
+
}
|
|
8527
|
+
return authority;
|
|
8528
|
+
}
|
|
8529
|
+
|
|
8456
8530
|
/**
|
|
8457
8531
|
* Absurd scheduler backend: persist the vendored engine's task id on the run
|
|
8458
8532
|
* row after `spawn`. Idempotent (last write wins); a re-submit under the same
|
|
@@ -4,9 +4,11 @@ import { staticPipelineStorageCodec } from './static-pipeline-storage-codec';
|
|
|
4
4
|
|
|
5
5
|
export {
|
|
6
6
|
retainedPlayPipelineFields,
|
|
7
|
+
WRITE_RETAINED_PIPELINE_GRAPHS,
|
|
7
8
|
type RetainedPlayContract,
|
|
8
9
|
type RetainedPipelineCarrier,
|
|
9
10
|
encodeRetainedPlayPipelines,
|
|
11
|
+
createRetainedPlayPipelinesForTransport,
|
|
10
12
|
hydrateRetainedPlayPipelines,
|
|
11
13
|
prepareRetainedPlayPipelinesForTransport,
|
|
12
14
|
type RetainedPipelineGraph,
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* This is an additive named representation. Readers must be deployed before
|
|
7
7
|
* WRITE_RETAINED_PIPELINE_GRAPHS is enabled in a separate release.
|
|
8
8
|
*/
|
|
9
|
-
export const WRITE_RETAINED_PIPELINE_GRAPHS =
|
|
9
|
+
export const WRITE_RETAINED_PIPELINE_GRAPHS = true;
|
|
10
10
|
|
|
11
11
|
type Scalar = null | boolean | number | string;
|
|
12
12
|
type GraphValue = Scalar | { ref: number };
|
|
@@ -255,12 +255,18 @@ function hydrateContractRoots(
|
|
|
255
255
|
return result;
|
|
256
256
|
}
|
|
257
257
|
|
|
258
|
-
/**
|
|
258
|
+
/** Create a graph only when the owner has selected reader-capable receivers. */
|
|
259
|
+
export function createRetainedPlayPipelinesForTransport<T>(input: T): T {
|
|
260
|
+
return WRITE_RETAINED_PIPELINE_GRAPHS
|
|
261
|
+
? encodeRetainedPlayPipelines(input)
|
|
262
|
+
: prepareRetainedPlayPipelinesForTransport(input);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** Forward an immutable launch in its original representation. */
|
|
259
266
|
export function prepareRetainedPlayPipelinesForTransport<T>(input: T): T {
|
|
260
|
-
//
|
|
261
|
-
//
|
|
262
|
-
return
|
|
263
|
-
(record(input) && input.retainedPipelineGraph !== undefined)
|
|
267
|
+
// A historical launch may still call an older pinned app after adoption.
|
|
268
|
+
// New workers and projectors must not upgrade that receiver's wire format.
|
|
269
|
+
return record(input) && input.retainedPipelineGraph !== undefined
|
|
264
270
|
? encodeRetainedPlayPipelines(input)
|
|
265
271
|
: input;
|
|
266
272
|
}
|
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
retainedPlayPipelineFields,
|
|
11
11
|
encodeRetainedPlayPipelines,
|
|
12
12
|
hydrateRetainedPlayPipelines,
|
|
13
|
-
WRITE_RETAINED_PIPELINE_GRAPHS,
|
|
14
13
|
type RetainedPipelineGraph,
|
|
15
14
|
type RetainedPipelineCarrier,
|
|
16
15
|
} from './retained-pipeline-graph';
|
|
@@ -29,7 +28,7 @@ function readExecutionPipeline(
|
|
|
29
28
|
return readStoredStaticPipeline(record);
|
|
30
29
|
}
|
|
31
30
|
|
|
32
|
-
/**
|
|
31
|
+
/** Preserve the representation selected by the immutable launch owner. */
|
|
33
32
|
function executionStorageFields(
|
|
34
33
|
pipeline: PlayStaticPipeline | null | undefined,
|
|
35
34
|
carrier?: RetainedPipelineCarrier,
|
|
@@ -38,11 +37,7 @@ function executionStorageFields(
|
|
|
38
37
|
staticPipelineV2?: ReturnType<typeof createStoredStaticPipelineV2>;
|
|
39
38
|
retainedPipelineGraph?: RetainedPipelineGraph;
|
|
40
39
|
} {
|
|
41
|
-
if (
|
|
42
|
-
(!WRITE_RETAINED_PIPELINE_GRAPHS &&
|
|
43
|
-
carrier?.retainedPipelineGraph === undefined) ||
|
|
44
|
-
pipeline === undefined
|
|
45
|
-
) {
|
|
40
|
+
if (carrier?.retainedPipelineGraph === undefined || pipeline === undefined) {
|
|
46
41
|
return createStaticPipelineStorageFields(pipeline);
|
|
47
42
|
}
|
|
48
43
|
const compact = encodeRetainedPlayPipelines(
|