deepline 0.3.288 → 0.3.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
- package/dist/cli/index.js +24 -11
- package/dist/cli/index.mjs +24 -11
- package/dist/index.js +3 -1
- package/dist/index.mjs +3 -1
- package/dist/install-integrity.json +3 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -320,6 +320,7 @@ export type PlayRunLedgerEvent =
|
|
|
320
320
|
lines: string[];
|
|
321
321
|
/** Search-only context aligned to `lines`; stripped before Convex. */
|
|
322
322
|
lineContexts?: Array<PlayLogContext | null>;
|
|
323
|
+
lineIds?: Array<string | null>;
|
|
323
324
|
/**
|
|
324
325
|
* Absolute per-run sequence (1-based) of `lines[0]`. Assigned by Run
|
|
325
326
|
* Log Stream ingestion in Convex only; producers never set it.
|
|
@@ -430,6 +431,7 @@ export type PlayRunLedgerStatusPatch = {
|
|
|
430
431
|
liveLogs?: readonly string[] | null;
|
|
431
432
|
/** Search-only context aligned to `liveLogs`; stripped before Convex. */
|
|
432
433
|
liveLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
434
|
+
liveLogIds?: readonly (string | null)[] | null;
|
|
433
435
|
/**
|
|
434
436
|
* Positional cursor for `liveLogs[0]` on this producer's log channel (the
|
|
435
437
|
* count of lines emitted before it). Computed by the producer-side cursor
|
|
@@ -2369,12 +2371,14 @@ function terminalEventTypeForStatus(
|
|
|
2369
2371
|
export function slicePositionalLogLines(input: {
|
|
2370
2372
|
bufferLines: readonly string[];
|
|
2371
2373
|
bufferContexts?: readonly (PlayLogContext | null)[];
|
|
2374
|
+
bufferIds?: readonly (string | null)[];
|
|
2372
2375
|
bufferTotalCount: number;
|
|
2373
2376
|
sentCount: number;
|
|
2374
2377
|
}): {
|
|
2375
2378
|
lines: string[];
|
|
2376
2379
|
channelOffset: number;
|
|
2377
2380
|
lineContexts?: Array<PlayLogContext | null>;
|
|
2381
|
+
lineIds?: Array<string | null>;
|
|
2378
2382
|
} | null {
|
|
2379
2383
|
if (
|
|
2380
2384
|
input.bufferContexts !== undefined &&
|
|
@@ -2382,6 +2386,9 @@ export function slicePositionalLogLines(input: {
|
|
|
2382
2386
|
) {
|
|
2383
2387
|
throw new TypeError('Run log contexts must align with the retained lines.');
|
|
2384
2388
|
}
|
|
2389
|
+
if (input.bufferIds && input.bufferIds.length !== input.bufferLines.length) {
|
|
2390
|
+
throw new TypeError('Run log identities must align with retained lines.');
|
|
2391
|
+
}
|
|
2385
2392
|
const total = Math.max(input.bufferTotalCount, input.bufferLines.length);
|
|
2386
2393
|
const sent = Math.max(0, input.sentCount);
|
|
2387
2394
|
if (total <= sent) {
|
|
@@ -2398,6 +2405,7 @@ export function slicePositionalLogLines(input: {
|
|
|
2398
2405
|
return {
|
|
2399
2406
|
lines: [...lines],
|
|
2400
2407
|
channelOffset: sendFromOffset,
|
|
2408
|
+
...(input.bufferIds ? { lineIds: input.bufferIds.slice(start) } : {}),
|
|
2401
2409
|
...(lineContexts?.some(Boolean) ? { lineContexts: [...lineContexts] } : {}),
|
|
2402
2410
|
};
|
|
2403
2411
|
}
|
|
@@ -2427,6 +2435,7 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
2427
2435
|
runId: string;
|
|
2428
2436
|
lines: readonly string[];
|
|
2429
2437
|
lineContexts?: readonly (PlayLogContext | null)[];
|
|
2438
|
+
lineIds?: readonly (string | null)[];
|
|
2430
2439
|
source?: PlayRunLedgerEventSource;
|
|
2431
2440
|
occurredAt?: number;
|
|
2432
2441
|
channelOffset?: number | null;
|
|
@@ -2448,10 +2457,14 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
2448
2457
|
) {
|
|
2449
2458
|
throw new TypeError('Run log contexts must align with their lines.');
|
|
2450
2459
|
}
|
|
2460
|
+
if (input.lineIds && input.lineIds.length !== input.lines.length) {
|
|
2461
|
+
throw new TypeError('Run log identities must align with their lines.');
|
|
2462
|
+
}
|
|
2451
2463
|
const entries = input.lines
|
|
2452
2464
|
.map((line, index) => ({
|
|
2453
2465
|
line: positional ? line.trim() || '(blank log line)' : line.trim(),
|
|
2454
2466
|
context: input.lineContexts?.[index] ?? null,
|
|
2467
|
+
entryId: input.lineIds?.[index] ?? null,
|
|
2455
2468
|
}))
|
|
2456
2469
|
.filter((entry) => positional || entry.line.length > 0);
|
|
2457
2470
|
const newLines = entries.map((entry) => entry.line);
|
|
@@ -2470,6 +2483,13 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
2470
2483
|
source,
|
|
2471
2484
|
occurredAt,
|
|
2472
2485
|
lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
|
|
2486
|
+
...(input.lineIds
|
|
2487
|
+
? {
|
|
2488
|
+
lineIds: entries
|
|
2489
|
+
.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT)
|
|
2490
|
+
.map((entry) => entry.entryId),
|
|
2491
|
+
}
|
|
2492
|
+
: {}),
|
|
2473
2493
|
...(entries
|
|
2474
2494
|
.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT)
|
|
2475
2495
|
.some((entry) => entry.context)
|
|
@@ -2507,6 +2527,7 @@ export function buildTerminalLogReplayEvents(input: {
|
|
|
2507
2527
|
runId: string;
|
|
2508
2528
|
lines: readonly string[];
|
|
2509
2529
|
lineContexts?: readonly (PlayLogContext | null)[];
|
|
2530
|
+
lineIds?: readonly (string | null)[];
|
|
2510
2531
|
source?: PlayRunLedgerEventSource;
|
|
2511
2532
|
occurredAt?: number;
|
|
2512
2533
|
liveLogTotalCount?: number;
|
|
@@ -2521,6 +2542,7 @@ export function buildTerminalLogReplayEvents(input: {
|
|
|
2521
2542
|
runId: input.runId,
|
|
2522
2543
|
lines: input.lines,
|
|
2523
2544
|
lineContexts: input.lineContexts,
|
|
2545
|
+
lineIds: input.lineIds,
|
|
2524
2546
|
source: 'coordinator',
|
|
2525
2547
|
occurredAt: input.occurredAt,
|
|
2526
2548
|
channelOffset,
|
|
@@ -2562,6 +2584,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
|
|
|
2562
2584
|
runId: patch.runId,
|
|
2563
2585
|
lines: patch.liveLogs,
|
|
2564
2586
|
lineContexts: patch.liveLogContexts ?? undefined,
|
|
2587
|
+
lineIds: patch.liveLogIds ?? undefined,
|
|
2565
2588
|
source,
|
|
2566
2589
|
occurredAt: checkpointAt,
|
|
2567
2590
|
channelOffset: patch.liveLogsChannelOffset ?? null,
|
|
@@ -25,6 +25,8 @@ export function boundRunLogLine(line: string): string {
|
|
|
25
25
|
*/
|
|
26
26
|
export class RunLogBuffer {
|
|
27
27
|
readonly lines: string[] = [];
|
|
28
|
+
readonly ids: string[] = [];
|
|
29
|
+
private readonly origin = crypto.randomUUID();
|
|
28
30
|
readonly contexts: Array<PlayLogContext | null> = [];
|
|
29
31
|
private bytes = 0;
|
|
30
32
|
private contextBytes = 0;
|
|
@@ -56,6 +58,7 @@ export class RunLogBuffer {
|
|
|
56
58
|
true,
|
|
57
59
|
);
|
|
58
60
|
}
|
|
61
|
+
this.ids.push(`${this.origin}:${this.lines.length}`);
|
|
59
62
|
this.lines.push(line);
|
|
60
63
|
const retainContext =
|
|
61
64
|
normalizedContext !== null &&
|
|
@@ -80,3 +83,30 @@ export class RunLogBuffer {
|
|
|
80
83
|
};
|
|
81
84
|
}
|
|
82
85
|
}
|
|
86
|
+
|
|
87
|
+
/** Optional origin identities: old producers omit them; never infer them from text. */
|
|
88
|
+
export function normalizeRunLogIds(
|
|
89
|
+
value: unknown,
|
|
90
|
+
lines: readonly unknown[],
|
|
91
|
+
): Array<string | null> | undefined {
|
|
92
|
+
if (value === undefined || value === null) return undefined;
|
|
93
|
+
if (
|
|
94
|
+
!Array.isArray(value) ||
|
|
95
|
+
value.length !== lines.length ||
|
|
96
|
+
value.some(
|
|
97
|
+
(id) =>
|
|
98
|
+
id !== null &&
|
|
99
|
+
(typeof id !== 'string' || !/^[A-Za-z0-9:_-]{1,128}$/.test(id)),
|
|
100
|
+
)
|
|
101
|
+
) {
|
|
102
|
+
throw new TypeError(
|
|
103
|
+
'Run log identities must be bounded strings aligned with their lines.',
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
const ids = value.filter((id): id is string => id !== null);
|
|
107
|
+
if (new Set(ids).size !== ids.length)
|
|
108
|
+
throw new TypeError(
|
|
109
|
+
'Run log identities must identify distinct occurrences.',
|
|
110
|
+
);
|
|
111
|
+
return value;
|
|
112
|
+
}
|
|
@@ -1903,6 +1903,14 @@ async function releaseRuntimeLeasesOnAppRunnerSettlement(input: {
|
|
|
1903
1903
|
}
|
|
1904
1904
|
}
|
|
1905
1905
|
|
|
1906
|
+
function terminalLogIdsWithPhaseSummary(ctx: {
|
|
1907
|
+
getLogIds(): string[];
|
|
1908
|
+
}): string[] {
|
|
1909
|
+
// Summaries are new occurrences at result construction. Terminal retries
|
|
1910
|
+
// reuse that result and its IDs, including the frozen summary bodies.
|
|
1911
|
+
return [...ctx.getLogIds(), crypto.randomUUID(), crypto.randomUUID()];
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1906
1914
|
function terminalLogsWithPhaseSummary(ctx: {
|
|
1907
1915
|
getLogs(): string[];
|
|
1908
1916
|
getRuntimeResourceSnapshot(): unknown;
|
|
@@ -2488,7 +2496,7 @@ async function run(
|
|
|
2488
2496
|
});
|
|
2489
2497
|
}
|
|
2490
2498
|
: undefined,
|
|
2491
|
-
onLog: (line, context) => {
|
|
2499
|
+
onLog: (line, context, entryId) => {
|
|
2492
2500
|
const timestampMatch = line.match(/^\[([^\]]+)\]/);
|
|
2493
2501
|
const timestamp = timestampMatch?.[1];
|
|
2494
2502
|
const at =
|
|
@@ -2501,6 +2509,9 @@ async function run(
|
|
|
2501
2509
|
at,
|
|
2502
2510
|
source: 'play',
|
|
2503
2511
|
line,
|
|
2512
|
+
...(config.context.runtimeLogIdentity === 'origin' && entryId
|
|
2513
|
+
? { entryId }
|
|
2514
|
+
: {}),
|
|
2504
2515
|
...(context ? { context } : {}),
|
|
2505
2516
|
},
|
|
2506
2517
|
options,
|
|
@@ -3128,6 +3139,9 @@ async function run(
|
|
|
3128
3139
|
error: runtimeLimitExceededCause(runtimeLimitSeconds),
|
|
3129
3140
|
errors: [runtimeLimitExceededFailure(runtimeLimitSeconds)],
|
|
3130
3141
|
logs: terminalLogsWithPhaseSummary(ctx),
|
|
3142
|
+
...(config.context.runtimeLogIdentity === 'origin'
|
|
3143
|
+
? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
|
|
3144
|
+
: {}),
|
|
3131
3145
|
logContexts: terminalLogContextsWithPhaseSummary(ctx),
|
|
3132
3146
|
stats: runtimeStats(),
|
|
3133
3147
|
steps: ctx.getSteps(),
|
|
@@ -3372,6 +3386,9 @@ async function run(
|
|
|
3372
3386
|
: {}),
|
|
3373
3387
|
outputRowCount,
|
|
3374
3388
|
logs,
|
|
3389
|
+
...(config.context.runtimeLogIdentity === 'origin'
|
|
3390
|
+
? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
|
|
3391
|
+
: {}),
|
|
3375
3392
|
logContexts: terminalLogContextsWithPhaseSummary(ctx),
|
|
3376
3393
|
...(runtimeDiagnostics.length > 0 ? { runtimeDiagnostics } : {}),
|
|
3377
3394
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
@@ -3427,6 +3444,9 @@ async function run(
|
|
|
3427
3444
|
status: 'suspended',
|
|
3428
3445
|
suspension: error.suspension,
|
|
3429
3446
|
logs: terminalLogsWithPhaseSummary(ctx),
|
|
3447
|
+
...(config.context.runtimeLogIdentity === 'origin'
|
|
3448
|
+
? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
|
|
3449
|
+
: {}),
|
|
3430
3450
|
logContexts: terminalLogContextsWithPhaseSummary(ctx),
|
|
3431
3451
|
stats: runtimeStats(),
|
|
3432
3452
|
steps: ctx.getSteps(),
|
|
@@ -3463,6 +3483,9 @@ async function run(
|
|
|
3463
3483
|
error: formatRunnerFailure(error),
|
|
3464
3484
|
errors: [failure],
|
|
3465
3485
|
logs: terminalLogsWithPhaseSummary(ctx),
|
|
3486
|
+
...(config.context.runtimeLogIdentity === 'origin'
|
|
3487
|
+
? { logIds: terminalLogIdsWithPhaseSummary(ctx) }
|
|
3488
|
+
: {}),
|
|
3466
3489
|
logContexts: terminalLogContextsWithPhaseSummary(ctx),
|
|
3467
3490
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
3468
3491
|
stats: runtimeStats(),
|
|
@@ -3483,11 +3506,23 @@ export async function executePlayRunner(
|
|
|
3483
3506
|
config: PlayRunnerExecutionConfig,
|
|
3484
3507
|
options?: ExecutePlayRunnerOptions,
|
|
3485
3508
|
): Promise<PlayRunnerResult> {
|
|
3486
|
-
const
|
|
3509
|
+
const emissionOptions: ExecutePlayRunnerOptions = {
|
|
3510
|
+
...options,
|
|
3511
|
+
emitEvent: (event) =>
|
|
3512
|
+
emitRunnerEvent(
|
|
3513
|
+
config.context.runtimeLogIdentity === 'origin' &&
|
|
3514
|
+
event.type === 'log' &&
|
|
3515
|
+
!event.entryId
|
|
3516
|
+
? { ...event, entryId: crypto.randomUUID() }
|
|
3517
|
+
: event,
|
|
3518
|
+
options,
|
|
3519
|
+
),
|
|
3520
|
+
};
|
|
3521
|
+
const result = await run(config, emissionOptions);
|
|
3487
3522
|
// Inline-executor mirror of the child terminal drain: the caller's finalize
|
|
3488
3523
|
// reads the observations table right after this returns, so land in-flight
|
|
3489
3524
|
// observation posts first (bounded, best-effort, docflow-gated by emptiness).
|
|
3490
|
-
await drainPendingDocflowObservationPosts(
|
|
3525
|
+
await drainPendingDocflowObservationPosts(emissionOptions);
|
|
3491
3526
|
return result;
|
|
3492
3527
|
}
|
|
3493
3528
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { PlayRunnerEvent } from '../protocol';
|
|
2
|
+
import { isDurableDirectRunnerLog } from '../log-provenance';
|
|
2
3
|
import { jsonByteLengthUpTo } from '../output-size-limits';
|
|
3
4
|
import { RUNTIME_RUNNER_TERMINAL_PROGRESS_MAX_EVENTS } from '../runtime-contract';
|
|
4
5
|
|
|
@@ -116,9 +117,7 @@ type PendingEvent = {
|
|
|
116
117
|
};
|
|
117
118
|
|
|
118
119
|
function isDurableLog(event: PlayRunnerEvent): boolean {
|
|
119
|
-
return (
|
|
120
|
-
event.type === 'log' && event.source !== 'worker' && event.line.length > 0
|
|
121
|
-
);
|
|
120
|
+
return event.type === 'log' && isDurableDirectRunnerLog(event);
|
|
122
121
|
}
|
|
123
122
|
|
|
124
123
|
export function createRunnerProgressPusher(
|
|
@@ -269,12 +269,18 @@ function appendTerminalDiagnosticsMarker(
|
|
|
269
269
|
logs: string[],
|
|
270
270
|
truncatedFields: string[],
|
|
271
271
|
logContexts?: Array<PlayLogContext | null>,
|
|
272
|
-
|
|
272
|
+
logIds?: string[],
|
|
273
|
+
): {
|
|
274
|
+
logs: string[];
|
|
275
|
+
logContexts?: Array<PlayLogContext | null>;
|
|
276
|
+
logIds?: string[];
|
|
277
|
+
} {
|
|
273
278
|
const marker =
|
|
274
279
|
`[deepline] terminal diagnostic metadata was bounded (${truncatedFields.join(', ')}); ` +
|
|
275
280
|
'the authored play output is complete.';
|
|
276
281
|
const marked = [marker, ...logs];
|
|
277
282
|
const markedContexts = logContexts ? [null, ...logContexts] : undefined;
|
|
283
|
+
const markedIds = logIds ? [crypto.randomUUID(), ...logIds] : undefined;
|
|
278
284
|
while (
|
|
279
285
|
marked.length > 1 &&
|
|
280
286
|
!fitsJsonByteLimit(marked, TERMINAL_TRANSPORT_LOG_TAIL_MAX_BYTES)
|
|
@@ -289,6 +295,7 @@ function appendTerminalDiagnosticsMarker(
|
|
|
289
295
|
const removeAt = removableIndex === -1 ? 1 : removableIndex;
|
|
290
296
|
marked.splice(removeAt, 1);
|
|
291
297
|
markedContexts?.splice(removeAt, 1);
|
|
298
|
+
markedIds?.splice(removeAt, 1);
|
|
292
299
|
}
|
|
293
300
|
const logsFit = fitsJsonByteLimit(
|
|
294
301
|
marked,
|
|
@@ -298,6 +305,9 @@ function appendTerminalDiagnosticsMarker(
|
|
|
298
305
|
const retainedContexts = logsFit ? markedContexts : [null];
|
|
299
306
|
return {
|
|
300
307
|
logs: retainedLogs,
|
|
308
|
+
...(markedIds
|
|
309
|
+
? { logIds: logsFit ? markedIds : markedIds.slice(0, 1) }
|
|
310
|
+
: {}),
|
|
301
311
|
...(retainedContexts?.some(Boolean)
|
|
302
312
|
? { logContexts: retainedContexts }
|
|
303
313
|
: {}),
|
|
@@ -325,6 +335,11 @@ function boundSettledTerminalTransportDiagnostics(
|
|
|
325
335
|
const logContexts = result.logContexts?.length
|
|
326
336
|
? logProjection.indexes.map((index) => result.logContexts![index] ?? null)
|
|
327
337
|
: undefined;
|
|
338
|
+
if (result.logIds && result.logIds.length !== result.logs.length)
|
|
339
|
+
throw new TypeError('Terminal log identities must align with result logs.');
|
|
340
|
+
const logIds = result.logIds
|
|
341
|
+
? logProjection.indexes.map((index) => result.logIds![index]!)
|
|
342
|
+
: undefined;
|
|
328
343
|
const statsProjection = retainRecordWithinJsonByteLimit(
|
|
329
344
|
result.stats,
|
|
330
345
|
TERMINAL_TRANSPORT_STATS_MAX_BYTES,
|
|
@@ -368,15 +383,18 @@ function boundSettledTerminalTransportDiagnostics(
|
|
|
368
383
|
logProjection.values,
|
|
369
384
|
truncatedFields,
|
|
370
385
|
logContexts,
|
|
386
|
+
logIds,
|
|
371
387
|
)
|
|
372
388
|
: {
|
|
373
389
|
logs: logProjection.values,
|
|
390
|
+
...(logIds ? { logIds } : {}),
|
|
374
391
|
...(logContexts?.some(Boolean) ? { logContexts } : {}),
|
|
375
392
|
};
|
|
376
393
|
return {
|
|
377
394
|
...result,
|
|
378
395
|
logs: boundedLogs.logs,
|
|
379
396
|
logContexts: boundedLogs.logContexts,
|
|
397
|
+
logIds: boundedLogs.logIds,
|
|
380
398
|
stats: statsProjection.value,
|
|
381
399
|
steps: stepsProjection.values,
|
|
382
400
|
...(result.outputWarnings
|
|
@@ -408,9 +426,11 @@ function boundSettledTerminalTransportDiagnostics(
|
|
|
408
426
|
logProjection.values,
|
|
409
427
|
truncatedFields,
|
|
410
428
|
logContexts,
|
|
429
|
+
logIds,
|
|
411
430
|
)
|
|
412
431
|
: {
|
|
413
432
|
logs: logProjection.values,
|
|
433
|
+
...(logIds ? { logIds } : {}),
|
|
414
434
|
...(logContexts?.some(Boolean) ? { logContexts } : {}),
|
|
415
435
|
};
|
|
416
436
|
return {
|
|
@@ -418,6 +438,7 @@ function boundSettledTerminalTransportDiagnostics(
|
|
|
418
438
|
error: error.value,
|
|
419
439
|
logs: boundedLogs.logs,
|
|
420
440
|
logContexts: boundedLogs.logContexts,
|
|
441
|
+
logIds: boundedLogs.logIds,
|
|
421
442
|
stats: statsProjection.value,
|
|
422
443
|
steps: stepsProjection.values,
|
|
423
444
|
...(result.errors ? { errors: errorsProjection.values } : {}),
|
|
@@ -90,7 +90,7 @@ const DAYTONA_PUBLIC_STATUS_TRANSIENT_RETRY_PATTERN =
|
|
|
90
90
|
const DAYTONA_RUNNER_STARTUP_TRANSPORT_RETRY_PATTERN =
|
|
91
91
|
/\bDaytonaRunnerInitializationError:\s*RUNTIME_SANDBOX_START_FAILED:.*\bScheduler diagnosis:\s*runner_heartbeat_not_observed\..*"schedulerReadinessObserved":false.*"commandFound":true.*"exitCode":null.*"sessionFound":true.*"exitCodeFile":null.*"runnerStartup":\{"phase":"heartbeat_transport","outcome":"network_error","errorCode":"(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT)"/i;
|
|
92
92
|
const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
|
|
93
|
-
/\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|Daytona sandbox create failed across|Daytona sandbox create did not complete|Daytona readiness baseline)\b/i;
|
|
93
|
+
/\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|The socket connection was closed unexpectedly|Daytona sandbox create failed across|Daytona sandbox create did not complete|Daytona readiness baseline)\b/i;
|
|
94
94
|
|
|
95
95
|
type DaytonaExecution = { exitCode: number | null; result: string };
|
|
96
96
|
export type DaytonaLookupFailure = {
|
|
@@ -86,6 +86,8 @@ export type PlaySchedulerSubmitInput = RetainedPipelineCarrier & {
|
|
|
86
86
|
* Omitted for legacy launches whose Play Run row was persisted synchronously.
|
|
87
87
|
*/
|
|
88
88
|
requiresRunCreatedProjection?: boolean;
|
|
89
|
+
/** Admission-owned producer pin. Absent keeps launch-pinned legacy logging. */
|
|
90
|
+
runtimeLogIdentity?: 'origin';
|
|
89
91
|
/** Optional inline artifact for the Node scheduler. */
|
|
90
92
|
runtimeArtifact?: unknown;
|
|
91
93
|
artifactHash: string;
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import {
|
|
3
|
+
PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS,
|
|
4
|
+
PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES,
|
|
5
|
+
PLAY_ADMISSION_MAX_PARALLELISM,
|
|
6
|
+
} from '../admission-transport-policy';
|
|
7
|
+
import {
|
|
8
|
+
RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS,
|
|
9
|
+
RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS,
|
|
10
|
+
resolveWorkflowRuntimePostgresUrl,
|
|
11
|
+
type RuntimeSchedulerPoolOptions,
|
|
12
|
+
} from './postgres-pool';
|
|
13
|
+
|
|
14
|
+
// All queues in one worker share this single process pool. Scheduler queries
|
|
15
|
+
// are short and batched; execution concurrency belongs to Daytona/Fly slots,
|
|
16
|
+
// not to a growing set of database clients. Keeping this at two makes a
|
|
17
|
+
// horizontally scaled fleet predictable against the shared PgBouncer budget.
|
|
18
|
+
export const ABSURD_RUNTIME_POOL_MAX_CLIENTS =
|
|
19
|
+
RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Vercel Fluid invocations share a process and its scheduler pool. A single
|
|
23
|
+
* client serializes unrelated launches, reads, and critical lease lookups;
|
|
24
|
+
* one held connection can make every other request hit its acquire timeout.
|
|
25
|
+
* Use the shared two-client control-plane budget so another request can make
|
|
26
|
+
* progress while keeping each instance's PgBouncer footprint bounded.
|
|
27
|
+
*/
|
|
28
|
+
export const ABSURD_SERVERLESS_POOL_MAX_CLIENTS =
|
|
29
|
+
RUNTIME_SCHEDULER_CONTROL_PLANE_MAX_CLIENTS;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A Vercel isolate may survive long after a webhook has returned. Keep its
|
|
33
|
+
* scheduler clients only long enough to reuse them for an immediate follow-up,
|
|
34
|
+
* then return idle PgBouncer clients before the source's delayed Workpool
|
|
35
|
+
* recovery is due. The worker fleet keeps the ordinary 30-second idle lifetime.
|
|
36
|
+
*/
|
|
37
|
+
export const ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS =
|
|
38
|
+
RUNTIME_SCHEDULER_SERVERLESS_IDLE_TIMEOUT_MS;
|
|
39
|
+
|
|
40
|
+
// Ordinary Vercel requests share the bounded control-plane pool above. The
|
|
41
|
+
// private batch adapter is different: Workpool elects one global up-to-1,024-
|
|
42
|
+
// source collection before entering this context, so one batch request may use
|
|
43
|
+
// its larger bounded scheduler capacity without multiplying it per source.
|
|
44
|
+
// Other Workpool adapters can still overlap during rollout.
|
|
45
|
+
type ServerlessAdmissionPoolContext = {
|
|
46
|
+
maxClients: number;
|
|
47
|
+
maxAcquireQueueDepth: number;
|
|
48
|
+
maxAcquireQueueWaitMs: number;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const serverlessAdmissionPoolContext =
|
|
52
|
+
new AsyncLocalStorage<ServerlessAdmissionPoolContext>();
|
|
53
|
+
|
|
54
|
+
export async function withServerlessAdmissionPoolMaxClients<T>(input: {
|
|
55
|
+
maxClients: number;
|
|
56
|
+
maxAcquireQueueDepth?: number;
|
|
57
|
+
maxAcquireQueueWaitMs?: number;
|
|
58
|
+
fn: () => Promise<T>;
|
|
59
|
+
}): Promise<T> {
|
|
60
|
+
if (
|
|
61
|
+
!Number.isInteger(input.maxClients) ||
|
|
62
|
+
input.maxClients < 1 ||
|
|
63
|
+
input.maxClients > PLAY_ADMISSION_MAX_PARALLELISM
|
|
64
|
+
) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
'Serverless admission pool capacity is outside the supported range.',
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
const maxAcquireQueueDepth =
|
|
70
|
+
input.maxAcquireQueueDepth ?? PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES;
|
|
71
|
+
const maxAcquireQueueWaitMs =
|
|
72
|
+
input.maxAcquireQueueWaitMs ?? PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS;
|
|
73
|
+
if (
|
|
74
|
+
!Number.isSafeInteger(maxAcquireQueueDepth) ||
|
|
75
|
+
maxAcquireQueueDepth < 1 ||
|
|
76
|
+
maxAcquireQueueDepth > PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES
|
|
77
|
+
) {
|
|
78
|
+
throw new Error(
|
|
79
|
+
'Serverless admission acquire queue capacity is outside the supported range.',
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
if (
|
|
83
|
+
!Number.isSafeInteger(maxAcquireQueueWaitMs) ||
|
|
84
|
+
maxAcquireQueueWaitMs < 5_000 ||
|
|
85
|
+
maxAcquireQueueWaitMs > PLAY_ADMISSION_MAX_ACQUIRE_QUEUE_WAIT_MS
|
|
86
|
+
) {
|
|
87
|
+
throw new Error(
|
|
88
|
+
'Serverless admission acquire queue wait is outside the supported range.',
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
return await serverlessAdmissionPoolContext.run(
|
|
92
|
+
{
|
|
93
|
+
maxClients: input.maxClients,
|
|
94
|
+
maxAcquireQueueDepth,
|
|
95
|
+
maxAcquireQueueWaitMs,
|
|
96
|
+
},
|
|
97
|
+
input.fn,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The private batch adapter owns this request-local scheduler pool. Reserve
|
|
103
|
+
* one client for executor-token and other critical control-plane traffic; the
|
|
104
|
+
* rest may submit the independently idempotent batch sources in parallel.
|
|
105
|
+
*/
|
|
106
|
+
export function serverlessAdmissionSubmitMaxActive(): number | null {
|
|
107
|
+
const context = serverlessAdmissionPoolContext.getStore();
|
|
108
|
+
return context === undefined ? null : Math.max(1, context.maxClients - 1);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Resolve the runtime scheduler Postgres pool options for the absurd control
|
|
113
|
+
* plane. Loud failure over silent fallback: the absurd worker and scheduler
|
|
114
|
+
* only run under Node where `WORKFLOW_RUNTIME_POSTGRES` is configured.
|
|
115
|
+
*/
|
|
116
|
+
export function resolveAbsurdPoolOptions(
|
|
117
|
+
explicitUrl?: string | null,
|
|
118
|
+
): RuntimeSchedulerPoolOptions {
|
|
119
|
+
const url = explicitUrl?.trim() || resolveWorkflowRuntimePostgresUrl();
|
|
120
|
+
if (!url) {
|
|
121
|
+
throw new Error(
|
|
122
|
+
'absurd scheduler backend requires WORKFLOW_RUNTIME_POSTGRES (runtime ' +
|
|
123
|
+
'scheduler Postgres URL). Refusing to fall back to local execution.',
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
const serverless = process.env.VERCEL?.trim();
|
|
127
|
+
const context = serverlessAdmissionPoolContext.getStore();
|
|
128
|
+
return {
|
|
129
|
+
url,
|
|
130
|
+
applicationName: 'deepline-absurd-runtime',
|
|
131
|
+
maxClients:
|
|
132
|
+
serverless || context !== undefined
|
|
133
|
+
? (context?.maxClients ?? ABSURD_SERVERLESS_POOL_MAX_CLIENTS)
|
|
134
|
+
: ABSURD_RUNTIME_POOL_MAX_CLIENTS,
|
|
135
|
+
...(serverless || context !== undefined
|
|
136
|
+
? { idleTimeoutMs: ABSURD_SERVERLESS_POOL_IDLE_TIMEOUT_MS }
|
|
137
|
+
: {}),
|
|
138
|
+
...(context
|
|
139
|
+
? {
|
|
140
|
+
maxAcquireQueueDepth: context.maxAcquireQueueDepth,
|
|
141
|
+
maxAcquireQueueWaitMs: context.maxAcquireQueueWaitMs,
|
|
142
|
+
}
|
|
143
|
+
: {}),
|
|
144
|
+
};
|
|
145
|
+
}
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -4463,7 +4463,6 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
|
|
|
4463
4463
|
);
|
|
4464
4464
|
});
|
|
4465
4465
|
};
|
|
4466
|
-
let projectionStartedAtMs: number | undefined;
|
|
4467
4466
|
return createPlayRunAttemptWorkerTerminalProjection({
|
|
4468
4467
|
async observeFenceLost() {
|
|
4469
4468
|
// A lost fence means this attempt is dead, which a cancelled run is the
|
|
@@ -4479,7 +4478,6 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
|
|
|
4479
4478
|
});
|
|
4480
4479
|
},
|
|
4481
4480
|
async reconcileTerminalTruth() {
|
|
4482
|
-
projectionStartedAtMs ??= Date.now();
|
|
4483
4481
|
// ADR 0016 rule 3: detached Daytona runs may provide no live events, so
|
|
4484
4482
|
// fold stored docflow/runtime-sheet facts into progress after truth wins.
|
|
4485
4483
|
await reconcileTerminalTruthIntoReporter({
|
|
@@ -4564,9 +4562,7 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
|
|
|
4564
4562
|
runId: input.runId,
|
|
4565
4563
|
options: input.options,
|
|
4566
4564
|
trigger: 'committed_terminal',
|
|
4567
|
-
|
|
4568
|
-
? {}
|
|
4569
|
-
: { projectionStartedAtMs }),
|
|
4565
|
+
projectionStartedAtMs: Date.now(),
|
|
4570
4566
|
});
|
|
4571
4567
|
},
|
|
4572
4568
|
});
|