deepline 0.1.212 → 0.1.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +167 -329
- package/dist/bundling-sources/sdk/src/client.ts +2 -2
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +49 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -123
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +30 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +160 -2
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +54 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +477 -85
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
- package/dist/cli/index.js +50 -436
- package/dist/cli/index.mjs +50 -436
- package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
- package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
- package/dist/index.d.mts +11 -8
- package/dist/index.d.ts +11 -8
- package/dist/index.js +4 -4
- package/dist/index.mjs +4 -4
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/package.json +1 -1
|
@@ -24,6 +24,7 @@ import type { PreviousCell } from './cell-staleness';
|
|
|
24
24
|
import type { MapRowOutcome } from './durability-store';
|
|
25
25
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
26
26
|
import type { RunExecutionScope } from './run-execution-scope';
|
|
27
|
+
import type { PlayCallExecution } from './play-call-execution';
|
|
27
28
|
|
|
28
29
|
export interface RowState {
|
|
29
30
|
results: Map<string, unknown>;
|
|
@@ -69,12 +70,26 @@ export interface RuntimeStepReceipt {
|
|
|
69
70
|
claimState?: 'claimed' | 'existing';
|
|
70
71
|
}
|
|
71
72
|
|
|
73
|
+
export type AcquireRuntimeReceiptExecutionLockInput = {
|
|
74
|
+
key: string;
|
|
75
|
+
runId: string;
|
|
76
|
+
ownerExecutionId: string;
|
|
77
|
+
ttlMs: number;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
export type ReleaseRuntimeReceiptExecutionLockInput = {
|
|
81
|
+
key: string;
|
|
82
|
+
runId: string;
|
|
83
|
+
ownerExecutionId: string;
|
|
84
|
+
};
|
|
85
|
+
|
|
72
86
|
export interface GetRuntimeStepReceiptInput {
|
|
73
87
|
key: string;
|
|
74
88
|
}
|
|
75
89
|
|
|
76
90
|
export interface ClaimRuntimeStepReceiptInput {
|
|
77
91
|
key: string;
|
|
92
|
+
leaseId?: string;
|
|
78
93
|
runId: string;
|
|
79
94
|
runAttempt?: number | null;
|
|
80
95
|
leaseAware?: boolean;
|
|
@@ -128,6 +143,7 @@ export interface GetRuntimeStepReceiptsInput {
|
|
|
128
143
|
|
|
129
144
|
export interface ClaimRuntimeStepReceiptsInput {
|
|
130
145
|
keys: string[];
|
|
146
|
+
leaseIds?: string[];
|
|
131
147
|
runId: string;
|
|
132
148
|
runAttempt?: number | null;
|
|
133
149
|
leaseAware?: boolean;
|
|
@@ -281,6 +297,7 @@ export type CustomerDbQueryHandler = (
|
|
|
281
297
|
|
|
282
298
|
export interface PlayCallOptions {
|
|
283
299
|
description?: string;
|
|
300
|
+
execution?: PlayCallExecution;
|
|
284
301
|
}
|
|
285
302
|
|
|
286
303
|
export interface StepOptions {
|
|
@@ -597,7 +614,10 @@ export interface ContextOptions {
|
|
|
597
614
|
rows: MapRowOutcome[];
|
|
598
615
|
outputFields: string[];
|
|
599
616
|
staticPipeline?: PlayStaticPipeline | null;
|
|
600
|
-
}) => Promise<void
|
|
617
|
+
}) => Promise<void | {
|
|
618
|
+
updated: number;
|
|
619
|
+
staleDroppedKeys?: string[];
|
|
620
|
+
}>;
|
|
601
621
|
playId?: string;
|
|
602
622
|
runId?: string;
|
|
603
623
|
/** Physical executor run that owns leases for an inline child context. */
|
|
@@ -634,6 +654,7 @@ export interface ContextOptions {
|
|
|
634
654
|
getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
|
|
635
655
|
getToolRetryPolicy?: (toolId: string) => Promise<{
|
|
636
656
|
retrySafeTransientHttp?: boolean;
|
|
657
|
+
requiresExecutionFence?: boolean;
|
|
637
658
|
} | null>;
|
|
638
659
|
getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
|
|
639
660
|
getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
|
|
@@ -672,6 +693,12 @@ export interface ContextOptions {
|
|
|
672
693
|
getRuntimeStepReceipt?: (
|
|
673
694
|
input: GetRuntimeStepReceiptInput,
|
|
674
695
|
) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
|
|
696
|
+
acquireRuntimeReceiptExecutionLock?: (
|
|
697
|
+
input: AcquireRuntimeReceiptExecutionLockInput,
|
|
698
|
+
) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
|
|
699
|
+
releaseRuntimeReceiptExecutionLock?: (
|
|
700
|
+
input: ReleaseRuntimeReceiptExecutionLockInput,
|
|
701
|
+
) => Promise<boolean>;
|
|
675
702
|
getRuntimeStepReceipts?: (
|
|
676
703
|
input: GetRuntimeStepReceiptsInput,
|
|
677
704
|
) =>
|
|
@@ -932,6 +959,7 @@ export type PlayStep =
|
|
|
932
959
|
| {
|
|
933
960
|
type: 'play_call';
|
|
934
961
|
playId: string;
|
|
962
|
+
execution?: PlayCallExecution;
|
|
935
963
|
results?: PlayStepRowResult[];
|
|
936
964
|
nestedSteps: PlayStep[];
|
|
937
965
|
description?: string;
|
|
@@ -971,6 +999,7 @@ export type PlayDatasetSubstep =
|
|
|
971
999
|
| {
|
|
972
1000
|
type: 'play_call';
|
|
973
1001
|
playId: string;
|
|
1002
|
+
execution?: PlayCallExecution;
|
|
974
1003
|
results?: PlayStepRowResult[];
|
|
975
1004
|
nestedSteps: PlayStep[];
|
|
976
1005
|
description?: string;
|
|
@@ -25,6 +25,9 @@ const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
|
|
|
25
25
|
const TOOL_RECEIPT_DEFAULT_WAIT_MS = 300_000;
|
|
26
26
|
const TOOL_RECEIPT_COMPLETION_BUFFER_MS = 30_000;
|
|
27
27
|
const TOOL_RECEIPT_MAX_WAIT_MS = 30 * 60_000;
|
|
28
|
+
const DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS = 15_000;
|
|
29
|
+
export const COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID =
|
|
30
|
+
'__deepline_completed_cache_insert__';
|
|
28
31
|
|
|
29
32
|
export class RuntimeReceiptWaitTimeoutError extends Error {
|
|
30
33
|
constructor(key: string) {
|
|
@@ -179,6 +182,15 @@ export type DurableReceiptExecutionStore = {
|
|
|
179
182
|
): Promise<RuntimeStepReceipt | null>;
|
|
180
183
|
canPersistFailure: boolean;
|
|
181
184
|
canPersistCompletion: boolean;
|
|
185
|
+
acquireExecutionLock?: (input: {
|
|
186
|
+
receiptKey: string;
|
|
187
|
+
ownerExecutionId: string;
|
|
188
|
+
ttlMs: number;
|
|
189
|
+
}) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
|
|
190
|
+
releaseExecutionLock?: (input: {
|
|
191
|
+
receiptKey: string;
|
|
192
|
+
ownerExecutionId: string;
|
|
193
|
+
}) => Promise<boolean>;
|
|
182
194
|
};
|
|
183
195
|
|
|
184
196
|
async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
|
|
@@ -243,7 +255,7 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
|
|
|
243
255
|
rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
|
|
244
256
|
});
|
|
245
257
|
|
|
246
|
-
//
|
|
258
|
+
// The claim is the authoritative pre-dispatch ownership fence. Renewal is
|
|
247
259
|
// only needed when the lease approaches expiry; an immediate read/renewal
|
|
248
260
|
// adds a round trip to every short tool call without extending ownership.
|
|
249
261
|
// Completion remains conditional on this lease, and long work still renews.
|
|
@@ -390,6 +402,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
390
402
|
shouldPersistFailure?: (error: unknown) => boolean;
|
|
391
403
|
heartbeatIntervalMs?: number;
|
|
392
404
|
markRunningBeforeExecute?: boolean;
|
|
405
|
+
/** New receipt-cache path: read completed facts, execute without creating an
|
|
406
|
+
* in-flight receipt, then atomically insert the completed fact. */
|
|
407
|
+
completedCacheOnly?: boolean;
|
|
408
|
+
requiresExecutionLock?: boolean;
|
|
409
|
+
executionLockTtlMs?: number;
|
|
393
410
|
formatError: (error: unknown) => string;
|
|
394
411
|
log: (message: string) => void;
|
|
395
412
|
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
@@ -426,6 +443,143 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
426
443
|
return recovered;
|
|
427
444
|
};
|
|
428
445
|
|
|
446
|
+
if (input.completedCacheOnly === true) {
|
|
447
|
+
if (input.force !== true) {
|
|
448
|
+
const cached = await input.store.get(input.receiptKey);
|
|
449
|
+
if (cached?.status === 'completed' || cached?.status === 'skipped') {
|
|
450
|
+
return await recoverCompletedReceipt(cached);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
const needsLock = input.requiresExecutionLock === true;
|
|
454
|
+
const ownerExecutionId = `${input.runId}:${crypto.randomUUID()}`;
|
|
455
|
+
let ownsLock = false;
|
|
456
|
+
let lockSupervisor: ReturnType<
|
|
457
|
+
typeof createRuntimeReceiptHeartbeatSupervisor
|
|
458
|
+
> | null = null;
|
|
459
|
+
if (needsLock) {
|
|
460
|
+
if (
|
|
461
|
+
!input.store.acquireExecutionLock ||
|
|
462
|
+
!input.store.releaseExecutionLock
|
|
463
|
+
) {
|
|
464
|
+
throw new Error(
|
|
465
|
+
`ctx.${input.operation}(${input.id}): non-idempotent provider requires the receipt execution-lock backend.`,
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
const deadline = Date.now() + DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS;
|
|
469
|
+
while (!ownsLock) {
|
|
470
|
+
const lock = await input.store.acquireExecutionLock({
|
|
471
|
+
receiptKey: input.receiptKey,
|
|
472
|
+
ownerExecutionId,
|
|
473
|
+
ttlMs: input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS,
|
|
474
|
+
});
|
|
475
|
+
ownsLock = lock?.ownerExecutionId === ownerExecutionId;
|
|
476
|
+
if (ownsLock) break;
|
|
477
|
+
const winner = await input.store.get(input.receiptKey);
|
|
478
|
+
if (
|
|
479
|
+
input.force !== true &&
|
|
480
|
+
(winner?.status === 'completed' || winner?.status === 'skipped')
|
|
481
|
+
) {
|
|
482
|
+
return await recoverCompletedReceipt(winner, 'in_flight');
|
|
483
|
+
}
|
|
484
|
+
if (Date.now() >= deadline) {
|
|
485
|
+
throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
|
|
486
|
+
}
|
|
487
|
+
await sleepReceiptWait(DURABLE_RECEIPT_WAIT_DELAY_MS);
|
|
488
|
+
}
|
|
489
|
+
const afterLock = await input.store.get(input.receiptKey);
|
|
490
|
+
if (
|
|
491
|
+
input.force !== true &&
|
|
492
|
+
(afterLock?.status === 'completed' ||
|
|
493
|
+
afterLock?.status === 'skipped')
|
|
494
|
+
) {
|
|
495
|
+
await input.store.releaseExecutionLock({
|
|
496
|
+
receiptKey: input.receiptKey,
|
|
497
|
+
ownerExecutionId,
|
|
498
|
+
});
|
|
499
|
+
return await recoverCompletedReceipt(afterLock, 'in_flight');
|
|
500
|
+
}
|
|
501
|
+
const lockTtlMs =
|
|
502
|
+
input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS;
|
|
503
|
+
lockSupervisor = createRuntimeReceiptHeartbeatSupervisor({
|
|
504
|
+
intervalMs: Math.max(1_000, Math.floor(lockTtlMs / 3)),
|
|
505
|
+
heartbeat: async () => {
|
|
506
|
+
const renewed = await input.store.acquireExecutionLock!({
|
|
507
|
+
receiptKey: input.receiptKey,
|
|
508
|
+
ownerExecutionId,
|
|
509
|
+
ttlMs: lockTtlMs,
|
|
510
|
+
});
|
|
511
|
+
if (renewed?.ownerExecutionId !== ownerExecutionId) {
|
|
512
|
+
throw new RuntimeReceiptLeaseLostError({
|
|
513
|
+
receiptKey: input.receiptKey,
|
|
514
|
+
runId: input.runId,
|
|
515
|
+
leaseId: ownerExecutionId,
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
return 'active';
|
|
519
|
+
},
|
|
520
|
+
isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
|
|
521
|
+
onLeaseLost: (error) => {
|
|
522
|
+
input.log(
|
|
523
|
+
`ctx.${input.operation}(${input.id}): dangerous side-effect execution fence lost: ${error instanceof Error ? error.message : String(error)}`,
|
|
524
|
+
);
|
|
525
|
+
},
|
|
526
|
+
});
|
|
527
|
+
lockSupervisor.start();
|
|
528
|
+
}
|
|
529
|
+
const releaseOwnLock = async (): Promise<void> => {
|
|
530
|
+
lockSupervisor?.stop();
|
|
531
|
+
lockSupervisor = null;
|
|
532
|
+
if (ownsLock) {
|
|
533
|
+
ownsLock = false;
|
|
534
|
+
await input.store.releaseExecutionLock!({
|
|
535
|
+
receiptKey: input.receiptKey,
|
|
536
|
+
ownerExecutionId,
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
let executed: T;
|
|
541
|
+
try {
|
|
542
|
+
executed = await input.execute({ leaseId: null });
|
|
543
|
+
} catch (error) {
|
|
544
|
+
await releaseOwnLock();
|
|
545
|
+
throw error;
|
|
546
|
+
}
|
|
547
|
+
try {
|
|
548
|
+
const result = input.onClaimedResult
|
|
549
|
+
? input.onClaimedResult(executed, input.receiptKey)
|
|
550
|
+
: executed;
|
|
551
|
+
assertNoSecretTaint(result, `ctx.${input.operation} result`);
|
|
552
|
+
// Force bypasses the immutable cache. It does not grant permission to
|
|
553
|
+
// replace the completed fact already published under this semantic key.
|
|
554
|
+
if (input.force === true) {
|
|
555
|
+
return result;
|
|
556
|
+
}
|
|
557
|
+
const completed = await input.store.complete(
|
|
558
|
+
input.receiptKey,
|
|
559
|
+
input.runId,
|
|
560
|
+
isToolExecuteResult(result)
|
|
561
|
+
? serializeToolExecuteResult(result)
|
|
562
|
+
: result,
|
|
563
|
+
COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
|
|
564
|
+
);
|
|
565
|
+
if (
|
|
566
|
+
completed?.status === 'completed' ||
|
|
567
|
+
completed?.status === 'skipped'
|
|
568
|
+
) {
|
|
569
|
+
return await recoverCompletedReceipt(completed, 'owner');
|
|
570
|
+
}
|
|
571
|
+
const winner = await input.store.get(input.receiptKey);
|
|
572
|
+
if (winner?.status === 'completed' || winner?.status === 'skipped') {
|
|
573
|
+
return await recoverCompletedReceipt(winner, 'owner');
|
|
574
|
+
}
|
|
575
|
+
throw new Error(
|
|
576
|
+
`ctx.${input.operation}(${input.id}): completed receipt cache insert failed.`,
|
|
577
|
+
);
|
|
578
|
+
} finally {
|
|
579
|
+
await releaseOwnLock();
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
429
583
|
let ownedLeaseId: string | null = null;
|
|
430
584
|
let ownedLeaseExpiresAt: string | null = null;
|
|
431
585
|
|
|
@@ -554,7 +708,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
554
708
|
|
|
555
709
|
let result: T;
|
|
556
710
|
try {
|
|
557
|
-
|
|
711
|
+
// New runners treat a claimed, nonterminal receipt as active. Keep the
|
|
712
|
+
// mark-running transition as an explicit compatibility option for older
|
|
713
|
+
// stores during their migration window, but do not pay the extra durable
|
|
714
|
+
// mutation on the normal claim -> terminal path.
|
|
715
|
+
if (input.markRunningBeforeExecute === true && input.store.markRunning) {
|
|
558
716
|
const markRunningStartedAt = Date.now();
|
|
559
717
|
const running = await input.store.markRunning(
|
|
560
718
|
input.receiptKey,
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
import { createSingleFlight } from './single-flight';
|
|
4
|
+
|
|
5
|
+
export type GatewayAuthSession<TAuth, TLaunch> = {
|
|
6
|
+
auth: TAuth;
|
|
7
|
+
launch: TLaunch;
|
|
8
|
+
source: 'hit' | 'miss' | 'coalesced';
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
type StoredSession<TAuth, TLaunch> = {
|
|
12
|
+
auth: TAuth;
|
|
13
|
+
launch: TLaunch;
|
|
14
|
+
expiresAt: number;
|
|
15
|
+
runId: string;
|
|
16
|
+
runAttempt: number | null;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/** Briefly reuse an authenticated immutable launch, bounded by token identity. */
|
|
20
|
+
export function createGatewayAuthSessionCache<TAuth, TLaunch>(options: {
|
|
21
|
+
ttlMs: number;
|
|
22
|
+
maxEntries: number;
|
|
23
|
+
identity: (auth: TAuth) => {
|
|
24
|
+
runId: string | null;
|
|
25
|
+
runAttempt: number | null;
|
|
26
|
+
};
|
|
27
|
+
now?: () => number;
|
|
28
|
+
}) {
|
|
29
|
+
const sessions = new Map<string, StoredSession<TAuth, TLaunch>>();
|
|
30
|
+
const flights = createSingleFlight<string, StoredSession<TAuth, TLaunch>>();
|
|
31
|
+
const now = options.now ?? Date.now;
|
|
32
|
+
const fingerprint = (token: string) =>
|
|
33
|
+
createHash('sha256').update(token).digest('base64url');
|
|
34
|
+
|
|
35
|
+
const prune = (at: number) => {
|
|
36
|
+
for (const [key, session] of sessions) {
|
|
37
|
+
if (session.expiresAt <= at) sessions.delete(key);
|
|
38
|
+
}
|
|
39
|
+
while (sessions.size >= options.maxEntries) {
|
|
40
|
+
const oldest = sessions.keys().next().value;
|
|
41
|
+
if (!oldest) break;
|
|
42
|
+
sessions.delete(oldest);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
async getOrEstablish(input: {
|
|
48
|
+
token: string;
|
|
49
|
+
tokenExpiresAt: number;
|
|
50
|
+
establish: () => Promise<{ auth: TAuth; launch: TLaunch }>;
|
|
51
|
+
}): Promise<GatewayAuthSession<TAuth, TLaunch>> {
|
|
52
|
+
const key = fingerprint(input.token);
|
|
53
|
+
const at = now();
|
|
54
|
+
const cached = sessions.get(key);
|
|
55
|
+
if (cached && cached.expiresAt > at) {
|
|
56
|
+
const identity = options.identity(cached.auth);
|
|
57
|
+
if (
|
|
58
|
+
identity.runId === cached.runId &&
|
|
59
|
+
identity.runAttempt === cached.runAttempt
|
|
60
|
+
) {
|
|
61
|
+
return { ...cached, source: 'hit' };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (cached) sessions.delete(key);
|
|
65
|
+
|
|
66
|
+
const coalesced = flights.has(key);
|
|
67
|
+
const session = await flights.run(key, async () => {
|
|
68
|
+
const established = await input.establish();
|
|
69
|
+
const identity = options.identity(established.auth);
|
|
70
|
+
if (!identity.runId) {
|
|
71
|
+
throw new Error('Gateway authentication produced no run identity.');
|
|
72
|
+
}
|
|
73
|
+
const establishedAt = now();
|
|
74
|
+
const stored: StoredSession<TAuth, TLaunch> = {
|
|
75
|
+
...established,
|
|
76
|
+
expiresAt: Math.min(
|
|
77
|
+
establishedAt + options.ttlMs,
|
|
78
|
+
input.tokenExpiresAt,
|
|
79
|
+
),
|
|
80
|
+
runId: identity.runId,
|
|
81
|
+
runAttempt: identity.runAttempt,
|
|
82
|
+
};
|
|
83
|
+
if (stored.expiresAt > establishedAt) {
|
|
84
|
+
prune(establishedAt);
|
|
85
|
+
sessions.set(key, stored);
|
|
86
|
+
}
|
|
87
|
+
return stored;
|
|
88
|
+
});
|
|
89
|
+
return { ...session, source: coalesced ? 'coalesced' : 'miss' };
|
|
90
|
+
},
|
|
91
|
+
size: () => sessions.size,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export type PlayCallExecution = 'inline' | 'child-workflow';
|
|
@@ -54,7 +54,7 @@ export const PLAY_RUNTIME_PROVIDERS: Record<
|
|
|
54
54
|
};
|
|
55
55
|
|
|
56
56
|
export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
|
|
57
|
-
return PLAY_RUNTIME_PROVIDERS.
|
|
57
|
+
return PLAY_RUNTIME_PROVIDERS.absurd;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
60
|
export function resolvePlayRuntimeProvider(
|
|
@@ -7,6 +7,7 @@ export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_SIZE = 200;
|
|
|
7
7
|
export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_FLUSH_MS = 10;
|
|
8
8
|
export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BUFFERED_BYTES =
|
|
9
9
|
RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES;
|
|
10
|
+
export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_BYTES = 1_000_000;
|
|
10
11
|
export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_INPUT_BYTES =
|
|
11
12
|
RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
|
|
12
13
|
|
|
@@ -22,6 +23,8 @@ export interface RuntimeReceiptCompletionSinkOptions<TInput, TOutput> {
|
|
|
22
23
|
maxBatchSize?: number;
|
|
23
24
|
maxFlushMs?: number;
|
|
24
25
|
maxBufferedBytes?: number;
|
|
26
|
+
/** Flush trigger and maximum ordinary transport batch size. */
|
|
27
|
+
maxBatchBytes?: number;
|
|
25
28
|
maxInputBytes?: number;
|
|
26
29
|
estimateBytes?: (input: TInput) => number;
|
|
27
30
|
completeMany: (inputs: TInput[]) => Promise<TOutput[]>;
|
|
@@ -83,12 +86,14 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
83
86
|
readonly maxBatchSize: number;
|
|
84
87
|
readonly maxFlushMs: number;
|
|
85
88
|
readonly maxBufferedBytes: number;
|
|
89
|
+
readonly maxBatchBytes: number;
|
|
86
90
|
readonly maxInputBytes: number;
|
|
87
91
|
|
|
88
92
|
#pending: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
|
|
89
93
|
#pendingHead = 0;
|
|
90
94
|
/** Pending plus in-flight bytes. Released only after transport settles. */
|
|
91
95
|
#residentBytes = 0;
|
|
96
|
+
#pendingBytes = 0;
|
|
92
97
|
#capacityWaiters: CapacityWaiter[] = [];
|
|
93
98
|
#flushTimer: ReceiptCompletionSinkTimer | null = null;
|
|
94
99
|
#flushPromise: Promise<void> | null = null;
|
|
@@ -116,6 +121,10 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
116
121
|
options.maxBufferedBytes,
|
|
117
122
|
DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BUFFERED_BYTES,
|
|
118
123
|
);
|
|
124
|
+
this.maxBatchBytes = normalizePositiveInteger(
|
|
125
|
+
options.maxBatchBytes,
|
|
126
|
+
DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_BYTES,
|
|
127
|
+
);
|
|
119
128
|
this.maxInputBytes = normalizePositiveInteger(
|
|
120
129
|
options.maxInputBytes,
|
|
121
130
|
DEFAULT_RECEIPT_COMPLETION_SINK_MAX_INPUT_BYTES,
|
|
@@ -194,12 +203,16 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
194
203
|
}
|
|
195
204
|
this.#throwIfUnavailable();
|
|
196
205
|
this.#residentBytes += bytes;
|
|
206
|
+
this.#pendingBytes += bytes;
|
|
197
207
|
const completion = new Promise<TOutput>((resolve, reject) => {
|
|
198
208
|
this.#pending.push({ input, bytes, resolve, reject });
|
|
199
209
|
if (this.#pendingCount() >= this.maxBatchSize) {
|
|
200
210
|
void this.flush().catch(() => undefined);
|
|
201
211
|
}
|
|
202
212
|
});
|
|
213
|
+
if (this.#pendingBytes >= this.maxBatchBytes) {
|
|
214
|
+
void this.flush().catch(() => undefined);
|
|
215
|
+
}
|
|
203
216
|
this.#scheduleFlush();
|
|
204
217
|
return await completion;
|
|
205
218
|
}
|
|
@@ -237,12 +250,13 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
237
250
|
let batchBytes = 0;
|
|
238
251
|
while (this.#pendingCount() > 0 && batch.length < this.maxBatchSize) {
|
|
239
252
|
const next = this.#pending[this.#pendingHead]!;
|
|
240
|
-
if (batch.length > 0 && batchBytes + next.bytes > this.
|
|
253
|
+
if (batch.length > 0 && batchBytes + next.bytes > this.maxBatchBytes) {
|
|
241
254
|
break;
|
|
242
255
|
}
|
|
243
256
|
this.#pendingHead += 1;
|
|
244
257
|
batch.push(next);
|
|
245
258
|
batchBytes += next.bytes;
|
|
259
|
+
this.#pendingBytes = Math.max(0, this.#pendingBytes - next.bytes);
|
|
246
260
|
}
|
|
247
261
|
if (
|
|
248
262
|
this.#pendingHead > 1_024 &&
|
|
@@ -321,6 +335,7 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
321
335
|
while (this.#pendingCount() > 0) {
|
|
322
336
|
const pending = this.#pending[this.#pendingHead++]!;
|
|
323
337
|
this.#residentBytes = Math.max(0, this.#residentBytes - pending.bytes);
|
|
338
|
+
this.#pendingBytes = Math.max(0, this.#pendingBytes - pending.bytes);
|
|
324
339
|
pending.reject(this.#failure);
|
|
325
340
|
}
|
|
326
341
|
this.#wakeCapacityWaiters();
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { RuntimeReceiptCompletionSink } from './receipt-completion-sink';
|
|
2
2
|
|
|
3
|
+
/** Claims/state transitions are small; flush once a 512 KB SQL payload forms. */
|
|
4
|
+
export const DEFAULT_RECEIPT_STATE_SINK_MAX_BATCH_BYTES = 512_000;
|
|
5
|
+
|
|
3
6
|
export interface RuntimeReceiptStateSinkOptions<TInput, TOutput> {
|
|
4
7
|
maxBatchSize?: number;
|
|
5
8
|
maxFlushMs?: number;
|
|
9
|
+
maxBatchBytes?: number;
|
|
10
|
+
maxBufferedBytes?: number;
|
|
11
|
+
maxInputBytes?: number;
|
|
6
12
|
estimateBytes?: (input: TInput) => number;
|
|
7
13
|
applyMany: (inputs: TInput[]) => Promise<TOutput[]>;
|
|
8
14
|
onFlushStart?: (event: { count: number; bytes: number }) => void;
|
|
@@ -33,6 +39,10 @@ export class RuntimeReceiptStateSink<TInput, TOutput> {
|
|
|
33
39
|
this.#sink = new RuntimeReceiptCompletionSink<TInput, TOutput>({
|
|
34
40
|
maxBatchSize: options.maxBatchSize,
|
|
35
41
|
maxFlushMs: options.maxFlushMs,
|
|
42
|
+
maxBatchBytes:
|
|
43
|
+
options.maxBatchBytes ?? DEFAULT_RECEIPT_STATE_SINK_MAX_BATCH_BYTES,
|
|
44
|
+
maxBufferedBytes: options.maxBufferedBytes,
|
|
45
|
+
maxInputBytes: options.maxInputBytes,
|
|
36
46
|
estimateBytes: options.estimateBytes,
|
|
37
47
|
completeMany: options.applyMany,
|
|
38
48
|
onFlushStart: options.onFlushStart,
|
|
@@ -30,7 +30,10 @@ export type RuntimeResourceSnapshot = {
|
|
|
30
30
|
};
|
|
31
31
|
sheetFlush: {
|
|
32
32
|
acquired: number;
|
|
33
|
+
active: number;
|
|
34
|
+
waiting: number;
|
|
33
35
|
bytes: number;
|
|
36
|
+
admissionWaitMsEwma: number | null;
|
|
34
37
|
latencyMsEwma: number | null;
|
|
35
38
|
};
|
|
36
39
|
providers: Record<
|
|
@@ -90,8 +93,74 @@ function createEwma(alpha = 0.2): Ewma {
|
|
|
90
93
|
};
|
|
91
94
|
}
|
|
92
95
|
|
|
93
|
-
|
|
94
|
-
|
|
96
|
+
// Sheet persistence has its own local admission lane. Receipt lifecycle calls
|
|
97
|
+
// are admitted by the receipt sink/gateway and must not queue behind a large
|
|
98
|
+
// Runtime Sheet flush in the runner. Two flushes keep the database busy while
|
|
99
|
+
// bounding resident payloads and gateway/pool pressure (ADR 0012).
|
|
100
|
+
const MAX_CONCURRENT_SHEET_FLUSHES = 2;
|
|
101
|
+
|
|
102
|
+
type SheetFlushWaiter = {
|
|
103
|
+
resolve: (lease: RuntimeResourceLease) => void;
|
|
104
|
+
reject: (error: unknown) => void;
|
|
105
|
+
signal?: AbortSignal;
|
|
106
|
+
onAbort?: () => void;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
class SheetFlushAdmission {
|
|
110
|
+
#active = 0;
|
|
111
|
+
#waiters: SheetFlushWaiter[] = [];
|
|
112
|
+
|
|
113
|
+
get active(): number {
|
|
114
|
+
return this.#active;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
get waiting(): number {
|
|
118
|
+
return this.#waiters.length;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
acquire(signal?: AbortSignal): Promise<RuntimeResourceLease> {
|
|
122
|
+
if (signal?.aborted) {
|
|
123
|
+
return Promise.reject(signal.reason ?? new Error('Sheet flush aborted'));
|
|
124
|
+
}
|
|
125
|
+
if (this.#active < MAX_CONCURRENT_SHEET_FLUSHES) {
|
|
126
|
+
this.#active += 1;
|
|
127
|
+
return Promise.resolve(this.#lease());
|
|
128
|
+
}
|
|
129
|
+
return new Promise((resolve, reject) => {
|
|
130
|
+
const waiter: SheetFlushWaiter = {
|
|
131
|
+
resolve,
|
|
132
|
+
reject,
|
|
133
|
+
...(signal ? { signal } : {}),
|
|
134
|
+
};
|
|
135
|
+
if (signal) {
|
|
136
|
+
waiter.onAbort = () => {
|
|
137
|
+
const index = this.#waiters.indexOf(waiter);
|
|
138
|
+
if (index >= 0) this.#waiters.splice(index, 1);
|
|
139
|
+
reject(signal.reason ?? new Error('Sheet flush aborted'));
|
|
140
|
+
};
|
|
141
|
+
signal.addEventListener('abort', waiter.onAbort, { once: true });
|
|
142
|
+
}
|
|
143
|
+
this.#waiters.push(waiter);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
#lease(): RuntimeResourceLease {
|
|
148
|
+
let released = false;
|
|
149
|
+
return {
|
|
150
|
+
release: () => {
|
|
151
|
+
if (released) return;
|
|
152
|
+
released = true;
|
|
153
|
+
while (this.#waiters.length > 0) {
|
|
154
|
+
const waiter = this.#waiters.shift()!;
|
|
155
|
+
waiter.signal?.removeEventListener('abort', waiter.onAbort!);
|
|
156
|
+
if (waiter.signal?.aborted) continue;
|
|
157
|
+
waiter.resolve(this.#lease());
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
this.#active = Math.max(0, this.#active - 1);
|
|
161
|
+
},
|
|
162
|
+
};
|
|
163
|
+
}
|
|
95
164
|
}
|
|
96
165
|
|
|
97
166
|
function elapsedSince(startedAt: number): number {
|
|
@@ -109,6 +178,8 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
109
178
|
const rowAdmissionWait = createEwma();
|
|
110
179
|
const toolAdmissionWait = createEwma();
|
|
111
180
|
const sheetFlushLatency = createEwma();
|
|
181
|
+
const sheetFlushAdmissionWait = createEwma();
|
|
182
|
+
const sheetFlushAdmission = new SheetFlushAdmission();
|
|
112
183
|
const providers = new Map<
|
|
113
184
|
string,
|
|
114
185
|
{
|
|
@@ -219,11 +290,14 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
219
290
|
},
|
|
220
291
|
|
|
221
292
|
async acquireSheetFlush(sheetInput) {
|
|
293
|
+
const startedAt = Date.now();
|
|
294
|
+
const lease = await sheetFlushAdmission.acquire(sheetInput?.signal);
|
|
222
295
|
sheetFlushCounters.acquired += 1;
|
|
296
|
+
sheetFlushAdmissionWait.observe(elapsedSince(startedAt));
|
|
223
297
|
observe({
|
|
224
298
|
sheetFlushBytes: sheetInput?.estimatedBytes ?? null,
|
|
225
299
|
});
|
|
226
|
-
return
|
|
300
|
+
return lease;
|
|
227
301
|
},
|
|
228
302
|
|
|
229
303
|
suggestedToolParallelism(toolId, fallback) {
|
|
@@ -258,7 +332,10 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
258
332
|
},
|
|
259
333
|
sheetFlush: {
|
|
260
334
|
acquired: sheetFlushCounters.acquired,
|
|
335
|
+
active: sheetFlushAdmission.active,
|
|
336
|
+
waiting: sheetFlushAdmission.waiting,
|
|
261
337
|
bytes: sheetFlushCounters.bytes,
|
|
338
|
+
admissionWaitMsEwma: sheetFlushAdmissionWait.value,
|
|
262
339
|
latencyMsEwma: sheetFlushLatency.value,
|
|
263
340
|
},
|
|
264
341
|
providers: Object.fromEntries(
|