deepline 0.1.212 → 0.1.214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +167 -329
  2. package/dist/bundling-sources/sdk/src/client.ts +2 -2
  3. package/dist/bundling-sources/sdk/src/index.ts +2 -0
  4. package/dist/bundling-sources/sdk/src/play.ts +8 -1
  5. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
  6. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  7. package/dist/bundling-sources/sdk/src/types.ts +3 -3
  8. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +49 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
  11. package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -123
  13. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +30 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +160 -2
  16. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
  22. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +54 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +477 -85
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -2
  27. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
  28. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
  29. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
  30. package/dist/cli/index.js +50 -436
  31. package/dist/cli/index.mjs +50 -436
  32. package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
  33. package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
  34. package/dist/index.d.mts +11 -8
  35. package/dist/index.d.ts +11 -8
  36. package/dist/index.js +4 -4
  37. package/dist/index.mjs +4 -4
  38. package/dist/plays/bundle-play-file.d.mts +2 -2
  39. package/dist/plays/bundle-play-file.d.ts +2 -2
  40. package/dist/plays/bundle-play-file.mjs +1 -1
  41. package/package.json +1 -1
@@ -24,6 +24,7 @@ import type { PreviousCell } from './cell-staleness';
24
24
  import type { MapRowOutcome } from './durability-store';
25
25
  import type { WorkReceiptFailureKind } from './work-receipts';
26
26
  import type { RunExecutionScope } from './run-execution-scope';
27
+ import type { PlayCallExecution } from './play-call-execution';
27
28
 
28
29
  export interface RowState {
29
30
  results: Map<string, unknown>;
@@ -69,12 +70,26 @@ export interface RuntimeStepReceipt {
69
70
  claimState?: 'claimed' | 'existing';
70
71
  }
71
72
 
73
+ export type AcquireRuntimeReceiptExecutionLockInput = {
74
+ key: string;
75
+ runId: string;
76
+ ownerExecutionId: string;
77
+ ttlMs: number;
78
+ };
79
+
80
+ export type ReleaseRuntimeReceiptExecutionLockInput = {
81
+ key: string;
82
+ runId: string;
83
+ ownerExecutionId: string;
84
+ };
85
+
72
86
  export interface GetRuntimeStepReceiptInput {
73
87
  key: string;
74
88
  }
75
89
 
76
90
  export interface ClaimRuntimeStepReceiptInput {
77
91
  key: string;
92
+ leaseId?: string;
78
93
  runId: string;
79
94
  runAttempt?: number | null;
80
95
  leaseAware?: boolean;
@@ -128,6 +143,7 @@ export interface GetRuntimeStepReceiptsInput {
128
143
 
129
144
  export interface ClaimRuntimeStepReceiptsInput {
130
145
  keys: string[];
146
+ leaseIds?: string[];
131
147
  runId: string;
132
148
  runAttempt?: number | null;
133
149
  leaseAware?: boolean;
@@ -281,6 +297,7 @@ export type CustomerDbQueryHandler = (
281
297
 
282
298
  export interface PlayCallOptions {
283
299
  description?: string;
300
+ execution?: PlayCallExecution;
284
301
  }
285
302
 
286
303
  export interface StepOptions {
@@ -597,7 +614,10 @@ export interface ContextOptions {
597
614
  rows: MapRowOutcome[];
598
615
  outputFields: string[];
599
616
  staticPipeline?: PlayStaticPipeline | null;
600
- }) => Promise<void>;
617
+ }) => Promise<void | {
618
+ updated: number;
619
+ staleDroppedKeys?: string[];
620
+ }>;
601
621
  playId?: string;
602
622
  runId?: string;
603
623
  /** Physical executor run that owns leases for an inline child context. */
@@ -634,6 +654,7 @@ export interface ContextOptions {
634
654
  getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
635
655
  getToolRetryPolicy?: (toolId: string) => Promise<{
636
656
  retrySafeTransientHttp?: boolean;
657
+ requiresExecutionFence?: boolean;
637
658
  } | null>;
638
659
  getToolActionCacheVersion?: (toolId: string) => Promise<string> | string;
639
660
  getToolAuthScopeDigest?: (toolId: string) => Promise<string> | string;
@@ -672,6 +693,12 @@ export interface ContextOptions {
672
693
  getRuntimeStepReceipt?: (
673
694
  input: GetRuntimeStepReceiptInput,
674
695
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
696
+ acquireRuntimeReceiptExecutionLock?: (
697
+ input: AcquireRuntimeReceiptExecutionLockInput,
698
+ ) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
699
+ releaseRuntimeReceiptExecutionLock?: (
700
+ input: ReleaseRuntimeReceiptExecutionLockInput,
701
+ ) => Promise<boolean>;
675
702
  getRuntimeStepReceipts?: (
676
703
  input: GetRuntimeStepReceiptsInput,
677
704
  ) =>
@@ -932,6 +959,7 @@ export type PlayStep =
932
959
  | {
933
960
  type: 'play_call';
934
961
  playId: string;
962
+ execution?: PlayCallExecution;
935
963
  results?: PlayStepRowResult[];
936
964
  nestedSteps: PlayStep[];
937
965
  description?: string;
@@ -971,6 +999,7 @@ export type PlayDatasetSubstep =
971
999
  | {
972
1000
  type: 'play_call';
973
1001
  playId: string;
1002
+ execution?: PlayCallExecution;
974
1003
  results?: PlayStepRowResult[];
975
1004
  nestedSteps: PlayStep[];
976
1005
  description?: string;
@@ -25,6 +25,9 @@ const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
25
25
  const TOOL_RECEIPT_DEFAULT_WAIT_MS = 300_000;
26
26
  const TOOL_RECEIPT_COMPLETION_BUFFER_MS = 30_000;
27
27
  const TOOL_RECEIPT_MAX_WAIT_MS = 30 * 60_000;
28
+ const DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS = 15_000;
29
+ export const COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID =
30
+ '__deepline_completed_cache_insert__';
28
31
 
29
32
  export class RuntimeReceiptWaitTimeoutError extends Error {
30
33
  constructor(key: string) {
@@ -179,6 +182,15 @@ export type DurableReceiptExecutionStore = {
179
182
  ): Promise<RuntimeStepReceipt | null>;
180
183
  canPersistFailure: boolean;
181
184
  canPersistCompletion: boolean;
185
+ acquireExecutionLock?: (input: {
186
+ receiptKey: string;
187
+ ownerExecutionId: string;
188
+ ttlMs: number;
189
+ }) => Promise<{ ownerExecutionId: string; expiresAt: string } | null>;
190
+ releaseExecutionLock?: (input: {
191
+ receiptKey: string;
192
+ ownerExecutionId: string;
193
+ }) => Promise<boolean>;
182
194
  };
183
195
 
184
196
  async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
@@ -243,7 +255,7 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
243
255
  rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
244
256
  });
245
257
 
246
- // markRunning is the authoritative pre-dispatch ownership fence. Renewal is
258
+ // The claim is the authoritative pre-dispatch ownership fence. Renewal is
247
259
  // only needed when the lease approaches expiry; an immediate read/renewal
248
260
  // adds a round trip to every short tool call without extending ownership.
249
261
  // Completion remains conditional on this lease, and long work still renews.
@@ -390,6 +402,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
390
402
  shouldPersistFailure?: (error: unknown) => boolean;
391
403
  heartbeatIntervalMs?: number;
392
404
  markRunningBeforeExecute?: boolean;
405
+ /** New receipt-cache path: read completed facts, execute without creating an
406
+ * in-flight receipt, then atomically insert the completed fact. */
407
+ completedCacheOnly?: boolean;
408
+ requiresExecutionLock?: boolean;
409
+ executionLockTtlMs?: number;
393
410
  formatError: (error: unknown) => string;
394
411
  log: (message: string) => void;
395
412
  execute: (context: { leaseId: string | null }) => Promise<T>;
@@ -426,6 +443,143 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
426
443
  return recovered;
427
444
  };
428
445
 
446
+ if (input.completedCacheOnly === true) {
447
+ if (input.force !== true) {
448
+ const cached = await input.store.get(input.receiptKey);
449
+ if (cached?.status === 'completed' || cached?.status === 'skipped') {
450
+ return await recoverCompletedReceipt(cached);
451
+ }
452
+ }
453
+ const needsLock = input.requiresExecutionLock === true;
454
+ const ownerExecutionId = `${input.runId}:${crypto.randomUUID()}`;
455
+ let ownsLock = false;
456
+ let lockSupervisor: ReturnType<
457
+ typeof createRuntimeReceiptHeartbeatSupervisor
458
+ > | null = null;
459
+ if (needsLock) {
460
+ if (
461
+ !input.store.acquireExecutionLock ||
462
+ !input.store.releaseExecutionLock
463
+ ) {
464
+ throw new Error(
465
+ `ctx.${input.operation}(${input.id}): non-idempotent provider requires the receipt execution-lock backend.`,
466
+ );
467
+ }
468
+ const deadline = Date.now() + DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS;
469
+ while (!ownsLock) {
470
+ const lock = await input.store.acquireExecutionLock({
471
+ receiptKey: input.receiptKey,
472
+ ownerExecutionId,
473
+ ttlMs: input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS,
474
+ });
475
+ ownsLock = lock?.ownerExecutionId === ownerExecutionId;
476
+ if (ownsLock) break;
477
+ const winner = await input.store.get(input.receiptKey);
478
+ if (
479
+ input.force !== true &&
480
+ (winner?.status === 'completed' || winner?.status === 'skipped')
481
+ ) {
482
+ return await recoverCompletedReceipt(winner, 'in_flight');
483
+ }
484
+ if (Date.now() >= deadline) {
485
+ throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
486
+ }
487
+ await sleepReceiptWait(DURABLE_RECEIPT_WAIT_DELAY_MS);
488
+ }
489
+ const afterLock = await input.store.get(input.receiptKey);
490
+ if (
491
+ input.force !== true &&
492
+ (afterLock?.status === 'completed' ||
493
+ afterLock?.status === 'skipped')
494
+ ) {
495
+ await input.store.releaseExecutionLock({
496
+ receiptKey: input.receiptKey,
497
+ ownerExecutionId,
498
+ });
499
+ return await recoverCompletedReceipt(afterLock, 'in_flight');
500
+ }
501
+ const lockTtlMs =
502
+ input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS;
503
+ lockSupervisor = createRuntimeReceiptHeartbeatSupervisor({
504
+ intervalMs: Math.max(1_000, Math.floor(lockTtlMs / 3)),
505
+ heartbeat: async () => {
506
+ const renewed = await input.store.acquireExecutionLock!({
507
+ receiptKey: input.receiptKey,
508
+ ownerExecutionId,
509
+ ttlMs: lockTtlMs,
510
+ });
511
+ if (renewed?.ownerExecutionId !== ownerExecutionId) {
512
+ throw new RuntimeReceiptLeaseLostError({
513
+ receiptKey: input.receiptKey,
514
+ runId: input.runId,
515
+ leaseId: ownerExecutionId,
516
+ });
517
+ }
518
+ return 'active';
519
+ },
520
+ isLeaseLost: (error) => error instanceof RuntimeReceiptLeaseLostError,
521
+ onLeaseLost: (error) => {
522
+ input.log(
523
+ `ctx.${input.operation}(${input.id}): dangerous side-effect execution fence lost: ${error instanceof Error ? error.message : String(error)}`,
524
+ );
525
+ },
526
+ });
527
+ lockSupervisor.start();
528
+ }
529
+ const releaseOwnLock = async (): Promise<void> => {
530
+ lockSupervisor?.stop();
531
+ lockSupervisor = null;
532
+ if (ownsLock) {
533
+ ownsLock = false;
534
+ await input.store.releaseExecutionLock!({
535
+ receiptKey: input.receiptKey,
536
+ ownerExecutionId,
537
+ });
538
+ }
539
+ };
540
+ let executed: T;
541
+ try {
542
+ executed = await input.execute({ leaseId: null });
543
+ } catch (error) {
544
+ await releaseOwnLock();
545
+ throw error;
546
+ }
547
+ try {
548
+ const result = input.onClaimedResult
549
+ ? input.onClaimedResult(executed, input.receiptKey)
550
+ : executed;
551
+ assertNoSecretTaint(result, `ctx.${input.operation} result`);
552
+ // Force bypasses the immutable cache. It does not grant permission to
553
+ // replace the completed fact already published under this semantic key.
554
+ if (input.force === true) {
555
+ return result;
556
+ }
557
+ const completed = await input.store.complete(
558
+ input.receiptKey,
559
+ input.runId,
560
+ isToolExecuteResult(result)
561
+ ? serializeToolExecuteResult(result)
562
+ : result,
563
+ COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
564
+ );
565
+ if (
566
+ completed?.status === 'completed' ||
567
+ completed?.status === 'skipped'
568
+ ) {
569
+ return await recoverCompletedReceipt(completed, 'owner');
570
+ }
571
+ const winner = await input.store.get(input.receiptKey);
572
+ if (winner?.status === 'completed' || winner?.status === 'skipped') {
573
+ return await recoverCompletedReceipt(winner, 'owner');
574
+ }
575
+ throw new Error(
576
+ `ctx.${input.operation}(${input.id}): completed receipt cache insert failed.`,
577
+ );
578
+ } finally {
579
+ await releaseOwnLock();
580
+ }
581
+ }
582
+
429
583
  let ownedLeaseId: string | null = null;
430
584
  let ownedLeaseExpiresAt: string | null = null;
431
585
 
@@ -554,7 +708,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
554
708
 
555
709
  let result: T;
556
710
  try {
557
- if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
711
+ // New runners treat a claimed, nonterminal receipt as active. Keep the
712
+ // mark-running transition as an explicit compatibility option for older
713
+ // stores during their migration window, but do not pay the extra durable
714
+ // mutation on the normal claim -> terminal path.
715
+ if (input.markRunningBeforeExecute === true && input.store.markRunning) {
558
716
  const markRunningStartedAt = Date.now();
559
717
  const running = await input.store.markRunning(
560
718
  input.receiptKey,
@@ -0,0 +1,93 @@
1
+ import { createHash } from 'node:crypto';
2
+
3
+ import { createSingleFlight } from './single-flight';
4
+
5
+ export type GatewayAuthSession<TAuth, TLaunch> = {
6
+ auth: TAuth;
7
+ launch: TLaunch;
8
+ source: 'hit' | 'miss' | 'coalesced';
9
+ };
10
+
11
+ type StoredSession<TAuth, TLaunch> = {
12
+ auth: TAuth;
13
+ launch: TLaunch;
14
+ expiresAt: number;
15
+ runId: string;
16
+ runAttempt: number | null;
17
+ };
18
+
19
+ /** Briefly reuse an authenticated immutable launch, bounded by token identity. */
20
+ export function createGatewayAuthSessionCache<TAuth, TLaunch>(options: {
21
+ ttlMs: number;
22
+ maxEntries: number;
23
+ identity: (auth: TAuth) => {
24
+ runId: string | null;
25
+ runAttempt: number | null;
26
+ };
27
+ now?: () => number;
28
+ }) {
29
+ const sessions = new Map<string, StoredSession<TAuth, TLaunch>>();
30
+ const flights = createSingleFlight<string, StoredSession<TAuth, TLaunch>>();
31
+ const now = options.now ?? Date.now;
32
+ const fingerprint = (token: string) =>
33
+ createHash('sha256').update(token).digest('base64url');
34
+
35
+ const prune = (at: number) => {
36
+ for (const [key, session] of sessions) {
37
+ if (session.expiresAt <= at) sessions.delete(key);
38
+ }
39
+ while (sessions.size >= options.maxEntries) {
40
+ const oldest = sessions.keys().next().value;
41
+ if (!oldest) break;
42
+ sessions.delete(oldest);
43
+ }
44
+ };
45
+
46
+ return {
47
+ async getOrEstablish(input: {
48
+ token: string;
49
+ tokenExpiresAt: number;
50
+ establish: () => Promise<{ auth: TAuth; launch: TLaunch }>;
51
+ }): Promise<GatewayAuthSession<TAuth, TLaunch>> {
52
+ const key = fingerprint(input.token);
53
+ const at = now();
54
+ const cached = sessions.get(key);
55
+ if (cached && cached.expiresAt > at) {
56
+ const identity = options.identity(cached.auth);
57
+ if (
58
+ identity.runId === cached.runId &&
59
+ identity.runAttempt === cached.runAttempt
60
+ ) {
61
+ return { ...cached, source: 'hit' };
62
+ }
63
+ }
64
+ if (cached) sessions.delete(key);
65
+
66
+ const coalesced = flights.has(key);
67
+ const session = await flights.run(key, async () => {
68
+ const established = await input.establish();
69
+ const identity = options.identity(established.auth);
70
+ if (!identity.runId) {
71
+ throw new Error('Gateway authentication produced no run identity.');
72
+ }
73
+ const establishedAt = now();
74
+ const stored: StoredSession<TAuth, TLaunch> = {
75
+ ...established,
76
+ expiresAt: Math.min(
77
+ establishedAt + options.ttlMs,
78
+ input.tokenExpiresAt,
79
+ ),
80
+ runId: identity.runId,
81
+ runAttempt: identity.runAttempt,
82
+ };
83
+ if (stored.expiresAt > establishedAt) {
84
+ prune(establishedAt);
85
+ sessions.set(key, stored);
86
+ }
87
+ return stored;
88
+ });
89
+ return { ...session, source: coalesced ? 'coalesced' : 'miss' };
90
+ },
91
+ size: () => sessions.size,
92
+ };
93
+ }
@@ -0,0 +1 @@
1
+ export type PlayCallExecution = 'inline' | 'child-workflow';
@@ -54,7 +54,7 @@ export const PLAY_RUNTIME_PROVIDERS: Record<
54
54
  };
55
55
 
56
56
  export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
57
- return PLAY_RUNTIME_PROVIDERS.workers_edge;
57
+ return PLAY_RUNTIME_PROVIDERS.absurd;
58
58
  }
59
59
 
60
60
  export function resolvePlayRuntimeProvider(
@@ -7,6 +7,7 @@ export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_SIZE = 200;
7
7
  export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_FLUSH_MS = 10;
8
8
  export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BUFFERED_BYTES =
9
9
  RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES;
10
+ export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_BYTES = 1_000_000;
10
11
  export const DEFAULT_RECEIPT_COMPLETION_SINK_MAX_INPUT_BYTES =
11
12
  RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
12
13
 
@@ -22,6 +23,8 @@ export interface RuntimeReceiptCompletionSinkOptions<TInput, TOutput> {
22
23
  maxBatchSize?: number;
23
24
  maxFlushMs?: number;
24
25
  maxBufferedBytes?: number;
26
+ /** Flush trigger and maximum ordinary transport batch size. */
27
+ maxBatchBytes?: number;
25
28
  maxInputBytes?: number;
26
29
  estimateBytes?: (input: TInput) => number;
27
30
  completeMany: (inputs: TInput[]) => Promise<TOutput[]>;
@@ -83,12 +86,14 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
83
86
  readonly maxBatchSize: number;
84
87
  readonly maxFlushMs: number;
85
88
  readonly maxBufferedBytes: number;
89
+ readonly maxBatchBytes: number;
86
90
  readonly maxInputBytes: number;
87
91
 
88
92
  #pending: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
89
93
  #pendingHead = 0;
90
94
  /** Pending plus in-flight bytes. Released only after transport settles. */
91
95
  #residentBytes = 0;
96
+ #pendingBytes = 0;
92
97
  #capacityWaiters: CapacityWaiter[] = [];
93
98
  #flushTimer: ReceiptCompletionSinkTimer | null = null;
94
99
  #flushPromise: Promise<void> | null = null;
@@ -116,6 +121,10 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
116
121
  options.maxBufferedBytes,
117
122
  DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BUFFERED_BYTES,
118
123
  );
124
+ this.maxBatchBytes = normalizePositiveInteger(
125
+ options.maxBatchBytes,
126
+ DEFAULT_RECEIPT_COMPLETION_SINK_MAX_BATCH_BYTES,
127
+ );
119
128
  this.maxInputBytes = normalizePositiveInteger(
120
129
  options.maxInputBytes,
121
130
  DEFAULT_RECEIPT_COMPLETION_SINK_MAX_INPUT_BYTES,
@@ -194,12 +203,16 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
194
203
  }
195
204
  this.#throwIfUnavailable();
196
205
  this.#residentBytes += bytes;
206
+ this.#pendingBytes += bytes;
197
207
  const completion = new Promise<TOutput>((resolve, reject) => {
198
208
  this.#pending.push({ input, bytes, resolve, reject });
199
209
  if (this.#pendingCount() >= this.maxBatchSize) {
200
210
  void this.flush().catch(() => undefined);
201
211
  }
202
212
  });
213
+ if (this.#pendingBytes >= this.maxBatchBytes) {
214
+ void this.flush().catch(() => undefined);
215
+ }
203
216
  this.#scheduleFlush();
204
217
  return await completion;
205
218
  }
@@ -237,12 +250,13 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
237
250
  let batchBytes = 0;
238
251
  while (this.#pendingCount() > 0 && batch.length < this.maxBatchSize) {
239
252
  const next = this.#pending[this.#pendingHead]!;
240
- if (batch.length > 0 && batchBytes + next.bytes > this.maxBufferedBytes) {
253
+ if (batch.length > 0 && batchBytes + next.bytes > this.maxBatchBytes) {
241
254
  break;
242
255
  }
243
256
  this.#pendingHead += 1;
244
257
  batch.push(next);
245
258
  batchBytes += next.bytes;
259
+ this.#pendingBytes = Math.max(0, this.#pendingBytes - next.bytes);
246
260
  }
247
261
  if (
248
262
  this.#pendingHead > 1_024 &&
@@ -321,6 +335,7 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
321
335
  while (this.#pendingCount() > 0) {
322
336
  const pending = this.#pending[this.#pendingHead++]!;
323
337
  this.#residentBytes = Math.max(0, this.#residentBytes - pending.bytes);
338
+ this.#pendingBytes = Math.max(0, this.#pendingBytes - pending.bytes);
324
339
  pending.reject(this.#failure);
325
340
  }
326
341
  this.#wakeCapacityWaiters();
@@ -1,8 +1,14 @@
1
1
  import { RuntimeReceiptCompletionSink } from './receipt-completion-sink';
2
2
 
3
+ /** Claims/state transitions are small; flush once a 512 KB SQL payload forms. */
4
+ export const DEFAULT_RECEIPT_STATE_SINK_MAX_BATCH_BYTES = 512_000;
5
+
3
6
  export interface RuntimeReceiptStateSinkOptions<TInput, TOutput> {
4
7
  maxBatchSize?: number;
5
8
  maxFlushMs?: number;
9
+ maxBatchBytes?: number;
10
+ maxBufferedBytes?: number;
11
+ maxInputBytes?: number;
6
12
  estimateBytes?: (input: TInput) => number;
7
13
  applyMany: (inputs: TInput[]) => Promise<TOutput[]>;
8
14
  onFlushStart?: (event: { count: number; bytes: number }) => void;
@@ -33,6 +39,10 @@ export class RuntimeReceiptStateSink<TInput, TOutput> {
33
39
  this.#sink = new RuntimeReceiptCompletionSink<TInput, TOutput>({
34
40
  maxBatchSize: options.maxBatchSize,
35
41
  maxFlushMs: options.maxFlushMs,
42
+ maxBatchBytes:
43
+ options.maxBatchBytes ?? DEFAULT_RECEIPT_STATE_SINK_MAX_BATCH_BYTES,
44
+ maxBufferedBytes: options.maxBufferedBytes,
45
+ maxInputBytes: options.maxInputBytes,
36
46
  estimateBytes: options.estimateBytes,
37
47
  completeMany: options.applyMany,
38
48
  onFlushStart: options.onFlushStart,
@@ -30,7 +30,10 @@ export type RuntimeResourceSnapshot = {
30
30
  };
31
31
  sheetFlush: {
32
32
  acquired: number;
33
+ active: number;
34
+ waiting: number;
33
35
  bytes: number;
36
+ admissionWaitMsEwma: number | null;
34
37
  latencyMsEwma: number | null;
35
38
  };
36
39
  providers: Record<
@@ -90,8 +93,74 @@ function createEwma(alpha = 0.2): Ewma {
90
93
  };
91
94
  }
92
95
 
93
- function noOpLease(): RuntimeResourceLease {
94
- return { release() {} };
96
+ // Sheet persistence has its own local admission lane. Receipt lifecycle calls
97
+ // are admitted by the receipt sink/gateway and must not queue behind a large
98
+ // Runtime Sheet flush in the runner. Two flushes keep the database busy while
99
+ // bounding resident payloads and gateway/pool pressure (ADR 0012).
100
+ const MAX_CONCURRENT_SHEET_FLUSHES = 2;
101
+
102
+ type SheetFlushWaiter = {
103
+ resolve: (lease: RuntimeResourceLease) => void;
104
+ reject: (error: unknown) => void;
105
+ signal?: AbortSignal;
106
+ onAbort?: () => void;
107
+ };
108
+
109
+ class SheetFlushAdmission {
110
+ #active = 0;
111
+ #waiters: SheetFlushWaiter[] = [];
112
+
113
+ get active(): number {
114
+ return this.#active;
115
+ }
116
+
117
+ get waiting(): number {
118
+ return this.#waiters.length;
119
+ }
120
+
121
+ acquire(signal?: AbortSignal): Promise<RuntimeResourceLease> {
122
+ if (signal?.aborted) {
123
+ return Promise.reject(signal.reason ?? new Error('Sheet flush aborted'));
124
+ }
125
+ if (this.#active < MAX_CONCURRENT_SHEET_FLUSHES) {
126
+ this.#active += 1;
127
+ return Promise.resolve(this.#lease());
128
+ }
129
+ return new Promise((resolve, reject) => {
130
+ const waiter: SheetFlushWaiter = {
131
+ resolve,
132
+ reject,
133
+ ...(signal ? { signal } : {}),
134
+ };
135
+ if (signal) {
136
+ waiter.onAbort = () => {
137
+ const index = this.#waiters.indexOf(waiter);
138
+ if (index >= 0) this.#waiters.splice(index, 1);
139
+ reject(signal.reason ?? new Error('Sheet flush aborted'));
140
+ };
141
+ signal.addEventListener('abort', waiter.onAbort, { once: true });
142
+ }
143
+ this.#waiters.push(waiter);
144
+ });
145
+ }
146
+
147
+ #lease(): RuntimeResourceLease {
148
+ let released = false;
149
+ return {
150
+ release: () => {
151
+ if (released) return;
152
+ released = true;
153
+ while (this.#waiters.length > 0) {
154
+ const waiter = this.#waiters.shift()!;
155
+ waiter.signal?.removeEventListener('abort', waiter.onAbort!);
156
+ if (waiter.signal?.aborted) continue;
157
+ waiter.resolve(this.#lease());
158
+ return;
159
+ }
160
+ this.#active = Math.max(0, this.#active - 1);
161
+ },
162
+ };
163
+ }
95
164
  }
96
165
 
97
166
  function elapsedSince(startedAt: number): number {
@@ -109,6 +178,8 @@ export function createRuntimeResourceGovernor(input: {
109
178
  const rowAdmissionWait = createEwma();
110
179
  const toolAdmissionWait = createEwma();
111
180
  const sheetFlushLatency = createEwma();
181
+ const sheetFlushAdmissionWait = createEwma();
182
+ const sheetFlushAdmission = new SheetFlushAdmission();
112
183
  const providers = new Map<
113
184
  string,
114
185
  {
@@ -219,11 +290,14 @@ export function createRuntimeResourceGovernor(input: {
219
290
  },
220
291
 
221
292
  async acquireSheetFlush(sheetInput) {
293
+ const startedAt = Date.now();
294
+ const lease = await sheetFlushAdmission.acquire(sheetInput?.signal);
222
295
  sheetFlushCounters.acquired += 1;
296
+ sheetFlushAdmissionWait.observe(elapsedSince(startedAt));
223
297
  observe({
224
298
  sheetFlushBytes: sheetInput?.estimatedBytes ?? null,
225
299
  });
226
- return noOpLease();
300
+ return lease;
227
301
  },
228
302
 
229
303
  suggestedToolParallelism(toolId, fallback) {
@@ -258,7 +332,10 @@ export function createRuntimeResourceGovernor(input: {
258
332
  },
259
333
  sheetFlush: {
260
334
  acquired: sheetFlushCounters.acquired,
335
+ active: sheetFlushAdmission.active,
336
+ waiting: sheetFlushAdmission.waiting,
261
337
  bytes: sheetFlushCounters.bytes,
338
+ admissionWaitMsEwma: sheetFlushAdmissionWait.value,
262
339
  latencyMsEwma: sheetFlushLatency.value,
263
340
  },
264
341
  providers: Object.fromEntries(