@opengeni/db 6.0.1 → 6.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/dist/browser-sessions.d.ts +26 -0
  2. package/dist/canonical-human-identities.js +3 -3
  3. package/dist/capability-reconciliation.js +2 -2
  4. package/dist/{chunk-4RDUZ5UV.js → chunk-4ZQFT5II.js} +2 -2
  5. package/dist/{chunk-6ORJPCYW.js → chunk-5UOWNKPP.js} +2 -2
  6. package/dist/{chunk-4CCFFLDZ.js → chunk-BG46J5HF.js} +13 -2
  7. package/dist/chunk-BG46J5HF.js.map +1 -0
  8. package/dist/{chunk-CSVHCF3R.js → chunk-IB3SW3JW.js} +11 -4
  9. package/dist/{chunk-CSVHCF3R.js.map → chunk-IB3SW3JW.js.map} +1 -1
  10. package/dist/{chunk-GSB4ISHE.js → chunk-IIPWNNVR.js} +4 -1
  11. package/dist/chunk-IIPWNNVR.js.map +1 -0
  12. package/dist/{chunk-MAMBI2HR.js → chunk-JPYN5XKV.js} +39 -3
  13. package/dist/chunk-JPYN5XKV.js.map +1 -0
  14. package/dist/{chunk-LSLNMENV.js → chunk-KZXWUXWV.js} +4 -4
  15. package/dist/{chunk-LO6BVXXH.js → chunk-LD52IFS3.js} +2 -2
  16. package/dist/{chunk-JILAWE26.js → chunk-M5Z6II2P.js} +2 -2
  17. package/dist/{chunk-PGIFHEBA.js → chunk-RVEOGRMF.js} +2 -2
  18. package/dist/{chunk-H7BC444C.js → chunk-SRAMNVHT.js} +17 -1
  19. package/dist/chunk-SRAMNVHT.js.map +1 -0
  20. package/dist/{chunk-XK6HXON2.js → chunk-UXFBXCHM.js} +2 -2
  21. package/dist/{chunk-RXH5CFG3.js → chunk-VE2XK2KD.js} +12 -8
  22. package/dist/{chunk-RXH5CFG3.js.map → chunk-VE2XK2KD.js.map} +1 -1
  23. package/dist/{chunk-FULLE5CF.js → chunk-WT7HG43Y.js} +3 -3
  24. package/dist/{chunk-I7HQL6FK.js → chunk-ZM5BA3TD.js} +3 -3
  25. package/dist/connect-attempts.js +3 -3
  26. package/dist/database.d.ts +8 -0
  27. package/dist/editable-artifact-durable-export.js +2 -2
  28. package/dist/editable-artifacts.js +3 -3
  29. package/dist/index.d.ts +69 -20
  30. package/dist/index.js +633 -202
  31. package/dist/index.js.map +1 -1
  32. package/dist/knowledge-entries.d.ts +2 -0
  33. package/dist/knowledge-indexing.d.ts +17 -0
  34. package/dist/managed-auth-session-sets.js +3 -3
  35. package/dist/managed-user-setup.d.ts +1 -0
  36. package/dist/mcp-operations.js +2 -2
  37. package/dist/migrate.d.ts +10 -1
  38. package/dist/migrate.js +3 -1
  39. package/dist/organization-integration-policy.js +3 -3
  40. package/dist/organization-membership-lifecycle.d.ts +1 -0
  41. package/dist/provision-roles.js +1 -1
  42. package/dist/retained-provider-commands.d.ts +4 -2
  43. package/dist/retained-provider-commands.js +6 -6
  44. package/dist/runtime-posture.d.ts +1 -1
  45. package/dist/schema.d.ts +25 -8
  46. package/dist/schema.js +1 -1
  47. package/dist/session-background-commands.js +4 -4
  48. package/dist/session-command-output.js +5 -5
  49. package/dist/session-event-slices.js +3 -3
  50. package/dist/session-mcp-credential-rotation.js +4 -4
  51. package/dist/session-tenancy.js +3 -3
  52. package/dist/video-generation.js +3 -3
  53. package/dist/workspace-tool-defaults.js +4 -4
  54. package/drizzle/0508_deadline_command_workspace_capture.sql +75 -0
  55. package/drizzle/0509_verified_signup_trial_credits.sql +76 -0
  56. package/drizzle/0510_knowledge_index_funding_wait.sql +174 -0
  57. package/drizzle/0511_knowledge_visible_index_status.sql +56 -0
  58. package/drizzle/0512_insights_fact_reads_hash_join.sql +31 -0
  59. package/drizzle/0513_message_fork_prefix_compaction.sql +85 -0
  60. package/drizzle/0514_failed_session_variable_set_attach.sql +25 -0
  61. package/package.json +5 -5
  62. package/src/browser-sessions.ts +71 -1
  63. package/src/database.ts +18 -0
  64. package/src/index.ts +706 -252
  65. package/src/insights.ts +44 -36
  66. package/src/knowledge-entries.ts +43 -2
  67. package/src/knowledge-indexing.ts +79 -1
  68. package/src/managed-user-setup.ts +18 -1
  69. package/src/migrate.ts +58 -1
  70. package/src/organization-membership-lifecycle.ts +11 -0
  71. package/src/provision-roles.ts +12 -0
  72. package/src/retained-provider-commands.ts +16 -6
  73. package/src/runtime-posture.ts +4 -0
  74. package/src/schema.ts +3 -0
  75. package/dist/chunk-4CCFFLDZ.js.map +0 -1
  76. package/dist/chunk-GSB4ISHE.js.map +0 -1
  77. package/dist/chunk-H7BC444C.js.map +0 -1
  78. package/dist/chunk-MAMBI2HR.js.map +0 -1
  79. /package/dist/{chunk-4RDUZ5UV.js.map → chunk-4ZQFT5II.js.map} +0 -0
  80. /package/dist/{chunk-6ORJPCYW.js.map → chunk-5UOWNKPP.js.map} +0 -0
  81. /package/dist/{chunk-LSLNMENV.js.map → chunk-KZXWUXWV.js.map} +0 -0
  82. /package/dist/{chunk-LO6BVXXH.js.map → chunk-LD52IFS3.js.map} +0 -0
  83. /package/dist/{chunk-JILAWE26.js.map → chunk-M5Z6II2P.js.map} +0 -0
  84. /package/dist/{chunk-PGIFHEBA.js.map → chunk-RVEOGRMF.js.map} +0 -0
  85. /package/dist/{chunk-XK6HXON2.js.map → chunk-UXFBXCHM.js.map} +0 -0
  86. /package/dist/{chunk-FULLE5CF.js.map → chunk-WT7HG43Y.js.map} +0 -0
  87. /package/dist/{chunk-I7HQL6FK.js.map → chunk-ZM5BA3TD.js.map} +0 -0
package/src/index.ts CHANGED
@@ -406,6 +406,7 @@ import {
406
406
  import {
407
407
  environmentsEncryptionKeyBytes,
408
408
  SANDBOX_LIFECYCLE_TRANSITION_MAX_WAIT_MS,
409
+ SANDBOX_DEADLINE_COMMAND_STOP_GRACE_MS,
409
410
  WORKSPACE_OPENROUTER_CONNECTION_DOMAIN,
410
411
  WORKSPACE_OPENROUTER_CONNECTION_ROLE,
411
412
  VERCEL_AI_GATEWAY_CONNECTION_DOMAIN,
@@ -480,9 +481,11 @@ import {
480
481
  type NewSessionDraftSnapshot,
481
482
  } from "./new-session-drafts";
482
483
  import {
484
+ isRetryableDatabaseTransportFailure,
483
485
  nestedPostgresSqlState,
484
486
  runIdempotentPersistenceTransaction,
485
487
  safeDatabaseErrorFacts,
488
+ SessionEventPersistenceError,
486
489
  type IdempotentPersistenceTransactionOptions,
487
490
  } from "./persistence-errors";
488
491
  import {
@@ -752,7 +755,7 @@ export {
752
755
  type UserLookup,
753
756
  type UserProfileLookup,
754
757
  } from "./database";
755
- export { withSessionRlsActorContext } from "./database";
758
+ export { currentSessionRlsActorIdentityKey, withSessionRlsActorContext } from "./database";
756
759
  export { withDatabaseTimingObserver, type DatabaseTimingObservation } from "./database-timing";
757
760
  export { normalizedCredentialHeaders } from "./connection-token-resolver";
758
761
  import {
@@ -5318,6 +5321,82 @@ export async function applyCreditDebitUpToBalance(
5318
5321
  );
5319
5322
  }
5320
5323
 
5324
+ /**
5325
+ * Settle an already-consumed, OpenGeni-funded resource for its full cost.
5326
+ * Admission happens before use; an in-flight operation can take the account
5327
+ * below zero. Future grants and purchased top-ups net against that balance.
5328
+ * Never use this helper to admit a new operation or for externally paid use.
5329
+ */
5330
+ export async function applyCreditDebitAfterUse(
5331
+ db: Database,
5332
+ input: {
5333
+ accountId: string;
5334
+ workspaceId?: string | null;
5335
+ type: string;
5336
+ amountMicros: number;
5337
+ sourceType: string;
5338
+ sourceId: string;
5339
+ idempotencyKey: string;
5340
+ metadata?: Record<string, unknown>;
5341
+ occurredAt?: Date;
5342
+ },
5343
+ ): Promise<{ balance: BillingBalance; debitedMicros: number }> {
5344
+ if (!Number.isSafeInteger(input.amountMicros) || input.amountMicros <= 0) {
5345
+ throw new Error("post-use credit debit requires a positive, safe integer micro amount");
5346
+ }
5347
+ return await withRlsContext(
5348
+ db,
5349
+ { accountId: input.accountId, workspaceId: input.workspaceId ?? null },
5350
+ async (scopedDb) => {
5351
+ const inserted = await scopedDb
5352
+ .insert(schema.creditLedgerEntries)
5353
+ .values({
5354
+ accountId: input.accountId,
5355
+ workspaceId: input.workspaceId ?? null,
5356
+ type: input.type,
5357
+ amountMicros: -input.amountMicros,
5358
+ sourceType: input.sourceType,
5359
+ sourceId: input.sourceId,
5360
+ idempotencyKey: input.idempotencyKey,
5361
+ metadata: input.metadata ?? {},
5362
+ occurredAt: input.occurredAt ?? new Date(),
5363
+ })
5364
+ .onConflictDoNothing({ target: schema.creditLedgerEntries.idempotencyKey })
5365
+ .returning({ id: schema.creditLedgerEntries.id });
5366
+ if (inserted.length === 0) {
5367
+ const [prior] = await scopedDb
5368
+ .select({
5369
+ accountId: schema.creditLedgerEntries.accountId,
5370
+ workspaceId: schema.creditLedgerEntries.workspaceId,
5371
+ type: schema.creditLedgerEntries.type,
5372
+ amountMicros: schema.creditLedgerEntries.amountMicros,
5373
+ sourceType: schema.creditLedgerEntries.sourceType,
5374
+ sourceId: schema.creditLedgerEntries.sourceId,
5375
+ })
5376
+ .from(schema.creditLedgerEntries)
5377
+ .where(eq(schema.creditLedgerEntries.idempotencyKey, input.idempotencyKey));
5378
+ if (
5379
+ !prior ||
5380
+ prior.accountId !== input.accountId ||
5381
+ prior.workspaceId !== (input.workspaceId ?? null) ||
5382
+ prior.type !== input.type ||
5383
+ prior.amountMicros !== -input.amountMicros ||
5384
+ prior.sourceType !== input.sourceType ||
5385
+ prior.sourceId !== input.sourceId
5386
+ ) {
5387
+ throw new Error(
5388
+ "post-use credit debit idempotency key conflicts with a different charge",
5389
+ );
5390
+ }
5391
+ }
5392
+ return {
5393
+ balance: await getBillingBalance(scopedDb, input.accountId),
5394
+ debitedMicros: inserted.length ? input.amountMicros : 0,
5395
+ };
5396
+ },
5397
+ );
5398
+ }
5399
+
5321
5400
  export async function hasCreditLedgerEntry(
5322
5401
  db: Database,
5323
5402
  accountId: string,
@@ -7330,6 +7409,39 @@ export async function getRetainedScreenshotArtifact(
7330
7409
  });
7331
7410
  }
7332
7411
 
7412
+ /** Recover a legacy damaged receipt only when its tool call has one exact
7413
+ * screenshot in this session. Never infer an artifact across fork ancestry. */
7414
+ export async function getRetainedScreenshotArtifactForToolCall(
7415
+ db: Database,
7416
+ workspaceId: string,
7417
+ sessionId: string,
7418
+ toolCallId: string,
7419
+ ): Promise<RetainedScreenshotArtifact | null> {
7420
+ return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
7421
+ const rows = await scopedDb
7422
+ .select({ artifact: schema.retainedScreenshotArtifacts, file: schema.files })
7423
+ .from(schema.retainedScreenshotArtifacts)
7424
+ .innerJoin(
7425
+ schema.files,
7426
+ and(
7427
+ eq(schema.files.workspaceId, schema.retainedScreenshotArtifacts.workspaceId),
7428
+ eq(schema.files.id, schema.retainedScreenshotArtifacts.artifactId),
7429
+ ),
7430
+ )
7431
+ .where(
7432
+ and(
7433
+ eq(schema.retainedScreenshotArtifacts.workspaceId, workspaceId),
7434
+ eq(schema.retainedScreenshotArtifacts.sessionId, sessionId),
7435
+ eq(schema.retainedScreenshotArtifacts.toolCallId, toolCallId),
7436
+ ),
7437
+ )
7438
+ .limit(2);
7439
+ return rows.length === 1
7440
+ ? mapRetainedScreenshotArtifact(rows[0]!.artifact, rows[0]!.file)
7441
+ : null;
7442
+ });
7443
+ }
7444
+
7333
7445
  export type RetainedScreenshotMaintenanceClaim = {
7334
7446
  action: "reconcile" | "delete";
7335
7447
  claimId: string;
@@ -41354,101 +41466,183 @@ function assertPendingToolOutputPolicyMatches(
41354
41466
  }
41355
41467
  }
41356
41468
 
41469
+ // Receipt inputs have already passed the strict JSON storage boundary. Compare
41470
+ // exact keys structurally: locale collation can equate distinct Unicode keys.
41471
+ function pendingToolCallItemsEqual(left: unknown, right: unknown): boolean {
41472
+ if (left === right) return true;
41473
+ if (left === null || right === null || typeof left !== "object" || typeof right !== "object")
41474
+ return false;
41475
+ if (Array.isArray(left) || Array.isArray(right)) {
41476
+ return (
41477
+ Array.isArray(left) &&
41478
+ Array.isArray(right) &&
41479
+ left.length === right.length &&
41480
+ left.every((value, index) => pendingToolCallItemsEqual(value, right[index]))
41481
+ );
41482
+ }
41483
+ const keys = Object.keys(left);
41484
+ return (
41485
+ keys.length === Object.keys(right).length &&
41486
+ keys.every(
41487
+ (key) =>
41488
+ Object.hasOwn(right, key) &&
41489
+ pendingToolCallItemsEqual(
41490
+ (left as Record<string, unknown>)[key],
41491
+ (right as Record<string, unknown>)[key],
41492
+ ),
41493
+ )
41494
+ );
41495
+ }
41496
+
41357
41497
  /**
41358
41498
  * Durably capture the raw SDK call item at the exact attempt boundary. This is
41359
41499
  * model-facing truth, deliberately separate from the session-event timeline.
41360
41500
  * The receipt belongs to the logical turn so an approval resume can
41361
41501
  * settle it from a newer attempt. Duplicate SDK delivery converges on the
41362
41502
  * unique (turn, call) identity.
41503
+ * Pass the root database handle so each rollback retry starts a fresh
41504
+ * transaction, not a savepoint in a caller-owned transaction.
41363
41505
  */
41364
41506
  export async function registerPendingSessionToolCall(
41365
41507
  db: Database,
41366
41508
  input: PendingSessionToolCallInput,
41367
41509
  ): Promise<{ accepted: boolean; registered: boolean }> {
41368
- return await withRlsContext(
41369
- db,
41370
- { accountId: input.accountId, workspaceId: input.workspaceId },
41371
- async (scopedDb) =>
41372
- await scopedDb.transaction(async (tx) => {
41373
- const fence = await lockTurnAttemptWriteFenceTx(tx, {
41374
- workspaceId: input.workspaceId,
41375
- sessionId: input.sessionId,
41376
- turnId: input.turnId,
41377
- executionGeneration: input.executionGeneration,
41378
- attemptId: input.attemptId,
41379
- });
41380
- if (!fence.allowed) return { accepted: false, registered: false };
41381
- const inserted = await tx
41382
- .insert(schema.sessionPendingToolCalls)
41383
- .values(
41384
- withLosslessContentWriteVersion(
41385
- {
41386
- accountId: input.accountId,
41387
- workspaceId: input.workspaceId,
41388
- sessionId: input.sessionId,
41389
- turnId: input.turnId,
41390
- executionGeneration: input.executionGeneration,
41391
- attemptId: input.attemptId,
41392
- callId: input.callId,
41393
- callType: input.callType,
41394
- callItem: input.callItem,
41395
- modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens ?? null,
41396
- },
41397
- "callItem",
41398
- "callItemCodecVersion",
41399
- ),
41400
- )
41401
- .onConflictDoNothing({
41402
- target: [
41403
- schema.sessionPendingToolCalls.workspaceId,
41404
- schema.sessionPendingToolCalls.turnId,
41405
- schema.sessionPendingToolCalls.callId,
41406
- ],
41407
- })
41408
- .returning({ id: schema.sessionPendingToolCalls.id });
41409
- if (inserted.length === 0) {
41410
- const [pending] = await tx
41411
- .select({
41412
- id: schema.sessionPendingToolCalls.id,
41413
- modelToolOutputTruncationTokens:
41414
- schema.sessionPendingToolCalls.modelToolOutputTruncationTokens,
41415
- })
41416
- .from(schema.sessionPendingToolCalls)
41417
- .where(
41418
- and(
41419
- eq(schema.sessionPendingToolCalls.workspaceId, input.workspaceId),
41420
- eq(schema.sessionPendingToolCalls.sessionId, input.sessionId),
41421
- eq(schema.sessionPendingToolCalls.turnId, input.turnId),
41422
- eq(schema.sessionPendingToolCalls.callId, input.callId),
41423
- ),
41424
- )
41425
- .for("update")
41426
- .limit(1);
41427
- if (!pending) {
41428
- throw new Error(
41429
- `Pending SDK tool call disappeared during registration: ${input.callId}`,
41430
- );
41431
- }
41432
- assertPendingToolOutputPolicyMatches(
41433
- pending.modelToolOutputTruncationTokens,
41434
- input.modelToolOutputTruncationTokens,
41435
- input.callId,
41436
- );
41437
- if (
41438
- pending.modelToolOutputTruncationTokens === null &&
41439
- input.modelToolOutputTruncationTokens !== undefined
41440
- ) {
41441
- await tx
41442
- .update(schema.sessionPendingToolCalls)
41443
- .set({
41444
- modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens,
41510
+ const correlationId = crypto.randomUUID();
41511
+ const registerOnce = async (attempt: number) => {
41512
+ try {
41513
+ return await withRlsContext(
41514
+ db,
41515
+ { accountId: input.accountId, workspaceId: input.workspaceId },
41516
+ async (scopedDb) =>
41517
+ await scopedDb.transaction(async (tx) => {
41518
+ const fence = await lockTurnAttemptWriteFenceTx(tx, {
41519
+ workspaceId: input.workspaceId,
41520
+ sessionId: input.sessionId,
41521
+ turnId: input.turnId,
41522
+ executionGeneration: input.executionGeneration,
41523
+ attemptId: input.attemptId,
41524
+ });
41525
+ if (!fence.allowed) return { accepted: false, registered: false };
41526
+ const inserted = await tx
41527
+ .insert(schema.sessionPendingToolCalls)
41528
+ .values(
41529
+ withLosslessContentWriteVersion(
41530
+ {
41531
+ accountId: input.accountId,
41532
+ workspaceId: input.workspaceId,
41533
+ sessionId: input.sessionId,
41534
+ turnId: input.turnId,
41535
+ executionGeneration: input.executionGeneration,
41536
+ attemptId: input.attemptId,
41537
+ callId: input.callId,
41538
+ callType: input.callType,
41539
+ callItem: input.callItem,
41540
+ modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens ?? null,
41541
+ },
41542
+ "callItem",
41543
+ "callItemCodecVersion",
41544
+ ),
41545
+ )
41546
+ .onConflictDoNothing({
41547
+ target: [
41548
+ schema.sessionPendingToolCalls.workspaceId,
41549
+ schema.sessionPendingToolCalls.turnId,
41550
+ schema.sessionPendingToolCalls.callId,
41551
+ ],
41445
41552
  })
41446
- .where(eq(schema.sessionPendingToolCalls.id, pending.id));
41447
- }
41448
- }
41449
- return { accepted: true, registered: inserted.length === 1 };
41450
- }),
41553
+ .returning({ id: schema.sessionPendingToolCalls.id });
41554
+ if (inserted.length === 0) {
41555
+ const [pending] = await tx
41556
+ .select({
41557
+ id: schema.sessionPendingToolCalls.id,
41558
+ accountId: schema.sessionPendingToolCalls.accountId,
41559
+ callType: schema.sessionPendingToolCalls.callType,
41560
+ callItem: schema.sessionPendingToolCalls.callItem,
41561
+ callItemCodecVersion: schema.sessionPendingToolCalls.callItemCodecVersion,
41562
+ modelToolOutputTruncationTokens:
41563
+ schema.sessionPendingToolCalls.modelToolOutputTruncationTokens,
41564
+ })
41565
+ .from(schema.sessionPendingToolCalls)
41566
+ .where(
41567
+ and(
41568
+ eq(schema.sessionPendingToolCalls.workspaceId, input.workspaceId),
41569
+ eq(schema.sessionPendingToolCalls.sessionId, input.sessionId),
41570
+ eq(schema.sessionPendingToolCalls.turnId, input.turnId),
41571
+ eq(schema.sessionPendingToolCalls.callId, input.callId),
41572
+ ),
41573
+ )
41574
+ .for("update")
41575
+ .limit(1);
41576
+ if (
41577
+ !pending ||
41578
+ pending.accountId !== input.accountId ||
41579
+ pending.callType !== input.callType ||
41580
+ !pendingToolCallItemsEqual(
41581
+ fromPostgresLosslessJson(pending.callItem, pending.callItemCodecVersion),
41582
+ input.callItem,
41583
+ )
41584
+ ) {
41585
+ // A duplicate acknowledges only this exact call. It neither
41586
+ // replaces the originating attempt nor authorizes effect replay.
41587
+ throw new Error("Pending tool receipt conflicts with the registered call");
41588
+ }
41589
+ assertPendingToolOutputPolicyMatches(
41590
+ pending.modelToolOutputTruncationTokens,
41591
+ input.modelToolOutputTruncationTokens,
41592
+ input.callId,
41593
+ );
41594
+ if (
41595
+ pending.modelToolOutputTruncationTokens === null &&
41596
+ input.modelToolOutputTruncationTokens !== undefined
41597
+ ) {
41598
+ await tx
41599
+ .update(schema.sessionPendingToolCalls)
41600
+ .set({
41601
+ modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens,
41602
+ })
41603
+ .where(eq(schema.sessionPendingToolCalls.id, pending.id));
41604
+ }
41605
+ }
41606
+ return { accepted: true, registered: inserted.length === 1 };
41607
+ }),
41608
+ );
41609
+ } catch (error) {
41610
+ if (!isRetryableDatabaseTransportFailure(error)) throw error;
41611
+ // Stop before the shared SQLSTATE traversal can prefer an older nested
41612
+ // rollback over a transport/commit-ack failure. This private outcome is
41613
+ // converted to a sanitized error outside the retry helper, never success.
41614
+ return { transportFailure: error, attempt };
41615
+ }
41616
+ };
41617
+ // Only PostgreSQL-confirmed rollback is retryable. Re-enter RLS and acquire
41618
+ // the complete attempt fence each time; never include inference or tools.
41619
+ const result = await runIdempotentPersistenceTransaction(
41620
+ {
41621
+ stage: "pending_tool_registration",
41622
+ correlationId,
41623
+ maxAttempts: 3,
41624
+ onRetry: async ({ attempt }) => {
41625
+ await new Promise((resolve) => setTimeout(resolve, 25 * attempt));
41626
+ },
41627
+ },
41628
+ registerOnce,
41451
41629
  );
41630
+ if ("transportFailure" in result) {
41631
+ throw new SessionEventPersistenceError(
41632
+ {
41633
+ code: "db_failure",
41634
+ sqlState: null,
41635
+ stage: "pending_tool_registration",
41636
+ eventTypes: [],
41637
+ correlationId,
41638
+ attempts: result.attempt,
41639
+ retryOutcome: "not_retryable",
41640
+ database: safeDatabaseErrorFacts(result.transportFailure),
41641
+ },
41642
+ result.transportFailure,
41643
+ );
41644
+ }
41645
+ return result;
41452
41646
  }
41453
41647
 
41454
41648
  /** Record the raw SDK result without dropping the call receipt. */
@@ -44033,6 +44227,9 @@ export interface AcquireLeaseInput {
44033
44227
  * attach, mirrored into the scoped stream token's claims. */
44034
44228
  viewerAuthorityEpoch?: number;
44035
44229
  backend: string; // sessions.sandbox_backend
44230
+ /** Explicit compute policy at this admission boundary. Omission is legacy
44231
+ * usage-only; only a priced credits backend needs prepaid admission. */
44232
+ warmBilling?: { mode: "usage_only" | "shadow" | "credits"; rateMicrosPerSecond: number };
44036
44233
  os?: string; // default 'linux'
44037
44234
  // The container image this run resolves (Modal image ref / docker image). Stamped on
44038
44235
  // the cold-create + folded onto a warming/CAS; a warm/draining/warming box already
@@ -44454,6 +44651,94 @@ export class SandboxViewerAdmissionBlockedError extends Error {
44454
44651
  }
44455
44652
  }
44456
44653
 
44654
+ export class SandboxPaidComputeAdmissionError extends Error {
44655
+ readonly name = "SandboxPaidComputeAdmissionError";
44656
+ constructor(
44657
+ public readonly workspaceId: string,
44658
+ public readonly sandboxGroupId: string,
44659
+ ) {
44660
+ super("Insufficient OpenGeni credits to admit paid sandbox compute");
44661
+ }
44662
+ }
44663
+
44664
+ type SandboxWarmBillingSnapshot = {
44665
+ mode: "usage_only" | "shadow" | "credits";
44666
+ rateMicrosPerSecond: number;
44667
+ accountId: string;
44668
+ workspaceId: string;
44669
+ stopChargeAt?: string;
44670
+ };
44671
+
44672
+ function warmBillingSnapshot(
44673
+ row: Pick<LeaseRow, "resume_state" | "account_id" | "workspace_id">,
44674
+ ): SandboxWarmBillingSnapshot | null {
44675
+ const value = row.resume_state?.opengeniWarmBilling;
44676
+ if (!value || typeof value !== "object") return null;
44677
+ const snapshot = value as SandboxWarmBillingSnapshot;
44678
+ return (snapshot.mode === "usage_only" ||
44679
+ snapshot.mode === "shadow" ||
44680
+ snapshot.mode === "credits") &&
44681
+ Number.isFinite(snapshot.rateMicrosPerSecond) &&
44682
+ snapshot.rateMicrosPerSecond >= 0 &&
44683
+ (snapshot.stopChargeAt === undefined ||
44684
+ (typeof snapshot.stopChargeAt === "string" &&
44685
+ Number.isFinite(Date.parse(snapshot.stopChargeAt)))) &&
44686
+ snapshot.accountId === row.account_id &&
44687
+ snapshot.workspaceId === row.workspace_id
44688
+ ? snapshot
44689
+ : null;
44690
+ }
44691
+
44692
+ /**
44693
+ * Fence the warm-time meter BEFORE requesting provider stop, including free
44694
+ * and shadow usage. The immutable cutoff survives a stop-success/settlement-
44695
+ * failure retry. A failed stop can undercount an idle retry, never count time
44696
+ * after the provider stopped.
44697
+ */
44698
+ export async function markWarmBillingStopCutoff(
44699
+ db: Database,
44700
+ input: { accountId: string; workspaceId: string; sandboxGroupId: string; expectedEpoch: number },
44701
+ ): Promise<Date | null> {
44702
+ return withRlsContext(
44703
+ db,
44704
+ { accountId: input.accountId, workspaceId: input.workspaceId },
44705
+ async (scopedDb) =>
44706
+ scopedDb.transaction(async (txRaw) => {
44707
+ const tx = txRaw as unknown as Database;
44708
+ const [row] = await tx.execute<LeaseRow>(sql`
44709
+ select * from sandbox_leases where workspace_id = ${input.workspaceId}
44710
+ and sandbox_group_id = ${input.sandboxGroupId} for update
44711
+ `);
44712
+ if (
44713
+ !row ||
44714
+ Number(row.lease_epoch) !== input.expectedEpoch ||
44715
+ row.liveness !== "draining" ||
44716
+ (Number(row.refcount) !== 0 && !row.unobservable_command_drain_ids?.length)
44717
+ )
44718
+ return null;
44719
+ const snapshot = warmBillingSnapshot(row);
44720
+ if (row.resume_state?.opengeniWarmBilling && !snapshot)
44721
+ throw new Error("sandbox warm billing snapshot is invalid");
44722
+ if (snapshot?.stopChargeAt) return new Date(snapshot.stopChargeAt);
44723
+ const [stamped] = await tx.execute<{ cutoff: string }>(sql`
44724
+ update sandbox_leases set
44725
+ resume_state = jsonb_set(coalesce(resume_state, '{}'::jsonb),
44726
+ '{opengeniWarmBilling}',
44727
+ coalesce(resume_state->'opengeniWarmBilling',
44728
+ jsonb_build_object('mode', 'usage_only', 'rateMicrosPerSecond', 0,
44729
+ 'accountId', account_id, 'workspaceId', workspace_id)) ||
44730
+ jsonb_build_object('stopChargeAt', clock_timestamp()), true),
44731
+ updated_at = now()
44732
+ where id = ${row.id}
44733
+ returning resume_state #>> '{opengeniWarmBilling,stopChargeAt}' as cutoff
44734
+ `);
44735
+ if (!stamped || !Number.isFinite(Date.parse(stamped.cutoff)))
44736
+ throw new Error("sandbox warm billing stop cutoff was not saved");
44737
+ return new Date(stamped.cutoff);
44738
+ }),
44739
+ );
44740
+ }
44741
+
44457
44742
  export class SandboxLeaseRecoveryBlockedError extends Error {
44458
44743
  readonly name = "SandboxLeaseRecoveryBlockedError";
44459
44744
 
@@ -45207,6 +45492,10 @@ async function acquireLeaseOnce(
45207
45492
  const os = input.os ?? "linux";
45208
45493
  const subjectId = input.subjectId ?? null;
45209
45494
  const warmingLeaseTtlMs = input.warmingLeaseTtlMs ?? input.leaseTtlMs;
45495
+ const warmPolicy = input.warmBilling ?? { mode: "usage_only" as const, rateMicrosPerSecond: 0 };
45496
+ if (!Number.isFinite(warmPolicy.rateMicrosPerSecond) || warmPolicy.rateMicrosPerSecond < 0) {
45497
+ throw new Error("sandbox warm rate must be finite and non-negative");
45498
+ }
45210
45499
  return await withRlsContext(
45211
45500
  db,
45212
45501
  { accountId, workspaceId },
@@ -45254,6 +45543,41 @@ async function acquireLeaseOnce(
45254
45543
  if (!row) throw new Error(`Lease row vanished post-insert: ${sandboxGroupId}`);
45255
45544
 
45256
45545
  const liveness = row.liveness;
45546
+ const existingSnapshot = warmBillingSnapshot(row);
45547
+ if (row.resume_state?.opengeniWarmBilling && !existingSnapshot) {
45548
+ throw new Error("sandbox warm billing snapshot is invalid");
45549
+ }
45550
+
45551
+ // Every new paid holder, including direct operations and interaction
45552
+ // controllers, passes the same account funding boundary before it can
45553
+ // elect a spawner, re-arm a drain or prolong a warm lease. Replaying an
45554
+ // existing exact holder does not create new compute authority.
45555
+ const activeMode = input.warmBilling?.mode ?? existingSnapshot?.mode ?? "usage_only";
45556
+ // Reconfiguration may not substitute a new (possibly zero) rate for
45557
+ // an already-running paid provider's immutable admitted tariff.
45558
+ const activeRate =
45559
+ liveness !== "cold" && existingSnapshot?.mode === "credits"
45560
+ ? existingSnapshot.rateMicrosPerSecond
45561
+ : (input.warmBilling?.rateMicrosPerSecond ??
45562
+ existingSnapshot?.rateMicrosPerSecond ??
45563
+ 0);
45564
+ if (
45565
+ activeMode === "credits" &&
45566
+ activeRate > 0 &&
45567
+ (liveness === "cold" ||
45568
+ (existingSnapshot?.mode === "credits" && existingSnapshot.rateMicrosPerSecond > 0))
45569
+ ) {
45570
+ const alreadyHeld = await tx.execute<{ held: boolean }>(sql`
45571
+ select exists (select 1 from sandbox_lease_holders
45572
+ where lease_id = ${row.id} and kind = ${kind} and holder_id = ${holderId}) as held
45573
+ `);
45574
+ if (!alreadyHeld[0]?.held) {
45575
+ const balance = await getBillingBalance(tx, accountId);
45576
+ if (balance.balanceMicros <= 0) {
45577
+ throw new SandboxPaidComputeAdmissionError(workspaceId, sandboxGroupId);
45578
+ }
45579
+ }
45580
+ }
45257
45581
 
45258
45582
  // A workspace-wide billing/cap drain survives this lease becoming cold.
45259
45583
  // Viewer-only admission must consult it while holding the exact lease
@@ -45270,7 +45594,7 @@ async function acquireLeaseOnce(
45270
45594
  where id = ${workspaceId}
45271
45595
  `);
45272
45596
  const reason = workspaceRows[0]?.sandbox_viewer_force_drain_reason ?? null;
45273
- if (reason !== null) {
45597
+ if (reason === "warm_cap") {
45274
45598
  throw new SandboxViewerAdmissionBlockedError(workspaceId, sandboxGroupId, reason);
45275
45599
  }
45276
45600
  }
@@ -45387,6 +45711,17 @@ async function acquireLeaseOnce(
45387
45711
  // The row lock serializes this holder insertion against the reaper claim:
45388
45712
  // whichever wins first owns the next state without an availability gap.
45389
45713
  if (liveness === "draining") {
45714
+ // Capture may have failed after an earlier stop attempt recorded a
45715
+ // cutoff. This provider was never stopped; a re-armed box must have
45716
+ // a fresh charge horizon when its next drain reaches provider stop.
45717
+ if (warmBillingSnapshot(row)?.stopChargeAt) {
45718
+ await tx.execute(sql`
45719
+ update sandbox_leases
45720
+ set resume_state = resume_state #- '{opengeniWarmBilling,stopChargeAt}',
45721
+ updated_at = now()
45722
+ where id = ${row.id}
45723
+ `);
45724
+ }
45390
45725
  await upsertLeaseHolder(tx, row.id, accountId, workspaceId, kind, holderId, subjectId, {
45391
45726
  subjectId: input.viewerSubjectId ?? null,
45392
45727
  authorityEpoch: input.viewerAuthorityEpoch ?? null,
@@ -45418,6 +45753,17 @@ async function acquireLeaseOnce(
45418
45753
  const casRows = await tx.execute<{ id: string }>(sql`
45419
45754
  update sandbox_leases set
45420
45755
  liveness = 'warming',
45756
+ ${
45757
+ input.warmBilling
45758
+ ? sql`resume_state = coalesce(resume_state, '{}'::jsonb) ||
45759
+ jsonb_build_object('opengeniWarmBilling', ${JSON.stringify({
45760
+ mode: warmPolicy.mode,
45761
+ rateMicrosPerSecond: warmPolicy.rateMicrosPerSecond,
45762
+ accountId,
45763
+ workspaceId,
45764
+ })}::jsonb),`
45765
+ : sql``
45766
+ }
45421
45767
  ${image !== null ? sql`image = ${image},` : sql``}
45422
45768
  ${rigVersionId !== null ? sql`rig_version_id = ${rigVersionId},` : sql``}
45423
45769
  updated_at = now()
@@ -46921,7 +47267,13 @@ export async function commitWarmingToWarm(
46921
47267
  };
46922
47268
  }
46923
47269
 
46924
- const completedAt = new Date().toISOString();
47270
+ // Warm-time billing and stop cutoffs share the database clock, even
47271
+ // when the worker host's wall clock differs from PostgreSQL's.
47272
+ const [warmClock] = await tx.execute<{ completed_at: Date | string }>(sql`
47273
+ select clock_timestamp() as completed_at
47274
+ `);
47275
+ if (!warmClock) throw new Error("sandbox warm-transition clock unavailable");
47276
+ const completedAt = new Date(warmClock.completed_at).toISOString();
46925
47277
  const recovery: SandboxRecoveryState = rematerialization
46926
47278
  ? {
46927
47279
  provider: {
@@ -46967,7 +47319,17 @@ export async function commitWarmingToWarm(
46967
47319
  input.resumeState ?? null,
46968
47320
  row.resume_state,
46969
47321
  );
46970
- const resumeStateJson = JSON.stringify(resumeStateWithRecovery(withArchives, recovery));
47322
+ const resumeStateJson = JSON.stringify(
47323
+ resumeStateWithRecovery(
47324
+ {
47325
+ ...(withArchives ?? {}),
47326
+ ...(row.resume_state?.opengeniWarmBilling
47327
+ ? { opengeniWarmBilling: row.resume_state.opengeniWarmBilling }
47328
+ : {}),
47329
+ },
47330
+ recovery,
47331
+ ),
47332
+ );
46971
47333
  const updated = await tx.execute<LeaseRow>(sql`
46972
47334
  update sandbox_leases set
46973
47335
  liveness = 'warm',
@@ -47094,7 +47456,17 @@ export async function recordWarmingSandboxCreated(
47094
47456
  input.resumeState ?? null,
47095
47457
  row.resume_state,
47096
47458
  );
47097
- const resumeStateJson = JSON.stringify(resumeStateWithRecovery(withArchives, recovery));
47459
+ const resumeStateJson = JSON.stringify(
47460
+ resumeStateWithRecovery(
47461
+ {
47462
+ ...(withArchives ?? {}),
47463
+ ...(row.resume_state?.opengeniWarmBilling
47464
+ ? { opengeniWarmBilling: row.resume_state.opengeniWarmBilling }
47465
+ : {}),
47466
+ },
47467
+ recovery,
47468
+ ),
47469
+ );
47098
47470
  const warmingTtlMs = input.warmingLeaseTtlMs ?? input.leaseTtlMs;
47099
47471
  const providerCreatedAt = input.providerCreatedAt ?? null;
47100
47472
  const providerDeadlineAt = input.providerDeadlineAt ?? null;
@@ -49559,7 +49931,7 @@ export type LeaseHolderHeartbeatStatus =
49559
49931
  | {
49560
49932
  holderAlive: true;
49561
49933
  leaseExtended: false;
49562
- fence: "epoch" | "liveness" | "rotation_requested";
49934
+ fence: "epoch" | "liveness" | "rotation_requested" | "funding";
49563
49935
  };
49564
49936
 
49565
49937
  export async function heartbeatLeaseHolderStatus(
@@ -49572,6 +49944,7 @@ export async function heartbeatLeaseHolderStatus(
49572
49944
  holderId: string;
49573
49945
  leaseTtlMs: number;
49574
49946
  expectedEpoch: number;
49947
+ billingMode?: "usage_only" | "shadow" | "credits";
49575
49948
  },
49576
49949
  ): Promise<LeaseHolderHeartbeatStatus> {
49577
49950
  if (input.kind === "process") {
@@ -49592,11 +49965,14 @@ export async function heartbeatLeaseHolderStatus(
49592
49965
  // against stale-holder deletion.
49593
49966
  const leases = await tx.execute<{
49594
49967
  id: string;
49968
+ account_id: string;
49969
+ workspace_id: string;
49970
+ resume_state: Record<string, unknown> | null;
49595
49971
  lease_epoch: number | string;
49596
49972
  liveness: SandboxLeaseLiveness;
49597
49973
  rotation_requested_at: Date | string | null;
49598
49974
  }>(sql`
49599
- select id, lease_epoch, liveness, rotation_requested_at from sandbox_leases
49975
+ select id, account_id, workspace_id, resume_state, lease_epoch, liveness, rotation_requested_at from sandbox_leases
49600
49976
  where workspace_id = ${input.workspaceId}
49601
49977
  and sandbox_group_id = ${input.sandboxGroupId}
49602
49978
  for update
@@ -49629,6 +50005,28 @@ export async function heartbeatLeaseHolderStatus(
49629
50005
  fence: "holder_gone",
49630
50006
  };
49631
50007
  }
50008
+ const snapshot = warmBillingSnapshot(lease);
50009
+ if (lease.resume_state?.opengeniWarmBilling && !snapshot) {
50010
+ throw new Error("sandbox warm billing snapshot is invalid");
50011
+ }
50012
+ // An exhausted balance must not mask an older epoch or a drain/rotation
50013
+ // that requires the client to detach and negotiate a new viewer.
50014
+ if (Number(lease.lease_epoch) !== input.expectedEpoch)
50015
+ return { holderAlive: true, leaseExtended: false, fence: "epoch" };
50016
+ if (lease.liveness !== "warm" && lease.liveness !== "warming")
50017
+ return { holderAlive: true, leaseExtended: false, fence: "liveness" };
50018
+ if (lease.rotation_requested_at !== null)
50019
+ return { holderAlive: true, leaseExtended: false, fence: "rotation_requested" };
50020
+ if (
50021
+ (input.billingMode ?? snapshot?.mode) === "credits" &&
50022
+ snapshot?.mode === "credits" &&
50023
+ snapshot.rateMicrosPerSecond > 0 &&
50024
+ (await getBillingBalance(tx, input.accountId)).balanceMicros <= 0
50025
+ ) {
50026
+ // Touching the holder above is essential: the running writer remains
50027
+ // protected while it winds down. The provider/lease TTL is not renewed.
50028
+ return { holderAlive: true, leaseExtended: false, fence: "funding" };
50029
+ }
49632
50030
  // Recheck the same fence in the write even though its row lock is held,
49633
50031
  // keeping the mutation independently auditable and future-proof.
49634
50032
  const leaseRows = await tx.execute<{ id: string }>(sql`
@@ -49671,6 +50069,7 @@ export async function heartbeatLeaseHolder(
49671
50069
  holderId: string;
49672
50070
  leaseTtlMs: number;
49673
50071
  expectedEpoch: number;
50072
+ billingMode?: "usage_only" | "shadow" | "credits";
49674
50073
  },
49675
50074
  ): Promise<boolean> {
49676
50075
  return (await heartbeatLeaseHolderStatus(db, input)).leaseExtended;
@@ -50300,6 +50699,13 @@ export async function enrollUnobservableCommandIdleDrain(
50300
50699
  'quarantined_provider_binding_mismatch')
50301
50700
  `);
50302
50701
  }
50702
+ // A deadline gives a legacy command two minutes after cancellation intent
50703
+ // to exit. Even when the provider cannot signal or observe that command,
50704
+ // capture its current files while the old box still exists. Ordinary idle
50705
+ // containment retains its longer grace.
50706
+ const deadlineStopGraceMs = SANDBOX_DEADLINE_COMMAND_STOP_GRACE_MS;
50707
+ const deadlineRotation =
50708
+ initial.rotationReason === "provider_deadline" && initial.rotationRequestedAt !== null;
50303
50709
  // Preserve process -> admission -> lease ordering used by settlement.
50304
50710
  const processes = await rawRows<{
50305
50711
  id: string;
@@ -50333,10 +50739,43 @@ export async function enrollUnobservableCommandIdleDrain(
50333
50739
  (${input.idleGraceMs}::bigint * interval '1 millisecond')
50334
50740
  )
50335
50741
  )
50742
+ or (
50743
+ ${deadlineRotation}
50744
+ and process.deadline_cancellation_requested_at < now() -
50745
+ (${deadlineStopGraceMs}::bigint * interval '1 millisecond')
50746
+ and process.reconcile_attempts >= 1
50747
+ and (
50748
+ (process.owner_actor_kind = 'turn' and attempt.state = 'closed'
50749
+ and attempt.quiesced_at is not null
50750
+ and greatest(attempt.quiesced_at, process.started_at) < now() -
50751
+ (${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
50752
+ or (process.owner_actor_kind = 'direct' and process.owner_attempt_id is null
50753
+ and process.started_at < now() -
50754
+ (${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
50755
+ )
50756
+ )
50336
50757
  )
50337
- and attempt.state = 'closed'
50338
- and greatest(coalesce(attempt.quiesced_at, attempt.closed_at, attempt.updated_at), process.started_at) < now() -
50339
- (${input.idleGraceMs}::bigint * interval '1 millisecond')) as eligible
50758
+ and (attempt.state = 'closed' or (
50759
+ ${deadlineRotation}
50760
+ and process.owner_actor_kind = 'direct' and process.owner_attempt_id is null))
50761
+ and (
50762
+ (attempt.state = 'closed' and
50763
+ greatest(coalesce(attempt.quiesced_at, attempt.closed_at, attempt.updated_at), process.started_at) < now() -
50764
+ (${input.idleGraceMs}::bigint * interval '1 millisecond'))
50765
+ or (
50766
+ ${deadlineRotation}
50767
+ and process.deadline_cancellation_requested_at < now() -
50768
+ (${deadlineStopGraceMs}::bigint * interval '1 millisecond')
50769
+ and (
50770
+ (process.owner_actor_kind = 'turn' and attempt.quiesced_at is not null
50771
+ and greatest(attempt.quiesced_at, process.started_at) < now() -
50772
+ (${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
50773
+ or (process.owner_actor_kind = 'direct' and process.owner_attempt_id is null
50774
+ and process.started_at < now() -
50775
+ (${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
50776
+ )
50777
+ )
50778
+ )) as eligible
50340
50779
  from sandbox_retained_processes process
50341
50780
  left join session_turn_attempts attempt on attempt.id = process.owner_attempt_id
50342
50781
  and attempt.workspace_id = process.workspace_id and attempt.session_id = process.session_id
@@ -50387,7 +50826,13 @@ export async function enrollUnobservableCommandIdleDrain(
50387
50826
  )
50388
50827
  return null;
50389
50828
  if (!enrolled) {
50390
- if (lease.archive_capture_id !== null || (await hasSandboxGroupAttemptActivityTx(tx, input)))
50829
+ if (
50830
+ lease.archive_capture_id !== null ||
50831
+ (await hasSandboxGroupAttemptActivityTx(tx, {
50832
+ ...input,
50833
+ idleGraceMs: deadlineRotation ? deadlineStopGraceMs : input.idleGraceMs,
50834
+ }))
50835
+ )
50391
50836
  return null;
50392
50837
  await tx.execute(sql`
50393
50838
  update sandbox_leases set unobservable_command_drain_ids = ${`{${ids.join(",")}}`}::uuid[],
@@ -50699,15 +51144,40 @@ export async function reArmDrainingLease(
50699
51144
  workspaceId: string;
50700
51145
  sandboxGroupId: string;
50701
51146
  leaseTtlMs: number;
51147
+ warmBilling?: AcquireLeaseInput["warmBilling"];
50702
51148
  },
50703
51149
  ): Promise<{ rearmed: boolean }> {
50704
51150
  return await withRlsContext(
50705
51151
  db,
50706
51152
  { accountId: input.accountId, workspaceId: input.workspaceId },
50707
- async (scopedDb) => {
50708
- const rows = await scopedDb.execute<{ id: string }>(sql`
51153
+ async (scopedDb) =>
51154
+ await scopedDb.transaction(async (txRaw) => {
51155
+ const tx = txRaw as unknown as Database;
51156
+ const [lease] = await tx.execute<LeaseRow>(sql`
51157
+ select * from sandbox_leases
51158
+ where workspace_id = ${input.workspaceId} and sandbox_group_id = ${input.sandboxGroupId}
51159
+ for update
51160
+ `);
51161
+ if (!lease || lease.liveness !== "draining") return { rearmed: false };
51162
+ const snapshot = warmBillingSnapshot(lease);
51163
+ if (lease.resume_state?.opengeniWarmBilling && !snapshot) {
51164
+ throw new Error("sandbox warm billing snapshot is invalid");
51165
+ }
51166
+ const activeMode = input.warmBilling?.mode ?? snapshot?.mode ?? "usage_only";
51167
+ if (
51168
+ activeMode === "credits" &&
51169
+ (snapshot
51170
+ ? snapshot.mode === "credits" && snapshot.rateMicrosPerSecond > 0
51171
+ : input.warmBilling?.rateMicrosPerSecond !== undefined &&
51172
+ input.warmBilling.rateMicrosPerSecond > 0) &&
51173
+ (await getBillingBalance(tx, input.accountId)).balanceMicros <= 0
51174
+ ) {
51175
+ throw new SandboxPaidComputeAdmissionError(input.workspaceId, input.sandboxGroupId);
51176
+ }
51177
+ const rows = await tx.execute<{ id: string }>(sql`
50709
51178
  update sandbox_leases set
50710
51179
  liveness = 'warm',
51180
+ resume_state = resume_state #- '{opengeniWarmBilling,stopChargeAt}',
50711
51181
  expires_at = now() + (${String(input.leaseTtlMs)} || ' milliseconds')::interval,
50712
51182
  updated_at = now()
50713
51183
  where workspace_id = ${input.workspaceId} and sandbox_group_id = ${input.sandboxGroupId}
@@ -50733,8 +51203,8 @@ export async function reArmDrainingLease(
50733
51203
  )
50734
51204
  returning id
50735
51205
  `);
50736
- return { rearmed: rows.length > 0 };
50737
- },
51206
+ return { rearmed: rows.length > 0 };
51207
+ }),
50738
51208
  );
50739
51209
  }
50740
51210
 
@@ -51106,6 +51576,9 @@ export type SandboxRetainedProcess = {
51106
51576
  reconcileClaimedAt: string | null;
51107
51577
  reconcileAttempts: number;
51108
51578
  lastReconcileOutcome: string | null;
51579
+ cancellationRequestedAt: string | null;
51580
+ cancellationReason: string | null;
51581
+ deadlineCancellationRequestedAt: string | null;
51109
51582
  reconcileProofOutcome: "exited" | "lost" | null;
51110
51583
  reconcileProofExitCode: number | null;
51111
51584
  reconcileProofReason:
@@ -51454,6 +51927,9 @@ function mapRetainedProcess(
51454
51927
  reconcileClaimedAt: row.reconcileClaimedAt?.toISOString() ?? null,
51455
51928
  reconcileAttempts: row.reconcileAttempts,
51456
51929
  lastReconcileOutcome: row.lastReconcileOutcome ?? null,
51930
+ cancellationRequestedAt: row.cancellationRequestedAt?.toISOString() ?? null,
51931
+ cancellationReason: row.cancellationReason ?? null,
51932
+ deadlineCancellationRequestedAt: row.deadlineCancellationRequestedAt?.toISOString() ?? null,
51457
51933
  reconcileProofOutcome: row.reconcileProofOutcome ?? null,
51458
51934
  reconcileProofExitCode: row.reconcileProofExitCode ?? null,
51459
51935
  reconcileProofReason:
@@ -60668,7 +61144,7 @@ export async function revokeViewer(
60668
61144
  // re-dispatched/overlapping tick at the same (group,epoch,tick) can never
60669
61145
  // double-charge (recordUsageEvent is onConflictDoNothing on idempotencyKey).
60670
61146
  //
60671
- // Cursor advance + usage insert are ATOMIC: both run inside ONE FOR UPDATE txn on
61147
+ // Cursor advance + usage insert + paid settlement are ATOMIC: all run inside ONE FOR UPDATE txn on
60672
61148
  // the lease row (the M3 cross-statement-atomicity fix). The insert uses ON
60673
61149
  // CONFLICT DO NOTHING on idempotency_key (matching recordUsageEvent), and the
60674
61150
  // cursor (last_meter_at/last_meter_tick) is advanced in the SAME txn — so the tick
@@ -60684,23 +61160,18 @@ export interface AccrueWarmSecondsResult {
60684
61160
  seconds: number;
60685
61161
  /** The monotonic tick index this accrual was recorded under. */
60686
61162
  tick: number;
60687
- /** usd_micros charged for this tick (0 when rate is 0). */
61163
+ /** Internal usd_micros estimate for this tick (0 in usage_only mode). */
60688
61164
  costMicros: number;
60689
61165
  }
60690
61166
 
60691
61167
  /**
60692
61168
  * Accrue warm-seconds for the elapsed wall-clock since the lease's last meter
60693
61169
  * cursor, idempotent on (sandbox_group_id, lease_epoch, tick). EPOCH-FENCED +
60694
- * liveness-guarded (warm only): a stale-epoch tick or a draining/cold lease is a
60695
- * no-op, so a superseded writer that re-fires cannot mis-meter. The FIRST tick on
60696
- * a never-metered lease (last_meter_at IS NULL) only SEEDS the cursor — it
60697
- * accrues nothing (there is no prior cursor to diff against), matching the
60698
- * "delta since last tick" contract. warmRateMicrosPerSecond > 0 also records a
60699
- * sandbox.warm_cost event (cost = seconds x rate) AND debits the same micros from
60700
- * the credit balance via applyCreditDebitUpToBalance (the model-cost precedent),
60701
- * idempotent on the SAME (group, epoch, tick) key. The usage event is the
60702
- * REQUESTED cost; the ledger is the ACTUAL debit (they legitimately differ when
60703
- * balance is low — M2). Set debitCredits:false to meter without debiting.
61170
+ * liveness-guarded (warm, or draining for a final provider-stop tick). The first
61171
+ * interval begins at the warm transition's recovery timestamp. A draining tick
61172
+ * is only allowed by the provider-stop path, never by the periodic reaper list.
61173
+ * In credits mode the signed full-cost ledger debit, both usage rows and cursor
61174
+ * commit together; a settlement failure rolls back the tick for retry.
60704
61175
  */
60705
61176
  export async function accrueWarmSeconds(
60706
61177
  db: Database,
@@ -60715,9 +61186,9 @@ export async function accrueWarmSeconds(
60715
61186
  /** Optional attribution: the founding/observing session (visibility only — the
60716
61187
  * group meter key makes the workspace charge correct regardless). */
60717
61188
  subjectId?: string | null;
60718
- /** Debit credits for warm-cost (default true). The force-drain at 0 balance
60719
- * depends on this decrementing the balance. */
60720
- debitCredits?: boolean;
61189
+ billingMode?: "usage_only" | "shadow" | "credits";
61190
+ /** Final provider-stop tick, after provider termination and before cold CAS. */
61191
+ finalDrain?: boolean;
60721
61192
  },
60722
61193
  ): Promise<AccrueWarmSecondsResult> {
60723
61194
  const none: AccrueWarmSecondsResult = {
@@ -60735,10 +61206,8 @@ export async function accrueWarmSeconds(
60735
61206
  // Lock the group's lease row so the cursor advance + the usage insert are
60736
61207
  // one atomic step (no other tick can interleave between the diff and the
60737
61208
  // cursor write).
60738
- const rows = await tx.execute<LeaseRow & { meter_elapsed_s: number | null }>(sql`
60739
- select *,
60740
- case when last_meter_at is null then null
60741
- else floor(extract(epoch from (now() - last_meter_at)))::int end as meter_elapsed_s
61209
+ const rows = await tx.execute<LeaseRow>(sql`
61210
+ select *
60742
61211
  from sandbox_leases
60743
61212
  where workspace_id = ${input.workspaceId} and sandbox_group_id = ${input.sandboxGroupId}
60744
61213
  for update
@@ -60746,30 +61215,78 @@ export async function accrueWarmSeconds(
60746
61215
  const row = rows[0];
60747
61216
  if (!row) return none;
60748
61217
 
60749
- // Epoch fence + liveness guard: only a live-epoch warm box meters. A stale
60750
- // (superseded) tick or a draining/cold/warming lease is a no-op.
60751
- if (Number(row.lease_epoch) !== input.expectedEpoch || row.liveness !== "warm") {
61218
+ // An ordinary heartbeat must never charge a box already draining.
61219
+ if (
61220
+ Number(row.lease_epoch) !== input.expectedEpoch ||
61221
+ (row.liveness !== "warm" && !(input.finalDrain && row.liveness === "draining"))
61222
+ ) {
60752
61223
  return none;
60753
61224
  }
60754
61225
 
60755
- // First tick on a never-metered lease: SEED the cursor, accrue nothing.
60756
- if (row.last_meter_at == null) {
61226
+ // commitWarmingToWarm stores this timestamp in the recovery envelope
61227
+ // before changing the lease to warm. The legacy cursor is nullable and
61228
+ // survives a new epoch: never start a new epoch at the previous box's
61229
+ // cursor. Refuse an invalid/missing transition timestamp instead of
61230
+ // charging an arbitrary cold/warming interval.
61231
+ const recovery = row.resume_state?.opengeniRecovery as
61232
+ | { restore?: { completedAt?: unknown } }
61233
+ | undefined;
61234
+ const warmStartedAt = recovery?.restore?.completedAt;
61235
+ const warmStartMs = typeof warmStartedAt === "string" ? Date.parse(warmStartedAt) : NaN;
61236
+ const snapshot = warmBillingSnapshot(row);
61237
+ if (row.resume_state?.opengeniWarmBilling && !snapshot) {
61238
+ throw new Error("sandbox warm billing snapshot is invalid");
61239
+ }
61240
+ // A current usage_only/shadow policy is a rollback switch for a paid
61241
+ // lease. In credits mode, only a box explicitly snapshotted as paid may
61242
+ // debit; pre-activation legacy boxes cannot acquire surprise charges.
61243
+ const mode =
61244
+ input.billingMode === "credits"
61245
+ ? (snapshot?.mode ?? "usage_only")
61246
+ : (input.billingMode ?? snapshot?.mode ?? "usage_only");
61247
+ const rate = snapshot?.rateMicrosPerSecond ?? input.warmRateMicrosPerSecond;
61248
+ if (!Number.isFinite(warmStartMs) && mode === "credits" && rate > 0) {
61249
+ throw new Error("paid warm lease has no valid warm-transition timestamp");
61250
+ }
61251
+ if (!Number.isFinite(warmStartMs) && row.last_meter_at == null) {
61252
+ // Legacy unpriced leases cannot prove their first warm instant. Seed
61253
+ // only these rows; new warm transitions carry an exact recovery clock.
60757
61254
  await tx.execute(sql`
60758
- update sandbox_leases set last_meter_at = now(), updated_at = now()
60759
- where id = ${row.id}
60760
- `);
61255
+ update sandbox_leases set last_meter_at = now(), updated_at = now()
61256
+ where id = ${row.id}
61257
+ `);
60761
61258
  return none;
60762
61259
  }
61260
+ const cursorMs = row.last_meter_at ? new Date(row.last_meter_at).getTime() : 0;
61261
+ const startMs = Number.isFinite(warmStartMs) ? Math.max(cursorMs, warmStartMs) : cursorMs;
61262
+ if (input.finalDrain && !snapshot?.stopChargeAt) {
61263
+ throw new Error("sandbox final tick has no durable stop cutoff");
61264
+ }
61265
+ const upperBound =
61266
+ snapshot?.stopChargeAt && input.finalDrain
61267
+ ? sql`${snapshot.stopChargeAt}::timestamptz`
61268
+ : sql`clock_timestamp()`;
61269
+ const elapsedRows = await tx.execute<{ elapsed_ms: number | string }>(sql`
61270
+ select greatest(0, floor(extract(epoch from (${upperBound} -
61271
+ ${new Date(startMs).toISOString()}::timestamptz)) * 1000))::bigint as elapsed_ms
61272
+ `);
61273
+ const elapsedMs = Number(elapsedRows[0]?.elapsed_ms ?? 0);
61274
+ const elapsedS = Math.floor(elapsedMs / 1000);
60763
61275
 
60764
- const elapsedS = Number(row.meter_elapsed_s ?? 0);
60765
- if (elapsedS <= 0) {
61276
+ if (elapsedS <= 0 && !(input.finalDrain && elapsedMs > 0)) {
60766
61277
  // No whole second elapsed yet — leave the cursor untouched so the
60767
61278
  // remainder accrues on the next tick (no silent seconds loss).
60768
61279
  return none;
60769
61280
  }
60770
61281
 
60771
61282
  const tick = Number(row.last_meter_tick) + 1;
60772
- const costMicros = Math.round(elapsedS * Math.max(0, input.warmRateMicrosPerSecond));
61283
+ const costMicros =
61284
+ mode === "usage_only"
61285
+ ? 0
61286
+ : Math.round((input.finalDrain ? elapsedMs / 1000 : elapsedS) * rate);
61287
+ if (!Number.isSafeInteger(costMicros) || costMicros < 0) {
61288
+ throw new Error("warm-cost estimate exceeds safe integer micros");
61289
+ }
60773
61290
 
60774
61291
  // (1) The warm-seconds meter — GROUP+epoch+tick keyed, ON CONFLICT DO
60775
61292
  // NOTHING (the idempotency that makes a shared box one stream + a re-fire a
@@ -60777,19 +61294,22 @@ export async function accrueWarmSeconds(
60777
61294
  await tx.execute(sql`
60778
61295
  insert into usage_events
60779
61296
  (account_id, workspace_id, subject_id, event_type, quantity, unit,
60780
- source_resource_type, source_resource_id, idempotency_key, occurred_at)
61297
+ source_resource_type, source_resource_id, idempotency_key, initiator_context, occurred_at)
60781
61298
  values
60782
61299
  (${input.accountId}, ${input.workspaceId}, ${input.subjectId ?? null},
60783
61300
  'sandbox.warm_seconds', ${elapsedS}, 'seconds',
60784
61301
  'sandbox_lease', ${`${input.sandboxGroupId}:${input.expectedEpoch}`},
60785
61302
  ${`usage:sandbox.warm_seconds:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`},
60786
- now())
61303
+ ${JSON.stringify(mode === "shadow" ? { warmCostShadowMicros: costMicros } : {})}::jsonb, now())
60787
61304
  on conflict (idempotency_key) do nothing
60788
61305
  `);
60789
61306
 
60790
- // (2) The warm-cost meter (only when a rate is configured). Same keying.
60791
- if (costMicros > 0) {
60792
- await tx.execute(sql`
61307
+ // (2) Only credits mode publishes actual warm spend. Shadow's internal
61308
+ // estimate rides the seconds event, never a public spend event.
61309
+ let settledCostMicros = costMicros;
61310
+ if (costMicros > 0 && mode === "credits") {
61311
+ const costKey = `usage:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`;
61312
+ const inserted = await tx.execute<{ quantity: number | string }>(sql`
60793
61313
  insert into usage_events
60794
61314
  (account_id, workspace_id, subject_id, event_type, quantity, unit,
60795
61315
  source_resource_type, source_resource_id, idempotency_key, occurred_at)
@@ -60797,62 +61317,65 @@ export async function accrueWarmSeconds(
60797
61317
  (${input.accountId}, ${input.workspaceId}, ${input.subjectId ?? null},
60798
61318
  'sandbox.warm_cost', ${costMicros}, 'usd_micros',
60799
61319
  'sandbox_lease', ${`${input.sandboxGroupId}:${input.expectedEpoch}`},
60800
- ${`usage:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`},
61320
+ ${costKey},
60801
61321
  now())
60802
61322
  on conflict (idempotency_key) do nothing
61323
+ returning quantity
60803
61324
  `);
61325
+ // Replayed historical cursor writes must settle the originally
61326
+ // recorded tick, not today's recalculated interval or new tariff.
61327
+ const previous = inserted[0]
61328
+ ? null
61329
+ : await tx.execute<{ quantity: number | string }>(sql`
61330
+ select quantity from usage_events
61331
+ where idempotency_key = ${costKey}
61332
+ and account_id = ${input.accountId}
61333
+ and workspace_id = ${input.workspaceId}
61334
+ and event_type = 'sandbox.warm_cost'
61335
+ `);
61336
+ settledCostMicros = Number((inserted[0] ?? previous?.[0])?.quantity ?? 0);
61337
+ if (!Number.isSafeInteger(settledCostMicros) || settledCostMicros <= 0) {
61338
+ throw new Error("warm-cost tick idempotency key conflicts with another usage event");
61339
+ }
61340
+ }
61341
+
61342
+ if (costMicros > 0 && mode === "credits") {
61343
+ await applyCreditDebitAfterUse(tx, {
61344
+ accountId: input.accountId,
61345
+ workspaceId: input.workspaceId,
61346
+ type: "sandbox.warm_cost",
61347
+ amountMicros: settledCostMicros,
61348
+ sourceType: "sandbox_lease",
61349
+ sourceId: `${input.sandboxGroupId}:${input.expectedEpoch}`,
61350
+ idempotencyKey: `debit:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`,
61351
+ });
60804
61352
  }
60805
61353
 
60806
61354
  // (3) Advance the cursor IN THE SAME TXN — the atomicity that makes the tick
60807
61355
  // index and the metered seconds inseparable.
60808
61356
  await tx.execute(sql`
60809
61357
  update sandbox_leases set
60810
- last_meter_at = now(), last_meter_tick = ${tick}, updated_at = now()
61358
+ last_meter_at = ${new Date(startMs).toISOString()}::timestamptz + (${String(input.finalDrain ? elapsedMs : elapsedS * 1000)} || ' milliseconds')::interval,
61359
+ last_meter_tick = ${tick}, updated_at = now()
60811
61360
  where id = ${row.id}
60812
61361
  `);
60813
61362
 
60814
- return { accrued: true, seconds: elapsedS, tick, costMicros };
61363
+ return { accrued: true, seconds: elapsedS, tick, costMicros: settledCostMicros };
60815
61364
  }),
60816
61365
  );
60817
61366
 
60818
- // Debit credits for the warm-cost OUTSIDE the lease-row txn (applyCreditDebit
60819
- // takes its own per-account advisory lock — never nest it under the lease row
60820
- // lock). Idempotent on the SAME (group, epoch, tick) key so a re-fire of an
60821
- // already-committed tick cannot double-debit. The ledger records the ACTUAL
60822
- // debit (min(requested, balance)); the warm_cost usage event above is the
60823
- // requested cost.
60824
- if (result.accrued && result.costMicros > 0 && (input.debitCredits ?? true)) {
60825
- await applyCreditDebitUpToBalance(db, {
60826
- accountId: input.accountId,
60827
- workspaceId: input.workspaceId,
60828
- type: "sandbox.warm_cost",
60829
- requestedAmountMicros: result.costMicros,
60830
- sourceType: "sandbox_lease",
60831
- sourceId: `${input.sandboxGroupId}:${input.expectedEpoch}`,
60832
- idempotencyKey: `debit:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${result.tick}`,
60833
- }).catch(() => undefined);
60834
- }
60835
-
60836
61367
  return result;
60837
61368
  }
60838
61369
 
60839
- // §2.2/2.3 — the per-workspace warm-cap + force-drain. Under the EXISTING usage
60840
- // lock (withWorkspaceUsageLock — NOT a bare count, so two concurrent ticks in
60841
- // different sessions of one workspace can't both read "under cap" and race past
60842
- // it). A workspace at 0 balance OR over its warm-second cap force-drains its
60843
- // VIEWER-ONLY boxes: CAS warm->draining guarded `AND turn_holders = 0` so a box
60844
- // with a running (paying) turn is NEVER killed. The reaper then issues the
60845
- // provider stop() at refcount 0 (this fn is DB-only — no provider call).
60846
- //
60847
- // Group-wide force-drain on workspace balance exhaustion is deliberate (one
60848
- // balance drains a multi-session box): the workspace, not the session, is the
60849
- // billing unit — correctness (charged once) is automatic from the group meter key.
61370
+ // §2.2/2.3 — independent optional warm quota. Exhausted credits are enforced
61371
+ // at paid admission; they never cancel an already active holder. The cap can
61372
+ // close new viewer admission but only idle, holderless boxes may drain.
60850
61373
  export interface ForceDrainResult {
60851
- /** Whether the workspace was over a limit (0 balance or over the warm cap). */
61374
+ /** Whether the workspace exceeded its optional warm cap. */
60852
61375
  overLimit: boolean;
60853
61376
  /** The reason, for observability. */
60854
61377
  reason: SandboxViewerForceDrainReason | null;
60855
- /** The (workspaceId, sandboxGroupId) viewer-only boxes CASed warm->draining. */
61378
+ /** The (workspaceId, sandboxGroupId) holderless boxes CASed warm->draining. */
60856
61379
  drained: { workspaceId: string; sandboxGroupId: string }[];
60857
61380
  }
60858
61381
 
@@ -60867,8 +61390,7 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
60867
61390
  db: Database,
60868
61391
  input: {
60869
61392
  workspaceId: string;
60870
- /** account balance gate: when <= 0 (and a billing/managed mode is on) drain. */
60871
- balanceMicros: number;
61393
+ /** Legacy caller option; balance belongs to paid admission, not draining. */
60872
61394
  enforceBalance: boolean;
60873
61395
  /** warm-second cap (cumulative this UTC month). 0 = unbounded (no cap gate). */
60874
61396
  maxWarmSecondsPerWorkspace: number;
@@ -60883,9 +61405,7 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
60883
61405
  // Determine over-limit under the lock (so the cap read + the drain are one
60884
61406
  // serialized critical section per workspace).
60885
61407
  let reason: SandboxViewerForceDrainReason | null = null;
60886
- if (input.enforceBalance && input.balanceMicros <= 0) {
60887
- reason = "balance";
60888
- } else if (input.maxWarmSecondsPerWorkspace > 0) {
61408
+ if (input.maxWarmSecondsPerWorkspace > 0) {
60889
61409
  const since = input.capWindowStart ?? startOfUtcMonthDefault();
60890
61410
  const [{ total } = { total: 0 }] = await scopedDb
60891
61411
  .select({
@@ -60918,10 +61438,8 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
60918
61438
  return { overLimit: false, reason: null, drained: [] };
60919
61439
  }
60920
61440
 
60921
- // Publish the workspace-wide admission gate before touching any holder.
60922
- // A concurrent viewer that already owns the lease lock may complete first;
60923
- // the locked drain below then deletes that holder. Every later viewer sees
60924
- // this committed intent before it can re-arm or spawn a provider box.
61441
+ // Publish a cap-only gate; no paid-balance gate survives a top-up or mode
61442
+ // change. An already active holder keeps the provider alive.
60925
61443
  await scopedDb.execute(sql`
60926
61444
  update workspaces set
60927
61445
  sandbox_viewer_force_drain_reason = ${reason},
@@ -60939,94 +61457,30 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
60939
61457
  )
60940
61458
  `);
60941
61459
 
60942
- // Canonical holder-mutation order is lease -> holder. Lock the exact
60943
- // passive-only candidates before deleting their holder rows so this path
60944
- // cannot deadlock with the global reaper or race a concurrent acquire.
61460
+ // Lock only holderless candidates. Neither the cap nor a zero balance may
61461
+ // mark an active Browser/ComputerSession, viewer, direct op or process lost.
60945
61462
  await scopedDb.execute(sql`
60946
- select id from sandbox_leases
60947
- where workspace_id = ${input.workspaceId}
60948
- and liveness = 'warm' and turn_holders = 0
61463
+ select lease.id from sandbox_leases lease
61464
+ where lease.workspace_id = ${input.workspaceId}
61465
+ and lease.liveness = 'warm' and lease.refcount = 0
61466
+ and not exists (select 1 from sandbox_lease_holders h where h.lease_id = lease.id)
60949
61467
  order by id
60950
61468
  for update
60951
61469
  `);
60952
-
60953
- // Force-drain passive-only warm boxes: CAS warm->draining guarded
60954
- // turn_holders = 0 (a paying turn is NEVER killed). Stamp the grace deadline
60955
- // so the reaper terminates at refcount 0 past the grace, exactly as a normal
60956
- // refcount->0 drain would.
60957
- // An interaction cannot keep a placement controller alive after the
60958
- // workspace cost boundary has force-drained it. Make BrowserSession and
60959
- // ComputerSession loss visible before removing their holders; later
60960
- // recovery is explicit admission, never a silent resurrection.
60961
- const lostBrowsers = await scopedDb.execute<{ id: string }>(sql`
60962
- update browser_sessions browser set
60963
- lifecycle = 'lost',
60964
- controller_id = null,
60965
- controller_generation = null,
60966
- placement_instance_id = null,
60967
- controller_heartbeat_at = null,
60968
- failure_code = 'workspace_force_drained',
60969
- updated_at = now()
60970
- where browser.workspace_id = ${input.workspaceId}
60971
- and browser.lifecycle in ('starting', 'active', 'suspending', 'restoring', 'ending')
60972
- and browser.controller_host_sandbox_group_id in (
60973
- select sandbox_group_id from sandbox_leases
60974
- where workspace_id = ${input.workspaceId}
60975
- and liveness = 'warm' and turn_holders = 0
60976
- )
60977
- returning browser.id
60978
- `);
60979
- const lostComputers = await scopedDb.execute<{ id: string }>(sql`
60980
- update computer_sessions computer set
60981
- lifecycle = 'lost',
60982
- controller_id = null,
60983
- controller_generation = null,
60984
- placement_instance_id = null,
60985
- controller_heartbeat_at = null,
60986
- failure_code = 'workspace_force_drained',
60987
- updated_at = now()
60988
- where computer.workspace_id = ${input.workspaceId}
60989
- and computer.lifecycle in ('starting', 'active', 'suspending', 'restoring', 'ending')
60990
- and computer.sandbox_group_id in (
60991
- select sandbox_group_id from sandbox_leases
60992
- where workspace_id = ${input.workspaceId}
60993
- and liveness = 'warm' and turn_holders = 0
60994
- )
60995
- returning computer.id
60996
- `);
60997
- if (lostBrowsers.length > 0 || lostComputers.length > 0) {
60998
- await scopedDb.execute(sql`
60999
- update workspace_interaction_revisions set
61000
- revision = revision + 1,
61001
- updated_at = now()
61002
- where workspace_id = ${input.workspaceId}
61003
- `);
61004
- }
61005
- // Drop viewer and interaction holders of every passive-only lease. Scoped
61006
- // through the locked warm rows so a turn-held box remains untouched.
61007
- await scopedDb.execute(sql`
61008
- delete from sandbox_lease_holders h
61009
- where h.kind in ('viewer', 'interaction')
61010
- and h.lease_id in (
61011
- select id from sandbox_leases
61012
- where workspace_id = ${input.workspaceId}
61013
- and liveness = 'warm' and turn_holders = 0
61014
- )
61015
- `);
61016
- // CAS the now-holderless leases warm→draining at refcount 0 with the grace
61017
- // deadline stamped — so the SAME reaper sweep's refcount=0 drain predicate
61018
- // then terminates the box.
61470
+ // Normal release/reaper already drains holderless boxes. This CAS is a
61471
+ // bounded accelerator only; it cannot delete an active holder.
61019
61472
  const drained = await rawRows<{ sandbox_group_id: string }>(
61020
61473
  scopedDb,
61021
61474
  sql`
61022
- update sandbox_leases set
61475
+ update sandbox_leases lease set
61023
61476
  liveness = 'draining',
61024
61477
  refcount = 0, turn_holders = 0, viewer_holders = 0,
61025
61478
  expires_at = now() + (${String(input.idleGraceMs)} || ' milliseconds')::interval,
61026
61479
  updated_at = now()
61027
- where workspace_id = ${input.workspaceId}
61028
- and liveness = 'warm' and turn_holders = 0
61029
- returning sandbox_group_id
61480
+ where lease.workspace_id = ${input.workspaceId}
61481
+ and lease.liveness = 'warm' and lease.refcount = 0
61482
+ and not exists (select 1 from sandbox_lease_holders h where h.lease_id = lease.id)
61483
+ returning lease.sandbox_group_id
61030
61484
  `,
61031
61485
  );
61032
61486