@opengeni/db 6.0.1 → 6.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-sessions.d.ts +26 -0
- package/dist/canonical-human-identities.js +3 -3
- package/dist/capability-reconciliation.js +2 -2
- package/dist/{chunk-4RDUZ5UV.js → chunk-4ZQFT5II.js} +2 -2
- package/dist/{chunk-6ORJPCYW.js → chunk-5UOWNKPP.js} +2 -2
- package/dist/{chunk-4CCFFLDZ.js → chunk-BG46J5HF.js} +13 -2
- package/dist/chunk-BG46J5HF.js.map +1 -0
- package/dist/{chunk-CSVHCF3R.js → chunk-IB3SW3JW.js} +11 -4
- package/dist/{chunk-CSVHCF3R.js.map → chunk-IB3SW3JW.js.map} +1 -1
- package/dist/{chunk-GSB4ISHE.js → chunk-IIPWNNVR.js} +4 -1
- package/dist/chunk-IIPWNNVR.js.map +1 -0
- package/dist/{chunk-MAMBI2HR.js → chunk-JPYN5XKV.js} +39 -3
- package/dist/chunk-JPYN5XKV.js.map +1 -0
- package/dist/{chunk-LSLNMENV.js → chunk-KZXWUXWV.js} +4 -4
- package/dist/{chunk-LO6BVXXH.js → chunk-LD52IFS3.js} +2 -2
- package/dist/{chunk-JILAWE26.js → chunk-M5Z6II2P.js} +2 -2
- package/dist/{chunk-PGIFHEBA.js → chunk-RVEOGRMF.js} +2 -2
- package/dist/{chunk-H7BC444C.js → chunk-SRAMNVHT.js} +17 -1
- package/dist/chunk-SRAMNVHT.js.map +1 -0
- package/dist/{chunk-XK6HXON2.js → chunk-UXFBXCHM.js} +2 -2
- package/dist/{chunk-RXH5CFG3.js → chunk-VE2XK2KD.js} +12 -8
- package/dist/{chunk-RXH5CFG3.js.map → chunk-VE2XK2KD.js.map} +1 -1
- package/dist/{chunk-FULLE5CF.js → chunk-WT7HG43Y.js} +3 -3
- package/dist/{chunk-I7HQL6FK.js → chunk-ZM5BA3TD.js} +3 -3
- package/dist/connect-attempts.js +3 -3
- package/dist/database.d.ts +8 -0
- package/dist/editable-artifact-durable-export.js +2 -2
- package/dist/editable-artifacts.js +3 -3
- package/dist/index.d.ts +69 -20
- package/dist/index.js +633 -202
- package/dist/index.js.map +1 -1
- package/dist/knowledge-entries.d.ts +2 -0
- package/dist/knowledge-indexing.d.ts +17 -0
- package/dist/managed-auth-session-sets.js +3 -3
- package/dist/managed-user-setup.d.ts +1 -0
- package/dist/mcp-operations.js +2 -2
- package/dist/migrate.d.ts +10 -1
- package/dist/migrate.js +3 -1
- package/dist/organization-integration-policy.js +3 -3
- package/dist/organization-membership-lifecycle.d.ts +1 -0
- package/dist/provision-roles.js +1 -1
- package/dist/retained-provider-commands.d.ts +4 -2
- package/dist/retained-provider-commands.js +6 -6
- package/dist/runtime-posture.d.ts +1 -1
- package/dist/schema.d.ts +25 -8
- package/dist/schema.js +1 -1
- package/dist/session-background-commands.js +4 -4
- package/dist/session-command-output.js +5 -5
- package/dist/session-event-slices.js +3 -3
- package/dist/session-mcp-credential-rotation.js +4 -4
- package/dist/session-tenancy.js +3 -3
- package/dist/video-generation.js +3 -3
- package/dist/workspace-tool-defaults.js +4 -4
- package/drizzle/0508_deadline_command_workspace_capture.sql +75 -0
- package/drizzle/0509_verified_signup_trial_credits.sql +76 -0
- package/drizzle/0510_knowledge_index_funding_wait.sql +174 -0
- package/drizzle/0511_knowledge_visible_index_status.sql +56 -0
- package/drizzle/0512_insights_fact_reads_hash_join.sql +31 -0
- package/drizzle/0513_message_fork_prefix_compaction.sql +85 -0
- package/drizzle/0514_failed_session_variable_set_attach.sql +25 -0
- package/package.json +5 -5
- package/src/browser-sessions.ts +71 -1
- package/src/database.ts +18 -0
- package/src/index.ts +706 -252
- package/src/insights.ts +44 -36
- package/src/knowledge-entries.ts +43 -2
- package/src/knowledge-indexing.ts +79 -1
- package/src/managed-user-setup.ts +18 -1
- package/src/migrate.ts +58 -1
- package/src/organization-membership-lifecycle.ts +11 -0
- package/src/provision-roles.ts +12 -0
- package/src/retained-provider-commands.ts +16 -6
- package/src/runtime-posture.ts +4 -0
- package/src/schema.ts +3 -0
- package/dist/chunk-4CCFFLDZ.js.map +0 -1
- package/dist/chunk-GSB4ISHE.js.map +0 -1
- package/dist/chunk-H7BC444C.js.map +0 -1
- package/dist/chunk-MAMBI2HR.js.map +0 -1
- /package/dist/{chunk-4RDUZ5UV.js.map → chunk-4ZQFT5II.js.map} +0 -0
- /package/dist/{chunk-6ORJPCYW.js.map → chunk-5UOWNKPP.js.map} +0 -0
- /package/dist/{chunk-LSLNMENV.js.map → chunk-KZXWUXWV.js.map} +0 -0
- /package/dist/{chunk-LO6BVXXH.js.map → chunk-LD52IFS3.js.map} +0 -0
- /package/dist/{chunk-JILAWE26.js.map → chunk-M5Z6II2P.js.map} +0 -0
- /package/dist/{chunk-PGIFHEBA.js.map → chunk-RVEOGRMF.js.map} +0 -0
- /package/dist/{chunk-XK6HXON2.js.map → chunk-UXFBXCHM.js.map} +0 -0
- /package/dist/{chunk-FULLE5CF.js.map → chunk-WT7HG43Y.js.map} +0 -0
- /package/dist/{chunk-I7HQL6FK.js.map → chunk-ZM5BA3TD.js.map} +0 -0
package/src/index.ts
CHANGED
|
@@ -406,6 +406,7 @@ import {
|
|
|
406
406
|
import {
|
|
407
407
|
environmentsEncryptionKeyBytes,
|
|
408
408
|
SANDBOX_LIFECYCLE_TRANSITION_MAX_WAIT_MS,
|
|
409
|
+
SANDBOX_DEADLINE_COMMAND_STOP_GRACE_MS,
|
|
409
410
|
WORKSPACE_OPENROUTER_CONNECTION_DOMAIN,
|
|
410
411
|
WORKSPACE_OPENROUTER_CONNECTION_ROLE,
|
|
411
412
|
VERCEL_AI_GATEWAY_CONNECTION_DOMAIN,
|
|
@@ -480,9 +481,11 @@ import {
|
|
|
480
481
|
type NewSessionDraftSnapshot,
|
|
481
482
|
} from "./new-session-drafts";
|
|
482
483
|
import {
|
|
484
|
+
isRetryableDatabaseTransportFailure,
|
|
483
485
|
nestedPostgresSqlState,
|
|
484
486
|
runIdempotentPersistenceTransaction,
|
|
485
487
|
safeDatabaseErrorFacts,
|
|
488
|
+
SessionEventPersistenceError,
|
|
486
489
|
type IdempotentPersistenceTransactionOptions,
|
|
487
490
|
} from "./persistence-errors";
|
|
488
491
|
import {
|
|
@@ -752,7 +755,7 @@ export {
|
|
|
752
755
|
type UserLookup,
|
|
753
756
|
type UserProfileLookup,
|
|
754
757
|
} from "./database";
|
|
755
|
-
export { withSessionRlsActorContext } from "./database";
|
|
758
|
+
export { currentSessionRlsActorIdentityKey, withSessionRlsActorContext } from "./database";
|
|
756
759
|
export { withDatabaseTimingObserver, type DatabaseTimingObservation } from "./database-timing";
|
|
757
760
|
export { normalizedCredentialHeaders } from "./connection-token-resolver";
|
|
758
761
|
import {
|
|
@@ -5318,6 +5321,82 @@ export async function applyCreditDebitUpToBalance(
|
|
|
5318
5321
|
);
|
|
5319
5322
|
}
|
|
5320
5323
|
|
|
5324
|
+
/**
|
|
5325
|
+
* Settle an already-consumed, OpenGeni-funded resource for its full cost.
|
|
5326
|
+
* Admission happens before use; an in-flight operation can take the account
|
|
5327
|
+
* below zero. Future grants and purchased top-ups net against that balance.
|
|
5328
|
+
* Never use this helper to admit a new operation or for externally paid use.
|
|
5329
|
+
*/
|
|
5330
|
+
export async function applyCreditDebitAfterUse(
|
|
5331
|
+
db: Database,
|
|
5332
|
+
input: {
|
|
5333
|
+
accountId: string;
|
|
5334
|
+
workspaceId?: string | null;
|
|
5335
|
+
type: string;
|
|
5336
|
+
amountMicros: number;
|
|
5337
|
+
sourceType: string;
|
|
5338
|
+
sourceId: string;
|
|
5339
|
+
idempotencyKey: string;
|
|
5340
|
+
metadata?: Record<string, unknown>;
|
|
5341
|
+
occurredAt?: Date;
|
|
5342
|
+
},
|
|
5343
|
+
): Promise<{ balance: BillingBalance; debitedMicros: number }> {
|
|
5344
|
+
if (!Number.isSafeInteger(input.amountMicros) || input.amountMicros <= 0) {
|
|
5345
|
+
throw new Error("post-use credit debit requires a positive, safe integer micro amount");
|
|
5346
|
+
}
|
|
5347
|
+
return await withRlsContext(
|
|
5348
|
+
db,
|
|
5349
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId ?? null },
|
|
5350
|
+
async (scopedDb) => {
|
|
5351
|
+
const inserted = await scopedDb
|
|
5352
|
+
.insert(schema.creditLedgerEntries)
|
|
5353
|
+
.values({
|
|
5354
|
+
accountId: input.accountId,
|
|
5355
|
+
workspaceId: input.workspaceId ?? null,
|
|
5356
|
+
type: input.type,
|
|
5357
|
+
amountMicros: -input.amountMicros,
|
|
5358
|
+
sourceType: input.sourceType,
|
|
5359
|
+
sourceId: input.sourceId,
|
|
5360
|
+
idempotencyKey: input.idempotencyKey,
|
|
5361
|
+
metadata: input.metadata ?? {},
|
|
5362
|
+
occurredAt: input.occurredAt ?? new Date(),
|
|
5363
|
+
})
|
|
5364
|
+
.onConflictDoNothing({ target: schema.creditLedgerEntries.idempotencyKey })
|
|
5365
|
+
.returning({ id: schema.creditLedgerEntries.id });
|
|
5366
|
+
if (inserted.length === 0) {
|
|
5367
|
+
const [prior] = await scopedDb
|
|
5368
|
+
.select({
|
|
5369
|
+
accountId: schema.creditLedgerEntries.accountId,
|
|
5370
|
+
workspaceId: schema.creditLedgerEntries.workspaceId,
|
|
5371
|
+
type: schema.creditLedgerEntries.type,
|
|
5372
|
+
amountMicros: schema.creditLedgerEntries.amountMicros,
|
|
5373
|
+
sourceType: schema.creditLedgerEntries.sourceType,
|
|
5374
|
+
sourceId: schema.creditLedgerEntries.sourceId,
|
|
5375
|
+
})
|
|
5376
|
+
.from(schema.creditLedgerEntries)
|
|
5377
|
+
.where(eq(schema.creditLedgerEntries.idempotencyKey, input.idempotencyKey));
|
|
5378
|
+
if (
|
|
5379
|
+
!prior ||
|
|
5380
|
+
prior.accountId !== input.accountId ||
|
|
5381
|
+
prior.workspaceId !== (input.workspaceId ?? null) ||
|
|
5382
|
+
prior.type !== input.type ||
|
|
5383
|
+
prior.amountMicros !== -input.amountMicros ||
|
|
5384
|
+
prior.sourceType !== input.sourceType ||
|
|
5385
|
+
prior.sourceId !== input.sourceId
|
|
5386
|
+
) {
|
|
5387
|
+
throw new Error(
|
|
5388
|
+
"post-use credit debit idempotency key conflicts with a different charge",
|
|
5389
|
+
);
|
|
5390
|
+
}
|
|
5391
|
+
}
|
|
5392
|
+
return {
|
|
5393
|
+
balance: await getBillingBalance(scopedDb, input.accountId),
|
|
5394
|
+
debitedMicros: inserted.length ? input.amountMicros : 0,
|
|
5395
|
+
};
|
|
5396
|
+
},
|
|
5397
|
+
);
|
|
5398
|
+
}
|
|
5399
|
+
|
|
5321
5400
|
export async function hasCreditLedgerEntry(
|
|
5322
5401
|
db: Database,
|
|
5323
5402
|
accountId: string,
|
|
@@ -7330,6 +7409,39 @@ export async function getRetainedScreenshotArtifact(
|
|
|
7330
7409
|
});
|
|
7331
7410
|
}
|
|
7332
7411
|
|
|
7412
|
+
/** Recover a legacy damaged receipt only when its tool call has one exact
|
|
7413
|
+
* screenshot in this session. Never infer an artifact across fork ancestry. */
|
|
7414
|
+
export async function getRetainedScreenshotArtifactForToolCall(
|
|
7415
|
+
db: Database,
|
|
7416
|
+
workspaceId: string,
|
|
7417
|
+
sessionId: string,
|
|
7418
|
+
toolCallId: string,
|
|
7419
|
+
): Promise<RetainedScreenshotArtifact | null> {
|
|
7420
|
+
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
7421
|
+
const rows = await scopedDb
|
|
7422
|
+
.select({ artifact: schema.retainedScreenshotArtifacts, file: schema.files })
|
|
7423
|
+
.from(schema.retainedScreenshotArtifacts)
|
|
7424
|
+
.innerJoin(
|
|
7425
|
+
schema.files,
|
|
7426
|
+
and(
|
|
7427
|
+
eq(schema.files.workspaceId, schema.retainedScreenshotArtifacts.workspaceId),
|
|
7428
|
+
eq(schema.files.id, schema.retainedScreenshotArtifacts.artifactId),
|
|
7429
|
+
),
|
|
7430
|
+
)
|
|
7431
|
+
.where(
|
|
7432
|
+
and(
|
|
7433
|
+
eq(schema.retainedScreenshotArtifacts.workspaceId, workspaceId),
|
|
7434
|
+
eq(schema.retainedScreenshotArtifacts.sessionId, sessionId),
|
|
7435
|
+
eq(schema.retainedScreenshotArtifacts.toolCallId, toolCallId),
|
|
7436
|
+
),
|
|
7437
|
+
)
|
|
7438
|
+
.limit(2);
|
|
7439
|
+
return rows.length === 1
|
|
7440
|
+
? mapRetainedScreenshotArtifact(rows[0]!.artifact, rows[0]!.file)
|
|
7441
|
+
: null;
|
|
7442
|
+
});
|
|
7443
|
+
}
|
|
7444
|
+
|
|
7333
7445
|
export type RetainedScreenshotMaintenanceClaim = {
|
|
7334
7446
|
action: "reconcile" | "delete";
|
|
7335
7447
|
claimId: string;
|
|
@@ -41354,101 +41466,183 @@ function assertPendingToolOutputPolicyMatches(
|
|
|
41354
41466
|
}
|
|
41355
41467
|
}
|
|
41356
41468
|
|
|
41469
|
+
// Receipt inputs have already passed the strict JSON storage boundary. Compare
|
|
41470
|
+
// exact keys structurally: locale collation can equate distinct Unicode keys.
|
|
41471
|
+
function pendingToolCallItemsEqual(left: unknown, right: unknown): boolean {
|
|
41472
|
+
if (left === right) return true;
|
|
41473
|
+
if (left === null || right === null || typeof left !== "object" || typeof right !== "object")
|
|
41474
|
+
return false;
|
|
41475
|
+
if (Array.isArray(left) || Array.isArray(right)) {
|
|
41476
|
+
return (
|
|
41477
|
+
Array.isArray(left) &&
|
|
41478
|
+
Array.isArray(right) &&
|
|
41479
|
+
left.length === right.length &&
|
|
41480
|
+
left.every((value, index) => pendingToolCallItemsEqual(value, right[index]))
|
|
41481
|
+
);
|
|
41482
|
+
}
|
|
41483
|
+
const keys = Object.keys(left);
|
|
41484
|
+
return (
|
|
41485
|
+
keys.length === Object.keys(right).length &&
|
|
41486
|
+
keys.every(
|
|
41487
|
+
(key) =>
|
|
41488
|
+
Object.hasOwn(right, key) &&
|
|
41489
|
+
pendingToolCallItemsEqual(
|
|
41490
|
+
(left as Record<string, unknown>)[key],
|
|
41491
|
+
(right as Record<string, unknown>)[key],
|
|
41492
|
+
),
|
|
41493
|
+
)
|
|
41494
|
+
);
|
|
41495
|
+
}
|
|
41496
|
+
|
|
41357
41497
|
/**
|
|
41358
41498
|
* Durably capture the raw SDK call item at the exact attempt boundary. This is
|
|
41359
41499
|
* model-facing truth, deliberately separate from the session-event timeline.
|
|
41360
41500
|
* The receipt belongs to the logical turn so an approval resume can
|
|
41361
41501
|
* settle it from a newer attempt. Duplicate SDK delivery converges on the
|
|
41362
41502
|
* unique (turn, call) identity.
|
|
41503
|
+
* Pass the root database handle so each rollback retry starts a fresh
|
|
41504
|
+
* transaction, not a savepoint in a caller-owned transaction.
|
|
41363
41505
|
*/
|
|
41364
41506
|
export async function registerPendingSessionToolCall(
|
|
41365
41507
|
db: Database,
|
|
41366
41508
|
input: PendingSessionToolCallInput,
|
|
41367
41509
|
): Promise<{ accepted: boolean; registered: boolean }> {
|
|
41368
|
-
|
|
41369
|
-
|
|
41370
|
-
{
|
|
41371
|
-
|
|
41372
|
-
|
|
41373
|
-
|
|
41374
|
-
|
|
41375
|
-
|
|
41376
|
-
|
|
41377
|
-
|
|
41378
|
-
|
|
41379
|
-
|
|
41380
|
-
|
|
41381
|
-
|
|
41382
|
-
|
|
41383
|
-
|
|
41384
|
-
|
|
41385
|
-
|
|
41386
|
-
|
|
41387
|
-
|
|
41388
|
-
|
|
41389
|
-
|
|
41390
|
-
|
|
41391
|
-
|
|
41392
|
-
|
|
41393
|
-
|
|
41394
|
-
|
|
41395
|
-
|
|
41396
|
-
|
|
41397
|
-
|
|
41398
|
-
|
|
41399
|
-
|
|
41400
|
-
|
|
41401
|
-
|
|
41402
|
-
|
|
41403
|
-
|
|
41404
|
-
|
|
41405
|
-
|
|
41406
|
-
|
|
41407
|
-
|
|
41408
|
-
|
|
41409
|
-
|
|
41410
|
-
const [pending] = await tx
|
|
41411
|
-
.select({
|
|
41412
|
-
id: schema.sessionPendingToolCalls.id,
|
|
41413
|
-
modelToolOutputTruncationTokens:
|
|
41414
|
-
schema.sessionPendingToolCalls.modelToolOutputTruncationTokens,
|
|
41415
|
-
})
|
|
41416
|
-
.from(schema.sessionPendingToolCalls)
|
|
41417
|
-
.where(
|
|
41418
|
-
and(
|
|
41419
|
-
eq(schema.sessionPendingToolCalls.workspaceId, input.workspaceId),
|
|
41420
|
-
eq(schema.sessionPendingToolCalls.sessionId, input.sessionId),
|
|
41421
|
-
eq(schema.sessionPendingToolCalls.turnId, input.turnId),
|
|
41422
|
-
eq(schema.sessionPendingToolCalls.callId, input.callId),
|
|
41423
|
-
),
|
|
41424
|
-
)
|
|
41425
|
-
.for("update")
|
|
41426
|
-
.limit(1);
|
|
41427
|
-
if (!pending) {
|
|
41428
|
-
throw new Error(
|
|
41429
|
-
`Pending SDK tool call disappeared during registration: ${input.callId}`,
|
|
41430
|
-
);
|
|
41431
|
-
}
|
|
41432
|
-
assertPendingToolOutputPolicyMatches(
|
|
41433
|
-
pending.modelToolOutputTruncationTokens,
|
|
41434
|
-
input.modelToolOutputTruncationTokens,
|
|
41435
|
-
input.callId,
|
|
41436
|
-
);
|
|
41437
|
-
if (
|
|
41438
|
-
pending.modelToolOutputTruncationTokens === null &&
|
|
41439
|
-
input.modelToolOutputTruncationTokens !== undefined
|
|
41440
|
-
) {
|
|
41441
|
-
await tx
|
|
41442
|
-
.update(schema.sessionPendingToolCalls)
|
|
41443
|
-
.set({
|
|
41444
|
-
modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens,
|
|
41510
|
+
const correlationId = crypto.randomUUID();
|
|
41511
|
+
const registerOnce = async (attempt: number) => {
|
|
41512
|
+
try {
|
|
41513
|
+
return await withRlsContext(
|
|
41514
|
+
db,
|
|
41515
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
41516
|
+
async (scopedDb) =>
|
|
41517
|
+
await scopedDb.transaction(async (tx) => {
|
|
41518
|
+
const fence = await lockTurnAttemptWriteFenceTx(tx, {
|
|
41519
|
+
workspaceId: input.workspaceId,
|
|
41520
|
+
sessionId: input.sessionId,
|
|
41521
|
+
turnId: input.turnId,
|
|
41522
|
+
executionGeneration: input.executionGeneration,
|
|
41523
|
+
attemptId: input.attemptId,
|
|
41524
|
+
});
|
|
41525
|
+
if (!fence.allowed) return { accepted: false, registered: false };
|
|
41526
|
+
const inserted = await tx
|
|
41527
|
+
.insert(schema.sessionPendingToolCalls)
|
|
41528
|
+
.values(
|
|
41529
|
+
withLosslessContentWriteVersion(
|
|
41530
|
+
{
|
|
41531
|
+
accountId: input.accountId,
|
|
41532
|
+
workspaceId: input.workspaceId,
|
|
41533
|
+
sessionId: input.sessionId,
|
|
41534
|
+
turnId: input.turnId,
|
|
41535
|
+
executionGeneration: input.executionGeneration,
|
|
41536
|
+
attemptId: input.attemptId,
|
|
41537
|
+
callId: input.callId,
|
|
41538
|
+
callType: input.callType,
|
|
41539
|
+
callItem: input.callItem,
|
|
41540
|
+
modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens ?? null,
|
|
41541
|
+
},
|
|
41542
|
+
"callItem",
|
|
41543
|
+
"callItemCodecVersion",
|
|
41544
|
+
),
|
|
41545
|
+
)
|
|
41546
|
+
.onConflictDoNothing({
|
|
41547
|
+
target: [
|
|
41548
|
+
schema.sessionPendingToolCalls.workspaceId,
|
|
41549
|
+
schema.sessionPendingToolCalls.turnId,
|
|
41550
|
+
schema.sessionPendingToolCalls.callId,
|
|
41551
|
+
],
|
|
41445
41552
|
})
|
|
41446
|
-
.
|
|
41447
|
-
|
|
41448
|
-
|
|
41449
|
-
|
|
41450
|
-
|
|
41553
|
+
.returning({ id: schema.sessionPendingToolCalls.id });
|
|
41554
|
+
if (inserted.length === 0) {
|
|
41555
|
+
const [pending] = await tx
|
|
41556
|
+
.select({
|
|
41557
|
+
id: schema.sessionPendingToolCalls.id,
|
|
41558
|
+
accountId: schema.sessionPendingToolCalls.accountId,
|
|
41559
|
+
callType: schema.sessionPendingToolCalls.callType,
|
|
41560
|
+
callItem: schema.sessionPendingToolCalls.callItem,
|
|
41561
|
+
callItemCodecVersion: schema.sessionPendingToolCalls.callItemCodecVersion,
|
|
41562
|
+
modelToolOutputTruncationTokens:
|
|
41563
|
+
schema.sessionPendingToolCalls.modelToolOutputTruncationTokens,
|
|
41564
|
+
})
|
|
41565
|
+
.from(schema.sessionPendingToolCalls)
|
|
41566
|
+
.where(
|
|
41567
|
+
and(
|
|
41568
|
+
eq(schema.sessionPendingToolCalls.workspaceId, input.workspaceId),
|
|
41569
|
+
eq(schema.sessionPendingToolCalls.sessionId, input.sessionId),
|
|
41570
|
+
eq(schema.sessionPendingToolCalls.turnId, input.turnId),
|
|
41571
|
+
eq(schema.sessionPendingToolCalls.callId, input.callId),
|
|
41572
|
+
),
|
|
41573
|
+
)
|
|
41574
|
+
.for("update")
|
|
41575
|
+
.limit(1);
|
|
41576
|
+
if (
|
|
41577
|
+
!pending ||
|
|
41578
|
+
pending.accountId !== input.accountId ||
|
|
41579
|
+
pending.callType !== input.callType ||
|
|
41580
|
+
!pendingToolCallItemsEqual(
|
|
41581
|
+
fromPostgresLosslessJson(pending.callItem, pending.callItemCodecVersion),
|
|
41582
|
+
input.callItem,
|
|
41583
|
+
)
|
|
41584
|
+
) {
|
|
41585
|
+
// A duplicate acknowledges only this exact call. It neither
|
|
41586
|
+
// replaces the originating attempt nor authorizes effect replay.
|
|
41587
|
+
throw new Error("Pending tool receipt conflicts with the registered call");
|
|
41588
|
+
}
|
|
41589
|
+
assertPendingToolOutputPolicyMatches(
|
|
41590
|
+
pending.modelToolOutputTruncationTokens,
|
|
41591
|
+
input.modelToolOutputTruncationTokens,
|
|
41592
|
+
input.callId,
|
|
41593
|
+
);
|
|
41594
|
+
if (
|
|
41595
|
+
pending.modelToolOutputTruncationTokens === null &&
|
|
41596
|
+
input.modelToolOutputTruncationTokens !== undefined
|
|
41597
|
+
) {
|
|
41598
|
+
await tx
|
|
41599
|
+
.update(schema.sessionPendingToolCalls)
|
|
41600
|
+
.set({
|
|
41601
|
+
modelToolOutputTruncationTokens: input.modelToolOutputTruncationTokens,
|
|
41602
|
+
})
|
|
41603
|
+
.where(eq(schema.sessionPendingToolCalls.id, pending.id));
|
|
41604
|
+
}
|
|
41605
|
+
}
|
|
41606
|
+
return { accepted: true, registered: inserted.length === 1 };
|
|
41607
|
+
}),
|
|
41608
|
+
);
|
|
41609
|
+
} catch (error) {
|
|
41610
|
+
if (!isRetryableDatabaseTransportFailure(error)) throw error;
|
|
41611
|
+
// Stop before the shared SQLSTATE traversal can prefer an older nested
|
|
41612
|
+
// rollback over a transport/commit-ack failure. This private outcome is
|
|
41613
|
+
// converted to a sanitized error outside the retry helper, never success.
|
|
41614
|
+
return { transportFailure: error, attempt };
|
|
41615
|
+
}
|
|
41616
|
+
};
|
|
41617
|
+
// Only PostgreSQL-confirmed rollback is retryable. Re-enter RLS and acquire
|
|
41618
|
+
// the complete attempt fence each time; never include inference or tools.
|
|
41619
|
+
const result = await runIdempotentPersistenceTransaction(
|
|
41620
|
+
{
|
|
41621
|
+
stage: "pending_tool_registration",
|
|
41622
|
+
correlationId,
|
|
41623
|
+
maxAttempts: 3,
|
|
41624
|
+
onRetry: async ({ attempt }) => {
|
|
41625
|
+
await new Promise((resolve) => setTimeout(resolve, 25 * attempt));
|
|
41626
|
+
},
|
|
41627
|
+
},
|
|
41628
|
+
registerOnce,
|
|
41451
41629
|
);
|
|
41630
|
+
if ("transportFailure" in result) {
|
|
41631
|
+
throw new SessionEventPersistenceError(
|
|
41632
|
+
{
|
|
41633
|
+
code: "db_failure",
|
|
41634
|
+
sqlState: null,
|
|
41635
|
+
stage: "pending_tool_registration",
|
|
41636
|
+
eventTypes: [],
|
|
41637
|
+
correlationId,
|
|
41638
|
+
attempts: result.attempt,
|
|
41639
|
+
retryOutcome: "not_retryable",
|
|
41640
|
+
database: safeDatabaseErrorFacts(result.transportFailure),
|
|
41641
|
+
},
|
|
41642
|
+
result.transportFailure,
|
|
41643
|
+
);
|
|
41644
|
+
}
|
|
41645
|
+
return result;
|
|
41452
41646
|
}
|
|
41453
41647
|
|
|
41454
41648
|
/** Record the raw SDK result without dropping the call receipt. */
|
|
@@ -44033,6 +44227,9 @@ export interface AcquireLeaseInput {
|
|
|
44033
44227
|
* attach, mirrored into the scoped stream token's claims. */
|
|
44034
44228
|
viewerAuthorityEpoch?: number;
|
|
44035
44229
|
backend: string; // sessions.sandbox_backend
|
|
44230
|
+
/** Explicit compute policy at this admission boundary. Omission is legacy
|
|
44231
|
+
* usage-only; only a priced credits backend needs prepaid admission. */
|
|
44232
|
+
warmBilling?: { mode: "usage_only" | "shadow" | "credits"; rateMicrosPerSecond: number };
|
|
44036
44233
|
os?: string; // default 'linux'
|
|
44037
44234
|
// The container image this run resolves (Modal image ref / docker image). Stamped on
|
|
44038
44235
|
// the cold-create + folded onto a warming/CAS; a warm/draining/warming box already
|
|
@@ -44454,6 +44651,94 @@ export class SandboxViewerAdmissionBlockedError extends Error {
|
|
|
44454
44651
|
}
|
|
44455
44652
|
}
|
|
44456
44653
|
|
|
44654
|
+
export class SandboxPaidComputeAdmissionError extends Error {
|
|
44655
|
+
readonly name = "SandboxPaidComputeAdmissionError";
|
|
44656
|
+
constructor(
|
|
44657
|
+
public readonly workspaceId: string,
|
|
44658
|
+
public readonly sandboxGroupId: string,
|
|
44659
|
+
) {
|
|
44660
|
+
super("Insufficient OpenGeni credits to admit paid sandbox compute");
|
|
44661
|
+
}
|
|
44662
|
+
}
|
|
44663
|
+
|
|
44664
|
+
type SandboxWarmBillingSnapshot = {
|
|
44665
|
+
mode: "usage_only" | "shadow" | "credits";
|
|
44666
|
+
rateMicrosPerSecond: number;
|
|
44667
|
+
accountId: string;
|
|
44668
|
+
workspaceId: string;
|
|
44669
|
+
stopChargeAt?: string;
|
|
44670
|
+
};
|
|
44671
|
+
|
|
44672
|
+
function warmBillingSnapshot(
|
|
44673
|
+
row: Pick<LeaseRow, "resume_state" | "account_id" | "workspace_id">,
|
|
44674
|
+
): SandboxWarmBillingSnapshot | null {
|
|
44675
|
+
const value = row.resume_state?.opengeniWarmBilling;
|
|
44676
|
+
if (!value || typeof value !== "object") return null;
|
|
44677
|
+
const snapshot = value as SandboxWarmBillingSnapshot;
|
|
44678
|
+
return (snapshot.mode === "usage_only" ||
|
|
44679
|
+
snapshot.mode === "shadow" ||
|
|
44680
|
+
snapshot.mode === "credits") &&
|
|
44681
|
+
Number.isFinite(snapshot.rateMicrosPerSecond) &&
|
|
44682
|
+
snapshot.rateMicrosPerSecond >= 0 &&
|
|
44683
|
+
(snapshot.stopChargeAt === undefined ||
|
|
44684
|
+
(typeof snapshot.stopChargeAt === "string" &&
|
|
44685
|
+
Number.isFinite(Date.parse(snapshot.stopChargeAt)))) &&
|
|
44686
|
+
snapshot.accountId === row.account_id &&
|
|
44687
|
+
snapshot.workspaceId === row.workspace_id
|
|
44688
|
+
? snapshot
|
|
44689
|
+
: null;
|
|
44690
|
+
}
|
|
44691
|
+
|
|
44692
|
+
/**
|
|
44693
|
+
* Fence the warm-time meter BEFORE requesting provider stop, including free
|
|
44694
|
+
* and shadow usage. The immutable cutoff survives a stop-success/settlement-
|
|
44695
|
+
* failure retry. A failed stop can undercount an idle retry, never count time
|
|
44696
|
+
* after the provider stopped.
|
|
44697
|
+
*/
|
|
44698
|
+
export async function markWarmBillingStopCutoff(
|
|
44699
|
+
db: Database,
|
|
44700
|
+
input: { accountId: string; workspaceId: string; sandboxGroupId: string; expectedEpoch: number },
|
|
44701
|
+
): Promise<Date | null> {
|
|
44702
|
+
return withRlsContext(
|
|
44703
|
+
db,
|
|
44704
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
44705
|
+
async (scopedDb) =>
|
|
44706
|
+
scopedDb.transaction(async (txRaw) => {
|
|
44707
|
+
const tx = txRaw as unknown as Database;
|
|
44708
|
+
const [row] = await tx.execute<LeaseRow>(sql`
|
|
44709
|
+
select * from sandbox_leases where workspace_id = ${input.workspaceId}
|
|
44710
|
+
and sandbox_group_id = ${input.sandboxGroupId} for update
|
|
44711
|
+
`);
|
|
44712
|
+
if (
|
|
44713
|
+
!row ||
|
|
44714
|
+
Number(row.lease_epoch) !== input.expectedEpoch ||
|
|
44715
|
+
row.liveness !== "draining" ||
|
|
44716
|
+
(Number(row.refcount) !== 0 && !row.unobservable_command_drain_ids?.length)
|
|
44717
|
+
)
|
|
44718
|
+
return null;
|
|
44719
|
+
const snapshot = warmBillingSnapshot(row);
|
|
44720
|
+
if (row.resume_state?.opengeniWarmBilling && !snapshot)
|
|
44721
|
+
throw new Error("sandbox warm billing snapshot is invalid");
|
|
44722
|
+
if (snapshot?.stopChargeAt) return new Date(snapshot.stopChargeAt);
|
|
44723
|
+
const [stamped] = await tx.execute<{ cutoff: string }>(sql`
|
|
44724
|
+
update sandbox_leases set
|
|
44725
|
+
resume_state = jsonb_set(coalesce(resume_state, '{}'::jsonb),
|
|
44726
|
+
'{opengeniWarmBilling}',
|
|
44727
|
+
coalesce(resume_state->'opengeniWarmBilling',
|
|
44728
|
+
jsonb_build_object('mode', 'usage_only', 'rateMicrosPerSecond', 0,
|
|
44729
|
+
'accountId', account_id, 'workspaceId', workspace_id)) ||
|
|
44730
|
+
jsonb_build_object('stopChargeAt', clock_timestamp()), true),
|
|
44731
|
+
updated_at = now()
|
|
44732
|
+
where id = ${row.id}
|
|
44733
|
+
returning resume_state #>> '{opengeniWarmBilling,stopChargeAt}' as cutoff
|
|
44734
|
+
`);
|
|
44735
|
+
if (!stamped || !Number.isFinite(Date.parse(stamped.cutoff)))
|
|
44736
|
+
throw new Error("sandbox warm billing stop cutoff was not saved");
|
|
44737
|
+
return new Date(stamped.cutoff);
|
|
44738
|
+
}),
|
|
44739
|
+
);
|
|
44740
|
+
}
|
|
44741
|
+
|
|
44457
44742
|
export class SandboxLeaseRecoveryBlockedError extends Error {
|
|
44458
44743
|
readonly name = "SandboxLeaseRecoveryBlockedError";
|
|
44459
44744
|
|
|
@@ -45207,6 +45492,10 @@ async function acquireLeaseOnce(
|
|
|
45207
45492
|
const os = input.os ?? "linux";
|
|
45208
45493
|
const subjectId = input.subjectId ?? null;
|
|
45209
45494
|
const warmingLeaseTtlMs = input.warmingLeaseTtlMs ?? input.leaseTtlMs;
|
|
45495
|
+
const warmPolicy = input.warmBilling ?? { mode: "usage_only" as const, rateMicrosPerSecond: 0 };
|
|
45496
|
+
if (!Number.isFinite(warmPolicy.rateMicrosPerSecond) || warmPolicy.rateMicrosPerSecond < 0) {
|
|
45497
|
+
throw new Error("sandbox warm rate must be finite and non-negative");
|
|
45498
|
+
}
|
|
45210
45499
|
return await withRlsContext(
|
|
45211
45500
|
db,
|
|
45212
45501
|
{ accountId, workspaceId },
|
|
@@ -45254,6 +45543,41 @@ async function acquireLeaseOnce(
|
|
|
45254
45543
|
if (!row) throw new Error(`Lease row vanished post-insert: ${sandboxGroupId}`);
|
|
45255
45544
|
|
|
45256
45545
|
const liveness = row.liveness;
|
|
45546
|
+
const existingSnapshot = warmBillingSnapshot(row);
|
|
45547
|
+
if (row.resume_state?.opengeniWarmBilling && !existingSnapshot) {
|
|
45548
|
+
throw new Error("sandbox warm billing snapshot is invalid");
|
|
45549
|
+
}
|
|
45550
|
+
|
|
45551
|
+
// Every new paid holder, including direct operations and interaction
|
|
45552
|
+
// controllers, passes the same account funding boundary before it can
|
|
45553
|
+
// elect a spawner, re-arm a drain or prolong a warm lease. Replaying an
|
|
45554
|
+
// existing exact holder does not create new compute authority.
|
|
45555
|
+
const activeMode = input.warmBilling?.mode ?? existingSnapshot?.mode ?? "usage_only";
|
|
45556
|
+
// Reconfiguration may not substitute a new (possibly zero) rate for
|
|
45557
|
+
// an already-running paid provider's immutable admitted tariff.
|
|
45558
|
+
const activeRate =
|
|
45559
|
+
liveness !== "cold" && existingSnapshot?.mode === "credits"
|
|
45560
|
+
? existingSnapshot.rateMicrosPerSecond
|
|
45561
|
+
: (input.warmBilling?.rateMicrosPerSecond ??
|
|
45562
|
+
existingSnapshot?.rateMicrosPerSecond ??
|
|
45563
|
+
0);
|
|
45564
|
+
if (
|
|
45565
|
+
activeMode === "credits" &&
|
|
45566
|
+
activeRate > 0 &&
|
|
45567
|
+
(liveness === "cold" ||
|
|
45568
|
+
(existingSnapshot?.mode === "credits" && existingSnapshot.rateMicrosPerSecond > 0))
|
|
45569
|
+
) {
|
|
45570
|
+
const alreadyHeld = await tx.execute<{ held: boolean }>(sql`
|
|
45571
|
+
select exists (select 1 from sandbox_lease_holders
|
|
45572
|
+
where lease_id = ${row.id} and kind = ${kind} and holder_id = ${holderId}) as held
|
|
45573
|
+
`);
|
|
45574
|
+
if (!alreadyHeld[0]?.held) {
|
|
45575
|
+
const balance = await getBillingBalance(tx, accountId);
|
|
45576
|
+
if (balance.balanceMicros <= 0) {
|
|
45577
|
+
throw new SandboxPaidComputeAdmissionError(workspaceId, sandboxGroupId);
|
|
45578
|
+
}
|
|
45579
|
+
}
|
|
45580
|
+
}
|
|
45257
45581
|
|
|
45258
45582
|
// A workspace-wide billing/cap drain survives this lease becoming cold.
|
|
45259
45583
|
// Viewer-only admission must consult it while holding the exact lease
|
|
@@ -45270,7 +45594,7 @@ async function acquireLeaseOnce(
|
|
|
45270
45594
|
where id = ${workspaceId}
|
|
45271
45595
|
`);
|
|
45272
45596
|
const reason = workspaceRows[0]?.sandbox_viewer_force_drain_reason ?? null;
|
|
45273
|
-
if (reason
|
|
45597
|
+
if (reason === "warm_cap") {
|
|
45274
45598
|
throw new SandboxViewerAdmissionBlockedError(workspaceId, sandboxGroupId, reason);
|
|
45275
45599
|
}
|
|
45276
45600
|
}
|
|
@@ -45387,6 +45711,17 @@ async function acquireLeaseOnce(
|
|
|
45387
45711
|
// The row lock serializes this holder insertion against the reaper claim:
|
|
45388
45712
|
// whichever wins first owns the next state without an availability gap.
|
|
45389
45713
|
if (liveness === "draining") {
|
|
45714
|
+
// Capture may have failed after an earlier stop attempt recorded a
|
|
45715
|
+
// cutoff. This provider was never stopped; a re-armed box must have
|
|
45716
|
+
// a fresh charge horizon when its next drain reaches provider stop.
|
|
45717
|
+
if (warmBillingSnapshot(row)?.stopChargeAt) {
|
|
45718
|
+
await tx.execute(sql`
|
|
45719
|
+
update sandbox_leases
|
|
45720
|
+
set resume_state = resume_state #- '{opengeniWarmBilling,stopChargeAt}',
|
|
45721
|
+
updated_at = now()
|
|
45722
|
+
where id = ${row.id}
|
|
45723
|
+
`);
|
|
45724
|
+
}
|
|
45390
45725
|
await upsertLeaseHolder(tx, row.id, accountId, workspaceId, kind, holderId, subjectId, {
|
|
45391
45726
|
subjectId: input.viewerSubjectId ?? null,
|
|
45392
45727
|
authorityEpoch: input.viewerAuthorityEpoch ?? null,
|
|
@@ -45418,6 +45753,17 @@ async function acquireLeaseOnce(
|
|
|
45418
45753
|
const casRows = await tx.execute<{ id: string }>(sql`
|
|
45419
45754
|
update sandbox_leases set
|
|
45420
45755
|
liveness = 'warming',
|
|
45756
|
+
${
|
|
45757
|
+
input.warmBilling
|
|
45758
|
+
? sql`resume_state = coalesce(resume_state, '{}'::jsonb) ||
|
|
45759
|
+
jsonb_build_object('opengeniWarmBilling', ${JSON.stringify({
|
|
45760
|
+
mode: warmPolicy.mode,
|
|
45761
|
+
rateMicrosPerSecond: warmPolicy.rateMicrosPerSecond,
|
|
45762
|
+
accountId,
|
|
45763
|
+
workspaceId,
|
|
45764
|
+
})}::jsonb),`
|
|
45765
|
+
: sql``
|
|
45766
|
+
}
|
|
45421
45767
|
${image !== null ? sql`image = ${image},` : sql``}
|
|
45422
45768
|
${rigVersionId !== null ? sql`rig_version_id = ${rigVersionId},` : sql``}
|
|
45423
45769
|
updated_at = now()
|
|
@@ -46921,7 +47267,13 @@ export async function commitWarmingToWarm(
|
|
|
46921
47267
|
};
|
|
46922
47268
|
}
|
|
46923
47269
|
|
|
46924
|
-
|
|
47270
|
+
// Warm-time billing and stop cutoffs share the database clock, even
|
|
47271
|
+
// when the worker host's wall clock differs from PostgreSQL's.
|
|
47272
|
+
const [warmClock] = await tx.execute<{ completed_at: Date | string }>(sql`
|
|
47273
|
+
select clock_timestamp() as completed_at
|
|
47274
|
+
`);
|
|
47275
|
+
if (!warmClock) throw new Error("sandbox warm-transition clock unavailable");
|
|
47276
|
+
const completedAt = new Date(warmClock.completed_at).toISOString();
|
|
46925
47277
|
const recovery: SandboxRecoveryState = rematerialization
|
|
46926
47278
|
? {
|
|
46927
47279
|
provider: {
|
|
@@ -46967,7 +47319,17 @@ export async function commitWarmingToWarm(
|
|
|
46967
47319
|
input.resumeState ?? null,
|
|
46968
47320
|
row.resume_state,
|
|
46969
47321
|
);
|
|
46970
|
-
const resumeStateJson = JSON.stringify(
|
|
47322
|
+
const resumeStateJson = JSON.stringify(
|
|
47323
|
+
resumeStateWithRecovery(
|
|
47324
|
+
{
|
|
47325
|
+
...(withArchives ?? {}),
|
|
47326
|
+
...(row.resume_state?.opengeniWarmBilling
|
|
47327
|
+
? { opengeniWarmBilling: row.resume_state.opengeniWarmBilling }
|
|
47328
|
+
: {}),
|
|
47329
|
+
},
|
|
47330
|
+
recovery,
|
|
47331
|
+
),
|
|
47332
|
+
);
|
|
46971
47333
|
const updated = await tx.execute<LeaseRow>(sql`
|
|
46972
47334
|
update sandbox_leases set
|
|
46973
47335
|
liveness = 'warm',
|
|
@@ -47094,7 +47456,17 @@ export async function recordWarmingSandboxCreated(
|
|
|
47094
47456
|
input.resumeState ?? null,
|
|
47095
47457
|
row.resume_state,
|
|
47096
47458
|
);
|
|
47097
|
-
const resumeStateJson = JSON.stringify(
|
|
47459
|
+
const resumeStateJson = JSON.stringify(
|
|
47460
|
+
resumeStateWithRecovery(
|
|
47461
|
+
{
|
|
47462
|
+
...(withArchives ?? {}),
|
|
47463
|
+
...(row.resume_state?.opengeniWarmBilling
|
|
47464
|
+
? { opengeniWarmBilling: row.resume_state.opengeniWarmBilling }
|
|
47465
|
+
: {}),
|
|
47466
|
+
},
|
|
47467
|
+
recovery,
|
|
47468
|
+
),
|
|
47469
|
+
);
|
|
47098
47470
|
const warmingTtlMs = input.warmingLeaseTtlMs ?? input.leaseTtlMs;
|
|
47099
47471
|
const providerCreatedAt = input.providerCreatedAt ?? null;
|
|
47100
47472
|
const providerDeadlineAt = input.providerDeadlineAt ?? null;
|
|
@@ -49559,7 +49931,7 @@ export type LeaseHolderHeartbeatStatus =
|
|
|
49559
49931
|
| {
|
|
49560
49932
|
holderAlive: true;
|
|
49561
49933
|
leaseExtended: false;
|
|
49562
|
-
fence: "epoch" | "liveness" | "rotation_requested";
|
|
49934
|
+
fence: "epoch" | "liveness" | "rotation_requested" | "funding";
|
|
49563
49935
|
};
|
|
49564
49936
|
|
|
49565
49937
|
export async function heartbeatLeaseHolderStatus(
|
|
@@ -49572,6 +49944,7 @@ export async function heartbeatLeaseHolderStatus(
|
|
|
49572
49944
|
holderId: string;
|
|
49573
49945
|
leaseTtlMs: number;
|
|
49574
49946
|
expectedEpoch: number;
|
|
49947
|
+
billingMode?: "usage_only" | "shadow" | "credits";
|
|
49575
49948
|
},
|
|
49576
49949
|
): Promise<LeaseHolderHeartbeatStatus> {
|
|
49577
49950
|
if (input.kind === "process") {
|
|
@@ -49592,11 +49965,14 @@ export async function heartbeatLeaseHolderStatus(
|
|
|
49592
49965
|
// against stale-holder deletion.
|
|
49593
49966
|
const leases = await tx.execute<{
|
|
49594
49967
|
id: string;
|
|
49968
|
+
account_id: string;
|
|
49969
|
+
workspace_id: string;
|
|
49970
|
+
resume_state: Record<string, unknown> | null;
|
|
49595
49971
|
lease_epoch: number | string;
|
|
49596
49972
|
liveness: SandboxLeaseLiveness;
|
|
49597
49973
|
rotation_requested_at: Date | string | null;
|
|
49598
49974
|
}>(sql`
|
|
49599
|
-
select id, lease_epoch, liveness, rotation_requested_at from sandbox_leases
|
|
49975
|
+
select id, account_id, workspace_id, resume_state, lease_epoch, liveness, rotation_requested_at from sandbox_leases
|
|
49600
49976
|
where workspace_id = ${input.workspaceId}
|
|
49601
49977
|
and sandbox_group_id = ${input.sandboxGroupId}
|
|
49602
49978
|
for update
|
|
@@ -49629,6 +50005,28 @@ export async function heartbeatLeaseHolderStatus(
|
|
|
49629
50005
|
fence: "holder_gone",
|
|
49630
50006
|
};
|
|
49631
50007
|
}
|
|
50008
|
+
const snapshot = warmBillingSnapshot(lease);
|
|
50009
|
+
if (lease.resume_state?.opengeniWarmBilling && !snapshot) {
|
|
50010
|
+
throw new Error("sandbox warm billing snapshot is invalid");
|
|
50011
|
+
}
|
|
50012
|
+
// An exhausted balance must not mask an older epoch or a drain/rotation
|
|
50013
|
+
// that requires the client to detach and negotiate a new viewer.
|
|
50014
|
+
if (Number(lease.lease_epoch) !== input.expectedEpoch)
|
|
50015
|
+
return { holderAlive: true, leaseExtended: false, fence: "epoch" };
|
|
50016
|
+
if (lease.liveness !== "warm" && lease.liveness !== "warming")
|
|
50017
|
+
return { holderAlive: true, leaseExtended: false, fence: "liveness" };
|
|
50018
|
+
if (lease.rotation_requested_at !== null)
|
|
50019
|
+
return { holderAlive: true, leaseExtended: false, fence: "rotation_requested" };
|
|
50020
|
+
if (
|
|
50021
|
+
(input.billingMode ?? snapshot?.mode) === "credits" &&
|
|
50022
|
+
snapshot?.mode === "credits" &&
|
|
50023
|
+
snapshot.rateMicrosPerSecond > 0 &&
|
|
50024
|
+
(await getBillingBalance(tx, input.accountId)).balanceMicros <= 0
|
|
50025
|
+
) {
|
|
50026
|
+
// Touching the holder above is essential: the running writer remains
|
|
50027
|
+
// protected while it winds down. The provider/lease TTL is not renewed.
|
|
50028
|
+
return { holderAlive: true, leaseExtended: false, fence: "funding" };
|
|
50029
|
+
}
|
|
49632
50030
|
// Recheck the same fence in the write even though its row lock is held,
|
|
49633
50031
|
// keeping the mutation independently auditable and future-proof.
|
|
49634
50032
|
const leaseRows = await tx.execute<{ id: string }>(sql`
|
|
@@ -49671,6 +50069,7 @@ export async function heartbeatLeaseHolder(
|
|
|
49671
50069
|
holderId: string;
|
|
49672
50070
|
leaseTtlMs: number;
|
|
49673
50071
|
expectedEpoch: number;
|
|
50072
|
+
billingMode?: "usage_only" | "shadow" | "credits";
|
|
49674
50073
|
},
|
|
49675
50074
|
): Promise<boolean> {
|
|
49676
50075
|
return (await heartbeatLeaseHolderStatus(db, input)).leaseExtended;
|
|
@@ -50300,6 +50699,13 @@ export async function enrollUnobservableCommandIdleDrain(
|
|
|
50300
50699
|
'quarantined_provider_binding_mismatch')
|
|
50301
50700
|
`);
|
|
50302
50701
|
}
|
|
50702
|
+
// A deadline gives a legacy command two minutes after cancellation intent
|
|
50703
|
+
// to exit. Even when the provider cannot signal or observe that command,
|
|
50704
|
+
// capture its current files while the old box still exists. Ordinary idle
|
|
50705
|
+
// containment retains its longer grace.
|
|
50706
|
+
const deadlineStopGraceMs = SANDBOX_DEADLINE_COMMAND_STOP_GRACE_MS;
|
|
50707
|
+
const deadlineRotation =
|
|
50708
|
+
initial.rotationReason === "provider_deadline" && initial.rotationRequestedAt !== null;
|
|
50303
50709
|
// Preserve process -> admission -> lease ordering used by settlement.
|
|
50304
50710
|
const processes = await rawRows<{
|
|
50305
50711
|
id: string;
|
|
@@ -50333,10 +50739,43 @@ export async function enrollUnobservableCommandIdleDrain(
|
|
|
50333
50739
|
(${input.idleGraceMs}::bigint * interval '1 millisecond')
|
|
50334
50740
|
)
|
|
50335
50741
|
)
|
|
50742
|
+
or (
|
|
50743
|
+
${deadlineRotation}
|
|
50744
|
+
and process.deadline_cancellation_requested_at < now() -
|
|
50745
|
+
(${deadlineStopGraceMs}::bigint * interval '1 millisecond')
|
|
50746
|
+
and process.reconcile_attempts >= 1
|
|
50747
|
+
and (
|
|
50748
|
+
(process.owner_actor_kind = 'turn' and attempt.state = 'closed'
|
|
50749
|
+
and attempt.quiesced_at is not null
|
|
50750
|
+
and greatest(attempt.quiesced_at, process.started_at) < now() -
|
|
50751
|
+
(${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
|
|
50752
|
+
or (process.owner_actor_kind = 'direct' and process.owner_attempt_id is null
|
|
50753
|
+
and process.started_at < now() -
|
|
50754
|
+
(${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
|
|
50755
|
+
)
|
|
50756
|
+
)
|
|
50336
50757
|
)
|
|
50337
|
-
and attempt.state = 'closed'
|
|
50338
|
-
|
|
50339
|
-
|
|
50758
|
+
and (attempt.state = 'closed' or (
|
|
50759
|
+
${deadlineRotation}
|
|
50760
|
+
and process.owner_actor_kind = 'direct' and process.owner_attempt_id is null))
|
|
50761
|
+
and (
|
|
50762
|
+
(attempt.state = 'closed' and
|
|
50763
|
+
greatest(coalesce(attempt.quiesced_at, attempt.closed_at, attempt.updated_at), process.started_at) < now() -
|
|
50764
|
+
(${input.idleGraceMs}::bigint * interval '1 millisecond'))
|
|
50765
|
+
or (
|
|
50766
|
+
${deadlineRotation}
|
|
50767
|
+
and process.deadline_cancellation_requested_at < now() -
|
|
50768
|
+
(${deadlineStopGraceMs}::bigint * interval '1 millisecond')
|
|
50769
|
+
and (
|
|
50770
|
+
(process.owner_actor_kind = 'turn' and attempt.quiesced_at is not null
|
|
50771
|
+
and greatest(attempt.quiesced_at, process.started_at) < now() -
|
|
50772
|
+
(${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
|
|
50773
|
+
or (process.owner_actor_kind = 'direct' and process.owner_attempt_id is null
|
|
50774
|
+
and process.started_at < now() -
|
|
50775
|
+
(${deadlineStopGraceMs}::bigint * interval '1 millisecond'))
|
|
50776
|
+
)
|
|
50777
|
+
)
|
|
50778
|
+
)) as eligible
|
|
50340
50779
|
from sandbox_retained_processes process
|
|
50341
50780
|
left join session_turn_attempts attempt on attempt.id = process.owner_attempt_id
|
|
50342
50781
|
and attempt.workspace_id = process.workspace_id and attempt.session_id = process.session_id
|
|
@@ -50387,7 +50826,13 @@ export async function enrollUnobservableCommandIdleDrain(
|
|
|
50387
50826
|
)
|
|
50388
50827
|
return null;
|
|
50389
50828
|
if (!enrolled) {
|
|
50390
|
-
if (
|
|
50829
|
+
if (
|
|
50830
|
+
lease.archive_capture_id !== null ||
|
|
50831
|
+
(await hasSandboxGroupAttemptActivityTx(tx, {
|
|
50832
|
+
...input,
|
|
50833
|
+
idleGraceMs: deadlineRotation ? deadlineStopGraceMs : input.idleGraceMs,
|
|
50834
|
+
}))
|
|
50835
|
+
)
|
|
50391
50836
|
return null;
|
|
50392
50837
|
await tx.execute(sql`
|
|
50393
50838
|
update sandbox_leases set unobservable_command_drain_ids = ${`{${ids.join(",")}}`}::uuid[],
|
|
@@ -50699,15 +51144,40 @@ export async function reArmDrainingLease(
|
|
|
50699
51144
|
workspaceId: string;
|
|
50700
51145
|
sandboxGroupId: string;
|
|
50701
51146
|
leaseTtlMs: number;
|
|
51147
|
+
warmBilling?: AcquireLeaseInput["warmBilling"];
|
|
50702
51148
|
},
|
|
50703
51149
|
): Promise<{ rearmed: boolean }> {
|
|
50704
51150
|
return await withRlsContext(
|
|
50705
51151
|
db,
|
|
50706
51152
|
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
50707
|
-
async (scopedDb) =>
|
|
50708
|
-
|
|
51153
|
+
async (scopedDb) =>
|
|
51154
|
+
await scopedDb.transaction(async (txRaw) => {
|
|
51155
|
+
const tx = txRaw as unknown as Database;
|
|
51156
|
+
const [lease] = await tx.execute<LeaseRow>(sql`
|
|
51157
|
+
select * from sandbox_leases
|
|
51158
|
+
where workspace_id = ${input.workspaceId} and sandbox_group_id = ${input.sandboxGroupId}
|
|
51159
|
+
for update
|
|
51160
|
+
`);
|
|
51161
|
+
if (!lease || lease.liveness !== "draining") return { rearmed: false };
|
|
51162
|
+
const snapshot = warmBillingSnapshot(lease);
|
|
51163
|
+
if (lease.resume_state?.opengeniWarmBilling && !snapshot) {
|
|
51164
|
+
throw new Error("sandbox warm billing snapshot is invalid");
|
|
51165
|
+
}
|
|
51166
|
+
const activeMode = input.warmBilling?.mode ?? snapshot?.mode ?? "usage_only";
|
|
51167
|
+
if (
|
|
51168
|
+
activeMode === "credits" &&
|
|
51169
|
+
(snapshot
|
|
51170
|
+
? snapshot.mode === "credits" && snapshot.rateMicrosPerSecond > 0
|
|
51171
|
+
: input.warmBilling?.rateMicrosPerSecond !== undefined &&
|
|
51172
|
+
input.warmBilling.rateMicrosPerSecond > 0) &&
|
|
51173
|
+
(await getBillingBalance(tx, input.accountId)).balanceMicros <= 0
|
|
51174
|
+
) {
|
|
51175
|
+
throw new SandboxPaidComputeAdmissionError(input.workspaceId, input.sandboxGroupId);
|
|
51176
|
+
}
|
|
51177
|
+
const rows = await tx.execute<{ id: string }>(sql`
|
|
50709
51178
|
update sandbox_leases set
|
|
50710
51179
|
liveness = 'warm',
|
|
51180
|
+
resume_state = resume_state #- '{opengeniWarmBilling,stopChargeAt}',
|
|
50711
51181
|
expires_at = now() + (${String(input.leaseTtlMs)} || ' milliseconds')::interval,
|
|
50712
51182
|
updated_at = now()
|
|
50713
51183
|
where workspace_id = ${input.workspaceId} and sandbox_group_id = ${input.sandboxGroupId}
|
|
@@ -50733,8 +51203,8 @@ export async function reArmDrainingLease(
|
|
|
50733
51203
|
)
|
|
50734
51204
|
returning id
|
|
50735
51205
|
`);
|
|
50736
|
-
|
|
50737
|
-
|
|
51206
|
+
return { rearmed: rows.length > 0 };
|
|
51207
|
+
}),
|
|
50738
51208
|
);
|
|
50739
51209
|
}
|
|
50740
51210
|
|
|
@@ -51106,6 +51576,9 @@ export type SandboxRetainedProcess = {
|
|
|
51106
51576
|
reconcileClaimedAt: string | null;
|
|
51107
51577
|
reconcileAttempts: number;
|
|
51108
51578
|
lastReconcileOutcome: string | null;
|
|
51579
|
+
cancellationRequestedAt: string | null;
|
|
51580
|
+
cancellationReason: string | null;
|
|
51581
|
+
deadlineCancellationRequestedAt: string | null;
|
|
51109
51582
|
reconcileProofOutcome: "exited" | "lost" | null;
|
|
51110
51583
|
reconcileProofExitCode: number | null;
|
|
51111
51584
|
reconcileProofReason:
|
|
@@ -51454,6 +51927,9 @@ function mapRetainedProcess(
|
|
|
51454
51927
|
reconcileClaimedAt: row.reconcileClaimedAt?.toISOString() ?? null,
|
|
51455
51928
|
reconcileAttempts: row.reconcileAttempts,
|
|
51456
51929
|
lastReconcileOutcome: row.lastReconcileOutcome ?? null,
|
|
51930
|
+
cancellationRequestedAt: row.cancellationRequestedAt?.toISOString() ?? null,
|
|
51931
|
+
cancellationReason: row.cancellationReason ?? null,
|
|
51932
|
+
deadlineCancellationRequestedAt: row.deadlineCancellationRequestedAt?.toISOString() ?? null,
|
|
51457
51933
|
reconcileProofOutcome: row.reconcileProofOutcome ?? null,
|
|
51458
51934
|
reconcileProofExitCode: row.reconcileProofExitCode ?? null,
|
|
51459
51935
|
reconcileProofReason:
|
|
@@ -60668,7 +61144,7 @@ export async function revokeViewer(
|
|
|
60668
61144
|
// re-dispatched/overlapping tick at the same (group,epoch,tick) can never
|
|
60669
61145
|
// double-charge (recordUsageEvent is onConflictDoNothing on idempotencyKey).
|
|
60670
61146
|
//
|
|
60671
|
-
// Cursor advance + usage insert are ATOMIC:
|
|
61147
|
+
// Cursor advance + usage insert + paid settlement are ATOMIC: all run inside ONE FOR UPDATE txn on
|
|
60672
61148
|
// the lease row (the M3 cross-statement-atomicity fix). The insert uses ON
|
|
60673
61149
|
// CONFLICT DO NOTHING on idempotency_key (matching recordUsageEvent), and the
|
|
60674
61150
|
// cursor (last_meter_at/last_meter_tick) is advanced in the SAME txn — so the tick
|
|
@@ -60684,23 +61160,18 @@ export interface AccrueWarmSecondsResult {
|
|
|
60684
61160
|
seconds: number;
|
|
60685
61161
|
/** The monotonic tick index this accrual was recorded under. */
|
|
60686
61162
|
tick: number;
|
|
60687
|
-
/** usd_micros
|
|
61163
|
+
/** Internal usd_micros estimate for this tick (0 in usage_only mode). */
|
|
60688
61164
|
costMicros: number;
|
|
60689
61165
|
}
|
|
60690
61166
|
|
|
60691
61167
|
/**
|
|
60692
61168
|
* Accrue warm-seconds for the elapsed wall-clock since the lease's last meter
|
|
60693
61169
|
* cursor, idempotent on (sandbox_group_id, lease_epoch, tick). EPOCH-FENCED +
|
|
60694
|
-
* liveness-guarded (warm
|
|
60695
|
-
*
|
|
60696
|
-
*
|
|
60697
|
-
*
|
|
60698
|
-
*
|
|
60699
|
-
* sandbox.warm_cost event (cost = seconds x rate) AND debits the same micros from
|
|
60700
|
-
* the credit balance via applyCreditDebitUpToBalance (the model-cost precedent),
|
|
60701
|
-
* idempotent on the SAME (group, epoch, tick) key. The usage event is the
|
|
60702
|
-
* REQUESTED cost; the ledger is the ACTUAL debit (they legitimately differ when
|
|
60703
|
-
* balance is low — M2). Set debitCredits:false to meter without debiting.
|
|
61170
|
+
* liveness-guarded (warm, or draining for a final provider-stop tick). The first
|
|
61171
|
+
* interval begins at the warm transition's recovery timestamp. A draining tick
|
|
61172
|
+
* is only allowed by the provider-stop path, never by the periodic reaper list.
|
|
61173
|
+
* In credits mode the signed full-cost ledger debit, both usage rows and cursor
|
|
61174
|
+
* commit together; a settlement failure rolls back the tick for retry.
|
|
60704
61175
|
*/
|
|
60705
61176
|
export async function accrueWarmSeconds(
|
|
60706
61177
|
db: Database,
|
|
@@ -60715,9 +61186,9 @@ export async function accrueWarmSeconds(
|
|
|
60715
61186
|
/** Optional attribution: the founding/observing session (visibility only — the
|
|
60716
61187
|
* group meter key makes the workspace charge correct regardless). */
|
|
60717
61188
|
subjectId?: string | null;
|
|
60718
|
-
|
|
60719
|
-
|
|
60720
|
-
|
|
61189
|
+
billingMode?: "usage_only" | "shadow" | "credits";
|
|
61190
|
+
/** Final provider-stop tick, after provider termination and before cold CAS. */
|
|
61191
|
+
finalDrain?: boolean;
|
|
60721
61192
|
},
|
|
60722
61193
|
): Promise<AccrueWarmSecondsResult> {
|
|
60723
61194
|
const none: AccrueWarmSecondsResult = {
|
|
@@ -60735,10 +61206,8 @@ export async function accrueWarmSeconds(
|
|
|
60735
61206
|
// Lock the group's lease row so the cursor advance + the usage insert are
|
|
60736
61207
|
// one atomic step (no other tick can interleave between the diff and the
|
|
60737
61208
|
// cursor write).
|
|
60738
|
-
const rows = await tx.execute<LeaseRow
|
|
60739
|
-
select
|
|
60740
|
-
case when last_meter_at is null then null
|
|
60741
|
-
else floor(extract(epoch from (now() - last_meter_at)))::int end as meter_elapsed_s
|
|
61209
|
+
const rows = await tx.execute<LeaseRow>(sql`
|
|
61210
|
+
select *
|
|
60742
61211
|
from sandbox_leases
|
|
60743
61212
|
where workspace_id = ${input.workspaceId} and sandbox_group_id = ${input.sandboxGroupId}
|
|
60744
61213
|
for update
|
|
@@ -60746,30 +61215,78 @@ export async function accrueWarmSeconds(
|
|
|
60746
61215
|
const row = rows[0];
|
|
60747
61216
|
if (!row) return none;
|
|
60748
61217
|
|
|
60749
|
-
//
|
|
60750
|
-
|
|
60751
|
-
|
|
61218
|
+
// An ordinary heartbeat must never charge a box already draining.
|
|
61219
|
+
if (
|
|
61220
|
+
Number(row.lease_epoch) !== input.expectedEpoch ||
|
|
61221
|
+
(row.liveness !== "warm" && !(input.finalDrain && row.liveness === "draining"))
|
|
61222
|
+
) {
|
|
60752
61223
|
return none;
|
|
60753
61224
|
}
|
|
60754
61225
|
|
|
60755
|
-
//
|
|
60756
|
-
|
|
61226
|
+
// commitWarmingToWarm stores this timestamp in the recovery envelope
|
|
61227
|
+
// before changing the lease to warm. The legacy cursor is nullable and
|
|
61228
|
+
// survives a new epoch: never start a new epoch at the previous box's
|
|
61229
|
+
// cursor. Refuse an invalid/missing transition timestamp instead of
|
|
61230
|
+
// charging an arbitrary cold/warming interval.
|
|
61231
|
+
const recovery = row.resume_state?.opengeniRecovery as
|
|
61232
|
+
| { restore?: { completedAt?: unknown } }
|
|
61233
|
+
| undefined;
|
|
61234
|
+
const warmStartedAt = recovery?.restore?.completedAt;
|
|
61235
|
+
const warmStartMs = typeof warmStartedAt === "string" ? Date.parse(warmStartedAt) : NaN;
|
|
61236
|
+
const snapshot = warmBillingSnapshot(row);
|
|
61237
|
+
if (row.resume_state?.opengeniWarmBilling && !snapshot) {
|
|
61238
|
+
throw new Error("sandbox warm billing snapshot is invalid");
|
|
61239
|
+
}
|
|
61240
|
+
// A current usage_only/shadow policy is a rollback switch for a paid
|
|
61241
|
+
// lease. In credits mode, only a box explicitly snapshotted as paid may
|
|
61242
|
+
// debit; pre-activation legacy boxes cannot acquire surprise charges.
|
|
61243
|
+
const mode =
|
|
61244
|
+
input.billingMode === "credits"
|
|
61245
|
+
? (snapshot?.mode ?? "usage_only")
|
|
61246
|
+
: (input.billingMode ?? snapshot?.mode ?? "usage_only");
|
|
61247
|
+
const rate = snapshot?.rateMicrosPerSecond ?? input.warmRateMicrosPerSecond;
|
|
61248
|
+
if (!Number.isFinite(warmStartMs) && mode === "credits" && rate > 0) {
|
|
61249
|
+
throw new Error("paid warm lease has no valid warm-transition timestamp");
|
|
61250
|
+
}
|
|
61251
|
+
if (!Number.isFinite(warmStartMs) && row.last_meter_at == null) {
|
|
61252
|
+
// Legacy unpriced leases cannot prove their first warm instant. Seed
|
|
61253
|
+
// only these rows; new warm transitions carry an exact recovery clock.
|
|
60757
61254
|
await tx.execute(sql`
|
|
60758
|
-
|
|
60759
|
-
|
|
60760
|
-
|
|
61255
|
+
update sandbox_leases set last_meter_at = now(), updated_at = now()
|
|
61256
|
+
where id = ${row.id}
|
|
61257
|
+
`);
|
|
60761
61258
|
return none;
|
|
60762
61259
|
}
|
|
61260
|
+
const cursorMs = row.last_meter_at ? new Date(row.last_meter_at).getTime() : 0;
|
|
61261
|
+
const startMs = Number.isFinite(warmStartMs) ? Math.max(cursorMs, warmStartMs) : cursorMs;
|
|
61262
|
+
if (input.finalDrain && !snapshot?.stopChargeAt) {
|
|
61263
|
+
throw new Error("sandbox final tick has no durable stop cutoff");
|
|
61264
|
+
}
|
|
61265
|
+
const upperBound =
|
|
61266
|
+
snapshot?.stopChargeAt && input.finalDrain
|
|
61267
|
+
? sql`${snapshot.stopChargeAt}::timestamptz`
|
|
61268
|
+
: sql`clock_timestamp()`;
|
|
61269
|
+
const elapsedRows = await tx.execute<{ elapsed_ms: number | string }>(sql`
|
|
61270
|
+
select greatest(0, floor(extract(epoch from (${upperBound} -
|
|
61271
|
+
${new Date(startMs).toISOString()}::timestamptz)) * 1000))::bigint as elapsed_ms
|
|
61272
|
+
`);
|
|
61273
|
+
const elapsedMs = Number(elapsedRows[0]?.elapsed_ms ?? 0);
|
|
61274
|
+
const elapsedS = Math.floor(elapsedMs / 1000);
|
|
60763
61275
|
|
|
60764
|
-
|
|
60765
|
-
if (elapsedS <= 0) {
|
|
61276
|
+
if (elapsedS <= 0 && !(input.finalDrain && elapsedMs > 0)) {
|
|
60766
61277
|
// No whole second elapsed yet — leave the cursor untouched so the
|
|
60767
61278
|
// remainder accrues on the next tick (no silent seconds loss).
|
|
60768
61279
|
return none;
|
|
60769
61280
|
}
|
|
60770
61281
|
|
|
60771
61282
|
const tick = Number(row.last_meter_tick) + 1;
|
|
60772
|
-
const costMicros =
|
|
61283
|
+
const costMicros =
|
|
61284
|
+
mode === "usage_only"
|
|
61285
|
+
? 0
|
|
61286
|
+
: Math.round((input.finalDrain ? elapsedMs / 1000 : elapsedS) * rate);
|
|
61287
|
+
if (!Number.isSafeInteger(costMicros) || costMicros < 0) {
|
|
61288
|
+
throw new Error("warm-cost estimate exceeds safe integer micros");
|
|
61289
|
+
}
|
|
60773
61290
|
|
|
60774
61291
|
// (1) The warm-seconds meter — GROUP+epoch+tick keyed, ON CONFLICT DO
|
|
60775
61292
|
// NOTHING (the idempotency that makes a shared box one stream + a re-fire a
|
|
@@ -60777,19 +61294,22 @@ export async function accrueWarmSeconds(
|
|
|
60777
61294
|
await tx.execute(sql`
|
|
60778
61295
|
insert into usage_events
|
|
60779
61296
|
(account_id, workspace_id, subject_id, event_type, quantity, unit,
|
|
60780
|
-
source_resource_type, source_resource_id, idempotency_key, occurred_at)
|
|
61297
|
+
source_resource_type, source_resource_id, idempotency_key, initiator_context, occurred_at)
|
|
60781
61298
|
values
|
|
60782
61299
|
(${input.accountId}, ${input.workspaceId}, ${input.subjectId ?? null},
|
|
60783
61300
|
'sandbox.warm_seconds', ${elapsedS}, 'seconds',
|
|
60784
61301
|
'sandbox_lease', ${`${input.sandboxGroupId}:${input.expectedEpoch}`},
|
|
60785
61302
|
${`usage:sandbox.warm_seconds:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`},
|
|
60786
|
-
now())
|
|
61303
|
+
${JSON.stringify(mode === "shadow" ? { warmCostShadowMicros: costMicros } : {})}::jsonb, now())
|
|
60787
61304
|
on conflict (idempotency_key) do nothing
|
|
60788
61305
|
`);
|
|
60789
61306
|
|
|
60790
|
-
// (2)
|
|
60791
|
-
|
|
60792
|
-
|
|
61307
|
+
// (2) Only credits mode publishes actual warm spend. Shadow's internal
|
|
61308
|
+
// estimate rides the seconds event, never a public spend event.
|
|
61309
|
+
let settledCostMicros = costMicros;
|
|
61310
|
+
if (costMicros > 0 && mode === "credits") {
|
|
61311
|
+
const costKey = `usage:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`;
|
|
61312
|
+
const inserted = await tx.execute<{ quantity: number | string }>(sql`
|
|
60793
61313
|
insert into usage_events
|
|
60794
61314
|
(account_id, workspace_id, subject_id, event_type, quantity, unit,
|
|
60795
61315
|
source_resource_type, source_resource_id, idempotency_key, occurred_at)
|
|
@@ -60797,62 +61317,65 @@ export async function accrueWarmSeconds(
|
|
|
60797
61317
|
(${input.accountId}, ${input.workspaceId}, ${input.subjectId ?? null},
|
|
60798
61318
|
'sandbox.warm_cost', ${costMicros}, 'usd_micros',
|
|
60799
61319
|
'sandbox_lease', ${`${input.sandboxGroupId}:${input.expectedEpoch}`},
|
|
60800
|
-
${
|
|
61320
|
+
${costKey},
|
|
60801
61321
|
now())
|
|
60802
61322
|
on conflict (idempotency_key) do nothing
|
|
61323
|
+
returning quantity
|
|
60803
61324
|
`);
|
|
61325
|
+
// Replayed historical cursor writes must settle the originally
|
|
61326
|
+
// recorded tick, not today's recalculated interval or new tariff.
|
|
61327
|
+
const previous = inserted[0]
|
|
61328
|
+
? null
|
|
61329
|
+
: await tx.execute<{ quantity: number | string }>(sql`
|
|
61330
|
+
select quantity from usage_events
|
|
61331
|
+
where idempotency_key = ${costKey}
|
|
61332
|
+
and account_id = ${input.accountId}
|
|
61333
|
+
and workspace_id = ${input.workspaceId}
|
|
61334
|
+
and event_type = 'sandbox.warm_cost'
|
|
61335
|
+
`);
|
|
61336
|
+
settledCostMicros = Number((inserted[0] ?? previous?.[0])?.quantity ?? 0);
|
|
61337
|
+
if (!Number.isSafeInteger(settledCostMicros) || settledCostMicros <= 0) {
|
|
61338
|
+
throw new Error("warm-cost tick idempotency key conflicts with another usage event");
|
|
61339
|
+
}
|
|
61340
|
+
}
|
|
61341
|
+
|
|
61342
|
+
if (costMicros > 0 && mode === "credits") {
|
|
61343
|
+
await applyCreditDebitAfterUse(tx, {
|
|
61344
|
+
accountId: input.accountId,
|
|
61345
|
+
workspaceId: input.workspaceId,
|
|
61346
|
+
type: "sandbox.warm_cost",
|
|
61347
|
+
amountMicros: settledCostMicros,
|
|
61348
|
+
sourceType: "sandbox_lease",
|
|
61349
|
+
sourceId: `${input.sandboxGroupId}:${input.expectedEpoch}`,
|
|
61350
|
+
idempotencyKey: `debit:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${tick}`,
|
|
61351
|
+
});
|
|
60804
61352
|
}
|
|
60805
61353
|
|
|
60806
61354
|
// (3) Advance the cursor IN THE SAME TXN — the atomicity that makes the tick
|
|
60807
61355
|
// index and the metered seconds inseparable.
|
|
60808
61356
|
await tx.execute(sql`
|
|
60809
61357
|
update sandbox_leases set
|
|
60810
|
-
last_meter_at =
|
|
61358
|
+
last_meter_at = ${new Date(startMs).toISOString()}::timestamptz + (${String(input.finalDrain ? elapsedMs : elapsedS * 1000)} || ' milliseconds')::interval,
|
|
61359
|
+
last_meter_tick = ${tick}, updated_at = now()
|
|
60811
61360
|
where id = ${row.id}
|
|
60812
61361
|
`);
|
|
60813
61362
|
|
|
60814
|
-
return { accrued: true, seconds: elapsedS, tick, costMicros };
|
|
61363
|
+
return { accrued: true, seconds: elapsedS, tick, costMicros: settledCostMicros };
|
|
60815
61364
|
}),
|
|
60816
61365
|
);
|
|
60817
61366
|
|
|
60818
|
-
// Debit credits for the warm-cost OUTSIDE the lease-row txn (applyCreditDebit
|
|
60819
|
-
// takes its own per-account advisory lock — never nest it under the lease row
|
|
60820
|
-
// lock). Idempotent on the SAME (group, epoch, tick) key so a re-fire of an
|
|
60821
|
-
// already-committed tick cannot double-debit. The ledger records the ACTUAL
|
|
60822
|
-
// debit (min(requested, balance)); the warm_cost usage event above is the
|
|
60823
|
-
// requested cost.
|
|
60824
|
-
if (result.accrued && result.costMicros > 0 && (input.debitCredits ?? true)) {
|
|
60825
|
-
await applyCreditDebitUpToBalance(db, {
|
|
60826
|
-
accountId: input.accountId,
|
|
60827
|
-
workspaceId: input.workspaceId,
|
|
60828
|
-
type: "sandbox.warm_cost",
|
|
60829
|
-
requestedAmountMicros: result.costMicros,
|
|
60830
|
-
sourceType: "sandbox_lease",
|
|
60831
|
-
sourceId: `${input.sandboxGroupId}:${input.expectedEpoch}`,
|
|
60832
|
-
idempotencyKey: `debit:sandbox.warm_cost:${input.sandboxGroupId}:${input.expectedEpoch}:${result.tick}`,
|
|
60833
|
-
}).catch(() => undefined);
|
|
60834
|
-
}
|
|
60835
|
-
|
|
60836
61367
|
return result;
|
|
60837
61368
|
}
|
|
60838
61369
|
|
|
60839
|
-
// §2.2/2.3 —
|
|
60840
|
-
//
|
|
60841
|
-
//
|
|
60842
|
-
// it). A workspace at 0 balance OR over its warm-second cap force-drains its
|
|
60843
|
-
// VIEWER-ONLY boxes: CAS warm->draining guarded `AND turn_holders = 0` so a box
|
|
60844
|
-
// with a running (paying) turn is NEVER killed. The reaper then issues the
|
|
60845
|
-
// provider stop() at refcount 0 (this fn is DB-only — no provider call).
|
|
60846
|
-
//
|
|
60847
|
-
// Group-wide force-drain on workspace balance exhaustion is deliberate (one
|
|
60848
|
-
// balance drains a multi-session box): the workspace, not the session, is the
|
|
60849
|
-
// billing unit — correctness (charged once) is automatic from the group meter key.
|
|
61370
|
+
// §2.2/2.3 — independent optional warm quota. Exhausted credits are enforced
|
|
61371
|
+
// at paid admission; they never cancel an already active holder. The cap can
|
|
61372
|
+
// close new viewer admission but only idle, holderless boxes may drain.
|
|
60850
61373
|
export interface ForceDrainResult {
|
|
60851
|
-
/** Whether the workspace
|
|
61374
|
+
/** Whether the workspace exceeded its optional warm cap. */
|
|
60852
61375
|
overLimit: boolean;
|
|
60853
61376
|
/** The reason, for observability. */
|
|
60854
61377
|
reason: SandboxViewerForceDrainReason | null;
|
|
60855
|
-
/** The (workspaceId, sandboxGroupId)
|
|
61378
|
+
/** The (workspaceId, sandboxGroupId) holderless boxes CASed warm->draining. */
|
|
60856
61379
|
drained: { workspaceId: string; sandboxGroupId: string }[];
|
|
60857
61380
|
}
|
|
60858
61381
|
|
|
@@ -60867,8 +61390,7 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
|
|
|
60867
61390
|
db: Database,
|
|
60868
61391
|
input: {
|
|
60869
61392
|
workspaceId: string;
|
|
60870
|
-
/**
|
|
60871
|
-
balanceMicros: number;
|
|
61393
|
+
/** Legacy caller option; balance belongs to paid admission, not draining. */
|
|
60872
61394
|
enforceBalance: boolean;
|
|
60873
61395
|
/** warm-second cap (cumulative this UTC month). 0 = unbounded (no cap gate). */
|
|
60874
61396
|
maxWarmSecondsPerWorkspace: number;
|
|
@@ -60883,9 +61405,7 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
|
|
|
60883
61405
|
// Determine over-limit under the lock (so the cap read + the drain are one
|
|
60884
61406
|
// serialized critical section per workspace).
|
|
60885
61407
|
let reason: SandboxViewerForceDrainReason | null = null;
|
|
60886
|
-
if (input.
|
|
60887
|
-
reason = "balance";
|
|
60888
|
-
} else if (input.maxWarmSecondsPerWorkspace > 0) {
|
|
61408
|
+
if (input.maxWarmSecondsPerWorkspace > 0) {
|
|
60889
61409
|
const since = input.capWindowStart ?? startOfUtcMonthDefault();
|
|
60890
61410
|
const [{ total } = { total: 0 }] = await scopedDb
|
|
60891
61411
|
.select({
|
|
@@ -60918,10 +61438,8 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
|
|
|
60918
61438
|
return { overLimit: false, reason: null, drained: [] };
|
|
60919
61439
|
}
|
|
60920
61440
|
|
|
60921
|
-
// Publish
|
|
60922
|
-
//
|
|
60923
|
-
// the locked drain below then deletes that holder. Every later viewer sees
|
|
60924
|
-
// this committed intent before it can re-arm or spawn a provider box.
|
|
61441
|
+
// Publish a cap-only gate; no paid-balance gate survives a top-up or mode
|
|
61442
|
+
// change. An already active holder keeps the provider alive.
|
|
60925
61443
|
await scopedDb.execute(sql`
|
|
60926
61444
|
update workspaces set
|
|
60927
61445
|
sandbox_viewer_force_drain_reason = ${reason},
|
|
@@ -60939,94 +61457,30 @@ export async function forceDrainOverLimitViewerOnlyBoxes(
|
|
|
60939
61457
|
)
|
|
60940
61458
|
`);
|
|
60941
61459
|
|
|
60942
|
-
//
|
|
60943
|
-
//
|
|
60944
|
-
// cannot deadlock with the global reaper or race a concurrent acquire.
|
|
61460
|
+
// Lock only holderless candidates. Neither the cap nor a zero balance may
|
|
61461
|
+
// mark an active Browser/ComputerSession, viewer, direct op or process lost.
|
|
60945
61462
|
await scopedDb.execute(sql`
|
|
60946
|
-
select id from sandbox_leases
|
|
60947
|
-
where workspace_id = ${input.workspaceId}
|
|
60948
|
-
and liveness = 'warm' and
|
|
61463
|
+
select lease.id from sandbox_leases lease
|
|
61464
|
+
where lease.workspace_id = ${input.workspaceId}
|
|
61465
|
+
and lease.liveness = 'warm' and lease.refcount = 0
|
|
61466
|
+
and not exists (select 1 from sandbox_lease_holders h where h.lease_id = lease.id)
|
|
60949
61467
|
order by id
|
|
60950
61468
|
for update
|
|
60951
61469
|
`);
|
|
60952
|
-
|
|
60953
|
-
//
|
|
60954
|
-
// turn_holders = 0 (a paying turn is NEVER killed). Stamp the grace deadline
|
|
60955
|
-
// so the reaper terminates at refcount 0 past the grace, exactly as a normal
|
|
60956
|
-
// refcount->0 drain would.
|
|
60957
|
-
// An interaction cannot keep a placement controller alive after the
|
|
60958
|
-
// workspace cost boundary has force-drained it. Make BrowserSession and
|
|
60959
|
-
// ComputerSession loss visible before removing their holders; later
|
|
60960
|
-
// recovery is explicit admission, never a silent resurrection.
|
|
60961
|
-
const lostBrowsers = await scopedDb.execute<{ id: string }>(sql`
|
|
60962
|
-
update browser_sessions browser set
|
|
60963
|
-
lifecycle = 'lost',
|
|
60964
|
-
controller_id = null,
|
|
60965
|
-
controller_generation = null,
|
|
60966
|
-
placement_instance_id = null,
|
|
60967
|
-
controller_heartbeat_at = null,
|
|
60968
|
-
failure_code = 'workspace_force_drained',
|
|
60969
|
-
updated_at = now()
|
|
60970
|
-
where browser.workspace_id = ${input.workspaceId}
|
|
60971
|
-
and browser.lifecycle in ('starting', 'active', 'suspending', 'restoring', 'ending')
|
|
60972
|
-
and browser.controller_host_sandbox_group_id in (
|
|
60973
|
-
select sandbox_group_id from sandbox_leases
|
|
60974
|
-
where workspace_id = ${input.workspaceId}
|
|
60975
|
-
and liveness = 'warm' and turn_holders = 0
|
|
60976
|
-
)
|
|
60977
|
-
returning browser.id
|
|
60978
|
-
`);
|
|
60979
|
-
const lostComputers = await scopedDb.execute<{ id: string }>(sql`
|
|
60980
|
-
update computer_sessions computer set
|
|
60981
|
-
lifecycle = 'lost',
|
|
60982
|
-
controller_id = null,
|
|
60983
|
-
controller_generation = null,
|
|
60984
|
-
placement_instance_id = null,
|
|
60985
|
-
controller_heartbeat_at = null,
|
|
60986
|
-
failure_code = 'workspace_force_drained',
|
|
60987
|
-
updated_at = now()
|
|
60988
|
-
where computer.workspace_id = ${input.workspaceId}
|
|
60989
|
-
and computer.lifecycle in ('starting', 'active', 'suspending', 'restoring', 'ending')
|
|
60990
|
-
and computer.sandbox_group_id in (
|
|
60991
|
-
select sandbox_group_id from sandbox_leases
|
|
60992
|
-
where workspace_id = ${input.workspaceId}
|
|
60993
|
-
and liveness = 'warm' and turn_holders = 0
|
|
60994
|
-
)
|
|
60995
|
-
returning computer.id
|
|
60996
|
-
`);
|
|
60997
|
-
if (lostBrowsers.length > 0 || lostComputers.length > 0) {
|
|
60998
|
-
await scopedDb.execute(sql`
|
|
60999
|
-
update workspace_interaction_revisions set
|
|
61000
|
-
revision = revision + 1,
|
|
61001
|
-
updated_at = now()
|
|
61002
|
-
where workspace_id = ${input.workspaceId}
|
|
61003
|
-
`);
|
|
61004
|
-
}
|
|
61005
|
-
// Drop viewer and interaction holders of every passive-only lease. Scoped
|
|
61006
|
-
// through the locked warm rows so a turn-held box remains untouched.
|
|
61007
|
-
await scopedDb.execute(sql`
|
|
61008
|
-
delete from sandbox_lease_holders h
|
|
61009
|
-
where h.kind in ('viewer', 'interaction')
|
|
61010
|
-
and h.lease_id in (
|
|
61011
|
-
select id from sandbox_leases
|
|
61012
|
-
where workspace_id = ${input.workspaceId}
|
|
61013
|
-
and liveness = 'warm' and turn_holders = 0
|
|
61014
|
-
)
|
|
61015
|
-
`);
|
|
61016
|
-
// CAS the now-holderless leases warm→draining at refcount 0 with the grace
|
|
61017
|
-
// deadline stamped — so the SAME reaper sweep's refcount=0 drain predicate
|
|
61018
|
-
// then terminates the box.
|
|
61470
|
+
// Normal release/reaper already drains holderless boxes. This CAS is a
|
|
61471
|
+
// bounded accelerator only; it cannot delete an active holder.
|
|
61019
61472
|
const drained = await rawRows<{ sandbox_group_id: string }>(
|
|
61020
61473
|
scopedDb,
|
|
61021
61474
|
sql`
|
|
61022
|
-
update sandbox_leases set
|
|
61475
|
+
update sandbox_leases lease set
|
|
61023
61476
|
liveness = 'draining',
|
|
61024
61477
|
refcount = 0, turn_holders = 0, viewer_holders = 0,
|
|
61025
61478
|
expires_at = now() + (${String(input.idleGraceMs)} || ' milliseconds')::interval,
|
|
61026
61479
|
updated_at = now()
|
|
61027
|
-
where workspace_id = ${input.workspaceId}
|
|
61028
|
-
and liveness = 'warm' and
|
|
61029
|
-
|
|
61480
|
+
where lease.workspace_id = ${input.workspaceId}
|
|
61481
|
+
and lease.liveness = 'warm' and lease.refcount = 0
|
|
61482
|
+
and not exists (select 1 from sandbox_lease_holders h where h.lease_id = lease.id)
|
|
61483
|
+
returning lease.sandbox_group_id
|
|
61030
61484
|
`,
|
|
61031
61485
|
);
|
|
61032
61486
|
|