@opengeni/db 0.12.1 → 0.12.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -23,6 +23,7 @@ import type {
23
23
  KnowledgeMemoryKind,
24
24
  KnowledgeMemoryStatus,
25
25
  KnowledgeSourceRef,
26
+ GitHubInstallationAuthorityKind,
26
27
  GitHubRepositoryScope,
27
28
  HostEventExport,
28
29
  HostEventExportBatch,
@@ -159,7 +160,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
159
160
  import postgres from "postgres";
160
161
  import { decryptEnvironmentValue } from "./environment-crypto";
161
162
  import { sanitizeEventPayload, sanitizeModelPayload } from "./event-payload-sanitizer";
162
- import { consumeNewSessionDraftInTransaction } from "./new-session-drafts";
163
+ import { seedNewSessionDraftInTransaction } from "./new-session-drafts";
163
164
  import {
164
165
  runIdempotentPersistenceTransaction,
165
166
  type IdempotentPersistenceTransactionOptions,
@@ -225,6 +226,7 @@ export {
225
226
  sanitizeModelPayload,
226
227
  } from "./event-payload-sanitizer";
227
228
  export * from "./persistence-errors";
229
+ export * from "./runtime-posture";
228
230
  export { sanitizeMemoryText } from "./memory-domain";
229
231
  // Re-exported so external consumers can `import { migrate } from "@opengeni/db"`.
230
232
  // The `@opengeni/db/migrate` subpath stays available too (internal callers + the
@@ -260,6 +262,16 @@ export * from "./memory-domain";
260
262
  // unaffected.
261
263
  export type Database = PgDatabase<any, typeof schema>;
262
264
 
265
+ /** Raised when a durable session tool-policy write lost its version fence. */
266
+ export class SessionToolPolicyVersionConflictError extends Error {
267
+ readonly code = "SESSION_TOOL_POLICY_CONFLICT";
268
+
269
+ constructor(readonly currentVersion: number) {
270
+ super("The session tool policy changed in another client");
271
+ this.name = "SessionToolPolicyVersionConflictError";
272
+ }
273
+ }
274
+
263
275
  export type DbClient = {
264
276
  db: Database;
265
277
  close: () => Promise<void>;
@@ -999,15 +1011,27 @@ export async function withRlsContext<T>(
999
1011
  // manufacturing a phantom "no active subscription" from a credential that is
1000
1012
  // in fact active. Convert that silent false into a loud, root-cause-bearing
1001
1013
  // error so the caller can retry rather than permanently mis-decide.
1002
- const applied = await tx.execute<{ account_id: string | null }>(
1003
- sql`select current_setting('opengeni.account_id', true) as account_id`,
1014
+ const applied = await tx.execute<{
1015
+ account_id: string | null;
1016
+ workspace_id: string | null;
1017
+ }>(
1018
+ sql`select
1019
+ current_setting('opengeni.account_id', true) as account_id,
1020
+ current_setting('opengeni.workspace_id', true) as workspace_id`,
1004
1021
  );
1005
1022
  const appliedAccountId = applied[0]?.account_id ?? "";
1023
+ const expectedWorkspaceId = context.workspaceId ?? "";
1024
+ const appliedWorkspaceId = applied[0]?.workspace_id ?? "";
1006
1025
  if (appliedAccountId !== context.accountId) {
1007
1026
  throw new Error(
1008
1027
  `RLS context not applied on the active backend: expected account ${context.accountId}, got "${appliedAccountId}"`,
1009
1028
  );
1010
1029
  }
1030
+ if (appliedWorkspaceId !== expectedWorkspaceId) {
1031
+ throw new Error(
1032
+ `RLS context not applied on the active backend: expected workspace "${expectedWorkspaceId}", got "${appliedWorkspaceId}"`,
1033
+ );
1034
+ }
1011
1035
  return await fn(scoped);
1012
1036
  }, transactionConfig);
1013
1037
  }
@@ -2044,10 +2068,17 @@ export type GitHubInstallation = {
2044
2068
  accountId: string;
2045
2069
  workspaceId: string;
2046
2070
  installationId: number;
2071
+ githubAccountId: number | null;
2047
2072
  accountLogin: string | null;
2048
2073
  accountType: string | null;
2049
2074
  repositoryScope: GitHubRepositoryScope;
2050
2075
  linkedBySubjectId: string | null;
2076
+ githubActorId: number | null;
2077
+ githubActorLogin: string | null;
2078
+ authorityKind: GitHubInstallationAuthorityKind | null;
2079
+ authorityCheckedAt: string | null;
2080
+ authorityExpiresAt: string | null;
2081
+ authorityNonce: string | null;
2051
2082
  createdAt: string;
2052
2083
  updatedAt: string;
2053
2084
  };
@@ -2056,6 +2087,59 @@ export type GitHubInstallationAccess = GitHubInstallation & {
2056
2087
  repositoryIds: number[];
2057
2088
  };
2058
2089
 
2090
+ export class GitHubInstallationAuthorityCommitError extends Error {
2091
+ constructor() {
2092
+ super("GitHub installation authority expired before the binding transaction completed");
2093
+ }
2094
+ }
2095
+
2096
+ const githubInstallationAuthorityMaxAgeMs = 10 * 60_000;
2097
+
2098
+ /**
2099
+ * True only for a binding created by the owner-authority transaction. Legacy
2100
+ * installation rows remain visible for audit/unlink, but can never make a
2101
+ * workspace healthy, enumerate repositories, or authorize a token mint.
2102
+ *
2103
+ * `authorityExpiresAt` is the expiry of the consumed proof, not of the durable
2104
+ * delegation. A binding remains delegated after that instant; GitHub's live
2105
+ * installation/repository checks govern its ongoing usability.
2106
+ */
2107
+ export function hasAuditableGitHubInstallationAuthority(
2108
+ installation: GitHubInstallationAccess,
2109
+ ): boolean {
2110
+ const checkedAt = installation.authorityCheckedAt
2111
+ ? Date.parse(installation.authorityCheckedAt)
2112
+ : Number.NaN;
2113
+ const expiresAt = installation.authorityExpiresAt
2114
+ ? Date.parse(installation.authorityExpiresAt)
2115
+ : Number.NaN;
2116
+ const common =
2117
+ installation.repositoryScope === "selected" &&
2118
+ installation.repositoryIds.length > 0 &&
2119
+ installation.repositoryIds.every((id) => Number.isSafeInteger(id) && id > 0) &&
2120
+ installation.githubAccountId !== null &&
2121
+ Number.isSafeInteger(installation.githubAccountId) &&
2122
+ installation.githubAccountId > 0 &&
2123
+ Boolean(installation.accountLogin) &&
2124
+ installation.githubActorId !== null &&
2125
+ Number.isSafeInteger(installation.githubActorId) &&
2126
+ installation.githubActorId > 0 &&
2127
+ Boolean(installation.githubActorLogin) &&
2128
+ Boolean(installation.linkedBySubjectId) &&
2129
+ Boolean(installation.authorityNonce) &&
2130
+ Number.isFinite(checkedAt) &&
2131
+ Number.isFinite(expiresAt) &&
2132
+ checkedAt < expiresAt;
2133
+ if (!common) {
2134
+ return false;
2135
+ }
2136
+ return installation.authorityKind === "personal_owner"
2137
+ ? installation.accountType === "User" &&
2138
+ installation.githubActorId === installation.githubAccountId
2139
+ : installation.authorityKind === "organization_owner" &&
2140
+ installation.accountType === "Organization";
2141
+ }
2142
+
2059
2143
  export async function upsertGitHubInstallation(
2060
2144
  db: Database,
2061
2145
  input: {
@@ -2095,6 +2179,9 @@ export async function upsertGitHubInstallation(
2095
2179
  : {}),
2096
2180
  updatedAt: new Date(),
2097
2181
  },
2182
+ // This legacy metadata helper cannot mutate an owner-authorized row.
2183
+ // New bindings use bindAuthorizedGitHubInstallationRepositories.
2184
+ setWhere: isNull(schema.githubInstallations.authorityNonce),
2098
2185
  })
2099
2186
  .returning();
2100
2187
  if (!row) {
@@ -2156,6 +2243,164 @@ export async function bindGitHubInstallationRepositories(
2156
2243
  linkedBySubjectId: input.linkedBySubjectId,
2157
2244
  updatedAt: new Date(),
2158
2245
  },
2246
+ // Preserve the immutable authority + allowlist audit boundary.
2247
+ setWhere: isNull(schema.githubInstallations.authorityNonce),
2248
+ })
2249
+ .returning();
2250
+ if (!row) {
2251
+ throw new Error("Failed to bind GitHub installation");
2252
+ }
2253
+ await tx
2254
+ .delete(schema.githubInstallationRepositories)
2255
+ .where(
2256
+ and(
2257
+ eq(schema.githubInstallationRepositories.workspaceId, input.workspaceId),
2258
+ eq(schema.githubInstallationRepositories.installationId, input.installationId),
2259
+ ),
2260
+ );
2261
+ for (let offset = 0; offset < repositoryIds.length; offset += 1_000) {
2262
+ await tx.insert(schema.githubInstallationRepositories).values(
2263
+ repositoryIds.slice(offset, offset + 1_000).map((repositoryId) => ({
2264
+ accountId: input.accountId,
2265
+ workspaceId: input.workspaceId,
2266
+ installationId: input.installationId,
2267
+ repositoryId,
2268
+ })),
2269
+ );
2270
+ }
2271
+ return { ...mapGitHubInstallation(row), repositoryIds };
2272
+ }),
2273
+ );
2274
+ }
2275
+
2276
+ export async function bindAuthorizedGitHubInstallationRepositories(
2277
+ db: Database,
2278
+ input: {
2279
+ accountId: string;
2280
+ workspaceId: string;
2281
+ installationId: number;
2282
+ githubAccountId: number;
2283
+ accountLogin: string | null;
2284
+ accountType: string | null;
2285
+ linkedBySubjectId: string;
2286
+ githubActorId: number;
2287
+ githubActorLogin: string;
2288
+ authorityKind: GitHubInstallationAuthorityKind;
2289
+ authorityCheckedAt: Date;
2290
+ authorityExpiresAt: Date;
2291
+ authorityNonce: string;
2292
+ repositoryIds: number[];
2293
+ },
2294
+ ): Promise<GitHubInstallationAccess | null> {
2295
+ if (!Number.isSafeInteger(input.installationId) || input.installationId <= 0) {
2296
+ throw new Error("GitHub installation id must be a positive safe integer");
2297
+ }
2298
+ if (!Number.isSafeInteger(input.githubAccountId) || input.githubAccountId <= 0) {
2299
+ throw new Error("GitHub account id must be a positive safe integer");
2300
+ }
2301
+ if (!Number.isSafeInteger(input.githubActorId) || input.githubActorId <= 0) {
2302
+ throw new Error("GitHub actor id must be a positive safe integer");
2303
+ }
2304
+ const authorityCheckedAtMs = input.authorityCheckedAt.getTime();
2305
+ const authorityExpiresAtMs = input.authorityExpiresAt.getTime();
2306
+ if (
2307
+ !Number.isFinite(authorityCheckedAtMs) ||
2308
+ !Number.isFinite(authorityExpiresAtMs) ||
2309
+ authorityCheckedAtMs >= authorityExpiresAtMs ||
2310
+ authorityExpiresAtMs - authorityCheckedAtMs > githubInstallationAuthorityMaxAgeMs ||
2311
+ !input.authorityNonce ||
2312
+ !input.githubActorLogin.trim() ||
2313
+ !input.accountLogin?.trim() ||
2314
+ !input.linkedBySubjectId.trim() ||
2315
+ (input.authorityKind !== "personal_owner" && input.authorityKind !== "organization_owner") ||
2316
+ (input.authorityKind === "personal_owner" &&
2317
+ (input.accountType !== "User" || input.githubActorId !== input.githubAccountId)) ||
2318
+ (input.authorityKind === "organization_owner" && input.accountType !== "Organization")
2319
+ ) {
2320
+ throw new Error("GitHub installation authority proof is invalid or expired");
2321
+ }
2322
+ const repositoryIds = [...new Set(input.repositoryIds)];
2323
+ if (
2324
+ repositoryIds.length === 0 ||
2325
+ repositoryIds.length !== input.repositoryIds.length ||
2326
+ repositoryIds.some((id) => !Number.isSafeInteger(id) || id <= 0)
2327
+ ) {
2328
+ throw new Error(
2329
+ "GitHub repository ids must be a nonempty, unique list of positive safe integers",
2330
+ );
2331
+ }
2332
+ return await withRlsContext(
2333
+ db,
2334
+ { accountId: input.accountId, workspaceId: input.workspaceId },
2335
+ async (scopedDb) =>
2336
+ await scopedDb.transaction(async (tx) => {
2337
+ await assertGitHubAuthorityWindowOpen(
2338
+ tx,
2339
+ input.authorityCheckedAt,
2340
+ input.authorityExpiresAt,
2341
+ );
2342
+ await tx
2343
+ .delete(schema.integrationOauthStateNonces)
2344
+ .where(
2345
+ and(
2346
+ eq(schema.integrationOauthStateNonces.workspaceId, input.workspaceId),
2347
+ lt(schema.integrationOauthStateNonces.expiresAt, input.authorityCheckedAt),
2348
+ ),
2349
+ );
2350
+ const consumed = await tx
2351
+ .insert(schema.integrationOauthStateNonces)
2352
+ .values({
2353
+ accountId: input.accountId,
2354
+ workspaceId: input.workspaceId,
2355
+ subjectId: input.linkedBySubjectId,
2356
+ nonce: input.authorityNonce,
2357
+ expiresAt: input.authorityExpiresAt,
2358
+ usedAt: input.authorityCheckedAt,
2359
+ })
2360
+ .onConflictDoNothing({ target: schema.integrationOauthStateNonces.nonce })
2361
+ .returning({ nonce: schema.integrationOauthStateNonces.nonce });
2362
+ if (consumed.length === 0) {
2363
+ return null;
2364
+ }
2365
+
2366
+ const [row] = await tx
2367
+ .insert(schema.githubInstallations)
2368
+ .values({
2369
+ accountId: input.accountId,
2370
+ workspaceId: input.workspaceId,
2371
+ installationId: input.installationId,
2372
+ githubAccountId: input.githubAccountId,
2373
+ accountLogin: input.accountLogin,
2374
+ accountType: input.accountType,
2375
+ repositoryScope: "selected",
2376
+ linkedBySubjectId: input.linkedBySubjectId,
2377
+ githubActorId: input.githubActorId,
2378
+ githubActorLogin: input.githubActorLogin,
2379
+ authorityKind: input.authorityKind,
2380
+ authorityCheckedAt: input.authorityCheckedAt,
2381
+ authorityExpiresAt: input.authorityExpiresAt,
2382
+ authorityNonce: input.authorityNonce,
2383
+ })
2384
+ .onConflictDoUpdate({
2385
+ target: [
2386
+ schema.githubInstallations.workspaceId,
2387
+ schema.githubInstallations.installationId,
2388
+ ],
2389
+ set: {
2390
+ accountId: input.accountId,
2391
+ githubAccountId: input.githubAccountId,
2392
+ accountLogin: input.accountLogin,
2393
+ accountType: input.accountType,
2394
+ repositoryScope: "selected",
2395
+ linkedBySubjectId: input.linkedBySubjectId,
2396
+ githubActorId: input.githubActorId,
2397
+ githubActorLogin: input.githubActorLogin,
2398
+ authorityKind: input.authorityKind,
2399
+ authorityCheckedAt: input.authorityCheckedAt,
2400
+ authorityExpiresAt: input.authorityExpiresAt,
2401
+ authorityNonce: input.authorityNonce,
2402
+ updatedAt: input.authorityCheckedAt,
2403
+ },
2159
2404
  })
2160
2405
  .returning();
2161
2406
  if (!row) {
@@ -2179,11 +2424,41 @@ export async function bindGitHubInstallationRepositories(
2179
2424
  })),
2180
2425
  );
2181
2426
  }
2427
+ // Recheck against the database clock after all writes. Throwing here
2428
+ // rolls the transaction back if the proof expired while it was being
2429
+ // committed, so no partial nonce or binding can survive.
2430
+ await assertGitHubAuthorityWindowOpen(
2431
+ tx,
2432
+ input.authorityCheckedAt,
2433
+ input.authorityExpiresAt,
2434
+ );
2182
2435
  return { ...mapGitHubInstallation(row), repositoryIds };
2183
2436
  }),
2184
2437
  );
2185
2438
  }
2186
2439
 
2440
+ async function assertGitHubAuthorityWindowOpen(
2441
+ tx: Database,
2442
+ checkedAt: Date,
2443
+ expiresAt: Date,
2444
+ ): Promise<void> {
2445
+ // postgres.js does not serialize JavaScript Date instances passed through
2446
+ // raw Drizzle SQL. Bind the already-validated ISO timestamp representation
2447
+ // and retain PostgreSQL's timestamptz/database-clock authority checks.
2448
+ const checkedAtIso = checkedAt.toISOString();
2449
+ const expiresAtIso = expiresAt.toISOString();
2450
+ const result = await tx.execute<{ valid: boolean }>(sql`
2451
+ select (
2452
+ ${checkedAtIso}::timestamptz <= clock_timestamp()
2453
+ and clock_timestamp() < ${expiresAtIso}::timestamptz
2454
+ and ${expiresAtIso}::timestamptz <= ${checkedAtIso}::timestamptz + interval '10 minutes'
2455
+ ) as valid
2456
+ `);
2457
+ if (result[0]?.valid !== true) {
2458
+ throw new GitHubInstallationAuthorityCommitError();
2459
+ }
2460
+ }
2461
+
2187
2462
  export async function listGitHubInstallationsForWorkspace(
2188
2463
  db: Database,
2189
2464
  workspaceId: string,
@@ -2256,7 +2531,7 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
2256
2531
  }
2257
2532
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
2258
2533
  const [installation] = await scopedDb
2259
- .select({ repositoryScope: schema.githubInstallations.repositoryScope })
2534
+ .select()
2260
2535
  .from(schema.githubInstallations)
2261
2536
  .where(
2262
2537
  and(
@@ -2268,9 +2543,6 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
2268
2543
  if (!installation) {
2269
2544
  return false;
2270
2545
  }
2271
- if (installation.repositoryScope === "all") {
2272
- return true;
2273
- }
2274
2546
  const allowed = await scopedDb
2275
2547
  .select({
2276
2548
  repositoryId: schema.githubInstallationRepositories.repositoryId,
@@ -2283,7 +2555,13 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
2283
2555
  inArray(schema.githubInstallationRepositories.repositoryId, requestedIds),
2284
2556
  ),
2285
2557
  );
2286
- return allowed.length === requestedIds.length;
2558
+ if (allowed.length !== requestedIds.length) {
2559
+ return false;
2560
+ }
2561
+ return hasAuditableGitHubInstallationAuthority({
2562
+ ...mapGitHubInstallation(installation),
2563
+ repositoryIds: allowed.map((row) => row.repositoryId),
2564
+ });
2287
2565
  });
2288
2566
  }
2289
2567
 
@@ -9516,12 +9794,13 @@ export type CodexCapacityWait = {
9516
9794
  accountId: string;
9517
9795
  workspaceId: string;
9518
9796
  sessionId: string;
9519
- goalId: string;
9797
+ goalId: string | null;
9520
9798
  blockedTurnId: string;
9799
+ blockedTurnGeneration: number;
9521
9800
  workflowId: string;
9522
9801
  generation: number;
9523
9802
  status: CodexCapacityWaitStatus;
9524
- goalVersion: number;
9803
+ goalVersion: number | null;
9525
9804
  policyHash: string | null;
9526
9805
  earliestResetAt: Date | null;
9527
9806
  nextCheckAt: Date;
@@ -9588,9 +9867,9 @@ export type ReconcileCodexCapacityWaitResult =
9588
9867
  | {
9589
9868
  action: "resumed";
9590
9869
  waiter: CodexCapacityWait;
9591
- update: SessionSystemUpdate;
9592
9870
  events: SessionEvent[];
9593
9871
  }
9872
+ | { action: "paused"; waiter: CodexCapacityWait; events: SessionEvent[] }
9594
9873
  | { action: "superseded"; waiter: CodexCapacityWait; events: SessionEvent[] }
9595
9874
  | {
9596
9875
  action: "stale";
@@ -9621,6 +9900,7 @@ function mapCodexCapacityWaiter(
9621
9900
  sessionId: row.sessionId,
9622
9901
  goalId: row.goalId,
9623
9902
  blockedTurnId: row.blockedTurnId,
9903
+ blockedTurnGeneration: row.blockedTurnGeneration,
9624
9904
  workflowId: row.workflowId,
9625
9905
  generation: row.generation,
9626
9906
  status: row.status as CodexCapacityWaitStatus,
@@ -9698,14 +9978,13 @@ function nextCodexCapacityCheckAt(
9698
9978
  }
9699
9979
 
9700
9980
  /**
9701
- * Atomically settle one all-unavailable turn and arm exactly one durable wait.
9702
- * Lock order is allocator rotation row -> workspace control -> actual workspace
9703
- * -> session -> exact turn -> exact attempt -> goal -> live lease (when a
9704
- * reactive failure owns one) -> waiter. The control share lock and effective
9705
- * control recheck prevent a capacity boundary from closing an attempt after a
9706
- * committed Pause. The failed turn, idle/capacity-paused session, durable
9707
- * events, lease release, and waiter generation commit together; a crash cannot
9708
- * leave only half of the boundary visible.
9981
+ * Atomically close one all-unavailable attempt and arm exactly one durable wait
9982
+ * for the same logical turn. Lock order is allocator rotation row -> workspace
9983
+ * control -> actual workspace -> session -> exact turn -> exact attempt ->
9984
+ * optional goal -> live lease (when a reactive failure owns one) -> waiter.
9985
+ * The waiting turn/session pointer, durable events, exact lease release, and
9986
+ * waiter generation commit together; a crash cannot leave only half of the
9987
+ * boundary visible.
9709
9988
  */
9710
9989
  export async function armCodexCapacityWait(
9711
9990
  db: Database,
@@ -9716,8 +9995,8 @@ export async function armCodexCapacityWait(
9716
9995
  turnId: string;
9717
9996
  attemptId: string;
9718
9997
  workflowId: string;
9719
- goalId: string;
9720
- goalVersion: number;
9998
+ goalId?: string | null;
9999
+ goalVersion?: number | null;
9721
10000
  policyHash?: string | null;
9722
10001
  earliestResetAt: Date | null;
9723
10002
  resetKind: CodexCapacityResetKind;
@@ -9730,6 +10009,14 @@ export async function armCodexCapacityWait(
9730
10009
  },
9731
10010
  ): Promise<ArmCodexCapacityWaitResult> {
9732
10011
  const now = input.now ?? new Date();
10012
+ const goalId = input.goalId ?? null;
10013
+ const goalVersion = input.goalVersion ?? null;
10014
+ if (
10015
+ (goalId === null) !== (goalVersion === null) ||
10016
+ (goalVersion !== null && (!Number.isSafeInteger(goalVersion) || goalVersion < 1))
10017
+ ) {
10018
+ throw new Error("Codex capacity goal fence must be absent or contain a positive version");
10019
+ }
9733
10020
  return await withRlsContext(
9734
10021
  db,
9735
10022
  { accountId: input.accountId, workspaceId: input.workspaceId },
@@ -9755,18 +10042,20 @@ export async function armCodexCapacityWait(
9755
10042
  workspaceControl: locks.control ?? undefined,
9756
10043
  })
9757
10044
  : null;
9758
- const [goal] = await tx
9759
- .select()
9760
- .from(schema.sessionGoals)
9761
- .where(
9762
- and(
9763
- eq(schema.sessionGoals.workspaceId, input.workspaceId),
9764
- eq(schema.sessionGoals.id, input.goalId),
9765
- eq(schema.sessionGoals.sessionId, input.sessionId),
9766
- ),
9767
- )
9768
- .for("update")
9769
- .limit(1);
10045
+ const [goal] = goalId
10046
+ ? await tx
10047
+ .select()
10048
+ .from(schema.sessionGoals)
10049
+ .where(
10050
+ and(
10051
+ eq(schema.sessionGoals.workspaceId, input.workspaceId),
10052
+ eq(schema.sessionGoals.id, goalId),
10053
+ eq(schema.sessionGoals.sessionId, input.sessionId),
10054
+ ),
10055
+ )
10056
+ .for("update")
10057
+ .limit(1)
10058
+ : [];
9770
10059
  const leaseRows = input.leaseFence
9771
10060
  ? await tx.execute(sql<{ holder_id: string; generation: number }>`
9772
10061
  select holder_id, generation
@@ -9807,7 +10096,10 @@ export async function armCodexCapacityWait(
9807
10096
  if (
9808
10097
  existing?.status === "waiting" &&
9809
10098
  existing.blockedTurnId === input.turnId &&
9810
- turn?.status === "failed"
10099
+ existing.blockedTurnGeneration === turn?.executionGeneration &&
10100
+ turn?.status === "waiting_capacity" &&
10101
+ session?.status === "waiting_capacity" &&
10102
+ session.activeTurnId === input.turnId
9811
10103
  ) {
9812
10104
  return {
9813
10105
  action: "waiting",
@@ -9825,13 +10117,14 @@ export async function armCodexCapacityWait(
9825
10117
  Number(lease.generation) === input.leaseFence.generation &&
9826
10118
  currentRedispatches === (input.expectedRedispatches ?? currentRedispatches));
9827
10119
  if (
9828
- !goal ||
10120
+ !session ||
10121
+ !turn ||
9829
10122
  effectiveControl?.state !== "active" ||
9830
10123
  effectiveControl.settlement !== null ||
9831
10124
  session.activeTurnId !== input.turnId ||
9832
10125
  session.status !== "running" ||
9833
- goal.status !== "active" ||
9834
- goal.version !== input.goalVersion ||
10126
+ (goalId !== null &&
10127
+ (!goal || goal.status !== "active" || goal.version !== goalVersion)) ||
9835
10128
  turn.status !== "running" ||
9836
10129
  turn.activeAttemptId !== input.attemptId ||
9837
10130
  !leaseFenceValid ||
@@ -9851,7 +10144,7 @@ export async function armCodexCapacityWait(
9851
10144
  sessionId: input.sessionId,
9852
10145
  turnId: input.turnId,
9853
10146
  executionGeneration: turn.executionGeneration,
9854
- outcome: "failed",
10147
+ outcome: "waiting_capacity",
9855
10148
  closedAt: now,
9856
10149
  });
9857
10150
 
@@ -9867,12 +10160,13 @@ export async function armCodexCapacityWait(
9867
10160
  accountId: input.accountId,
9868
10161
  workspaceId: input.workspaceId,
9869
10162
  sessionId: input.sessionId,
9870
- goalId: input.goalId,
10163
+ goalId,
9871
10164
  blockedTurnId: input.turnId,
10165
+ blockedTurnGeneration: turn.executionGeneration,
9872
10166
  workflowId: input.workflowId,
9873
10167
  generation,
9874
10168
  status: "waiting",
9875
- goalVersion: input.goalVersion,
10169
+ goalVersion,
9876
10170
  policyHash,
9877
10171
  earliestResetAt: input.earliestResetAt,
9878
10172
  nextCheckAt,
@@ -9917,32 +10211,17 @@ export async function armCodexCapacityWait(
9917
10211
  workspaceId: input.workspaceId,
9918
10212
  sessionId: input.sessionId,
9919
10213
  sequence: ++sequence,
9920
- type: "turn.failed",
10214
+ type: "codex.capacity.waiting",
9921
10215
  payload: sanitizeEventPayload({
9922
10216
  ...input.failurePayload,
9923
10217
  recovery: "codex_capacity",
9924
- retryable: false,
10218
+ retryable: true,
9925
10219
  rotated: true,
9926
- capacityWaiterId: waiterRow.id,
9927
- capacityWaitGeneration: waiterRow.generation,
9928
- }),
9929
- turnId: input.turnId,
9930
- turnGeneration: turn.executionGeneration,
9931
- turnAttemptId: input.attemptId,
9932
- turnAssociation: "current",
9933
- occurredAt: now,
9934
- },
9935
- {
9936
- accountId: input.accountId,
9937
- workspaceId: input.workspaceId,
9938
- sessionId: input.sessionId,
9939
- sequence: ++sequence,
9940
- type: "codex.capacity.waiting",
9941
- payload: sanitizeEventPayload({
9942
10220
  waiterId: waiterRow.id,
9943
10221
  generation: waiterRow.generation,
9944
- goalId: input.goalId,
9945
- goalVersion: input.goalVersion,
10222
+ goalId,
10223
+ goalVersion,
10224
+ blockedTurnGeneration: turn.executionGeneration,
9946
10225
  policyHash,
9947
10226
  resetKind: input.resetKind,
9948
10227
  earliestResetAt: input.earliestResetAt?.toISOString() ?? null,
@@ -9960,7 +10239,7 @@ export async function armCodexCapacityWait(
9960
10239
  sessionId: input.sessionId,
9961
10240
  sequence: ++sequence,
9962
10241
  type: "session.status.changed",
9963
- payload: { status: "idle", reason: "codex_capacity" },
10242
+ payload: { status: "waiting_capacity", reason: "codex_capacity" },
9964
10243
  turnId: input.turnId,
9965
10244
  turnGeneration: turn.executionGeneration,
9966
10245
  turnAttemptId: input.attemptId,
@@ -9969,13 +10248,14 @@ export async function armCodexCapacityWait(
9969
10248
  },
9970
10249
  ])
9971
10250
  .returning();
9972
- await tx
10251
+ const [waitingTurn] = await tx
9973
10252
  .update(schema.sessionTurns)
9974
10253
  .set({
9975
- status: "failed",
10254
+ status: "waiting_capacity",
9976
10255
  activeAttemptId: null,
10256
+ metadata: metadataWithoutTurnDispatchAttempt(turn.metadata),
9977
10257
  version: turn.version + 1,
9978
- finishedAt: now,
10258
+ finishedAt: null,
9979
10259
  updatedAt: now,
9980
10260
  })
9981
10261
  .where(
@@ -9983,13 +10263,18 @@ export async function armCodexCapacityWait(
9983
10263
  eq(schema.sessionTurns.workspaceId, input.workspaceId),
9984
10264
  eq(schema.sessionTurns.id, input.turnId),
9985
10265
  eq(schema.sessionTurns.status, "running"),
10266
+ eq(schema.sessionTurns.activeAttemptId, input.attemptId),
9986
10267
  ),
9987
- );
9988
- await tx
10268
+ )
10269
+ .returning({ id: schema.sessionTurns.id });
10270
+ if (!waitingTurn) {
10271
+ throw new Error("Codex capacity blocked turn changed during atomic arm");
10272
+ }
10273
+ const [waitingSession] = await tx
9989
10274
  .update(schema.sessions)
9990
10275
  .set({
9991
- status: "idle",
9992
- activeTurnId: null,
10276
+ status: "waiting_capacity",
10277
+ activeTurnId: input.turnId,
9993
10278
  lastSequence: sequence,
9994
10279
  updatedAt: now,
9995
10280
  })
@@ -9997,15 +10282,24 @@ export async function armCodexCapacityWait(
9997
10282
  and(
9998
10283
  eq(schema.sessions.workspaceId, input.workspaceId),
9999
10284
  eq(schema.sessions.id, input.sessionId),
10285
+ eq(schema.sessions.status, "running"),
10000
10286
  eq(schema.sessions.activeTurnId, input.turnId),
10001
10287
  ),
10002
- );
10003
- await tx.execute(sql`
10004
- delete from codex_credential_leases
10005
- where account_id = ${input.accountId}
10006
- and workspace_id = ${input.workspaceId}
10007
- and turn_id = ${input.turnId}
10008
- `);
10288
+ )
10289
+ .returning({ id: schema.sessions.id });
10290
+ if (!waitingSession) {
10291
+ throw new Error("Codex capacity session changed during atomic arm");
10292
+ }
10293
+ if (input.leaseFence) {
10294
+ await tx.execute(sql`
10295
+ delete from codex_credential_leases
10296
+ where account_id = ${input.accountId}
10297
+ and workspace_id = ${input.workspaceId}
10298
+ and turn_id = ${input.turnId}
10299
+ and holder_id = ${input.leaseFence.holderId}
10300
+ and generation = ${input.leaseFence.generation}
10301
+ `);
10302
+ }
10009
10303
  return {
10010
10304
  action: "waiting",
10011
10305
  waiter: mapCodexCapacityWaiter(waiterRow),
@@ -10037,8 +10331,8 @@ export async function getCodexCapacityWaitForSession(
10037
10331
  }
10038
10332
 
10039
10333
  /**
10040
- * Same-transaction capacity-mutation/outbox seam for account eligibility and
10041
- * policy-scope membership/default changes. The allocator rotation row is always the
10334
+ * Same-transaction capacity-mutation/outbox seam for eligibility and
10335
+ * membership/default changes. The allocator rotation row is always the
10042
10336
  * first lock. Mutations report whether capacity truth changed; only then are
10043
10337
  * matching waiter wake revisions advanced and returned for best-effort signal.
10044
10338
  */
@@ -10153,6 +10447,7 @@ async function supersedeCodexCapacityWaitInTransaction(
10153
10447
  tx: Database,
10154
10448
  input: {
10155
10449
  session: typeof schema.sessions.$inferSelect;
10450
+ blockedTurn: typeof schema.sessionTurns.$inferSelect;
10156
10451
  waiter: typeof schema.codexCapacityWaiters.$inferSelect;
10157
10452
  reason: string;
10158
10453
  now: Date;
@@ -10176,9 +10471,52 @@ async function supersedeCodexCapacityWaitInTransaction(
10176
10471
  if (!updated) {
10177
10472
  return { waiter: mapCodexCapacityWaiter(input.waiter), events: [] };
10178
10473
  }
10179
- const inserted = await tx
10180
- .insert(schema.sessionEvents)
10181
- .values({
10474
+ const turnWasCurrent = input.session.activeTurnId === input.blockedTurn.id;
10475
+ const turnStillWaiting = input.blockedTurn.status === "waiting_capacity";
10476
+ const terminalTurnStatus = input.session.status === "cancelled" ? "cancelled" : "superseded";
10477
+ if (turnStillWaiting) {
10478
+ const [supersededTurn] = await tx
10479
+ .update(schema.sessionTurns)
10480
+ .set({
10481
+ status: terminalTurnStatus,
10482
+ activeAttemptId: null,
10483
+ cancelledBy: "codex_capacity_reconcile",
10484
+ cancelReason: input.reason,
10485
+ version: input.blockedTurn.version + 1,
10486
+ finishedAt: input.now,
10487
+ updatedAt: input.now,
10488
+ })
10489
+ .where(
10490
+ and(
10491
+ eq(schema.sessionTurns.workspaceId, input.session.workspaceId),
10492
+ eq(schema.sessionTurns.id, input.blockedTurn.id),
10493
+ eq(schema.sessionTurns.status, "waiting_capacity"),
10494
+ isNull(schema.sessionTurns.activeAttemptId),
10495
+ eq(schema.sessionTurns.executionGeneration, input.waiter.blockedTurnGeneration),
10496
+ ),
10497
+ )
10498
+ .returning({ id: schema.sessionTurns.id });
10499
+ if (!supersededTurn) {
10500
+ throw new Error("Codex capacity blocked turn changed during atomic supersession");
10501
+ }
10502
+ }
10503
+ const [queued] = turnWasCurrent
10504
+ ? await tx
10505
+ .select({ id: schema.sessionTurns.id })
10506
+ .from(schema.sessionTurns)
10507
+ .where(
10508
+ and(
10509
+ eq(schema.sessionTurns.workspaceId, input.session.workspaceId),
10510
+ eq(schema.sessionTurns.sessionId, input.session.id),
10511
+ eq(schema.sessionTurns.status, "queued"),
10512
+ ),
10513
+ )
10514
+ .limit(1)
10515
+ : [];
10516
+ const nextSessionStatus =
10517
+ input.session.status === "cancelled" ? "cancelled" : queued ? "queued" : "idle";
10518
+ const eventValues: Array<typeof schema.sessionEvents.$inferInsert> = [
10519
+ {
10182
10520
  accountId: input.session.accountId,
10183
10521
  workspaceId: input.session.workspaceId,
10184
10522
  sessionId: input.session.id,
@@ -10190,18 +10528,45 @@ async function supersedeCodexCapacityWaitInTransaction(
10190
10528
  reason: input.reason,
10191
10529
  }),
10192
10530
  turnId: updated.blockedTurnId,
10531
+ turnGeneration: input.blockedTurn.executionGeneration,
10532
+ ...(turnWasCurrent ? { turnAssociation: "current" } : {}),
10193
10533
  occurredAt: input.now,
10194
- })
10195
- .returning();
10196
- await tx
10534
+ },
10535
+ ];
10536
+ if (turnWasCurrent && input.session.status !== nextSessionStatus) {
10537
+ eventValues.push({
10538
+ accountId: input.session.accountId,
10539
+ workspaceId: input.session.workspaceId,
10540
+ sessionId: input.session.id,
10541
+ sequence: input.session.lastSequence + 2,
10542
+ type: "session.status.changed",
10543
+ payload: { status: nextSessionStatus, reason: input.reason },
10544
+ turnId: updated.blockedTurnId,
10545
+ turnGeneration: input.blockedTurn.executionGeneration,
10546
+ turnAssociation: "current",
10547
+ occurredAt: input.now,
10548
+ });
10549
+ }
10550
+ const inserted = await tx.insert(schema.sessionEvents).values(eventValues).returning();
10551
+ const lastSequence = input.session.lastSequence + inserted.length;
10552
+ const [updatedSession] = await tx
10197
10553
  .update(schema.sessions)
10198
- .set({ lastSequence: input.session.lastSequence + 1, updatedAt: input.now })
10554
+ .set({
10555
+ ...(turnWasCurrent ? { status: nextSessionStatus, activeTurnId: null } : {}),
10556
+ lastSequence,
10557
+ updatedAt: input.now,
10558
+ })
10199
10559
  .where(
10200
10560
  and(
10201
10561
  eq(schema.sessions.workspaceId, input.session.workspaceId),
10202
10562
  eq(schema.sessions.id, input.session.id),
10563
+ ...(turnWasCurrent ? [eq(schema.sessions.activeTurnId, input.blockedTurn.id)] : []),
10203
10564
  ),
10204
- );
10565
+ )
10566
+ .returning({ id: schema.sessions.id });
10567
+ if (!updatedSession) {
10568
+ throw new Error("Codex capacity session changed during atomic supersession");
10569
+ }
10205
10570
  return {
10206
10571
  waiter: mapCodexCapacityWaiter(updated),
10207
10572
  events: inserted.map(mapEvent),
@@ -10211,10 +10576,11 @@ async function supersedeCodexCapacityWaitInTransaction(
10211
10576
  /**
10212
10577
  * Row-lock and re-evaluate one waiter. Availability is decided by a pure
10213
10578
  * caller supplied policy over the same rotation-row transaction as normal
10214
- * acquisition. If available, one system goal-continuation event and one turn
10215
- * are committed; duplicate timers/signals observe status=resumed and do no
10216
- * work. If any goal/control/policy/turn/queue fence changed, the waiter is
10217
- * superseded without inference.
10579
+ * acquisition. If available, the exact blocked turn becomes `recovering`;
10580
+ * duplicate timers/signals observe status=resumed and do no work. Effective
10581
+ * Pause leaves the waiter untouched, ordinary queued prompts remain behind the
10582
+ * current inference, and only an explicit semantic fence change supersedes the
10583
+ * waiter/blocked turn without inference.
10218
10584
  */
10219
10585
  export async function reconcileCodexCapacityWait<
10220
10586
  TPolicyScope = never,
@@ -10287,18 +10653,20 @@ export async function reconcileCodexCapacityWait<
10287
10653
  workspaceControl: prefix.control ?? undefined,
10288
10654
  })
10289
10655
  : null;
10290
- const [goal] = await tx
10291
- .select()
10292
- .from(schema.sessionGoals)
10293
- .where(
10294
- and(
10295
- eq(schema.sessionGoals.workspaceId, input.workspaceId),
10296
- eq(schema.sessionGoals.id, waiterRead.goalId),
10297
- eq(schema.sessionGoals.sessionId, input.sessionId),
10298
- ),
10299
- )
10300
- .for("update")
10301
- .limit(1);
10656
+ const [goal] = waiterRead.goalId
10657
+ ? await tx
10658
+ .select()
10659
+ .from(schema.sessionGoals)
10660
+ .where(
10661
+ and(
10662
+ eq(schema.sessionGoals.workspaceId, input.workspaceId),
10663
+ eq(schema.sessionGoals.id, waiterRead.goalId),
10664
+ eq(schema.sessionGoals.sessionId, input.sessionId),
10665
+ ),
10666
+ )
10667
+ .for("update")
10668
+ .limit(1)
10669
+ : [];
10302
10670
  const [waiter] = await tx
10303
10671
  .select()
10304
10672
  .from(schema.codexCapacityWaiters)
@@ -10307,7 +10675,6 @@ export async function reconcileCodexCapacityWait<
10307
10675
  .limit(1);
10308
10676
  if (
10309
10677
  !session ||
10310
- !goal ||
10311
10678
  !blockedTurn ||
10312
10679
  !waiter ||
10313
10680
  session.accountId !== input.accountId ||
@@ -10326,55 +10693,40 @@ export async function reconcileCodexCapacityWait<
10326
10693
  events: [],
10327
10694
  } as const;
10328
10695
  }
10696
+ if (effectiveControl?.state !== "active" || effectiveControl.settlement !== null) {
10697
+ return {
10698
+ action: "paused",
10699
+ waiter: mapCodexCapacityWaiter(waiter),
10700
+ events: [],
10701
+ } as const;
10702
+ }
10329
10703
 
10330
- const [pending] = await tx
10331
- .select({ id: schema.sessionTurns.id })
10332
- .from(schema.sessionTurns)
10333
- .where(
10334
- and(
10335
- eq(schema.sessionTurns.workspaceId, input.workspaceId),
10336
- eq(schema.sessionTurns.sessionId, input.sessionId),
10337
- inArray(schema.sessionTurns.status, [
10338
- "queued",
10339
- "running",
10340
- "requires_action",
10341
- "recovering",
10342
- "waiting_capacity",
10343
- ]),
10344
- ),
10345
- )
10346
- .limit(1);
10347
- const [laterTurn] = await tx
10348
- .select({ id: schema.sessionTurns.id })
10349
- .from(schema.sessionTurns)
10350
- .where(
10351
- and(
10352
- eq(schema.sessionTurns.workspaceId, input.workspaceId),
10353
- eq(schema.sessionTurns.sessionId, input.sessionId),
10354
- gt(schema.sessionTurns.position, blockedTurn.position),
10355
- ),
10356
- )
10357
- .limit(1);
10358
10704
  const currentPolicyHash = codexCapacityPolicyHashFromTurnMetadata(blockedTurn.metadata);
10359
10705
  let supersedeReason: string | null = null;
10360
- if (effectiveControl?.state !== "active" || effectiveControl.settlement !== null) {
10361
- supersedeReason = "control_changed";
10362
- } else if (goal.status !== "active" || goal.version !== waiter.goalVersion) {
10706
+ if (session.status === "cancelled") {
10707
+ supersedeReason = "session_cancelled";
10708
+ } else if (
10709
+ waiter.goalId !== null &&
10710
+ (!goal || goal.status !== "active" || goal.version !== waiter.goalVersion)
10711
+ ) {
10363
10712
  supersedeReason = "goal_changed";
10364
10713
  } else if (currentPolicyHash !== waiter.policyHash) {
10365
10714
  supersedeReason = "credential_policy_changed";
10366
- } else if (session.status !== "idle" || session.activeTurnId !== null) {
10367
- supersedeReason = "session_not_capacity_idle";
10368
- } else if (blockedTurn.status !== "failed") {
10715
+ } else if (session.activeTurnId !== blockedTurn.id) {
10716
+ supersedeReason = "active_turn_changed";
10717
+ } else if (session.status !== "waiting_capacity") {
10718
+ supersedeReason = "session_not_waiting_capacity";
10719
+ } else if (
10720
+ blockedTurn.status !== "waiting_capacity" ||
10721
+ blockedTurn.activeAttemptId !== null ||
10722
+ blockedTurn.executionGeneration !== waiter.blockedTurnGeneration
10723
+ ) {
10369
10724
  supersedeReason = "blocked_turn_changed";
10370
- } else if (pending) {
10371
- supersedeReason = "pending_work_exists";
10372
- } else if (laterTurn) {
10373
- supersedeReason = "newer_turn_exists";
10374
10725
  }
10375
10726
  if (supersedeReason) {
10376
10727
  const superseded = await supersedeCodexCapacityWaitInTransaction(tx, {
10377
10728
  session,
10729
+ blockedTurn,
10378
10730
  waiter,
10379
10731
  reason: supersedeReason,
10380
10732
  now,
@@ -10446,69 +10798,6 @@ export async function reconcileCodexCapacityWait<
10446
10798
  } as const;
10447
10799
  }
10448
10800
 
10449
- const prompt = [
10450
- "[CODEX CAPACITY RESUME] Codex subscription capacity is available again.",
10451
- `Continue the existing active goal from durable conversation history: ${goal.text}`,
10452
- `Success criteria: ${goal.successCriteria ?? "none specified"}.`,
10453
- "Do not replay completed tool side effects; verify any ambiguous in-flight effect before repeating it.",
10454
- "If the goal is complete, call opengeni__goal_complete. If blocked for another reason, call opengeni__goal_pause.",
10455
- ].join("\n");
10456
- const [update] = await tx
10457
- .insert(schema.sessionSystemUpdates)
10458
- .values({
10459
- accountId: input.accountId,
10460
- workspaceId: input.workspaceId,
10461
- sessionId: input.sessionId,
10462
- kind: "goal_continuation",
10463
- classification: "info",
10464
- sourceId: goal.id,
10465
- dedupeKey: `codex-capacity-resume:${waiter.id}:${waiter.generation}`,
10466
- summary: prompt,
10467
- payload: {
10468
- type: "goal_continuation",
10469
- goalId: goal.id,
10470
- goalVersion: goal.version,
10471
- prompt,
10472
- reason: "codex_capacity",
10473
- capacityWaiterId: waiter.id,
10474
- capacityWaitGeneration: waiter.generation,
10475
- policy: {
10476
- model: blockedTurn.model,
10477
- reasoningEffort: blockedTurn.reasoningEffort,
10478
- tools: blockedTurn.tools,
10479
- sandboxBackend: blockedTurn.sandboxBackend,
10480
- },
10481
- },
10482
- lineage: {
10483
- goalId: goal.id,
10484
- blockedTurnId: blockedTurn.id,
10485
- capacityWaiterId: waiter.id,
10486
- },
10487
- state: "pending",
10488
- })
10489
- .returning();
10490
- if (!update) {
10491
- throw new Error("Codex capacity resume did not create an internal update");
10492
- }
10493
- await tx
10494
- .insert(schema.usageEvents)
10495
- .values({
10496
- accountId: input.accountId,
10497
- workspaceId: input.workspaceId,
10498
- eventType: "agent_run.created",
10499
- quantity: 1,
10500
- unit: "run",
10501
- sourceResourceType: "session_system_update",
10502
- sourceResourceId: update.id,
10503
- sessionId: input.sessionId,
10504
- initiatorKind: "service",
10505
- initiatorSubjectId: "goal-continuation",
10506
- initiatorContext: { goalId: goal.id, reason: "codex_capacity" },
10507
- origin: "goal",
10508
- idempotencyKey: `agent_run.created:codex-capacity:${input.workspaceId}:${update.id}`,
10509
- occurredAt: now,
10510
- })
10511
- .onConflictDoNothing({ target: schema.usageEvents.idempotencyKey });
10512
10801
  const events = await tx
10513
10802
  .insert(schema.sessionEvents)
10514
10803
  .values([
@@ -10517,14 +10806,20 @@ export async function reconcileCodexCapacityWait<
10517
10806
  workspaceId: input.workspaceId,
10518
10807
  sessionId: input.sessionId,
10519
10808
  sequence: session.lastSequence + 1,
10520
- type: "system.update.pending",
10809
+ type: "codex.capacity.resumed",
10521
10810
  payload: sanitizeEventPayload({
10522
- updateId: update.id,
10523
- kind: update.kind,
10524
- classification: update.classification,
10525
- sourceId: update.sourceId,
10526
- summary: update.summary,
10811
+ waiterId: waiter.id,
10812
+ generation: waiter.generation,
10813
+ wakeRevision: waiter.wakeRevision,
10814
+ goalId: waiter.goalId,
10815
+ goalVersion: waiter.goalVersion,
10816
+ blockedTurnGeneration: waiter.blockedTurnGeneration,
10817
+ policyHash: waiter.policyHash,
10818
+ diagnostic: decision.diagnostic ?? null,
10527
10819
  }),
10820
+ turnId: blockedTurn.id,
10821
+ turnGeneration: blockedTurn.executionGeneration,
10822
+ turnAssociation: "current",
10528
10823
  occurredAt: now,
10529
10824
  },
10530
10825
  {
@@ -10532,18 +10827,11 @@ export async function reconcileCodexCapacityWait<
10532
10827
  workspaceId: input.workspaceId,
10533
10828
  sessionId: input.sessionId,
10534
10829
  sequence: session.lastSequence + 2,
10535
- type: "codex.capacity.resumed",
10536
- payload: sanitizeEventPayload({
10537
- waiterId: waiter.id,
10538
- generation: waiter.generation,
10539
- wakeRevision: waiter.wakeRevision,
10540
- goalId: goal.id,
10541
- goalVersion: goal.version,
10542
- policyHash: waiter.policyHash,
10543
- diagnostic: decision.diagnostic ?? null,
10544
- updateId: update.id,
10545
- }),
10830
+ type: "session.status.changed",
10831
+ payload: { status: "recovering", reason: "codex_capacity" },
10546
10832
  turnId: blockedTurn.id,
10833
+ turnGeneration: blockedTurn.executionGeneration,
10834
+ turnAssociation: "current",
10547
10835
  occurredAt: now,
10548
10836
  },
10549
10837
  ])
@@ -10552,7 +10840,7 @@ export async function reconcileCodexCapacityWait<
10552
10840
  .update(schema.codexCapacityWaiters)
10553
10841
  .set({
10554
10842
  status: "resumed",
10555
- resumedUpdateId: update.id,
10843
+ resumedUpdateId: null,
10556
10844
  observedWakeRevision: waiter.wakeRevision,
10557
10845
  lastWakeReason: "capacity_available",
10558
10846
  updatedAt: now,
@@ -10568,11 +10856,34 @@ export async function reconcileCodexCapacityWait<
10568
10856
  if (!updatedWaiter) {
10569
10857
  throw new Error("Codex capacity waiter changed during atomic resume");
10570
10858
  }
10571
- await tx
10859
+ const [recoveringTurn] = await tx
10860
+ .update(schema.sessionTurns)
10861
+ .set({
10862
+ status: "recovering",
10863
+ activeAttemptId: null,
10864
+ metadata: metadataWithoutTurnDispatchAttempt(blockedTurn.metadata),
10865
+ version: blockedTurn.version + 1,
10866
+ finishedAt: null,
10867
+ updatedAt: now,
10868
+ })
10869
+ .where(
10870
+ and(
10871
+ eq(schema.sessionTurns.workspaceId, input.workspaceId),
10872
+ eq(schema.sessionTurns.id, blockedTurn.id),
10873
+ eq(schema.sessionTurns.status, "waiting_capacity"),
10874
+ isNull(schema.sessionTurns.activeAttemptId),
10875
+ eq(schema.sessionTurns.executionGeneration, waiter.blockedTurnGeneration),
10876
+ ),
10877
+ )
10878
+ .returning({ id: schema.sessionTurns.id });
10879
+ if (!recoveringTurn) {
10880
+ throw new Error("Codex capacity blocked turn changed during atomic resume");
10881
+ }
10882
+ const [recoveringSession] = await tx
10572
10883
  .update(schema.sessions)
10573
10884
  .set({
10574
- status: "queued",
10575
- activeTurnId: null,
10885
+ status: "recovering",
10886
+ activeTurnId: blockedTurn.id,
10576
10887
  lastSequence: session.lastSequence + 2,
10577
10888
  updatedAt: now,
10578
10889
  })
@@ -10580,13 +10891,17 @@ export async function reconcileCodexCapacityWait<
10580
10891
  and(
10581
10892
  eq(schema.sessions.workspaceId, input.workspaceId),
10582
10893
  eq(schema.sessions.id, input.sessionId),
10583
- isNull(schema.sessions.activeTurnId),
10894
+ eq(schema.sessions.status, "waiting_capacity"),
10895
+ eq(schema.sessions.activeTurnId, blockedTurn.id),
10584
10896
  ),
10585
- );
10897
+ )
10898
+ .returning({ id: schema.sessions.id });
10899
+ if (!recoveringSession) {
10900
+ throw new Error("Codex capacity session changed during atomic resume");
10901
+ }
10586
10902
  return {
10587
10903
  action: "resumed",
10588
10904
  waiter: mapCodexCapacityWaiter(updatedWaiter),
10589
- update: mapSessionSystemUpdate(update),
10590
10905
  events: events.map(mapEvent),
10591
10906
  } as const;
10592
10907
  }),
@@ -14463,21 +14778,24 @@ export async function listSessionsForSubject(
14463
14778
  if (ordinaryIds.length > limit) {
14464
14779
  let snapshot = reusableSnapshot;
14465
14780
  if (!snapshot) {
14466
- const activeSnapshots = await tx
14467
- .select({ id: schema.sessionListSnapshots.id })
14468
- .from(schema.sessionListSnapshots)
14469
- .where(
14470
- and(
14471
- eq(schema.sessionListSnapshots.workspaceId, workspaceId),
14472
- eq(schema.sessionListSnapshots.subjectId, options.subjectId),
14473
- ),
14474
- )
14475
- .limit(SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT);
14476
- if (activeSnapshots.length >= SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT) {
14477
- throw new SessionListSnapshotLimitError(
14478
- "too many active session list snapshots; retry after an existing cursor expires",
14479
- );
14480
- }
14781
+ // Creation is serialized by the subject advisory lock above.
14782
+ // Retain only the newest N-1 before inserting so polling and
14783
+ // search identity churn cannot turn this bounded cache into a
14784
+ // user-visible 429. A continuation for an evicted row retains
14785
+ // the existing typed expiry/410 + client rebase contract.
14786
+ await tx.execute(sql`
14787
+ delete from ${schema.sessionListSnapshots} snapshot
14788
+ where snapshot.workspace_id = ${workspaceId}
14789
+ and snapshot.subject_id = ${options.subjectId}
14790
+ and snapshot.id in (
14791
+ select evicted.id
14792
+ from ${schema.sessionListSnapshots} evicted
14793
+ where evicted.workspace_id = ${workspaceId}
14794
+ and evicted.subject_id = ${options.subjectId}
14795
+ order by evicted.created_at desc, evicted.id desc
14796
+ offset ${SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT - 1}
14797
+ )
14798
+ `);
14481
14799
  const [workspace] = await tx
14482
14800
  .select({ accountId: schema.workspaces.accountId })
14483
14801
  .from(schema.workspaces)
@@ -19374,7 +19692,7 @@ export async function acquireLease(
19374
19692
  if (liveness === "cold") {
19375
19693
  const recovery = recoveryStateFromLeaseRow(row);
19376
19694
  if (
19377
- recovery.restore.status === "degraded" ||
19695
+ (recovery.restore.status === "degraded" && recovery.restore.retryable !== true) ||
19378
19696
  recovery.restore.status === "unrecoverable"
19379
19697
  ) {
19380
19698
  return {
@@ -20075,10 +20393,186 @@ export async function recordWarmingSandboxCreated(
20075
20393
  );
20076
20394
  }
20077
20395
 
20396
+ export type LostProviderWorkspaceSettlement = {
20397
+ processesLost: number;
20398
+ admissionsRejected: number;
20399
+ ptysClosed: number;
20400
+ processHoldersDeleted: number;
20401
+ };
20402
+
20078
20403
  export type MarkWarmLeaseInstanceLostResult =
20079
- | { status: "marked"; lease: LeaseSnapshot }
20404
+ | {
20405
+ status: "marked";
20406
+ lease: LeaseSnapshot;
20407
+ settlement: LostProviderWorkspaceSettlement;
20408
+ }
20080
20409
  | { status: "stale"; lease: LeaseSnapshot | null };
20081
20410
 
20411
+ const LOST_PROVIDER_PROCESS_REASON = "provider_instance_lost";
20412
+
20413
+ /** Lock the exact provider-owned blocker set before taking the lease row. Normal
20414
+ * retained-process settlement locks process -> admission -> lease, so provider
20415
+ * loss must not hold the lease while waiting for one of those rows. The lease
20416
+ * tuple is revalidated under FOR UPDATE after these locks are acquired. */
20417
+ async function lockExactLostProviderWorkspaceBlockersTx(
20418
+ tx: Database,
20419
+ input: {
20420
+ accountId: string;
20421
+ workspaceId: string;
20422
+ leaseId: string;
20423
+ sandboxGroupId: string;
20424
+ lostEpoch: number;
20425
+ lostInstanceId: string;
20426
+ },
20427
+ ): Promise<void> {
20428
+ await tx.execute(sql`
20429
+ select id from sandbox_retained_processes
20430
+ where account_id = ${input.accountId}
20431
+ and workspace_id = ${input.workspaceId}
20432
+ and lease_id = ${input.leaseId}
20433
+ and sandbox_group_id = ${input.sandboxGroupId}
20434
+ and lease_epoch = ${input.lostEpoch}
20435
+ and provider_instance_id = ${input.lostInstanceId}
20436
+ and state = 'active'
20437
+ order by id
20438
+ for update
20439
+ `);
20440
+ await tx.execute(sql`
20441
+ select id from sandbox_workspace_mutation_admissions
20442
+ where account_id = ${input.accountId}
20443
+ and workspace_id = ${input.workspaceId}
20444
+ and lease_id = ${input.leaseId}
20445
+ and sandbox_group_id = ${input.sandboxGroupId}
20446
+ and lease_epoch = ${input.lostEpoch}
20447
+ and provider_instance_id = ${input.lostInstanceId}
20448
+ and settled_at is null
20449
+ order by id
20450
+ for update
20451
+ `);
20452
+ await tx.execute(sql`
20453
+ select id from sandbox_pty_sessions
20454
+ where account_id = ${input.accountId}
20455
+ and workspace_id = ${input.workspaceId}
20456
+ and lease_id = ${input.leaseId}
20457
+ and sandbox_group_id = ${input.sandboxGroupId}
20458
+ and lease_epoch = ${input.lostEpoch}
20459
+ and provider_instance_id = ${input.lostInstanceId}
20460
+ and status = 'open'
20461
+ order by id
20462
+ for update
20463
+ `);
20464
+ }
20465
+
20466
+ async function settleExactLostProviderWorkspaceBlockersTx(
20467
+ tx: Database,
20468
+ input: {
20469
+ accountId: string;
20470
+ workspaceId: string;
20471
+ leaseId: string;
20472
+ sandboxGroupId: string;
20473
+ lostEpoch: number;
20474
+ lostInstanceId: string;
20475
+ },
20476
+ ): Promise<LostProviderWorkspaceSettlement> {
20477
+ const lostProcesses = await tx
20478
+ .update(schema.sandboxRetainedProcesses)
20479
+ .set({
20480
+ state: "lost",
20481
+ exitCode: null,
20482
+ settlementReason: LOST_PROVIDER_PROCESS_REASON,
20483
+ settledAt: new Date(),
20484
+ })
20485
+ .where(
20486
+ and(
20487
+ eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
20488
+ eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
20489
+ eq(schema.sandboxRetainedProcesses.leaseId, input.leaseId),
20490
+ eq(schema.sandboxRetainedProcesses.sandboxGroupId, input.sandboxGroupId),
20491
+ eq(schema.sandboxRetainedProcesses.leaseEpoch, input.lostEpoch),
20492
+ eq(schema.sandboxRetainedProcesses.providerInstanceId, input.lostInstanceId),
20493
+ eq(schema.sandboxRetainedProcesses.state, "active"),
20494
+ ),
20495
+ )
20496
+ .returning({
20497
+ id: schema.sandboxRetainedProcesses.id,
20498
+ holderId: schema.sandboxRetainedProcesses.holderId,
20499
+ });
20500
+
20501
+ const rejectedAdmissions = await tx
20502
+ .update(schema.sandboxWorkspaceMutationAdmissions)
20503
+ .set({ providerOutcome: "rejected", settledAt: new Date() })
20504
+ .where(
20505
+ and(
20506
+ eq(schema.sandboxWorkspaceMutationAdmissions.accountId, input.accountId),
20507
+ eq(schema.sandboxWorkspaceMutationAdmissions.workspaceId, input.workspaceId),
20508
+ eq(schema.sandboxWorkspaceMutationAdmissions.leaseId, input.leaseId),
20509
+ eq(schema.sandboxWorkspaceMutationAdmissions.sandboxGroupId, input.sandboxGroupId),
20510
+ eq(schema.sandboxWorkspaceMutationAdmissions.leaseEpoch, input.lostEpoch),
20511
+ eq(schema.sandboxWorkspaceMutationAdmissions.providerInstanceId, input.lostInstanceId),
20512
+ isNull(schema.sandboxWorkspaceMutationAdmissions.settledAt),
20513
+ ),
20514
+ )
20515
+ .returning({ id: schema.sandboxWorkspaceMutationAdmissions.id });
20516
+
20517
+ const closedPtys = await tx
20518
+ .update(schema.sandboxPtySessions)
20519
+ .set({ status: "closed", closedAt: new Date() })
20520
+ .where(
20521
+ and(
20522
+ eq(schema.sandboxPtySessions.accountId, input.accountId),
20523
+ eq(schema.sandboxPtySessions.workspaceId, input.workspaceId),
20524
+ eq(schema.sandboxPtySessions.leaseId, input.leaseId),
20525
+ eq(schema.sandboxPtySessions.sandboxGroupId, input.sandboxGroupId),
20526
+ eq(schema.sandboxPtySessions.leaseEpoch, input.lostEpoch),
20527
+ eq(schema.sandboxPtySessions.providerInstanceId, input.lostInstanceId),
20528
+ eq(schema.sandboxPtySessions.status, "open"),
20529
+ ),
20530
+ )
20531
+ .returning({ id: schema.sandboxPtySessions.id });
20532
+
20533
+ const deletedHolders =
20534
+ lostProcesses.length === 0
20535
+ ? []
20536
+ : await tx
20537
+ .delete(schema.sandboxLeaseHolders)
20538
+ .where(
20539
+ and(
20540
+ eq(schema.sandboxLeaseHolders.accountId, input.accountId),
20541
+ eq(schema.sandboxLeaseHolders.workspaceId, input.workspaceId),
20542
+ eq(schema.sandboxLeaseHolders.leaseId, input.leaseId),
20543
+ eq(schema.sandboxLeaseHolders.kind, "process"),
20544
+ inArray(
20545
+ schema.sandboxLeaseHolders.holderId,
20546
+ lostProcesses.map((process) => process.holderId),
20547
+ ),
20548
+ ),
20549
+ )
20550
+ .returning({ holderId: schema.sandboxLeaseHolders.holderId });
20551
+
20552
+ await tx.execute(sql`
20553
+ update sandbox_leases as lease set
20554
+ refcount = counts.total,
20555
+ turn_holders = counts.turns,
20556
+ viewer_holders = counts.viewers,
20557
+ updated_at = now()
20558
+ from (
20559
+ select count(*)::int as total,
20560
+ count(*) filter (where kind = 'turn')::int as turns,
20561
+ count(*) filter (where kind = 'viewer')::int as viewers
20562
+ from sandbox_lease_holders
20563
+ where lease_id = ${input.leaseId}
20564
+ ) as counts
20565
+ where lease.id = ${input.leaseId}
20566
+ `);
20567
+
20568
+ return {
20569
+ processesLost: lostProcesses.length,
20570
+ admissionsRejected: rejectedAdmissions.length,
20571
+ ptysClosed: closedPtys.length,
20572
+ processHoldersDeleted: deletedHolders.length,
20573
+ };
20574
+ }
20575
+
20082
20576
  /**
20083
20577
  * Atomically retire one exact warm provider instance after a resume-only caller
20084
20578
  * receives a provider NotFound. The epoch + instance predicates are the
@@ -20086,10 +20580,11 @@ export type MarkWarmLeaseInstanceLostResult =
20086
20580
  * box, but only the first one transitions the lease to cold and advances its
20087
20581
  * epoch. The next ordinary acquire elects one cold->warming spawner.
20088
20582
  *
20089
- * Holders remain intact because their logical work/viewer interest still exists.
20090
- * Only live provider identity is cleared. A persisted workspace archive is
20091
- * reduced to the same minimal cold envelope used by the drain/failure paths, so
20092
- * the elected replacement can hydrate it without carrying the dead box id.
20583
+ * Logical turn/viewer interest remains intact, while holders for active
20584
+ * processes that physically belonged to the lost provider are removed with
20585
+ * their terminal loss proof. A persisted workspace archive is reduced to the
20586
+ * same minimal cold envelope used by the drain/failure paths, so the elected
20587
+ * replacement can hydrate it without carrying the dead box id.
20093
20588
  */
20094
20589
  export async function markWarmLeaseInstanceLost(
20095
20590
  db: Database,
@@ -20109,6 +20604,33 @@ export async function markWarmLeaseInstanceLost(
20109
20604
  async (scopedDb) =>
20110
20605
  await scopedDb.transaction(async (txRaw) => {
20111
20606
  const tx = txRaw as unknown as Database;
20607
+ const observedRows = await tx.execute<LeaseRow>(sql`
20608
+ select * from sandbox_leases
20609
+ where workspace_id = ${input.workspaceId}
20610
+ and sandbox_group_id = ${input.sandboxGroupId}
20611
+ `);
20612
+ const observed = observedRows[0];
20613
+ if (
20614
+ !observed ||
20615
+ observed.liveness !== "warm" ||
20616
+ Number(observed.lease_epoch) !== input.expectedEpoch ||
20617
+ observed.instance_id !== input.expectedInstanceId
20618
+ ) {
20619
+ return {
20620
+ status: "stale" as const,
20621
+ lease: observed ? mapLeaseRow(observed) : null,
20622
+ };
20623
+ }
20624
+
20625
+ const blockerScope = {
20626
+ accountId: input.accountId,
20627
+ workspaceId: input.workspaceId,
20628
+ leaseId: observed.id,
20629
+ sandboxGroupId: input.sandboxGroupId,
20630
+ lostEpoch: input.expectedEpoch,
20631
+ lostInstanceId: input.expectedInstanceId,
20632
+ };
20633
+ await lockExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
20112
20634
  const currentRows = await tx.execute<LeaseRow>(sql`
20113
20635
  select * from sandbox_leases
20114
20636
  where workspace_id = ${input.workspaceId}
@@ -20118,6 +20640,7 @@ export async function markWarmLeaseInstanceLost(
20118
20640
  const current = currentRows[0];
20119
20641
  if (
20120
20642
  !current ||
20643
+ current.id !== observed.id ||
20121
20644
  current.liveness !== "warm" ||
20122
20645
  Number(current.lease_epoch) !== input.expectedEpoch ||
20123
20646
  current.instance_id !== input.expectedInstanceId
@@ -20128,6 +20651,8 @@ export async function markWarmLeaseInstanceLost(
20128
20651
  };
20129
20652
  }
20130
20653
 
20654
+ const settlement = await settleExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
20655
+
20131
20656
  const observedAt = new Date().toISOString();
20132
20657
  const before = recoveryStateFromLeaseRow(current);
20133
20658
  const archiveStatus = before.archive.status;
@@ -20198,7 +20723,126 @@ export async function markWarmLeaseInstanceLost(
20198
20723
  if (!updated) {
20199
20724
  throw new Error(`Warm sandbox lease vanished while retiring instance ${current.id}`);
20200
20725
  }
20201
- return { status: "marked" as const, lease: mapLeaseRow(updated) };
20726
+ return { status: "marked" as const, lease: mapLeaseRow(updated), settlement };
20727
+ }),
20728
+ );
20729
+ }
20730
+
20731
+ export type ReconcileColdLostLeaseInstanceBlockersResult =
20732
+ | {
20733
+ status: "reconciled";
20734
+ lease: LeaseSnapshot;
20735
+ settlement: LostProviderWorkspaceSettlement;
20736
+ }
20737
+ | { status: "stale"; lease: LeaseSnapshot | null };
20738
+
20739
+ /**
20740
+ * Settle blockers left by a provider-loss transition that predates exact loss
20741
+ * cleanup. This is deliberately narrower than markWarmLeaseInstanceLost: it
20742
+ * requires the already-cold lease to match the full operator-observed
20743
+ * generation/archive tuple, the current epoch to be exactly lostEpoch + 1, and
20744
+ * recovery truth to name the same missing provider. It never advances an epoch,
20745
+ * rematerializes a provider, or changes archive/recovery completeness.
20746
+ */
20747
+ export async function reconcileColdLostLeaseInstanceBlockers(
20748
+ db: Database,
20749
+ input: {
20750
+ accountId: string;
20751
+ workspaceId: string;
20752
+ sandboxGroupId: string;
20753
+ expectedCurrentEpoch: number;
20754
+ expectedLostEpoch: number;
20755
+ expectedLostInstanceId: string;
20756
+ expectedWorkspaceGeneration: number;
20757
+ expectedArchiveGeneration: number | null;
20758
+ expectedArchiveComplete: boolean;
20759
+ },
20760
+ ): Promise<ReconcileColdLostLeaseInstanceBlockersResult> {
20761
+ if (
20762
+ !Number.isSafeInteger(input.expectedCurrentEpoch) ||
20763
+ !Number.isSafeInteger(input.expectedLostEpoch) ||
20764
+ input.expectedCurrentEpoch !== input.expectedLostEpoch + 1
20765
+ ) {
20766
+ throw new Error("Cold lost-provider reconciliation requires currentEpoch = lostEpoch + 1");
20767
+ }
20768
+ return await withRlsContext(
20769
+ db,
20770
+ { accountId: input.accountId, workspaceId: input.workspaceId },
20771
+ async (scopedDb) =>
20772
+ await scopedDb.transaction(async (txRaw) => {
20773
+ const tx = txRaw as unknown as Database;
20774
+ const observedRows = await tx.execute<LeaseRow>(sql`
20775
+ select * from sandbox_leases
20776
+ where workspace_id = ${input.workspaceId}
20777
+ and sandbox_group_id = ${input.sandboxGroupId}
20778
+ `);
20779
+ const observed = observedRows[0];
20780
+ const observedRecovery = observed ? recoveryStateFromLeaseRow(observed) : null;
20781
+ if (
20782
+ !observed ||
20783
+ observed.liveness !== "cold" ||
20784
+ observed.instance_id !== null ||
20785
+ Number(observed.lease_epoch) !== input.expectedCurrentEpoch ||
20786
+ Number(observed.workspace_generation) !== input.expectedWorkspaceGeneration ||
20787
+ (observed.archive_generation === null ? null : Number(observed.archive_generation)) !==
20788
+ input.expectedArchiveGeneration ||
20789
+ hasCompleteWorkspaceArchive(observed) !== input.expectedArchiveComplete ||
20790
+ observedRecovery?.provider.status !== "missing" ||
20791
+ observedRecovery.provider.instanceId !== input.expectedLostInstanceId
20792
+ ) {
20793
+ return {
20794
+ status: "stale" as const,
20795
+ lease: observed ? mapLeaseRow(observed) : null,
20796
+ };
20797
+ }
20798
+
20799
+ const blockerScope = {
20800
+ accountId: input.accountId,
20801
+ workspaceId: input.workspaceId,
20802
+ leaseId: observed.id,
20803
+ sandboxGroupId: input.sandboxGroupId,
20804
+ lostEpoch: input.expectedLostEpoch,
20805
+ lostInstanceId: input.expectedLostInstanceId,
20806
+ };
20807
+ await lockExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
20808
+ const currentRows = await tx.execute<LeaseRow>(sql`
20809
+ select * from sandbox_leases
20810
+ where workspace_id = ${input.workspaceId}
20811
+ and sandbox_group_id = ${input.sandboxGroupId}
20812
+ for update
20813
+ `);
20814
+ const current = currentRows[0];
20815
+ const recovery = current ? recoveryStateFromLeaseRow(current) : null;
20816
+ if (
20817
+ !current ||
20818
+ current.id !== observed.id ||
20819
+ current.liveness !== "cold" ||
20820
+ current.instance_id !== null ||
20821
+ Number(current.lease_epoch) !== input.expectedCurrentEpoch ||
20822
+ Number(current.workspace_generation) !== input.expectedWorkspaceGeneration ||
20823
+ (current.archive_generation === null ? null : Number(current.archive_generation)) !==
20824
+ input.expectedArchiveGeneration ||
20825
+ hasCompleteWorkspaceArchive(current) !== input.expectedArchiveComplete ||
20826
+ recovery?.provider.status !== "missing" ||
20827
+ recovery.provider.instanceId !== input.expectedLostInstanceId
20828
+ ) {
20829
+ return {
20830
+ status: "stale" as const,
20831
+ lease: current ? mapLeaseRow(current) : null,
20832
+ };
20833
+ }
20834
+
20835
+ const settlement = await settleExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
20836
+ const refreshedRows = await tx.execute<LeaseRow>(sql`
20837
+ select * from sandbox_leases where id = ${current.id}
20838
+ `);
20839
+ const refreshed = refreshedRows[0];
20840
+ if (!refreshed) throw new Error("Cold sandbox lease vanished during reconciliation");
20841
+ return {
20842
+ status: "reconciled" as const,
20843
+ lease: mapLeaseRow(refreshed),
20844
+ settlement,
20845
+ };
20202
20846
  }),
20203
20847
  );
20204
20848
  }
@@ -21066,7 +21710,7 @@ export async function confirmDrainCold(
21066
21710
  // The fence is the split-brain guard: a stale-epoch reaper writes ZERO rows and
21067
21711
  // is told not to terminate.
21068
21712
  export class SandboxWorkspaceMutationFencedError extends Error {
21069
- readonly name = "SandboxWorkspaceMutationFencedError";
21713
+ readonly name: string = "SandboxWorkspaceMutationFencedError";
21070
21714
 
21071
21715
  constructor(
21072
21716
  public readonly code:
@@ -21132,8 +21776,131 @@ export type SandboxRetainedProcess = {
21132
21776
  settlementReason: string | null;
21133
21777
  startedAt: string;
21134
21778
  settledAt: string | null;
21779
+ reconcileAfter: string;
21780
+ reconcileClaimId: string | null;
21781
+ reconcileClaimedAt: string | null;
21782
+ reconcileAttempts: number;
21783
+ lastReconcileOutcome: string | null;
21784
+ reconcileProofOutcome: "exited" | "lost" | null;
21785
+ reconcileProofExitCode: number | null;
21786
+ reconcileProofReason:
21787
+ | "provider_exit_banner"
21788
+ | "provider_session_lost_banner"
21789
+ | "provider_instance_not_found"
21790
+ | null;
21791
+ reconcileProofObservedAt: string | null;
21135
21792
  };
21136
21793
 
21794
+ export type RetainedProcessProviderProof =
21795
+ | { outcome: "exited"; exitCode: number; reason: "provider_exit_banner" }
21796
+ | {
21797
+ outcome: "lost";
21798
+ exitCode: null;
21799
+ reason: "provider_session_lost_banner" | "provider_instance_not_found";
21800
+ };
21801
+
21802
+ export type SandboxRetainedProcessIdentity = Pick<
21803
+ SandboxRetainedProcess,
21804
+ | "leaseId"
21805
+ | "sandboxGroupId"
21806
+ | "parentAdmissionId"
21807
+ | "holderId"
21808
+ | "leaseEpoch"
21809
+ | "providerBackend"
21810
+ | "providerInstanceId"
21811
+ | "routeKind"
21812
+ | "routeTargetId"
21813
+ | "routeEpoch"
21814
+ | "providerSessionId"
21815
+ >;
21816
+
21817
+ export type SandboxRetainedProcessReconciliationClaim = {
21818
+ process: SandboxRetainedProcess;
21819
+ claimId: string;
21820
+ ownerState: string;
21821
+ ownerAttemptOutcome: string | null;
21822
+ };
21823
+
21824
+ export type ActiveRetainedProcessOwnerCount = {
21825
+ ownerState: string;
21826
+ activeCount: number;
21827
+ terminalOwnerCount: number;
21828
+ };
21829
+
21830
+ export type ExpiredDrainingSandboxLeaseCount = {
21831
+ backend: string;
21832
+ ageBucket: "lt_5m" | "5m_1h" | "1h_1d" | "gte_1d";
21833
+ count: number;
21834
+ };
21835
+
21836
+ export function retainedProcessSettlementIdentity(
21837
+ process: SandboxRetainedProcess,
21838
+ ): SandboxRetainedProcessIdentity {
21839
+ return {
21840
+ leaseId: process.leaseId,
21841
+ sandboxGroupId: process.sandboxGroupId,
21842
+ parentAdmissionId: process.parentAdmissionId,
21843
+ holderId: process.holderId,
21844
+ leaseEpoch: process.leaseEpoch,
21845
+ providerBackend: process.providerBackend,
21846
+ providerInstanceId: process.providerInstanceId,
21847
+ routeKind: process.routeKind,
21848
+ routeTargetId: process.routeTargetId,
21849
+ routeEpoch: process.routeEpoch,
21850
+ providerSessionId: process.providerSessionId,
21851
+ };
21852
+ }
21853
+
21854
+ export function retainedProcessReconciliationProof(
21855
+ process: SandboxRetainedProcess,
21856
+ ): RetainedProcessProviderProof | null {
21857
+ if (
21858
+ process.reconcileProofOutcome === "exited" &&
21859
+ process.reconcileProofReason === "provider_exit_banner" &&
21860
+ process.reconcileProofExitCode !== null
21861
+ ) {
21862
+ return {
21863
+ outcome: "exited",
21864
+ exitCode: process.reconcileProofExitCode,
21865
+ reason: "provider_exit_banner",
21866
+ };
21867
+ }
21868
+ if (
21869
+ process.reconcileProofOutcome === "lost" &&
21870
+ (process.reconcileProofReason === "provider_session_lost_banner" ||
21871
+ process.reconcileProofReason === "provider_instance_not_found")
21872
+ ) {
21873
+ return {
21874
+ outcome: "lost",
21875
+ exitCode: null,
21876
+ reason: process.reconcileProofReason,
21877
+ };
21878
+ }
21879
+ return null;
21880
+ }
21881
+
21882
+ /** Durable promotion committed before a mutable route/turn authority check
21883
+ * rejected the provider output. The process identity is safe to hand to the
21884
+ * exact-backend cancellation path; callers must still reject the output and
21885
+ * must never replay the provider mutation. */
21886
+ export class SandboxRetainedProcessPromotionFencedError extends SandboxWorkspaceMutationFencedError {
21887
+ readonly name = "SandboxRetainedProcessPromotionFencedError";
21888
+
21889
+ constructor(
21890
+ code:
21891
+ | "attempt_fenced"
21892
+ | "holder_fenced"
21893
+ | "lease_fenced"
21894
+ | "route_fenced"
21895
+ | "process_fenced"
21896
+ | "admission_fenced",
21897
+ message: string,
21898
+ public readonly process: SandboxRetainedProcess,
21899
+ ) {
21900
+ super(code, message);
21901
+ }
21902
+ }
21903
+
21137
21904
  type SandboxWorkspaceMutationSettlementResult =
21138
21905
  | { failure: null }
21139
21906
  | {
@@ -21246,6 +22013,18 @@ function normalizeRetainedProcessSettlementReason(reason: string): string {
21246
22013
  return bounded;
21247
22014
  }
21248
22015
 
22016
+ function normalizeRetainedProcessReconciliationOutcome(outcome: string): string {
22017
+ const normalized = outcome.trim();
22018
+ const byteLength = Buffer.byteLength(normalized, "utf8");
22019
+ if (byteLength < 1 || byteLength > 64) {
22020
+ throw new SandboxWorkspaceMutationFencedError(
22021
+ "process_fenced",
22022
+ "Retained process reconciliation outcome must contain between 1 and 64 UTF-8 bytes",
22023
+ );
22024
+ }
22025
+ return normalized;
22026
+ }
22027
+
21249
22028
  function mapWorkspaceMutationAdmission(row: {
21250
22029
  id: string;
21251
22030
  lease_id: string;
@@ -21311,9 +22090,38 @@ function mapRetainedProcess(
21311
22090
  settlementReason: row.settlementReason ?? null,
21312
22091
  startedAt: row.startedAt.toISOString(),
21313
22092
  settledAt: row.settledAt?.toISOString() ?? null,
22093
+ reconcileAfter: row.reconcileAfter.toISOString(),
22094
+ reconcileClaimId: row.reconcileClaimId ?? null,
22095
+ reconcileClaimedAt: row.reconcileClaimedAt?.toISOString() ?? null,
22096
+ reconcileAttempts: row.reconcileAttempts,
22097
+ lastReconcileOutcome: row.lastReconcileOutcome ?? null,
22098
+ reconcileProofOutcome: row.reconcileProofOutcome ?? null,
22099
+ reconcileProofExitCode: row.reconcileProofExitCode ?? null,
22100
+ reconcileProofReason:
22101
+ (row.reconcileProofReason as SandboxRetainedProcess["reconcileProofReason"]) ?? null,
22102
+ reconcileProofObservedAt: row.reconcileProofObservedAt?.toISOString() ?? null,
21314
22103
  };
21315
22104
  }
21316
22105
 
22106
+ function retainedProcessMatchesSettlementIdentity(
22107
+ process: typeof schema.sandboxRetainedProcesses.$inferSelect,
22108
+ expected: SandboxRetainedProcessIdentity,
22109
+ ): boolean {
22110
+ return (
22111
+ process.leaseId === expected.leaseId &&
22112
+ process.sandboxGroupId === expected.sandboxGroupId &&
22113
+ process.parentAdmissionId === expected.parentAdmissionId &&
22114
+ process.holderId === expected.holderId &&
22115
+ process.leaseEpoch === expected.leaseEpoch &&
22116
+ process.providerBackend === expected.providerBackend &&
22117
+ process.providerInstanceId === expected.providerInstanceId &&
22118
+ process.routeKind === expected.routeKind &&
22119
+ (process.routeTargetId ?? null) === expected.routeTargetId &&
22120
+ process.routeEpoch === expected.routeEpoch &&
22121
+ process.providerSessionId === expected.providerSessionId
22122
+ );
22123
+ }
22124
+
21317
22125
  async function lockWorkspaceMutationSessionTx(
21318
22126
  tx: Database,
21319
22127
  workspaceId: string,
@@ -22333,7 +23141,11 @@ export async function retainWorkspaceMutationProcess(
22333
23141
  }),
22334
23142
  );
22335
23143
  if (result.failure.failure !== null) {
22336
- throw new SandboxWorkspaceMutationFencedError(result.failure.failure, result.failure.detail);
23144
+ throw new SandboxRetainedProcessPromotionFencedError(
23145
+ result.failure.failure,
23146
+ result.failure.detail,
23147
+ result.process,
23148
+ );
22337
23149
  }
22338
23150
  return result.process;
22339
23151
  }
@@ -22360,6 +23172,293 @@ export async function getRetainedProcess(
22360
23172
  });
22361
23173
  }
22362
23174
 
23175
+ /** Claim a bounded, oldest-due batch of active retained processes whose exact
23176
+ * owner attempt is closed (or whose direct request already returned). Claim
23177
+ * expiry only recovers coordination after worker death; it is never provider
23178
+ * exit proof. Rows that settle between the global claim and scoped read are
23179
+ * deliberately omitted. */
23180
+ export async function claimTerminalRetainedProcesses(
23181
+ db: Database,
23182
+ input: { claimId: string; limit: number; claimTtlMs: number },
23183
+ ): Promise<SandboxRetainedProcessReconciliationClaim[]> {
23184
+ if (!Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) {
23185
+ throw new Error("Retained process reconciliation limit must be between 1 and 100");
23186
+ }
23187
+ if (
23188
+ !Number.isSafeInteger(input.claimTtlMs) ||
23189
+ input.claimTtlMs < 0 ||
23190
+ input.claimTtlMs > 3_600_000
23191
+ ) {
23192
+ throw new Error("Retained process reconciliation claim TTL is invalid");
23193
+ }
23194
+ const rows = await rawRows<{
23195
+ account_id: string;
23196
+ workspace_id: string;
23197
+ session_id: string;
23198
+ process_id: string;
23199
+ claim_id: string;
23200
+ owner_state: string;
23201
+ owner_attempt_outcome: string | null;
23202
+ }>(
23203
+ db,
23204
+ sql`
23205
+ select account_id, workspace_id, session_id, process_id, claim_id,
23206
+ owner_state, owner_attempt_outcome
23207
+ from opengeni_private.claim_terminal_retained_processes(
23208
+ ${input.claimId}::uuid, ${input.limit}::integer, ${input.claimTtlMs}::bigint
23209
+ )
23210
+ `,
23211
+ );
23212
+ const claims: SandboxRetainedProcessReconciliationClaim[] = [];
23213
+ for (const row of rows) {
23214
+ const process = await getRetainedProcess(db, {
23215
+ workspaceId: row.workspace_id,
23216
+ sessionId: row.session_id,
23217
+ processId: row.process_id,
23218
+ });
23219
+ if (
23220
+ !process ||
23221
+ process.accountId !== row.account_id ||
23222
+ process.state !== "active" ||
23223
+ process.reconcileClaimId !== row.claim_id
23224
+ ) {
23225
+ continue;
23226
+ }
23227
+ claims.push({
23228
+ process,
23229
+ claimId: row.claim_id,
23230
+ ownerState: row.owner_state,
23231
+ ownerAttemptOutcome: row.owner_attempt_outcome,
23232
+ });
23233
+ }
23234
+ return claims;
23235
+ }
23236
+
23237
+ /** Durably checkpoint exact provider exit/loss proof before canonical
23238
+ * settlement. This closes the worker-death window between provider observation
23239
+ * and settlement without converting owner state, age, timeout, or claim expiry
23240
+ * into physical proof. */
23241
+ export async function recordRetainedProcessReconciliationProof(
23242
+ db: Database,
23243
+ input: {
23244
+ accountId: string;
23245
+ workspaceId: string;
23246
+ sessionId: string;
23247
+ processId: string;
23248
+ expected: SandboxRetainedProcessIdentity;
23249
+ claimId: string;
23250
+ proof: RetainedProcessProviderProof;
23251
+ },
23252
+ ): Promise<SandboxRetainedProcess> {
23253
+ if (
23254
+ input.proof.outcome === "exited" &&
23255
+ (!Number.isSafeInteger(input.proof.exitCode) || input.proof.exitCode === null)
23256
+ ) {
23257
+ throw new SandboxWorkspaceMutationFencedError(
23258
+ "process_fenced",
23259
+ "Retained process reconciliation exit proof requires a safe integer exit code",
23260
+ );
23261
+ }
23262
+ return await withRlsContext(
23263
+ db,
23264
+ { accountId: input.accountId, workspaceId: input.workspaceId },
23265
+ async (scopedDb) =>
23266
+ await scopedDb.transaction(async (txRaw) => {
23267
+ const tx = txRaw as unknown as Database;
23268
+ const [process] = await tx
23269
+ .select()
23270
+ .from(schema.sandboxRetainedProcesses)
23271
+ .where(
23272
+ and(
23273
+ eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
23274
+ eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
23275
+ eq(schema.sandboxRetainedProcesses.sessionId, input.sessionId),
23276
+ eq(schema.sandboxRetainedProcesses.id, input.processId),
23277
+ ),
23278
+ )
23279
+ .for("update")
23280
+ .limit(1);
23281
+ if (
23282
+ !process ||
23283
+ process.state !== "active" ||
23284
+ !retainedProcessMatchesSettlementIdentity(process, input.expected)
23285
+ ) {
23286
+ throw new SandboxWorkspaceMutationFencedError(
23287
+ "process_fenced",
23288
+ "Retained process proof did not match an active copied durable identity",
23289
+ );
23290
+ }
23291
+ if (process.reconcileClaimId !== input.claimId) {
23292
+ throw new SandboxWorkspaceMutationFencedError(
23293
+ "process_fenced",
23294
+ "Retained process proof claim was lost or superseded",
23295
+ );
23296
+ }
23297
+ const existing = mapRetainedProcess(process);
23298
+ const existingProof = retainedProcessReconciliationProof(existing);
23299
+ if (existingProof) {
23300
+ if (
23301
+ existingProof.outcome !== input.proof.outcome ||
23302
+ existingProof.exitCode !== input.proof.exitCode ||
23303
+ existingProof.reason !== input.proof.reason
23304
+ ) {
23305
+ throw new SandboxWorkspaceMutationFencedError(
23306
+ "process_fenced",
23307
+ "Retained process already carries different provider proof",
23308
+ );
23309
+ }
23310
+ return existing;
23311
+ }
23312
+ const [updated] = await tx
23313
+ .update(schema.sandboxRetainedProcesses)
23314
+ .set({
23315
+ reconcileProofOutcome: input.proof.outcome,
23316
+ reconcileProofExitCode: input.proof.exitCode,
23317
+ reconcileProofReason: input.proof.reason,
23318
+ reconcileProofObservedAt: new Date(),
23319
+ lastReconcileOutcome: `proof_${input.proof.outcome}`,
23320
+ })
23321
+ .where(
23322
+ and(
23323
+ eq(schema.sandboxRetainedProcesses.id, process.id),
23324
+ eq(schema.sandboxRetainedProcesses.state, "active"),
23325
+ eq(schema.sandboxRetainedProcesses.reconcileClaimId, input.claimId),
23326
+ ),
23327
+ )
23328
+ .returning();
23329
+ if (!updated) {
23330
+ throw new SandboxWorkspaceMutationFencedError(
23331
+ "process_fenced",
23332
+ "Retained process changed while checkpointing provider proof",
23333
+ );
23334
+ }
23335
+ return mapRetainedProcess(updated);
23336
+ }),
23337
+ );
23338
+ }
23339
+
23340
+ /** Release one exact reconciliation claim after an ambiguous/running provider
23341
+ * observation. This schedules a bounded retry but never changes process,
23342
+ * admission, holder, lease, archive, snapshot, or workspace state. */
23343
+ export async function deferRetainedProcessReconciliation(
23344
+ db: Database,
23345
+ input: {
23346
+ accountId: string;
23347
+ workspaceId: string;
23348
+ sessionId: string;
23349
+ processId: string;
23350
+ expected: SandboxRetainedProcessIdentity;
23351
+ claimId: string;
23352
+ outcome: string;
23353
+ retryAfterMs: number;
23354
+ },
23355
+ ): Promise<boolean> {
23356
+ const outcome = normalizeRetainedProcessReconciliationOutcome(input.outcome);
23357
+ if (
23358
+ !Number.isSafeInteger(input.retryAfterMs) ||
23359
+ input.retryAfterMs < 0 ||
23360
+ input.retryAfterMs > 86_400_000
23361
+ ) {
23362
+ throw new SandboxWorkspaceMutationFencedError(
23363
+ "process_fenced",
23364
+ "Retained process reconciliation retry delay is invalid",
23365
+ );
23366
+ }
23367
+ return await withRlsContext(
23368
+ db,
23369
+ { accountId: input.accountId, workspaceId: input.workspaceId },
23370
+ async (scopedDb) =>
23371
+ await scopedDb.transaction(async (txRaw) => {
23372
+ const tx = txRaw as unknown as Database;
23373
+ const [process] = await tx
23374
+ .select()
23375
+ .from(schema.sandboxRetainedProcesses)
23376
+ .where(
23377
+ and(
23378
+ eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
23379
+ eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
23380
+ eq(schema.sandboxRetainedProcesses.sessionId, input.sessionId),
23381
+ eq(schema.sandboxRetainedProcesses.id, input.processId),
23382
+ ),
23383
+ )
23384
+ .for("update")
23385
+ .limit(1);
23386
+ if (!process || !retainedProcessMatchesSettlementIdentity(process, input.expected)) {
23387
+ throw new SandboxWorkspaceMutationFencedError(
23388
+ "process_fenced",
23389
+ "Retained process reconciliation did not match the copied durable identity",
23390
+ );
23391
+ }
23392
+ if (process.state !== "active") return false;
23393
+ if (process.reconcileClaimId !== input.claimId) {
23394
+ throw new SandboxWorkspaceMutationFencedError(
23395
+ "process_fenced",
23396
+ "Retained process reconciliation claim was lost or superseded",
23397
+ );
23398
+ }
23399
+ const [updated] = await tx
23400
+ .update(schema.sandboxRetainedProcesses)
23401
+ .set({
23402
+ reconcileAfter: new Date(Date.now() + input.retryAfterMs),
23403
+ reconcileClaimId: null,
23404
+ reconcileClaimedAt: null,
23405
+ lastReconcileOutcome: outcome,
23406
+ })
23407
+ .where(
23408
+ and(
23409
+ eq(schema.sandboxRetainedProcesses.id, process.id),
23410
+ eq(schema.sandboxRetainedProcesses.state, "active"),
23411
+ eq(schema.sandboxRetainedProcesses.reconcileClaimId, input.claimId),
23412
+ ),
23413
+ )
23414
+ .returning({ id: schema.sandboxRetainedProcesses.id });
23415
+ return Boolean(updated);
23416
+ }),
23417
+ );
23418
+ }
23419
+
23420
+ export async function countActiveRetainedProcessesByOwnerState(
23421
+ db: Database,
23422
+ ): Promise<ActiveRetainedProcessOwnerCount[]> {
23423
+ const rows = await rawRows<{
23424
+ owner_state: string;
23425
+ active_count: number | string;
23426
+ terminal_owner_count: number | string;
23427
+ }>(
23428
+ db,
23429
+ sql`
23430
+ select owner_state, active_count, terminal_owner_count
23431
+ from opengeni_private.count_active_retained_processes_by_owner_state()
23432
+ `,
23433
+ );
23434
+ return rows.map((row) => ({
23435
+ ownerState: row.owner_state,
23436
+ activeCount: Number(row.active_count),
23437
+ terminalOwnerCount: Number(row.terminal_owner_count),
23438
+ }));
23439
+ }
23440
+
23441
+ export async function countExpiredDrainingSandboxLeases(
23442
+ db: Database,
23443
+ ): Promise<ExpiredDrainingSandboxLeaseCount[]> {
23444
+ const rows = await rawRows<{
23445
+ backend: string;
23446
+ age_bucket: ExpiredDrainingSandboxLeaseCount["ageBucket"];
23447
+ count: number | string;
23448
+ }>(
23449
+ db,
23450
+ sql`
23451
+ select backend, age_bucket, count
23452
+ from opengeni_private.count_expired_draining_sandbox_leases()
23453
+ `,
23454
+ );
23455
+ return rows.map((row) => ({
23456
+ backend: row.backend,
23457
+ ageBucket: row.age_bucket,
23458
+ count: Number(row.count),
23459
+ }));
23460
+ }
23461
+
22363
23462
  /** Settle an exact retained process only after exit or definitive loss proof.
22364
23463
  * The process row, parent admission, non-TTL holder, and lease count transition
22365
23464
  * commit atomically. Duplicate identical proof is idempotent; conflicting proof
@@ -22371,6 +23470,8 @@ export async function settleRetainedProcess(
22371
23470
  workspaceId: string;
22372
23471
  sessionId: string;
22373
23472
  processId: string;
23473
+ expected: SandboxRetainedProcessIdentity;
23474
+ reconciliationClaimId?: string;
22374
23475
  outcome: "exited" | "lost";
22375
23476
  exitCode?: number | null;
22376
23477
  reason: string;
@@ -22411,6 +23512,12 @@ export async function settleRetainedProcess(
22411
23512
  "Retained process settlement did not match a durable process",
22412
23513
  );
22413
23514
  }
23515
+ if (!retainedProcessMatchesSettlementIdentity(process, input.expected)) {
23516
+ throw new SandboxWorkspaceMutationFencedError(
23517
+ "process_fenced",
23518
+ "Retained process settlement did not match the copied durable identity",
23519
+ );
23520
+ }
22414
23521
  if (process.state !== "active") {
22415
23522
  if (
22416
23523
  process.state !== input.outcome ||
@@ -22433,6 +23540,27 @@ export async function settleRetainedProcess(
22433
23540
  );
22434
23541
  return { settled: false, process: mapRetainedProcess(process) };
22435
23542
  }
23543
+ if (
23544
+ input.reconciliationClaimId !== undefined &&
23545
+ process.reconcileClaimId !== input.reconciliationClaimId
23546
+ ) {
23547
+ throw new SandboxWorkspaceMutationFencedError(
23548
+ "process_fenced",
23549
+ "Retained process settlement reconciliation claim was lost or superseded",
23550
+ );
23551
+ }
23552
+ const durableProof = retainedProcessReconciliationProof(mapRetainedProcess(process));
23553
+ if (
23554
+ durableProof &&
23555
+ (durableProof.outcome !== input.outcome ||
23556
+ durableProof.exitCode !== exitCode ||
23557
+ durableProof.reason !== reason)
23558
+ ) {
23559
+ throw new SandboxWorkspaceMutationFencedError(
23560
+ "process_fenced",
23561
+ "Retained process settlement conflicts with checkpointed provider proof",
23562
+ );
23563
+ }
22436
23564
  const admissions = await tx.execute<AdmissionIdentityRow>(sql`
22437
23565
  select * from sandbox_workspace_mutation_admissions
22438
23566
  where id = ${process.parentAdmissionId}
@@ -22440,6 +23568,13 @@ export async function settleRetainedProcess(
22440
23568
  and workspace_id = ${input.workspaceId}
22441
23569
  and session_id = ${input.sessionId}
22442
23570
  and lease_id = ${process.leaseId}
23571
+ and sandbox_group_id = ${process.sandboxGroupId}
23572
+ and lease_epoch = ${process.leaseEpoch}
23573
+ and provider_backend = ${process.providerBackend}
23574
+ and provider_instance_id = ${process.providerInstanceId}
23575
+ and route_kind = ${process.routeKind}
23576
+ and route_target_id is not distinct from ${process.routeTargetId}
23577
+ and route_epoch = ${process.routeEpoch}
22443
23578
  and provider_outcome = 'retained'
22444
23579
  and settled_at is null
22445
23580
  for update
@@ -22450,6 +23585,23 @@ export async function settleRetainedProcess(
22450
23585
  "Retained process parent admission is not open",
22451
23586
  );
22452
23587
  }
23588
+ const leases = await tx.execute<LeaseRow>(sql`
23589
+ select * from sandbox_leases
23590
+ where id = ${process.leaseId}
23591
+ and account_id = ${input.accountId}
23592
+ and workspace_id = ${input.workspaceId}
23593
+ and sandbox_group_id = ${process.sandboxGroupId}
23594
+ and lease_epoch = ${process.leaseEpoch}
23595
+ and backend = ${process.providerBackend}
23596
+ and instance_id = ${process.providerInstanceId}
23597
+ for update
23598
+ `);
23599
+ if (!leases[0]) {
23600
+ throw new SandboxWorkspaceMutationFencedError(
23601
+ "lease_fenced",
23602
+ "Retained process settlement cannot mutate a successor lease identity",
23603
+ );
23604
+ }
22453
23605
  await tx
22454
23606
  .update(schema.sandboxPtySessions)
22455
23607
  .set({ status: "closed", closedAt: new Date() })
@@ -22466,6 +23618,12 @@ export async function settleRetainedProcess(
22466
23618
  exitCode,
22467
23619
  settlementReason: reason,
22468
23620
  settledAt: new Date(),
23621
+ reconcileClaimId: null,
23622
+ reconcileClaimedAt: null,
23623
+ lastReconcileOutcome:
23624
+ input.reconciliationClaimId === undefined
23625
+ ? `owner_settled_${input.outcome}`
23626
+ : `reconciled_${input.outcome}`,
22469
23627
  })
22470
23628
  .where(
22471
23629
  and(
@@ -22490,6 +23648,8 @@ export async function settleRetainedProcess(
22490
23648
  await tx.execute(sql`
22491
23649
  delete from sandbox_lease_holders
22492
23650
  where lease_id = ${process.leaseId}
23651
+ and account_id = ${input.accountId}
23652
+ and workspace_id = ${input.workspaceId}
22493
23653
  and kind = 'process' and holder_id = ${process.holderId}
22494
23654
  `);
22495
23655
  const [counts] = await tx.execute<{
@@ -22518,6 +23678,10 @@ export async function settleRetainedProcess(
22518
23678
  }
22519
23679
  updated_at = now()
22520
23680
  where id = ${process.leaseId}
23681
+ and sandbox_group_id = ${process.sandboxGroupId}
23682
+ and lease_epoch = ${process.leaseEpoch}
23683
+ and backend = ${process.providerBackend}
23684
+ and instance_id = ${process.providerInstanceId}
22521
23685
  `);
22522
23686
  return { settled: true, process: mapRetainedProcess(updated) };
22523
23687
  }),
@@ -27521,7 +28685,7 @@ export async function initializeSessionStartAtomically(
27521
28685
  if (!goal) throw new Error("Failed to create initial session goal");
27522
28686
  }
27523
28687
 
27524
- let [userEvent] = await tx
28688
+ const existingUserEvents = await tx
27525
28689
  .select()
27526
28690
  .from(schema.sessionEvents)
27527
28691
  .where(
@@ -27533,6 +28697,7 @@ export async function initializeSessionStartAtomically(
27533
28697
  )
27534
28698
  .orderBy(asc(schema.sessionEvents.sequence))
27535
28699
  .limit(1);
28700
+ let userEvent: typeof schema.sessionEvents.$inferSelect | undefined = existingUserEvents[0];
27536
28701
  let sequence = session.lastSequence;
27537
28702
  const insertedEvents: Array<typeof schema.sessionEvents.$inferSelect> = [];
27538
28703
  const runnable = effectiveControl.state === "active";
@@ -27732,7 +28897,7 @@ export async function initializeSessionStartAtomically(
27732
28897
  tx as unknown as Database,
27733
28898
  input.consumeNewSessionDraft.subjectId,
27734
28899
  );
27735
- await consumeNewSessionDraftInTransaction(tx as unknown as Database, {
28900
+ await seedNewSessionDraftInTransaction(tx as unknown as Database, {
27736
28901
  workspaceId: input.workspaceId,
27737
28902
  subjectId: input.consumeNewSessionDraft.subjectId,
27738
28903
  expectedRevision: input.consumeNewSessionDraft.expectedRevision,
@@ -29568,7 +30733,7 @@ export async function peekSessionWork(
29568
30733
  if (
29569
30734
  latestInterruption &&
29570
30735
  latestInterruption.quiescedAt === null &&
29571
- ["settled", "rejected_stale"].includes(latestInterruption.interruptionState)
30736
+ latestInterruption.interruptionState === "settled"
29572
30737
  ) {
29573
30738
  return {
29574
30739
  kind: "cancellation-wait",
@@ -32894,6 +34059,9 @@ function sessionEventTypesAdvanceActivity(inputs: ReadonlyArray<{ type: string }
32894
34059
  function sessionMutationAdvancesActivity(update: {
32895
34060
  resources?: ResourceRef[];
32896
34061
  tools?: ToolRef[];
34062
+ toolPolicy?: SessionToolPolicy;
34063
+ toolPolicyVersion?: number;
34064
+ expectedToolPolicyVersion?: number;
32897
34065
  model?: string;
32898
34066
  metadata?: Record<string, unknown>;
32899
34067
  status?: SessionStatus;
@@ -33482,6 +34650,7 @@ type LockedSessionUpdateContext = {
33482
34650
  requireApproval: SessionMcpApprovalPolicy,
33483
34651
  ) => Promise<UpdateSessionMcpApprovalPolicyResult>;
33484
34652
  listPendingSessionTurns: () => Promise<SessionTurn[]>;
34653
+ getLockedSession: (sessionId: string) => Promise<Session | null>;
33485
34654
  };
33486
34655
 
33487
34656
  type LockedSessionUpdateResult = {
@@ -33489,6 +34658,9 @@ type LockedSessionUpdateResult = {
33489
34658
  update?: {
33490
34659
  resources?: ResourceRef[];
33491
34660
  tools?: ToolRef[];
34661
+ toolPolicy?: SessionToolPolicy;
34662
+ toolPolicyVersion?: number;
34663
+ expectedToolPolicyVersion?: number;
33492
34664
  model?: string;
33493
34665
  metadata?: Record<string, unknown>;
33494
34666
  status?: SessionStatus;
@@ -33504,18 +34676,39 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33504
34676
  session: Session,
33505
34677
  context: LockedSessionUpdateContext,
33506
34678
  ) => LockedSessionUpdateResult | Promise<LockedSessionUpdateResult>,
34679
+ options: { lockParentSession?: boolean } = {},
33507
34680
  ): Promise<SessionEvent[]> {
33508
34681
  return await withWorkspaceRls(
33509
34682
  db,
33510
34683
  workspaceId,
33511
34684
  async (scopedDb) =>
33512
34685
  await scopedDb.transaction(async (tx) => {
33513
- const locks = await lockSessionEventWriteRows(tx as unknown as Database, {
34686
+ const firstLocks = await lockSessionEventWriteRows(tx as unknown as Database, {
33514
34687
  workspaceId,
33515
34688
  controlLock: "share",
33516
34689
  sessionIds: [sessionId],
33517
34690
  });
33518
- const sessionRow = locks.sessions[0];
34691
+ const firstSessionRow = firstLocks.sessions.find((row) => row.id === sessionId);
34692
+ if (!firstSessionRow) {
34693
+ throw new Error(`Session not found: ${sessionId}`);
34694
+ }
34695
+ // Child-policy updates must serialize with a concurrent parent-policy
34696
+ // update. The target lock establishes the parent id, then the second
34697
+ // call acquires the complete UUID-ordered set under the already-held
34698
+ // workspace/control prefix.
34699
+ const lockSessionIds = options.lockParentSession
34700
+ ? [sessionId, firstSessionRow.parentSessionId].filter((id): id is string => Boolean(id))
34701
+ : [sessionId];
34702
+ const locks =
34703
+ lockSessionIds.length === 1
34704
+ ? firstLocks
34705
+ : await lockSessionEventWriteRows(tx as unknown as Database, {
34706
+ workspaceId,
34707
+ controlLock: "already_locked",
34708
+ workspaceLock: "already_locked",
34709
+ sessionIds: lockSessionIds,
34710
+ });
34711
+ const sessionRow = locks.sessions.find((row) => row.id === sessionId);
33519
34712
  if (!sessionRow) {
33520
34713
  throw new Error(`Session not found: ${sessionId}`);
33521
34714
  }
@@ -33554,6 +34747,18 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33554
34747
  .orderBy(asc(schema.sessionTurns.position), asc(schema.sessionTurns.createdAt));
33555
34748
  return rows.map(mapSessionTurn);
33556
34749
  },
34750
+ getLockedSession: async (lockedSessionId) => {
34751
+ const row = locks.sessions.find((candidate) => candidate.id === lockedSessionId);
34752
+ return row
34753
+ ? await mapSessionWithControl(
34754
+ tx as unknown as Database,
34755
+ row,
34756
+ [],
34757
+ undefined,
34758
+ locks.control ?? undefined,
34759
+ )
34760
+ : null;
34761
+ },
33557
34762
  });
33558
34763
  if (built.events.length === 0) {
33559
34764
  return [];
@@ -33591,12 +34796,16 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33591
34796
  const update = built.update ?? {};
33592
34797
  const advancesActivity =
33593
34798
  sessionMutationAdvancesActivity(update) || sessionEventTypesAdvanceActivity(values);
33594
- await tx
34799
+ const updated = await tx
33595
34800
  .update(schema.sessions)
33596
34801
  .set({
33597
34802
  lastSequence: sequence,
33598
34803
  ...(update.resources !== undefined ? { resources: update.resources } : {}),
33599
34804
  ...(update.tools !== undefined ? { tools: update.tools } : {}),
34805
+ ...(update.toolPolicy !== undefined ? { toolPolicy: update.toolPolicy } : {}),
34806
+ ...(update.toolPolicyVersion !== undefined
34807
+ ? { toolPolicyVersion: update.toolPolicyVersion }
34808
+ : {}),
33600
34809
  ...(update.model !== undefined ? { model: update.model } : {}),
33601
34810
  ...(update.metadata !== undefined ? { metadata: update.metadata } : {}),
33602
34811
  ...(update.status !== undefined ? { status: update.status } : {}),
@@ -33604,8 +34813,27 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33604
34813
  ...(advancesActivity ? { updatedAt: now } : {}),
33605
34814
  })
33606
34815
  .where(
33607
- and(eq(schema.sessions.workspaceId, workspaceId), eq(schema.sessions.id, sessionId)),
34816
+ and(
34817
+ eq(schema.sessions.workspaceId, workspaceId),
34818
+ eq(schema.sessions.id, sessionId),
34819
+ ...(update.expectedToolPolicyVersion !== undefined
34820
+ ? [eq(schema.sessions.toolPolicyVersion, update.expectedToolPolicyVersion)]
34821
+ : []),
34822
+ ),
34823
+ )
34824
+ .returning({ id: schema.sessions.id });
34825
+ if (updated.length === 0) {
34826
+ const [current] = await tx
34827
+ .select({ toolPolicyVersion: schema.sessions.toolPolicyVersion })
34828
+ .from(schema.sessions)
34829
+ .where(
34830
+ and(eq(schema.sessions.workspaceId, workspaceId), eq(schema.sessions.id, sessionId)),
34831
+ )
34832
+ .limit(1);
34833
+ throw new SessionToolPolicyVersionConflictError(
34834
+ Number(current?.toolPolicyVersion ?? update.expectedToolPolicyVersion ?? 1),
33608
34835
  );
34836
+ }
33609
34837
  return inserted.map(mapEvent);
33610
34838
  }),
33611
34839
  );
@@ -33666,6 +34894,7 @@ function mapSession(
33666
34894
  mode: "legacy",
33667
34895
  inheritedFromSessionId: null,
33668
34896
  },
34897
+ toolPolicyVersion: Number(row.toolPolicyVersion ?? 1),
33669
34898
  metadata: row.metadata,
33670
34899
  createdBy: initiatorFromStorage(
33671
34900
  row.createdByKind,
@@ -34270,10 +35499,17 @@ function mapGitHubInstallation(
34270
35499
  accountId: row.accountId,
34271
35500
  workspaceId: row.workspaceId,
34272
35501
  installationId: row.installationId,
35502
+ githubAccountId: row.githubAccountId,
34273
35503
  accountLogin: row.accountLogin,
34274
35504
  accountType: row.accountType,
34275
35505
  repositoryScope: row.repositoryScope as GitHubRepositoryScope,
34276
35506
  linkedBySubjectId: row.linkedBySubjectId,
35507
+ githubActorId: row.githubActorId,
35508
+ githubActorLogin: row.githubActorLogin,
35509
+ authorityKind: row.authorityKind as GitHubInstallationAuthorityKind | null,
35510
+ authorityCheckedAt: row.authorityCheckedAt?.toISOString() ?? null,
35511
+ authorityExpiresAt: row.authorityExpiresAt?.toISOString() ?? null,
35512
+ authorityNonce: row.authorityNonce,
34277
35513
  createdAt: row.createdAt.toISOString(),
34278
35514
  updatedAt: row.updatedAt.toISOString(),
34279
35515
  };