@opengeni/db 0.12.1 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/chunk-ELMUCYZF.js +906 -0
  2. package/dist/chunk-ELMUCYZF.js.map +1 -0
  3. package/dist/chunk-N4WTE6SB.js +4033 -0
  4. package/dist/chunk-N4WTE6SB.js.map +1 -0
  5. package/dist/index.d.ts +2 -2
  6. package/dist/index.js +3594 -2059
  7. package/dist/index.js.map +1 -1
  8. package/dist/provision-roles.d.ts +472 -43
  9. package/dist/provision-roles.js +1 -1
  10. package/dist/{schema-BejThLcd.d.ts → schema-Bh1Hr7xY.d.ts} +2507 -970
  11. package/dist/schema.d.ts +1 -1
  12. package/dist/schema.js +9 -1
  13. package/drizzle/0122_codex_capacity_same_turn.sql +59 -0
  14. package/drizzle/0123_session_tool_policy_version.sql +14 -0
  15. package/drizzle/0124_session_event_duplicate_lookup.sql +4 -0
  16. package/drizzle/0125_document_drops_visibility.sql +29 -0
  17. package/drizzle/0126_document_access_constraints.sql +138 -0
  18. package/drizzle/0127_document_default_base_index.sql +5 -0
  19. package/drizzle/0128_github_installation_authority.sql +69 -0
  20. package/drizzle/0129_retained_process_reconciliation.sql +356 -0
  21. package/drizzle/0130_workspace_instruction_policies.sql +330 -0
  22. package/drizzle/0131_slack_bot_install_and_post_idempotency.sql +133 -0
  23. package/package.json +4 -3
  24. package/src/event-payload-sanitizer.ts +167 -55
  25. package/src/index.ts +1850 -264
  26. package/src/new-session-drafts.ts +127 -6
  27. package/src/provision-roles.ts +184 -2
  28. package/src/runtime-posture-cli.ts +57 -0
  29. package/src/runtime-posture.ts +770 -0
  30. package/src/schema.ts +243 -3
  31. package/src/session-control.ts +1 -0
  32. package/src/workspace-instruction-policies-schema.ts +140 -0
  33. package/src/workspace-instruction-policies.ts +624 -0
  34. package/dist/chunk-BMFDXFPA.js +0 -155
  35. package/dist/chunk-BMFDXFPA.js.map +0 -1
  36. package/dist/chunk-VUKRIBO5.js +0 -3679
  37. package/dist/chunk-VUKRIBO5.js.map +0 -1
package/src/index.ts CHANGED
@@ -23,6 +23,7 @@ import type {
23
23
  KnowledgeMemoryKind,
24
24
  KnowledgeMemoryStatus,
25
25
  KnowledgeSourceRef,
26
+ GitHubInstallationAuthorityKind,
26
27
  GitHubRepositoryScope,
27
28
  HostEventExport,
28
29
  HostEventExportBatch,
@@ -159,7 +160,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
159
160
  import postgres from "postgres";
160
161
  import { decryptEnvironmentValue } from "./environment-crypto";
161
162
  import { sanitizeEventPayload, sanitizeModelPayload } from "./event-payload-sanitizer";
162
- import { consumeNewSessionDraftInTransaction } from "./new-session-drafts";
163
+ import { seedNewSessionDraftInTransaction } from "./new-session-drafts";
163
164
  import {
164
165
  runIdempotentPersistenceTransaction,
165
166
  type IdempotentPersistenceTransactionOptions,
@@ -213,6 +214,7 @@ export { sql as dbSql } from "drizzle-orm";
213
214
  export * from "./session-control";
214
215
  export * from "./session-queue-commands";
215
216
  export * from "./new-session-drafts";
217
+ export * from "./workspace-instruction-policies";
216
218
  export { interruptedToolCallResult } from "./session-tool-call-settlement";
217
219
  export { decryptEnvironmentValue, encryptEnvironmentValue } from "./environment-crypto";
218
220
  export {
@@ -225,6 +227,7 @@ export {
225
227
  sanitizeModelPayload,
226
228
  } from "./event-payload-sanitizer";
227
229
  export * from "./persistence-errors";
230
+ export * from "./runtime-posture";
228
231
  export { sanitizeMemoryText } from "./memory-domain";
229
232
  // Re-exported so external consumers can `import { migrate } from "@opengeni/db"`.
230
233
  // The `@opengeni/db/migrate` subpath stays available too (internal callers + the
@@ -260,6 +263,16 @@ export * from "./memory-domain";
260
263
  // unaffected.
261
264
  export type Database = PgDatabase<any, typeof schema>;
262
265
 
266
+ /** Raised when a durable session tool-policy write lost its version fence. */
267
+ export class SessionToolPolicyVersionConflictError extends Error {
268
+ readonly code = "SESSION_TOOL_POLICY_CONFLICT";
269
+
270
+ constructor(readonly currentVersion: number) {
271
+ super("The session tool policy changed in another client");
272
+ this.name = "SessionToolPolicyVersionConflictError";
273
+ }
274
+ }
275
+
263
276
  export type DbClient = {
264
277
  db: Database;
265
278
  close: () => Promise<void>;
@@ -999,15 +1012,27 @@ export async function withRlsContext<T>(
999
1012
  // manufacturing a phantom "no active subscription" from a credential that is
1000
1013
  // in fact active. Convert that silent false into a loud, root-cause-bearing
1001
1014
  // error so the caller can retry rather than permanently mis-decide.
1002
- const applied = await tx.execute<{ account_id: string | null }>(
1003
- sql`select current_setting('opengeni.account_id', true) as account_id`,
1015
+ const applied = await tx.execute<{
1016
+ account_id: string | null;
1017
+ workspace_id: string | null;
1018
+ }>(
1019
+ sql`select
1020
+ current_setting('opengeni.account_id', true) as account_id,
1021
+ current_setting('opengeni.workspace_id', true) as workspace_id`,
1004
1022
  );
1005
1023
  const appliedAccountId = applied[0]?.account_id ?? "";
1024
+ const expectedWorkspaceId = context.workspaceId ?? "";
1025
+ const appliedWorkspaceId = applied[0]?.workspace_id ?? "";
1006
1026
  if (appliedAccountId !== context.accountId) {
1007
1027
  throw new Error(
1008
1028
  `RLS context not applied on the active backend: expected account ${context.accountId}, got "${appliedAccountId}"`,
1009
1029
  );
1010
1030
  }
1031
+ if (appliedWorkspaceId !== expectedWorkspaceId) {
1032
+ throw new Error(
1033
+ `RLS context not applied on the active backend: expected workspace "${expectedWorkspaceId}", got "${appliedWorkspaceId}"`,
1034
+ );
1035
+ }
1011
1036
  return await fn(scoped);
1012
1037
  }, transactionConfig);
1013
1038
  }
@@ -2044,10 +2069,17 @@ export type GitHubInstallation = {
2044
2069
  accountId: string;
2045
2070
  workspaceId: string;
2046
2071
  installationId: number;
2072
+ githubAccountId: number | null;
2047
2073
  accountLogin: string | null;
2048
2074
  accountType: string | null;
2049
2075
  repositoryScope: GitHubRepositoryScope;
2050
2076
  linkedBySubjectId: string | null;
2077
+ githubActorId: number | null;
2078
+ githubActorLogin: string | null;
2079
+ authorityKind: GitHubInstallationAuthorityKind | null;
2080
+ authorityCheckedAt: string | null;
2081
+ authorityExpiresAt: string | null;
2082
+ authorityNonce: string | null;
2051
2083
  createdAt: string;
2052
2084
  updatedAt: string;
2053
2085
  };
@@ -2056,6 +2088,59 @@ export type GitHubInstallationAccess = GitHubInstallation & {
2056
2088
  repositoryIds: number[];
2057
2089
  };
2058
2090
 
2091
+ export class GitHubInstallationAuthorityCommitError extends Error {
2092
+ constructor() {
2093
+ super("GitHub installation authority expired before the binding transaction completed");
2094
+ }
2095
+ }
2096
+
2097
+ const githubInstallationAuthorityMaxAgeMs = 10 * 60_000;
2098
+
2099
+ /**
2100
+ * True only for a binding created by the owner-authority transaction. Legacy
2101
+ * installation rows remain visible for audit/unlink, but can never make a
2102
+ * workspace healthy, enumerate repositories, or authorize a token mint.
2103
+ *
2104
+ * `authorityExpiresAt` is the expiry of the consumed proof, not of the durable
2105
+ * delegation. A binding remains delegated after that instant; GitHub's live
2106
+ * installation/repository checks govern its ongoing usability.
2107
+ */
2108
+ export function hasAuditableGitHubInstallationAuthority(
2109
+ installation: GitHubInstallationAccess,
2110
+ ): boolean {
2111
+ const checkedAt = installation.authorityCheckedAt
2112
+ ? Date.parse(installation.authorityCheckedAt)
2113
+ : Number.NaN;
2114
+ const expiresAt = installation.authorityExpiresAt
2115
+ ? Date.parse(installation.authorityExpiresAt)
2116
+ : Number.NaN;
2117
+ const common =
2118
+ installation.repositoryScope === "selected" &&
2119
+ installation.repositoryIds.length > 0 &&
2120
+ installation.repositoryIds.every((id) => Number.isSafeInteger(id) && id > 0) &&
2121
+ installation.githubAccountId !== null &&
2122
+ Number.isSafeInteger(installation.githubAccountId) &&
2123
+ installation.githubAccountId > 0 &&
2124
+ Boolean(installation.accountLogin) &&
2125
+ installation.githubActorId !== null &&
2126
+ Number.isSafeInteger(installation.githubActorId) &&
2127
+ installation.githubActorId > 0 &&
2128
+ Boolean(installation.githubActorLogin) &&
2129
+ Boolean(installation.linkedBySubjectId) &&
2130
+ Boolean(installation.authorityNonce) &&
2131
+ Number.isFinite(checkedAt) &&
2132
+ Number.isFinite(expiresAt) &&
2133
+ checkedAt < expiresAt;
2134
+ if (!common) {
2135
+ return false;
2136
+ }
2137
+ return installation.authorityKind === "personal_owner"
2138
+ ? installation.accountType === "User" &&
2139
+ installation.githubActorId === installation.githubAccountId
2140
+ : installation.authorityKind === "organization_owner" &&
2141
+ installation.accountType === "Organization";
2142
+ }
2143
+
2059
2144
  export async function upsertGitHubInstallation(
2060
2145
  db: Database,
2061
2146
  input: {
@@ -2095,6 +2180,9 @@ export async function upsertGitHubInstallation(
2095
2180
  : {}),
2096
2181
  updatedAt: new Date(),
2097
2182
  },
2183
+ // This legacy metadata helper cannot mutate an owner-authorized row.
2184
+ // New bindings use bindAuthorizedGitHubInstallationRepositories.
2185
+ setWhere: isNull(schema.githubInstallations.authorityNonce),
2098
2186
  })
2099
2187
  .returning();
2100
2188
  if (!row) {
@@ -2156,6 +2244,164 @@ export async function bindGitHubInstallationRepositories(
2156
2244
  linkedBySubjectId: input.linkedBySubjectId,
2157
2245
  updatedAt: new Date(),
2158
2246
  },
2247
+ // Preserve the immutable authority + allowlist audit boundary.
2248
+ setWhere: isNull(schema.githubInstallations.authorityNonce),
2249
+ })
2250
+ .returning();
2251
+ if (!row) {
2252
+ throw new Error("Failed to bind GitHub installation");
2253
+ }
2254
+ await tx
2255
+ .delete(schema.githubInstallationRepositories)
2256
+ .where(
2257
+ and(
2258
+ eq(schema.githubInstallationRepositories.workspaceId, input.workspaceId),
2259
+ eq(schema.githubInstallationRepositories.installationId, input.installationId),
2260
+ ),
2261
+ );
2262
+ for (let offset = 0; offset < repositoryIds.length; offset += 1_000) {
2263
+ await tx.insert(schema.githubInstallationRepositories).values(
2264
+ repositoryIds.slice(offset, offset + 1_000).map((repositoryId) => ({
2265
+ accountId: input.accountId,
2266
+ workspaceId: input.workspaceId,
2267
+ installationId: input.installationId,
2268
+ repositoryId,
2269
+ })),
2270
+ );
2271
+ }
2272
+ return { ...mapGitHubInstallation(row), repositoryIds };
2273
+ }),
2274
+ );
2275
+ }
2276
+
2277
+ export async function bindAuthorizedGitHubInstallationRepositories(
2278
+ db: Database,
2279
+ input: {
2280
+ accountId: string;
2281
+ workspaceId: string;
2282
+ installationId: number;
2283
+ githubAccountId: number;
2284
+ accountLogin: string | null;
2285
+ accountType: string | null;
2286
+ linkedBySubjectId: string;
2287
+ githubActorId: number;
2288
+ githubActorLogin: string;
2289
+ authorityKind: GitHubInstallationAuthorityKind;
2290
+ authorityCheckedAt: Date;
2291
+ authorityExpiresAt: Date;
2292
+ authorityNonce: string;
2293
+ repositoryIds: number[];
2294
+ },
2295
+ ): Promise<GitHubInstallationAccess | null> {
2296
+ if (!Number.isSafeInteger(input.installationId) || input.installationId <= 0) {
2297
+ throw new Error("GitHub installation id must be a positive safe integer");
2298
+ }
2299
+ if (!Number.isSafeInteger(input.githubAccountId) || input.githubAccountId <= 0) {
2300
+ throw new Error("GitHub account id must be a positive safe integer");
2301
+ }
2302
+ if (!Number.isSafeInteger(input.githubActorId) || input.githubActorId <= 0) {
2303
+ throw new Error("GitHub actor id must be a positive safe integer");
2304
+ }
2305
+ const authorityCheckedAtMs = input.authorityCheckedAt.getTime();
2306
+ const authorityExpiresAtMs = input.authorityExpiresAt.getTime();
2307
+ if (
2308
+ !Number.isFinite(authorityCheckedAtMs) ||
2309
+ !Number.isFinite(authorityExpiresAtMs) ||
2310
+ authorityCheckedAtMs >= authorityExpiresAtMs ||
2311
+ authorityExpiresAtMs - authorityCheckedAtMs > githubInstallationAuthorityMaxAgeMs ||
2312
+ !input.authorityNonce ||
2313
+ !input.githubActorLogin.trim() ||
2314
+ !input.accountLogin?.trim() ||
2315
+ !input.linkedBySubjectId.trim() ||
2316
+ (input.authorityKind !== "personal_owner" && input.authorityKind !== "organization_owner") ||
2317
+ (input.authorityKind === "personal_owner" &&
2318
+ (input.accountType !== "User" || input.githubActorId !== input.githubAccountId)) ||
2319
+ (input.authorityKind === "organization_owner" && input.accountType !== "Organization")
2320
+ ) {
2321
+ throw new Error("GitHub installation authority proof is invalid or expired");
2322
+ }
2323
+ const repositoryIds = [...new Set(input.repositoryIds)];
2324
+ if (
2325
+ repositoryIds.length === 0 ||
2326
+ repositoryIds.length !== input.repositoryIds.length ||
2327
+ repositoryIds.some((id) => !Number.isSafeInteger(id) || id <= 0)
2328
+ ) {
2329
+ throw new Error(
2330
+ "GitHub repository ids must be a nonempty, unique list of positive safe integers",
2331
+ );
2332
+ }
2333
+ return await withRlsContext(
2334
+ db,
2335
+ { accountId: input.accountId, workspaceId: input.workspaceId },
2336
+ async (scopedDb) =>
2337
+ await scopedDb.transaction(async (tx) => {
2338
+ await assertGitHubAuthorityWindowOpen(
2339
+ tx,
2340
+ input.authorityCheckedAt,
2341
+ input.authorityExpiresAt,
2342
+ );
2343
+ await tx
2344
+ .delete(schema.integrationOauthStateNonces)
2345
+ .where(
2346
+ and(
2347
+ eq(schema.integrationOauthStateNonces.workspaceId, input.workspaceId),
2348
+ lt(schema.integrationOauthStateNonces.expiresAt, input.authorityCheckedAt),
2349
+ ),
2350
+ );
2351
+ const consumed = await tx
2352
+ .insert(schema.integrationOauthStateNonces)
2353
+ .values({
2354
+ accountId: input.accountId,
2355
+ workspaceId: input.workspaceId,
2356
+ subjectId: input.linkedBySubjectId,
2357
+ nonce: input.authorityNonce,
2358
+ expiresAt: input.authorityExpiresAt,
2359
+ usedAt: input.authorityCheckedAt,
2360
+ })
2361
+ .onConflictDoNothing({ target: schema.integrationOauthStateNonces.nonce })
2362
+ .returning({ nonce: schema.integrationOauthStateNonces.nonce });
2363
+ if (consumed.length === 0) {
2364
+ return null;
2365
+ }
2366
+
2367
+ const [row] = await tx
2368
+ .insert(schema.githubInstallations)
2369
+ .values({
2370
+ accountId: input.accountId,
2371
+ workspaceId: input.workspaceId,
2372
+ installationId: input.installationId,
2373
+ githubAccountId: input.githubAccountId,
2374
+ accountLogin: input.accountLogin,
2375
+ accountType: input.accountType,
2376
+ repositoryScope: "selected",
2377
+ linkedBySubjectId: input.linkedBySubjectId,
2378
+ githubActorId: input.githubActorId,
2379
+ githubActorLogin: input.githubActorLogin,
2380
+ authorityKind: input.authorityKind,
2381
+ authorityCheckedAt: input.authorityCheckedAt,
2382
+ authorityExpiresAt: input.authorityExpiresAt,
2383
+ authorityNonce: input.authorityNonce,
2384
+ })
2385
+ .onConflictDoUpdate({
2386
+ target: [
2387
+ schema.githubInstallations.workspaceId,
2388
+ schema.githubInstallations.installationId,
2389
+ ],
2390
+ set: {
2391
+ accountId: input.accountId,
2392
+ githubAccountId: input.githubAccountId,
2393
+ accountLogin: input.accountLogin,
2394
+ accountType: input.accountType,
2395
+ repositoryScope: "selected",
2396
+ linkedBySubjectId: input.linkedBySubjectId,
2397
+ githubActorId: input.githubActorId,
2398
+ githubActorLogin: input.githubActorLogin,
2399
+ authorityKind: input.authorityKind,
2400
+ authorityCheckedAt: input.authorityCheckedAt,
2401
+ authorityExpiresAt: input.authorityExpiresAt,
2402
+ authorityNonce: input.authorityNonce,
2403
+ updatedAt: input.authorityCheckedAt,
2404
+ },
2159
2405
  })
2160
2406
  .returning();
2161
2407
  if (!row) {
@@ -2179,11 +2425,41 @@ export async function bindGitHubInstallationRepositories(
2179
2425
  })),
2180
2426
  );
2181
2427
  }
2428
+ // Recheck against the database clock after all writes. Throwing here
2429
+ // rolls the transaction back if the proof expired while it was being
2430
+ // committed, so no partial nonce or binding can survive.
2431
+ await assertGitHubAuthorityWindowOpen(
2432
+ tx,
2433
+ input.authorityCheckedAt,
2434
+ input.authorityExpiresAt,
2435
+ );
2182
2436
  return { ...mapGitHubInstallation(row), repositoryIds };
2183
2437
  }),
2184
2438
  );
2185
2439
  }
2186
2440
 
2441
+ async function assertGitHubAuthorityWindowOpen(
2442
+ tx: Database,
2443
+ checkedAt: Date,
2444
+ expiresAt: Date,
2445
+ ): Promise<void> {
2446
+ // postgres.js does not serialize JavaScript Date instances passed through
2447
+ // raw Drizzle SQL. Bind the already-validated ISO timestamp representation
2448
+ // and retain PostgreSQL's timestamptz/database-clock authority checks.
2449
+ const checkedAtIso = checkedAt.toISOString();
2450
+ const expiresAtIso = expiresAt.toISOString();
2451
+ const result = await tx.execute<{ valid: boolean }>(sql`
2452
+ select (
2453
+ ${checkedAtIso}::timestamptz <= clock_timestamp()
2454
+ and clock_timestamp() < ${expiresAtIso}::timestamptz
2455
+ and ${expiresAtIso}::timestamptz <= ${checkedAtIso}::timestamptz + interval '10 minutes'
2456
+ ) as valid
2457
+ `);
2458
+ if (result[0]?.valid !== true) {
2459
+ throw new GitHubInstallationAuthorityCommitError();
2460
+ }
2461
+ }
2462
+
2187
2463
  export async function listGitHubInstallationsForWorkspace(
2188
2464
  db: Database,
2189
2465
  workspaceId: string,
@@ -2256,7 +2532,7 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
2256
2532
  }
2257
2533
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
2258
2534
  const [installation] = await scopedDb
2259
- .select({ repositoryScope: schema.githubInstallations.repositoryScope })
2535
+ .select()
2260
2536
  .from(schema.githubInstallations)
2261
2537
  .where(
2262
2538
  and(
@@ -2268,9 +2544,6 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
2268
2544
  if (!installation) {
2269
2545
  return false;
2270
2546
  }
2271
- if (installation.repositoryScope === "all") {
2272
- return true;
2273
- }
2274
2547
  const allowed = await scopedDb
2275
2548
  .select({
2276
2549
  repositoryId: schema.githubInstallationRepositories.repositoryId,
@@ -2283,7 +2556,13 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
2283
2556
  inArray(schema.githubInstallationRepositories.repositoryId, requestedIds),
2284
2557
  ),
2285
2558
  );
2286
- return allowed.length === requestedIds.length;
2559
+ if (allowed.length !== requestedIds.length) {
2560
+ return false;
2561
+ }
2562
+ return hasAuditableGitHubInstallationAuthority({
2563
+ ...mapGitHubInstallation(installation),
2564
+ repositoryIds: allowed.map((row) => row.repositoryId),
2565
+ });
2287
2566
  });
2288
2567
  }
2289
2568
 
@@ -2866,6 +3145,8 @@ export type CreateConnectionInput = {
2866
3145
  credentialEncrypted: string;
2867
3146
  grantedScopes?: string[];
2868
3147
  expiresAt?: Date | null;
3148
+ verifiedInstallAt?: Date | null;
3149
+ verifiedInstallVersion?: number | null;
2869
3150
  metadata?: Record<string, unknown>;
2870
3151
  createdBySubjectId?: string | null;
2871
3152
  updatedBySubjectId?: string | null;
@@ -2883,10 +3164,40 @@ export type UpdateConnectionInput = {
2883
3164
  credentialEncrypted?: string;
2884
3165
  grantedScopes?: string[];
2885
3166
  expiresAt?: Date | null;
3167
+ verifiedInstallAt?: Date | null;
3168
+ verifiedInstallVersion?: number | null;
2886
3169
  metadata?: Record<string, unknown>;
2887
3170
  updatedBySubjectId?: string | null;
2888
3171
  };
2889
3172
 
3173
+ /** Server-owned verification facts; public schemas expose them read-only and nullable. */
3174
+ export type ConnectionMetadataWithVerification = ConnectionMetadata & {
3175
+ verifiedInstallAt: string | null;
3176
+ verifiedInstallVersion: number | null;
3177
+ };
3178
+
3179
+ export type SlackBotPostOperation = {
3180
+ id: string;
3181
+ accountId: string;
3182
+ workspaceId: string;
3183
+ connectionId: string;
3184
+ operationId: string;
3185
+ clientMessageId: string;
3186
+ targetKind: "channel" | "user";
3187
+ targetId: string;
3188
+ requestDigest: string;
3189
+ status: "provider_started" | "completed";
3190
+ claimHolderId: string | null;
3191
+ claimExpiresAt: Date | null;
3192
+ attemptCount: number;
3193
+ lastFailureCode: string | null;
3194
+ slackChannelId: string | null;
3195
+ slackMessageTimestamp: string | null;
3196
+ completedAt: Date | null;
3197
+ createdAt: Date;
3198
+ updatedAt: Date;
3199
+ };
3200
+
2890
3201
  export type ConnectionCredentialForBroker = {
2891
3202
  id: string;
2892
3203
  accountId: string;
@@ -4445,6 +4756,8 @@ const connectionMetadataColumns = {
4445
4756
  lastUsedAt: schema.connections.lastUsedAt,
4446
4757
  lastError: schema.connections.lastError,
4447
4758
  version: schema.connections.version,
4759
+ verifiedInstallAt: schema.connections.verifiedInstallAt,
4760
+ verifiedInstallVersion: schema.connections.verifiedInstallVersion,
4448
4761
  metadata: schema.connections.metadata,
4449
4762
  createdBySubjectId: schema.connections.createdBySubjectId,
4450
4763
  updatedBySubjectId: schema.connections.updatedBySubjectId,
@@ -4461,7 +4774,7 @@ function connectionSubjectVisibility(subjectId?: string | null): SQL {
4461
4774
  export async function createConnection(
4462
4775
  db: Database,
4463
4776
  input: CreateConnectionInput,
4464
- ): Promise<ConnectionMetadata> {
4777
+ ): Promise<ConnectionMetadataWithVerification> {
4465
4778
  return await withRlsContext(
4466
4779
  db,
4467
4780
  { accountId: input.accountId, workspaceId: input.workspaceId },
@@ -4478,6 +4791,8 @@ export async function createConnection(
4478
4791
  credentialEncrypted: input.credentialEncrypted,
4479
4792
  grantedScopes: input.grantedScopes ?? [],
4480
4793
  expiresAt: input.expiresAt ?? null,
4794
+ verifiedInstallAt: input.verifiedInstallAt ?? null,
4795
+ verifiedInstallVersion: input.verifiedInstallVersion ?? null,
4481
4796
  metadata: input.metadata ?? {},
4482
4797
  createdBySubjectId: input.createdBySubjectId ?? null,
4483
4798
  updatedBySubjectId: input.updatedBySubjectId ?? input.createdBySubjectId ?? null,
@@ -4495,7 +4810,7 @@ export async function listConnectionsMetadata(
4495
4810
  db: Database,
4496
4811
  workspaceId: string,
4497
4812
  subjectId?: string | null,
4498
- ): Promise<ConnectionMetadata[]> {
4813
+ ): Promise<ConnectionMetadataWithVerification[]> {
4499
4814
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
4500
4815
  const rows = await scopedDb
4501
4816
  .select(connectionMetadataColumns)
@@ -4516,7 +4831,7 @@ export async function getConnectionMetadata(
4516
4831
  workspaceId: string,
4517
4832
  connectionId: string,
4518
4833
  subjectId?: string | null,
4519
- ): Promise<ConnectionMetadata | null> {
4834
+ ): Promise<ConnectionMetadataWithVerification | null> {
4520
4835
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
4521
4836
  const [row] = await scopedDb
4522
4837
  .select(connectionMetadataColumns)
@@ -4536,7 +4851,7 @@ export async function getConnectionMetadata(
4536
4851
  export async function updateConnection(
4537
4852
  db: Database,
4538
4853
  input: UpdateConnectionInput,
4539
- ): Promise<ConnectionMetadata | null> {
4854
+ ): Promise<ConnectionMetadataWithVerification | null> {
4540
4855
  return await withWorkspaceRls(db, input.workspaceId, async (scopedDb) => {
4541
4856
  const set = {
4542
4857
  updatedAt: new Date(),
@@ -4553,6 +4868,12 @@ export async function updateConnection(
4553
4868
  : {}),
4554
4869
  ...(input.grantedScopes !== undefined ? { grantedScopes: input.grantedScopes } : {}),
4555
4870
  ...(input.expiresAt !== undefined ? { expiresAt: input.expiresAt } : {}),
4871
+ ...(input.verifiedInstallAt !== undefined
4872
+ ? { verifiedInstallAt: input.verifiedInstallAt }
4873
+ : {}),
4874
+ ...(input.verifiedInstallVersion !== undefined
4875
+ ? { verifiedInstallVersion: input.verifiedInstallVersion }
4876
+ : {}),
4556
4877
  ...(input.metadata !== undefined ? { metadata: input.metadata } : {}),
4557
4878
  ...(input.updatedBySubjectId !== undefined
4558
4879
  ? { updatedBySubjectId: input.updatedBySubjectId }
@@ -4581,7 +4902,7 @@ export async function revokeConnection(
4581
4902
  workspaceId: string,
4582
4903
  connectionId: string,
4583
4904
  updatedBySubjectId?: string | null,
4584
- ): Promise<ConnectionMetadata | null> {
4905
+ ): Promise<ConnectionMetadataWithVerification | null> {
4585
4906
  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
4586
4907
  const [row] = await scopedDb
4587
4908
  .update(schema.connections)
@@ -4590,6 +4911,13 @@ export async function revokeConnection(
4590
4911
  // The version bump invalidates any in-flight refresh's (id, version) CAS,
4591
4912
  // so a racing refresh cannot commit and flip the row back to active.
4592
4913
  version: sql`${schema.connections.version} + 1`,
4914
+ // Status-only revocation does not replace the verified credential or bot
4915
+ // identity. Carry the marker to the same new CAS version so the dedicated
4916
+ // reinstall path can still recognize (but not use) the inactive row.
4917
+ verifiedInstallVersion: sql`case
4918
+ when ${schema.connections.verifiedInstallAt} is null then null
4919
+ else ${schema.connections.version} + 1
4920
+ end`,
4593
4921
  updatedBySubjectId: updatedBySubjectId ?? null,
4594
4922
  updatedAt: new Date(),
4595
4923
  })
@@ -4608,6 +4936,272 @@ export async function revokeConnection(
4608
4936
  });
4609
4937
  }
4610
4938
 
4939
+ export type ClaimSlackBotPostOperationResult =
4940
+ | { kind: "claimed" | "in_progress" | "completed"; operation: SlackBotPostOperation }
4941
+ | { kind: "conflict" | "connection_not_found" };
4942
+
4943
+ /**
4944
+ * Claims one durable Slack post identity. The insert occurs before any provider
4945
+ * call; retries retain the original client_msg_id and immutable request digest.
4946
+ * A live claim suppresses concurrent sends, while a released/expired claim can
4947
+ * be reclaimed after response loss or process death.
4948
+ */
4949
+ export async function claimSlackBotPostOperation(
4950
+ db: Database,
4951
+ input: {
4952
+ accountId: string;
4953
+ workspaceId: string;
4954
+ connectionId: string;
4955
+ operationId: string;
4956
+ targetKind: "channel" | "user";
4957
+ targetId: string;
4958
+ requestDigest: string;
4959
+ claimHolderId: string;
4960
+ claimLeaseMs: number;
4961
+ },
4962
+ ): Promise<ClaimSlackBotPostOperationResult> {
4963
+ const claimLeaseMs = Math.max(1, Math.min(Math.trunc(input.claimLeaseMs), 120_000));
4964
+ return await withRlsContext(
4965
+ db,
4966
+ { accountId: input.accountId, workspaceId: input.workspaceId },
4967
+ async (scopedDb) =>
4968
+ await scopedDb.transaction(async (txRaw) => {
4969
+ const tx = txRaw as unknown as Database;
4970
+ const [connection] = await tx
4971
+ .select({ id: schema.connections.id })
4972
+ .from(schema.connections)
4973
+ .where(
4974
+ and(
4975
+ eq(schema.connections.accountId, input.accountId),
4976
+ eq(schema.connections.workspaceId, input.workspaceId),
4977
+ eq(schema.connections.id, input.connectionId),
4978
+ ),
4979
+ )
4980
+ .for("share")
4981
+ .limit(1);
4982
+ if (!connection) return { kind: "connection_not_found" } as const;
4983
+
4984
+ const [created] = await tx
4985
+ .insert(schema.slackBotPostOperations)
4986
+ .values({
4987
+ accountId: input.accountId,
4988
+ workspaceId: input.workspaceId,
4989
+ connectionId: input.connectionId,
4990
+ operationId: input.operationId,
4991
+ clientMessageId: input.operationId,
4992
+ targetKind: input.targetKind,
4993
+ targetId: input.targetId,
4994
+ requestDigest: input.requestDigest,
4995
+ status: "provider_started",
4996
+ claimHolderId: input.claimHolderId,
4997
+ claimExpiresAt: sql`now() + (${claimLeaseMs} * interval '1 millisecond')`,
4998
+ attemptCount: 1,
4999
+ })
5000
+ .onConflictDoNothing({
5001
+ target: [
5002
+ schema.slackBotPostOperations.workspaceId,
5003
+ schema.slackBotPostOperations.connectionId,
5004
+ schema.slackBotPostOperations.operationId,
5005
+ ],
5006
+ })
5007
+ .returning();
5008
+ if (created) {
5009
+ return { kind: "claimed", operation: mapSlackBotPostOperation(created) } as const;
5010
+ }
5011
+
5012
+ const [existing] = await tx
5013
+ .select()
5014
+ .from(schema.slackBotPostOperations)
5015
+ .where(
5016
+ and(
5017
+ eq(schema.slackBotPostOperations.workspaceId, input.workspaceId),
5018
+ eq(schema.slackBotPostOperations.connectionId, input.connectionId),
5019
+ eq(schema.slackBotPostOperations.operationId, input.operationId),
5020
+ ),
5021
+ )
5022
+ .for("update")
5023
+ .limit(1);
5024
+ if (!existing) throw new Error("Slack post operation disappeared after conflict");
5025
+ if (
5026
+ existing.accountId !== input.accountId ||
5027
+ existing.targetKind !== input.targetKind ||
5028
+ existing.targetId !== input.targetId ||
5029
+ existing.requestDigest !== input.requestDigest ||
5030
+ existing.clientMessageId !== input.operationId
5031
+ ) {
5032
+ return { kind: "conflict" } as const;
5033
+ }
5034
+ if (existing.status === "completed") {
5035
+ return { kind: "completed", operation: mapSlackBotPostOperation(existing) } as const;
5036
+ }
5037
+
5038
+ const [reclaimed] = await tx
5039
+ .update(schema.slackBotPostOperations)
5040
+ .set({
5041
+ claimHolderId: input.claimHolderId,
5042
+ claimExpiresAt: sql`now() + (${claimLeaseMs} * interval '1 millisecond')`,
5043
+ attemptCount: sql`${schema.slackBotPostOperations.attemptCount} + 1`,
5044
+ lastFailureCode: null,
5045
+ updatedAt: sql`now()`,
5046
+ })
5047
+ .where(
5048
+ and(
5049
+ eq(schema.slackBotPostOperations.id, existing.id),
5050
+ or(
5051
+ isNull(schema.slackBotPostOperations.claimHolderId),
5052
+ lte(schema.slackBotPostOperations.claimExpiresAt, sql`now()`),
5053
+ ),
5054
+ ),
5055
+ )
5056
+ .returning();
5057
+ return reclaimed
5058
+ ? ({ kind: "claimed", operation: mapSlackBotPostOperation(reclaimed) } as const)
5059
+ : ({ kind: "in_progress", operation: mapSlackBotPostOperation(existing) } as const);
5060
+ }),
5061
+ );
5062
+ }
5063
+
5064
+ export async function releaseSlackBotPostOperationClaim(
5065
+ db: Database,
5066
+ input: {
5067
+ accountId: string;
5068
+ workspaceId: string;
5069
+ connectionId: string;
5070
+ operationId: string;
5071
+ claimHolderId: string;
5072
+ failureCode: string;
5073
+ },
5074
+ ): Promise<boolean> {
5075
+ return await withRlsContext(
5076
+ db,
5077
+ { accountId: input.accountId, workspaceId: input.workspaceId },
5078
+ async (scopedDb) => {
5079
+ const rows = await scopedDb
5080
+ .update(schema.slackBotPostOperations)
5081
+ .set({
5082
+ claimHolderId: null,
5083
+ claimExpiresAt: null,
5084
+ lastFailureCode: input.failureCode.slice(0, 128),
5085
+ updatedAt: sql`now()`,
5086
+ })
5087
+ .where(
5088
+ and(
5089
+ eq(schema.slackBotPostOperations.workspaceId, input.workspaceId),
5090
+ eq(schema.slackBotPostOperations.connectionId, input.connectionId),
5091
+ eq(schema.slackBotPostOperations.operationId, input.operationId),
5092
+ eq(schema.slackBotPostOperations.status, "provider_started"),
5093
+ eq(schema.slackBotPostOperations.claimHolderId, input.claimHolderId),
5094
+ ),
5095
+ )
5096
+ .returning({ id: schema.slackBotPostOperations.id });
5097
+ return rows.length === 1;
5098
+ },
5099
+ );
5100
+ }
5101
+
5102
+ export type CompleteSlackBotPostOperationResult =
5103
+ | { kind: "completed"; operation: SlackBotPostOperation; newlyCompleted: boolean }
5104
+ | { kind: "not_found" | "not_owned" };
5105
+
5106
+ /** Completion and the single success audit receipt commit atomically. */
5107
+ export async function completeSlackBotPostOperation(
5108
+ db: Database,
5109
+ input: {
5110
+ accountId: string;
5111
+ workspaceId: string;
5112
+ connectionId: string;
5113
+ operationId: string;
5114
+ claimHolderId: string;
5115
+ slackChannelId: string;
5116
+ slackMessageTimestamp: string;
5117
+ subjectId?: string | null;
5118
+ auditMetadata: Record<string, unknown>;
5119
+ },
5120
+ ): Promise<CompleteSlackBotPostOperationResult> {
5121
+ return await withRlsContext(
5122
+ db,
5123
+ { accountId: input.accountId, workspaceId: input.workspaceId },
5124
+ async (scopedDb) =>
5125
+ await scopedDb.transaction(async (txRaw) => {
5126
+ const tx = txRaw as unknown as Database;
5127
+ const [current] = await tx
5128
+ .select()
5129
+ .from(schema.slackBotPostOperations)
5130
+ .where(
5131
+ and(
5132
+ eq(schema.slackBotPostOperations.workspaceId, input.workspaceId),
5133
+ eq(schema.slackBotPostOperations.connectionId, input.connectionId),
5134
+ eq(schema.slackBotPostOperations.operationId, input.operationId),
5135
+ ),
5136
+ )
5137
+ .for("update")
5138
+ .limit(1);
5139
+ if (!current) return { kind: "not_found" } as const;
5140
+ if (current.status === "completed") {
5141
+ return {
5142
+ kind: "completed",
5143
+ operation: mapSlackBotPostOperation(current),
5144
+ newlyCompleted: false,
5145
+ } as const;
5146
+ }
5147
+ if (current.claimHolderId !== input.claimHolderId) {
5148
+ return { kind: "not_owned" } as const;
5149
+ }
5150
+ const [completed] = await tx
5151
+ .update(schema.slackBotPostOperations)
5152
+ .set({
5153
+ status: "completed",
5154
+ claimHolderId: null,
5155
+ claimExpiresAt: null,
5156
+ lastFailureCode: null,
5157
+ slackChannelId: input.slackChannelId,
5158
+ slackMessageTimestamp: input.slackMessageTimestamp,
5159
+ completedAt: sql`now()`,
5160
+ updatedAt: sql`now()`,
5161
+ })
5162
+ .where(eq(schema.slackBotPostOperations.id, current.id))
5163
+ .returning();
5164
+ if (!completed) throw new Error("Slack post completion returned no row");
5165
+ await tx.insert(schema.auditEvents).values({
5166
+ accountId: input.accountId,
5167
+ workspaceId: input.workspaceId,
5168
+ subjectId: input.subjectId ?? null,
5169
+ action: "slack_bot.message.post",
5170
+ targetType: "connection",
5171
+ targetId: input.connectionId,
5172
+ metadata: input.auditMetadata,
5173
+ });
5174
+ return {
5175
+ kind: "completed",
5176
+ operation: mapSlackBotPostOperation(completed),
5177
+ newlyCompleted: true,
5178
+ } as const;
5179
+ }),
5180
+ );
5181
+ }
5182
+
5183
+ export async function getSlackBotPostOperation(
5184
+ db: Database,
5185
+ workspaceId: string,
5186
+ connectionId: string,
5187
+ operationId: string,
5188
+ ): Promise<SlackBotPostOperation | null> {
5189
+ return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
5190
+ const [row] = await scopedDb
5191
+ .select()
5192
+ .from(schema.slackBotPostOperations)
5193
+ .where(
5194
+ and(
5195
+ eq(schema.slackBotPostOperations.workspaceId, workspaceId),
5196
+ eq(schema.slackBotPostOperations.connectionId, connectionId),
5197
+ eq(schema.slackBotPostOperations.operationId, operationId),
5198
+ ),
5199
+ )
5200
+ .limit(1);
5201
+ return row ? mapSlackBotPostOperation(row) : null;
5202
+ });
5203
+ }
5204
+
4611
5205
  export async function loadConnectionCredentialForBroker(
4612
5206
  db: Database,
4613
5207
  settings: Settings,
@@ -4741,6 +5335,10 @@ export async function setConnectionStatus(
4741
5335
  status,
4742
5336
  lastError,
4743
5337
  version: sql`${schema.connections.version} + 1`,
5338
+ verifiedInstallVersion: sql`case
5339
+ when ${schema.connections.verifiedInstallAt} is null then null
5340
+ else ${schema.connections.version} + 1
5341
+ end`,
4744
5342
  updatedAt: new Date(),
4745
5343
  })
4746
5344
  .where(
@@ -9516,12 +10114,13 @@ export type CodexCapacityWait = {
9516
10114
  accountId: string;
9517
10115
  workspaceId: string;
9518
10116
  sessionId: string;
9519
- goalId: string;
10117
+ goalId: string | null;
9520
10118
  blockedTurnId: string;
10119
+ blockedTurnGeneration: number;
9521
10120
  workflowId: string;
9522
10121
  generation: number;
9523
10122
  status: CodexCapacityWaitStatus;
9524
- goalVersion: number;
10123
+ goalVersion: number | null;
9525
10124
  policyHash: string | null;
9526
10125
  earliestResetAt: Date | null;
9527
10126
  nextCheckAt: Date;
@@ -9588,9 +10187,9 @@ export type ReconcileCodexCapacityWaitResult =
9588
10187
  | {
9589
10188
  action: "resumed";
9590
10189
  waiter: CodexCapacityWait;
9591
- update: SessionSystemUpdate;
9592
10190
  events: SessionEvent[];
9593
10191
  }
10192
+ | { action: "paused"; waiter: CodexCapacityWait; events: SessionEvent[] }
9594
10193
  | { action: "superseded"; waiter: CodexCapacityWait; events: SessionEvent[] }
9595
10194
  | {
9596
10195
  action: "stale";
@@ -9621,6 +10220,7 @@ function mapCodexCapacityWaiter(
9621
10220
  sessionId: row.sessionId,
9622
10221
  goalId: row.goalId,
9623
10222
  blockedTurnId: row.blockedTurnId,
10223
+ blockedTurnGeneration: row.blockedTurnGeneration,
9624
10224
  workflowId: row.workflowId,
9625
10225
  generation: row.generation,
9626
10226
  status: row.status as CodexCapacityWaitStatus,
@@ -9698,14 +10298,13 @@ function nextCodexCapacityCheckAt(
9698
10298
  }
9699
10299
 
9700
10300
  /**
9701
- * Atomically settle one all-unavailable turn and arm exactly one durable wait.
9702
- * Lock order is allocator rotation row -> workspace control -> actual workspace
9703
- * -> session -> exact turn -> exact attempt -> goal -> live lease (when a
9704
- * reactive failure owns one) -> waiter. The control share lock and effective
9705
- * control recheck prevent a capacity boundary from closing an attempt after a
9706
- * committed Pause. The failed turn, idle/capacity-paused session, durable
9707
- * events, lease release, and waiter generation commit together; a crash cannot
9708
- * leave only half of the boundary visible.
10301
+ * Atomically close one all-unavailable attempt and arm exactly one durable wait
10302
+ * for the same logical turn. Lock order is allocator rotation row -> workspace
10303
+ * control -> actual workspace -> session -> exact turn -> exact attempt ->
10304
+ * optional goal -> live lease (when a reactive failure owns one) -> waiter.
10305
+ * The waiting turn/session pointer, durable events, exact lease release, and
10306
+ * waiter generation commit together; a crash cannot leave only half of the
10307
+ * boundary visible.
9709
10308
  */
9710
10309
  export async function armCodexCapacityWait(
9711
10310
  db: Database,
@@ -9716,8 +10315,8 @@ export async function armCodexCapacityWait(
9716
10315
  turnId: string;
9717
10316
  attemptId: string;
9718
10317
  workflowId: string;
9719
- goalId: string;
9720
- goalVersion: number;
10318
+ goalId?: string | null;
10319
+ goalVersion?: number | null;
9721
10320
  policyHash?: string | null;
9722
10321
  earliestResetAt: Date | null;
9723
10322
  resetKind: CodexCapacityResetKind;
@@ -9730,6 +10329,14 @@ export async function armCodexCapacityWait(
9730
10329
  },
9731
10330
  ): Promise<ArmCodexCapacityWaitResult> {
9732
10331
  const now = input.now ?? new Date();
10332
+ const goalId = input.goalId ?? null;
10333
+ const goalVersion = input.goalVersion ?? null;
10334
+ if (
10335
+ (goalId === null) !== (goalVersion === null) ||
10336
+ (goalVersion !== null && (!Number.isSafeInteger(goalVersion) || goalVersion < 1))
10337
+ ) {
10338
+ throw new Error("Codex capacity goal fence must be absent or contain a positive version");
10339
+ }
9733
10340
  return await withRlsContext(
9734
10341
  db,
9735
10342
  { accountId: input.accountId, workspaceId: input.workspaceId },
@@ -9755,18 +10362,20 @@ export async function armCodexCapacityWait(
9755
10362
  workspaceControl: locks.control ?? undefined,
9756
10363
  })
9757
10364
  : null;
9758
- const [goal] = await tx
9759
- .select()
9760
- .from(schema.sessionGoals)
9761
- .where(
9762
- and(
9763
- eq(schema.sessionGoals.workspaceId, input.workspaceId),
9764
- eq(schema.sessionGoals.id, input.goalId),
9765
- eq(schema.sessionGoals.sessionId, input.sessionId),
9766
- ),
9767
- )
9768
- .for("update")
9769
- .limit(1);
10365
+ const [goal] = goalId
10366
+ ? await tx
10367
+ .select()
10368
+ .from(schema.sessionGoals)
10369
+ .where(
10370
+ and(
10371
+ eq(schema.sessionGoals.workspaceId, input.workspaceId),
10372
+ eq(schema.sessionGoals.id, goalId),
10373
+ eq(schema.sessionGoals.sessionId, input.sessionId),
10374
+ ),
10375
+ )
10376
+ .for("update")
10377
+ .limit(1)
10378
+ : [];
9770
10379
  const leaseRows = input.leaseFence
9771
10380
  ? await tx.execute(sql<{ holder_id: string; generation: number }>`
9772
10381
  select holder_id, generation
@@ -9807,7 +10416,10 @@ export async function armCodexCapacityWait(
9807
10416
  if (
9808
10417
  existing?.status === "waiting" &&
9809
10418
  existing.blockedTurnId === input.turnId &&
9810
- turn?.status === "failed"
10419
+ existing.blockedTurnGeneration === turn?.executionGeneration &&
10420
+ turn?.status === "waiting_capacity" &&
10421
+ session?.status === "waiting_capacity" &&
10422
+ session.activeTurnId === input.turnId
9811
10423
  ) {
9812
10424
  return {
9813
10425
  action: "waiting",
@@ -9825,13 +10437,14 @@ export async function armCodexCapacityWait(
9825
10437
  Number(lease.generation) === input.leaseFence.generation &&
9826
10438
  currentRedispatches === (input.expectedRedispatches ?? currentRedispatches));
9827
10439
  if (
9828
- !goal ||
10440
+ !session ||
10441
+ !turn ||
9829
10442
  effectiveControl?.state !== "active" ||
9830
10443
  effectiveControl.settlement !== null ||
9831
10444
  session.activeTurnId !== input.turnId ||
9832
10445
  session.status !== "running" ||
9833
- goal.status !== "active" ||
9834
- goal.version !== input.goalVersion ||
10446
+ (goalId !== null &&
10447
+ (!goal || goal.status !== "active" || goal.version !== goalVersion)) ||
9835
10448
  turn.status !== "running" ||
9836
10449
  turn.activeAttemptId !== input.attemptId ||
9837
10450
  !leaseFenceValid ||
@@ -9851,7 +10464,7 @@ export async function armCodexCapacityWait(
9851
10464
  sessionId: input.sessionId,
9852
10465
  turnId: input.turnId,
9853
10466
  executionGeneration: turn.executionGeneration,
9854
- outcome: "failed",
10467
+ outcome: "waiting_capacity",
9855
10468
  closedAt: now,
9856
10469
  });
9857
10470
 
@@ -9867,12 +10480,13 @@ export async function armCodexCapacityWait(
9867
10480
  accountId: input.accountId,
9868
10481
  workspaceId: input.workspaceId,
9869
10482
  sessionId: input.sessionId,
9870
- goalId: input.goalId,
10483
+ goalId,
9871
10484
  blockedTurnId: input.turnId,
10485
+ blockedTurnGeneration: turn.executionGeneration,
9872
10486
  workflowId: input.workflowId,
9873
10487
  generation,
9874
10488
  status: "waiting",
9875
- goalVersion: input.goalVersion,
10489
+ goalVersion,
9876
10490
  policyHash,
9877
10491
  earliestResetAt: input.earliestResetAt,
9878
10492
  nextCheckAt,
@@ -9917,32 +10531,17 @@ export async function armCodexCapacityWait(
9917
10531
  workspaceId: input.workspaceId,
9918
10532
  sessionId: input.sessionId,
9919
10533
  sequence: ++sequence,
9920
- type: "turn.failed",
10534
+ type: "codex.capacity.waiting",
9921
10535
  payload: sanitizeEventPayload({
9922
10536
  ...input.failurePayload,
9923
10537
  recovery: "codex_capacity",
9924
- retryable: false,
10538
+ retryable: true,
9925
10539
  rotated: true,
9926
- capacityWaiterId: waiterRow.id,
9927
- capacityWaitGeneration: waiterRow.generation,
9928
- }),
9929
- turnId: input.turnId,
9930
- turnGeneration: turn.executionGeneration,
9931
- turnAttemptId: input.attemptId,
9932
- turnAssociation: "current",
9933
- occurredAt: now,
9934
- },
9935
- {
9936
- accountId: input.accountId,
9937
- workspaceId: input.workspaceId,
9938
- sessionId: input.sessionId,
9939
- sequence: ++sequence,
9940
- type: "codex.capacity.waiting",
9941
- payload: sanitizeEventPayload({
9942
10540
  waiterId: waiterRow.id,
9943
10541
  generation: waiterRow.generation,
9944
- goalId: input.goalId,
9945
- goalVersion: input.goalVersion,
10542
+ goalId,
10543
+ goalVersion,
10544
+ blockedTurnGeneration: turn.executionGeneration,
9946
10545
  policyHash,
9947
10546
  resetKind: input.resetKind,
9948
10547
  earliestResetAt: input.earliestResetAt?.toISOString() ?? null,
@@ -9960,7 +10559,7 @@ export async function armCodexCapacityWait(
9960
10559
  sessionId: input.sessionId,
9961
10560
  sequence: ++sequence,
9962
10561
  type: "session.status.changed",
9963
- payload: { status: "idle", reason: "codex_capacity" },
10562
+ payload: { status: "waiting_capacity", reason: "codex_capacity" },
9964
10563
  turnId: input.turnId,
9965
10564
  turnGeneration: turn.executionGeneration,
9966
10565
  turnAttemptId: input.attemptId,
@@ -9969,13 +10568,14 @@ export async function armCodexCapacityWait(
9969
10568
  },
9970
10569
  ])
9971
10570
  .returning();
9972
- await tx
10571
+ const [waitingTurn] = await tx
9973
10572
  .update(schema.sessionTurns)
9974
10573
  .set({
9975
- status: "failed",
10574
+ status: "waiting_capacity",
9976
10575
  activeAttemptId: null,
10576
+ metadata: metadataWithoutTurnDispatchAttempt(turn.metadata),
9977
10577
  version: turn.version + 1,
9978
- finishedAt: now,
10578
+ finishedAt: null,
9979
10579
  updatedAt: now,
9980
10580
  })
9981
10581
  .where(
@@ -9983,13 +10583,18 @@ export async function armCodexCapacityWait(
9983
10583
  eq(schema.sessionTurns.workspaceId, input.workspaceId),
9984
10584
  eq(schema.sessionTurns.id, input.turnId),
9985
10585
  eq(schema.sessionTurns.status, "running"),
10586
+ eq(schema.sessionTurns.activeAttemptId, input.attemptId),
9986
10587
  ),
9987
- );
9988
- await tx
10588
+ )
10589
+ .returning({ id: schema.sessionTurns.id });
10590
+ if (!waitingTurn) {
10591
+ throw new Error("Codex capacity blocked turn changed during atomic arm");
10592
+ }
10593
+ const [waitingSession] = await tx
9989
10594
  .update(schema.sessions)
9990
10595
  .set({
9991
- status: "idle",
9992
- activeTurnId: null,
10596
+ status: "waiting_capacity",
10597
+ activeTurnId: input.turnId,
9993
10598
  lastSequence: sequence,
9994
10599
  updatedAt: now,
9995
10600
  })
@@ -9997,15 +10602,24 @@ export async function armCodexCapacityWait(
9997
10602
  and(
9998
10603
  eq(schema.sessions.workspaceId, input.workspaceId),
9999
10604
  eq(schema.sessions.id, input.sessionId),
10605
+ eq(schema.sessions.status, "running"),
10000
10606
  eq(schema.sessions.activeTurnId, input.turnId),
10001
10607
  ),
10002
- );
10003
- await tx.execute(sql`
10004
- delete from codex_credential_leases
10005
- where account_id = ${input.accountId}
10006
- and workspace_id = ${input.workspaceId}
10007
- and turn_id = ${input.turnId}
10008
- `);
10608
+ )
10609
+ .returning({ id: schema.sessions.id });
10610
+ if (!waitingSession) {
10611
+ throw new Error("Codex capacity session changed during atomic arm");
10612
+ }
10613
+ if (input.leaseFence) {
10614
+ await tx.execute(sql`
10615
+ delete from codex_credential_leases
10616
+ where account_id = ${input.accountId}
10617
+ and workspace_id = ${input.workspaceId}
10618
+ and turn_id = ${input.turnId}
10619
+ and holder_id = ${input.leaseFence.holderId}
10620
+ and generation = ${input.leaseFence.generation}
10621
+ `);
10622
+ }
10009
10623
  return {
10010
10624
  action: "waiting",
10011
10625
  waiter: mapCodexCapacityWaiter(waiterRow),
@@ -10037,8 +10651,8 @@ export async function getCodexCapacityWaitForSession(
10037
10651
  }
10038
10652
 
10039
10653
  /**
10040
- * Same-transaction capacity-mutation/outbox seam for account eligibility and
10041
- * policy-scope membership/default changes. The allocator rotation row is always the
10654
+ * Same-transaction capacity-mutation/outbox seam for eligibility and
10655
+ * membership/default changes. The allocator rotation row is always the
10042
10656
  * first lock. Mutations report whether capacity truth changed; only then are
10043
10657
  * matching waiter wake revisions advanced and returned for best-effort signal.
10044
10658
  */
@@ -10153,6 +10767,7 @@ async function supersedeCodexCapacityWaitInTransaction(
10153
10767
  tx: Database,
10154
10768
  input: {
10155
10769
  session: typeof schema.sessions.$inferSelect;
10770
+ blockedTurn: typeof schema.sessionTurns.$inferSelect;
10156
10771
  waiter: typeof schema.codexCapacityWaiters.$inferSelect;
10157
10772
  reason: string;
10158
10773
  now: Date;
@@ -10176,9 +10791,52 @@ async function supersedeCodexCapacityWaitInTransaction(
10176
10791
  if (!updated) {
10177
10792
  return { waiter: mapCodexCapacityWaiter(input.waiter), events: [] };
10178
10793
  }
10179
- const inserted = await tx
10180
- .insert(schema.sessionEvents)
10181
- .values({
10794
+ const turnWasCurrent = input.session.activeTurnId === input.blockedTurn.id;
10795
+ const turnStillWaiting = input.blockedTurn.status === "waiting_capacity";
10796
+ const terminalTurnStatus = input.session.status === "cancelled" ? "cancelled" : "superseded";
10797
+ if (turnStillWaiting) {
10798
+ const [supersededTurn] = await tx
10799
+ .update(schema.sessionTurns)
10800
+ .set({
10801
+ status: terminalTurnStatus,
10802
+ activeAttemptId: null,
10803
+ cancelledBy: "codex_capacity_reconcile",
10804
+ cancelReason: input.reason,
10805
+ version: input.blockedTurn.version + 1,
10806
+ finishedAt: input.now,
10807
+ updatedAt: input.now,
10808
+ })
10809
+ .where(
10810
+ and(
10811
+ eq(schema.sessionTurns.workspaceId, input.session.workspaceId),
10812
+ eq(schema.sessionTurns.id, input.blockedTurn.id),
10813
+ eq(schema.sessionTurns.status, "waiting_capacity"),
10814
+ isNull(schema.sessionTurns.activeAttemptId),
10815
+ eq(schema.sessionTurns.executionGeneration, input.waiter.blockedTurnGeneration),
10816
+ ),
10817
+ )
10818
+ .returning({ id: schema.sessionTurns.id });
10819
+ if (!supersededTurn) {
10820
+ throw new Error("Codex capacity blocked turn changed during atomic supersession");
10821
+ }
10822
+ }
10823
+ const [queued] = turnWasCurrent
10824
+ ? await tx
10825
+ .select({ id: schema.sessionTurns.id })
10826
+ .from(schema.sessionTurns)
10827
+ .where(
10828
+ and(
10829
+ eq(schema.sessionTurns.workspaceId, input.session.workspaceId),
10830
+ eq(schema.sessionTurns.sessionId, input.session.id),
10831
+ eq(schema.sessionTurns.status, "queued"),
10832
+ ),
10833
+ )
10834
+ .limit(1)
10835
+ : [];
10836
+ const nextSessionStatus =
10837
+ input.session.status === "cancelled" ? "cancelled" : queued ? "queued" : "idle";
10838
+ const eventValues: Array<typeof schema.sessionEvents.$inferInsert> = [
10839
+ {
10182
10840
  accountId: input.session.accountId,
10183
10841
  workspaceId: input.session.workspaceId,
10184
10842
  sessionId: input.session.id,
@@ -10190,18 +10848,45 @@ async function supersedeCodexCapacityWaitInTransaction(
10190
10848
  reason: input.reason,
10191
10849
  }),
10192
10850
  turnId: updated.blockedTurnId,
10851
+ turnGeneration: input.blockedTurn.executionGeneration,
10852
+ ...(turnWasCurrent ? { turnAssociation: "current" } : {}),
10193
10853
  occurredAt: input.now,
10194
- })
10195
- .returning();
10196
- await tx
10854
+ },
10855
+ ];
10856
+ if (turnWasCurrent && input.session.status !== nextSessionStatus) {
10857
+ eventValues.push({
10858
+ accountId: input.session.accountId,
10859
+ workspaceId: input.session.workspaceId,
10860
+ sessionId: input.session.id,
10861
+ sequence: input.session.lastSequence + 2,
10862
+ type: "session.status.changed",
10863
+ payload: { status: nextSessionStatus, reason: input.reason },
10864
+ turnId: updated.blockedTurnId,
10865
+ turnGeneration: input.blockedTurn.executionGeneration,
10866
+ turnAssociation: "current",
10867
+ occurredAt: input.now,
10868
+ });
10869
+ }
10870
+ const inserted = await tx.insert(schema.sessionEvents).values(eventValues).returning();
10871
+ const lastSequence = input.session.lastSequence + inserted.length;
10872
+ const [updatedSession] = await tx
10197
10873
  .update(schema.sessions)
10198
- .set({ lastSequence: input.session.lastSequence + 1, updatedAt: input.now })
10874
+ .set({
10875
+ ...(turnWasCurrent ? { status: nextSessionStatus, activeTurnId: null } : {}),
10876
+ lastSequence,
10877
+ updatedAt: input.now,
10878
+ })
10199
10879
  .where(
10200
10880
  and(
10201
10881
  eq(schema.sessions.workspaceId, input.session.workspaceId),
10202
10882
  eq(schema.sessions.id, input.session.id),
10883
+ ...(turnWasCurrent ? [eq(schema.sessions.activeTurnId, input.blockedTurn.id)] : []),
10203
10884
  ),
10204
- );
10885
+ )
10886
+ .returning({ id: schema.sessions.id });
10887
+ if (!updatedSession) {
10888
+ throw new Error("Codex capacity session changed during atomic supersession");
10889
+ }
10205
10890
  return {
10206
10891
  waiter: mapCodexCapacityWaiter(updated),
10207
10892
  events: inserted.map(mapEvent),
@@ -10211,10 +10896,11 @@ async function supersedeCodexCapacityWaitInTransaction(
10211
10896
  /**
10212
10897
  * Row-lock and re-evaluate one waiter. Availability is decided by a pure
10213
10898
  * caller supplied policy over the same rotation-row transaction as normal
10214
- * acquisition. If available, one system goal-continuation event and one turn
10215
- * are committed; duplicate timers/signals observe status=resumed and do no
10216
- * work. If any goal/control/policy/turn/queue fence changed, the waiter is
10217
- * superseded without inference.
10899
+ * acquisition. If available, the exact blocked turn becomes `recovering`;
10900
+ * duplicate timers/signals observe status=resumed and do no work. Effective
10901
+ * Pause leaves the waiter untouched, ordinary queued prompts remain behind the
10902
+ * current inference, and only an explicit semantic fence change supersedes the
10903
+ * waiter/blocked turn without inference.
10218
10904
  */
10219
10905
  export async function reconcileCodexCapacityWait<
10220
10906
  TPolicyScope = never,
@@ -10287,18 +10973,20 @@ export async function reconcileCodexCapacityWait<
10287
10973
  workspaceControl: prefix.control ?? undefined,
10288
10974
  })
10289
10975
  : null;
10290
- const [goal] = await tx
10291
- .select()
10292
- .from(schema.sessionGoals)
10293
- .where(
10294
- and(
10295
- eq(schema.sessionGoals.workspaceId, input.workspaceId),
10296
- eq(schema.sessionGoals.id, waiterRead.goalId),
10297
- eq(schema.sessionGoals.sessionId, input.sessionId),
10298
- ),
10299
- )
10300
- .for("update")
10301
- .limit(1);
10976
+ const [goal] = waiterRead.goalId
10977
+ ? await tx
10978
+ .select()
10979
+ .from(schema.sessionGoals)
10980
+ .where(
10981
+ and(
10982
+ eq(schema.sessionGoals.workspaceId, input.workspaceId),
10983
+ eq(schema.sessionGoals.id, waiterRead.goalId),
10984
+ eq(schema.sessionGoals.sessionId, input.sessionId),
10985
+ ),
10986
+ )
10987
+ .for("update")
10988
+ .limit(1)
10989
+ : [];
10302
10990
  const [waiter] = await tx
10303
10991
  .select()
10304
10992
  .from(schema.codexCapacityWaiters)
@@ -10307,7 +10995,6 @@ export async function reconcileCodexCapacityWait<
10307
10995
  .limit(1);
10308
10996
  if (
10309
10997
  !session ||
10310
- !goal ||
10311
10998
  !blockedTurn ||
10312
10999
  !waiter ||
10313
11000
  session.accountId !== input.accountId ||
@@ -10326,55 +11013,40 @@ export async function reconcileCodexCapacityWait<
10326
11013
  events: [],
10327
11014
  } as const;
10328
11015
  }
11016
+ if (effectiveControl?.state !== "active" || effectiveControl.settlement !== null) {
11017
+ return {
11018
+ action: "paused",
11019
+ waiter: mapCodexCapacityWaiter(waiter),
11020
+ events: [],
11021
+ } as const;
11022
+ }
10329
11023
 
10330
- const [pending] = await tx
10331
- .select({ id: schema.sessionTurns.id })
10332
- .from(schema.sessionTurns)
10333
- .where(
10334
- and(
10335
- eq(schema.sessionTurns.workspaceId, input.workspaceId),
10336
- eq(schema.sessionTurns.sessionId, input.sessionId),
10337
- inArray(schema.sessionTurns.status, [
10338
- "queued",
10339
- "running",
10340
- "requires_action",
10341
- "recovering",
10342
- "waiting_capacity",
10343
- ]),
10344
- ),
10345
- )
10346
- .limit(1);
10347
- const [laterTurn] = await tx
10348
- .select({ id: schema.sessionTurns.id })
10349
- .from(schema.sessionTurns)
10350
- .where(
10351
- and(
10352
- eq(schema.sessionTurns.workspaceId, input.workspaceId),
10353
- eq(schema.sessionTurns.sessionId, input.sessionId),
10354
- gt(schema.sessionTurns.position, blockedTurn.position),
10355
- ),
10356
- )
10357
- .limit(1);
10358
11024
  const currentPolicyHash = codexCapacityPolicyHashFromTurnMetadata(blockedTurn.metadata);
10359
11025
  let supersedeReason: string | null = null;
10360
- if (effectiveControl?.state !== "active" || effectiveControl.settlement !== null) {
10361
- supersedeReason = "control_changed";
10362
- } else if (goal.status !== "active" || goal.version !== waiter.goalVersion) {
11026
+ if (session.status === "cancelled") {
11027
+ supersedeReason = "session_cancelled";
11028
+ } else if (
11029
+ waiter.goalId !== null &&
11030
+ (!goal || goal.status !== "active" || goal.version !== waiter.goalVersion)
11031
+ ) {
10363
11032
  supersedeReason = "goal_changed";
10364
11033
  } else if (currentPolicyHash !== waiter.policyHash) {
10365
11034
  supersedeReason = "credential_policy_changed";
10366
- } else if (session.status !== "idle" || session.activeTurnId !== null) {
10367
- supersedeReason = "session_not_capacity_idle";
10368
- } else if (blockedTurn.status !== "failed") {
11035
+ } else if (session.activeTurnId !== blockedTurn.id) {
11036
+ supersedeReason = "active_turn_changed";
11037
+ } else if (session.status !== "waiting_capacity") {
11038
+ supersedeReason = "session_not_waiting_capacity";
11039
+ } else if (
11040
+ blockedTurn.status !== "waiting_capacity" ||
11041
+ blockedTurn.activeAttemptId !== null ||
11042
+ blockedTurn.executionGeneration !== waiter.blockedTurnGeneration
11043
+ ) {
10369
11044
  supersedeReason = "blocked_turn_changed";
10370
- } else if (pending) {
10371
- supersedeReason = "pending_work_exists";
10372
- } else if (laterTurn) {
10373
- supersedeReason = "newer_turn_exists";
10374
11045
  }
10375
11046
  if (supersedeReason) {
10376
11047
  const superseded = await supersedeCodexCapacityWaitInTransaction(tx, {
10377
11048
  session,
11049
+ blockedTurn,
10378
11050
  waiter,
10379
11051
  reason: supersedeReason,
10380
11052
  now,
@@ -10446,69 +11118,6 @@ export async function reconcileCodexCapacityWait<
10446
11118
  } as const;
10447
11119
  }
10448
11120
 
10449
- const prompt = [
10450
- "[CODEX CAPACITY RESUME] Codex subscription capacity is available again.",
10451
- `Continue the existing active goal from durable conversation history: ${goal.text}`,
10452
- `Success criteria: ${goal.successCriteria ?? "none specified"}.`,
10453
- "Do not replay completed tool side effects; verify any ambiguous in-flight effect before repeating it.",
10454
- "If the goal is complete, call opengeni__goal_complete. If blocked for another reason, call opengeni__goal_pause.",
10455
- ].join("\n");
10456
- const [update] = await tx
10457
- .insert(schema.sessionSystemUpdates)
10458
- .values({
10459
- accountId: input.accountId,
10460
- workspaceId: input.workspaceId,
10461
- sessionId: input.sessionId,
10462
- kind: "goal_continuation",
10463
- classification: "info",
10464
- sourceId: goal.id,
10465
- dedupeKey: `codex-capacity-resume:${waiter.id}:${waiter.generation}`,
10466
- summary: prompt,
10467
- payload: {
10468
- type: "goal_continuation",
10469
- goalId: goal.id,
10470
- goalVersion: goal.version,
10471
- prompt,
10472
- reason: "codex_capacity",
10473
- capacityWaiterId: waiter.id,
10474
- capacityWaitGeneration: waiter.generation,
10475
- policy: {
10476
- model: blockedTurn.model,
10477
- reasoningEffort: blockedTurn.reasoningEffort,
10478
- tools: blockedTurn.tools,
10479
- sandboxBackend: blockedTurn.sandboxBackend,
10480
- },
10481
- },
10482
- lineage: {
10483
- goalId: goal.id,
10484
- blockedTurnId: blockedTurn.id,
10485
- capacityWaiterId: waiter.id,
10486
- },
10487
- state: "pending",
10488
- })
10489
- .returning();
10490
- if (!update) {
10491
- throw new Error("Codex capacity resume did not create an internal update");
10492
- }
10493
- await tx
10494
- .insert(schema.usageEvents)
10495
- .values({
10496
- accountId: input.accountId,
10497
- workspaceId: input.workspaceId,
10498
- eventType: "agent_run.created",
10499
- quantity: 1,
10500
- unit: "run",
10501
- sourceResourceType: "session_system_update",
10502
- sourceResourceId: update.id,
10503
- sessionId: input.sessionId,
10504
- initiatorKind: "service",
10505
- initiatorSubjectId: "goal-continuation",
10506
- initiatorContext: { goalId: goal.id, reason: "codex_capacity" },
10507
- origin: "goal",
10508
- idempotencyKey: `agent_run.created:codex-capacity:${input.workspaceId}:${update.id}`,
10509
- occurredAt: now,
10510
- })
10511
- .onConflictDoNothing({ target: schema.usageEvents.idempotencyKey });
10512
11121
  const events = await tx
10513
11122
  .insert(schema.sessionEvents)
10514
11123
  .values([
@@ -10517,14 +11126,20 @@ export async function reconcileCodexCapacityWait<
10517
11126
  workspaceId: input.workspaceId,
10518
11127
  sessionId: input.sessionId,
10519
11128
  sequence: session.lastSequence + 1,
10520
- type: "system.update.pending",
11129
+ type: "codex.capacity.resumed",
10521
11130
  payload: sanitizeEventPayload({
10522
- updateId: update.id,
10523
- kind: update.kind,
10524
- classification: update.classification,
10525
- sourceId: update.sourceId,
10526
- summary: update.summary,
11131
+ waiterId: waiter.id,
11132
+ generation: waiter.generation,
11133
+ wakeRevision: waiter.wakeRevision,
11134
+ goalId: waiter.goalId,
11135
+ goalVersion: waiter.goalVersion,
11136
+ blockedTurnGeneration: waiter.blockedTurnGeneration,
11137
+ policyHash: waiter.policyHash,
11138
+ diagnostic: decision.diagnostic ?? null,
10527
11139
  }),
11140
+ turnId: blockedTurn.id,
11141
+ turnGeneration: blockedTurn.executionGeneration,
11142
+ turnAssociation: "current",
10528
11143
  occurredAt: now,
10529
11144
  },
10530
11145
  {
@@ -10532,18 +11147,11 @@ export async function reconcileCodexCapacityWait<
10532
11147
  workspaceId: input.workspaceId,
10533
11148
  sessionId: input.sessionId,
10534
11149
  sequence: session.lastSequence + 2,
10535
- type: "codex.capacity.resumed",
10536
- payload: sanitizeEventPayload({
10537
- waiterId: waiter.id,
10538
- generation: waiter.generation,
10539
- wakeRevision: waiter.wakeRevision,
10540
- goalId: goal.id,
10541
- goalVersion: goal.version,
10542
- policyHash: waiter.policyHash,
10543
- diagnostic: decision.diagnostic ?? null,
10544
- updateId: update.id,
10545
- }),
11150
+ type: "session.status.changed",
11151
+ payload: { status: "recovering", reason: "codex_capacity" },
10546
11152
  turnId: blockedTurn.id,
11153
+ turnGeneration: blockedTurn.executionGeneration,
11154
+ turnAssociation: "current",
10547
11155
  occurredAt: now,
10548
11156
  },
10549
11157
  ])
@@ -10552,7 +11160,7 @@ export async function reconcileCodexCapacityWait<
10552
11160
  .update(schema.codexCapacityWaiters)
10553
11161
  .set({
10554
11162
  status: "resumed",
10555
- resumedUpdateId: update.id,
11163
+ resumedUpdateId: null,
10556
11164
  observedWakeRevision: waiter.wakeRevision,
10557
11165
  lastWakeReason: "capacity_available",
10558
11166
  updatedAt: now,
@@ -10568,11 +11176,34 @@ export async function reconcileCodexCapacityWait<
10568
11176
  if (!updatedWaiter) {
10569
11177
  throw new Error("Codex capacity waiter changed during atomic resume");
10570
11178
  }
10571
- await tx
11179
+ const [recoveringTurn] = await tx
11180
+ .update(schema.sessionTurns)
11181
+ .set({
11182
+ status: "recovering",
11183
+ activeAttemptId: null,
11184
+ metadata: metadataWithoutTurnDispatchAttempt(blockedTurn.metadata),
11185
+ version: blockedTurn.version + 1,
11186
+ finishedAt: null,
11187
+ updatedAt: now,
11188
+ })
11189
+ .where(
11190
+ and(
11191
+ eq(schema.sessionTurns.workspaceId, input.workspaceId),
11192
+ eq(schema.sessionTurns.id, blockedTurn.id),
11193
+ eq(schema.sessionTurns.status, "waiting_capacity"),
11194
+ isNull(schema.sessionTurns.activeAttemptId),
11195
+ eq(schema.sessionTurns.executionGeneration, waiter.blockedTurnGeneration),
11196
+ ),
11197
+ )
11198
+ .returning({ id: schema.sessionTurns.id });
11199
+ if (!recoveringTurn) {
11200
+ throw new Error("Codex capacity blocked turn changed during atomic resume");
11201
+ }
11202
+ const [recoveringSession] = await tx
10572
11203
  .update(schema.sessions)
10573
11204
  .set({
10574
- status: "queued",
10575
- activeTurnId: null,
11205
+ status: "recovering",
11206
+ activeTurnId: blockedTurn.id,
10576
11207
  lastSequence: session.lastSequence + 2,
10577
11208
  updatedAt: now,
10578
11209
  })
@@ -10580,13 +11211,17 @@ export async function reconcileCodexCapacityWait<
10580
11211
  and(
10581
11212
  eq(schema.sessions.workspaceId, input.workspaceId),
10582
11213
  eq(schema.sessions.id, input.sessionId),
10583
- isNull(schema.sessions.activeTurnId),
11214
+ eq(schema.sessions.status, "waiting_capacity"),
11215
+ eq(schema.sessions.activeTurnId, blockedTurn.id),
10584
11216
  ),
10585
- );
11217
+ )
11218
+ .returning({ id: schema.sessions.id });
11219
+ if (!recoveringSession) {
11220
+ throw new Error("Codex capacity session changed during atomic resume");
11221
+ }
10586
11222
  return {
10587
11223
  action: "resumed",
10588
11224
  waiter: mapCodexCapacityWaiter(updatedWaiter),
10589
- update: mapSessionSystemUpdate(update),
10590
11225
  events: events.map(mapEvent),
10591
11226
  } as const;
10592
11227
  }),
@@ -14463,21 +15098,24 @@ export async function listSessionsForSubject(
14463
15098
  if (ordinaryIds.length > limit) {
14464
15099
  let snapshot = reusableSnapshot;
14465
15100
  if (!snapshot) {
14466
- const activeSnapshots = await tx
14467
- .select({ id: schema.sessionListSnapshots.id })
14468
- .from(schema.sessionListSnapshots)
14469
- .where(
14470
- and(
14471
- eq(schema.sessionListSnapshots.workspaceId, workspaceId),
14472
- eq(schema.sessionListSnapshots.subjectId, options.subjectId),
14473
- ),
14474
- )
14475
- .limit(SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT);
14476
- if (activeSnapshots.length >= SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT) {
14477
- throw new SessionListSnapshotLimitError(
14478
- "too many active session list snapshots; retry after an existing cursor expires",
14479
- );
14480
- }
15101
+ // Creation is serialized by the subject advisory lock above.
15102
+ // Retain only the newest N-1 before inserting so polling and
15103
+ // search identity churn cannot turn this bounded cache into a
15104
+ // user-visible 429. A continuation for an evicted row retains
15105
+ // the existing typed expiry/410 + client rebase contract.
15106
+ await tx.execute(sql`
15107
+ delete from ${schema.sessionListSnapshots} snapshot
15108
+ where snapshot.workspace_id = ${workspaceId}
15109
+ and snapshot.subject_id = ${options.subjectId}
15110
+ and snapshot.id in (
15111
+ select evicted.id
15112
+ from ${schema.sessionListSnapshots} evicted
15113
+ where evicted.workspace_id = ${workspaceId}
15114
+ and evicted.subject_id = ${options.subjectId}
15115
+ order by evicted.created_at desc, evicted.id desc
15116
+ offset ${SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT - 1}
15117
+ )
15118
+ `);
14481
15119
  const [workspace] = await tx
14482
15120
  .select({ accountId: schema.workspaces.accountId })
14483
15121
  .from(schema.workspaces)
@@ -19374,7 +20012,7 @@ export async function acquireLease(
19374
20012
  if (liveness === "cold") {
19375
20013
  const recovery = recoveryStateFromLeaseRow(row);
19376
20014
  if (
19377
- recovery.restore.status === "degraded" ||
20015
+ (recovery.restore.status === "degraded" && recovery.restore.retryable !== true) ||
19378
20016
  recovery.restore.status === "unrecoverable"
19379
20017
  ) {
19380
20018
  return {
@@ -20075,10 +20713,186 @@ export async function recordWarmingSandboxCreated(
20075
20713
  );
20076
20714
  }
20077
20715
 
20716
+ export type LostProviderWorkspaceSettlement = {
20717
+ processesLost: number;
20718
+ admissionsRejected: number;
20719
+ ptysClosed: number;
20720
+ processHoldersDeleted: number;
20721
+ };
20722
+
20078
20723
  export type MarkWarmLeaseInstanceLostResult =
20079
- | { status: "marked"; lease: LeaseSnapshot }
20724
+ | {
20725
+ status: "marked";
20726
+ lease: LeaseSnapshot;
20727
+ settlement: LostProviderWorkspaceSettlement;
20728
+ }
20080
20729
  | { status: "stale"; lease: LeaseSnapshot | null };
20081
20730
 
20731
+ const LOST_PROVIDER_PROCESS_REASON = "provider_instance_lost";
20732
+
20733
+ /** Lock the exact provider-owned blocker set before taking the lease row. Normal
20734
+ * retained-process settlement locks process -> admission -> lease, so provider
20735
+ * loss must not hold the lease while waiting for one of those rows. The lease
20736
+ * tuple is revalidated under FOR UPDATE after these locks are acquired. */
20737
+ async function lockExactLostProviderWorkspaceBlockersTx(
20738
+ tx: Database,
20739
+ input: {
20740
+ accountId: string;
20741
+ workspaceId: string;
20742
+ leaseId: string;
20743
+ sandboxGroupId: string;
20744
+ lostEpoch: number;
20745
+ lostInstanceId: string;
20746
+ },
20747
+ ): Promise<void> {
20748
+ await tx.execute(sql`
20749
+ select id from sandbox_retained_processes
20750
+ where account_id = ${input.accountId}
20751
+ and workspace_id = ${input.workspaceId}
20752
+ and lease_id = ${input.leaseId}
20753
+ and sandbox_group_id = ${input.sandboxGroupId}
20754
+ and lease_epoch = ${input.lostEpoch}
20755
+ and provider_instance_id = ${input.lostInstanceId}
20756
+ and state = 'active'
20757
+ order by id
20758
+ for update
20759
+ `);
20760
+ await tx.execute(sql`
20761
+ select id from sandbox_workspace_mutation_admissions
20762
+ where account_id = ${input.accountId}
20763
+ and workspace_id = ${input.workspaceId}
20764
+ and lease_id = ${input.leaseId}
20765
+ and sandbox_group_id = ${input.sandboxGroupId}
20766
+ and lease_epoch = ${input.lostEpoch}
20767
+ and provider_instance_id = ${input.lostInstanceId}
20768
+ and settled_at is null
20769
+ order by id
20770
+ for update
20771
+ `);
20772
+ await tx.execute(sql`
20773
+ select id from sandbox_pty_sessions
20774
+ where account_id = ${input.accountId}
20775
+ and workspace_id = ${input.workspaceId}
20776
+ and lease_id = ${input.leaseId}
20777
+ and sandbox_group_id = ${input.sandboxGroupId}
20778
+ and lease_epoch = ${input.lostEpoch}
20779
+ and provider_instance_id = ${input.lostInstanceId}
20780
+ and status = 'open'
20781
+ order by id
20782
+ for update
20783
+ `);
20784
+ }
20785
+
20786
+ async function settleExactLostProviderWorkspaceBlockersTx(
20787
+ tx: Database,
20788
+ input: {
20789
+ accountId: string;
20790
+ workspaceId: string;
20791
+ leaseId: string;
20792
+ sandboxGroupId: string;
20793
+ lostEpoch: number;
20794
+ lostInstanceId: string;
20795
+ },
20796
+ ): Promise<LostProviderWorkspaceSettlement> {
20797
+ const lostProcesses = await tx
20798
+ .update(schema.sandboxRetainedProcesses)
20799
+ .set({
20800
+ state: "lost",
20801
+ exitCode: null,
20802
+ settlementReason: LOST_PROVIDER_PROCESS_REASON,
20803
+ settledAt: new Date(),
20804
+ })
20805
+ .where(
20806
+ and(
20807
+ eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
20808
+ eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
20809
+ eq(schema.sandboxRetainedProcesses.leaseId, input.leaseId),
20810
+ eq(schema.sandboxRetainedProcesses.sandboxGroupId, input.sandboxGroupId),
20811
+ eq(schema.sandboxRetainedProcesses.leaseEpoch, input.lostEpoch),
20812
+ eq(schema.sandboxRetainedProcesses.providerInstanceId, input.lostInstanceId),
20813
+ eq(schema.sandboxRetainedProcesses.state, "active"),
20814
+ ),
20815
+ )
20816
+ .returning({
20817
+ id: schema.sandboxRetainedProcesses.id,
20818
+ holderId: schema.sandboxRetainedProcesses.holderId,
20819
+ });
20820
+
20821
+ const rejectedAdmissions = await tx
20822
+ .update(schema.sandboxWorkspaceMutationAdmissions)
20823
+ .set({ providerOutcome: "rejected", settledAt: new Date() })
20824
+ .where(
20825
+ and(
20826
+ eq(schema.sandboxWorkspaceMutationAdmissions.accountId, input.accountId),
20827
+ eq(schema.sandboxWorkspaceMutationAdmissions.workspaceId, input.workspaceId),
20828
+ eq(schema.sandboxWorkspaceMutationAdmissions.leaseId, input.leaseId),
20829
+ eq(schema.sandboxWorkspaceMutationAdmissions.sandboxGroupId, input.sandboxGroupId),
20830
+ eq(schema.sandboxWorkspaceMutationAdmissions.leaseEpoch, input.lostEpoch),
20831
+ eq(schema.sandboxWorkspaceMutationAdmissions.providerInstanceId, input.lostInstanceId),
20832
+ isNull(schema.sandboxWorkspaceMutationAdmissions.settledAt),
20833
+ ),
20834
+ )
20835
+ .returning({ id: schema.sandboxWorkspaceMutationAdmissions.id });
20836
+
20837
+ const closedPtys = await tx
20838
+ .update(schema.sandboxPtySessions)
20839
+ .set({ status: "closed", closedAt: new Date() })
20840
+ .where(
20841
+ and(
20842
+ eq(schema.sandboxPtySessions.accountId, input.accountId),
20843
+ eq(schema.sandboxPtySessions.workspaceId, input.workspaceId),
20844
+ eq(schema.sandboxPtySessions.leaseId, input.leaseId),
20845
+ eq(schema.sandboxPtySessions.sandboxGroupId, input.sandboxGroupId),
20846
+ eq(schema.sandboxPtySessions.leaseEpoch, input.lostEpoch),
20847
+ eq(schema.sandboxPtySessions.providerInstanceId, input.lostInstanceId),
20848
+ eq(schema.sandboxPtySessions.status, "open"),
20849
+ ),
20850
+ )
20851
+ .returning({ id: schema.sandboxPtySessions.id });
20852
+
20853
+ const deletedHolders =
20854
+ lostProcesses.length === 0
20855
+ ? []
20856
+ : await tx
20857
+ .delete(schema.sandboxLeaseHolders)
20858
+ .where(
20859
+ and(
20860
+ eq(schema.sandboxLeaseHolders.accountId, input.accountId),
20861
+ eq(schema.sandboxLeaseHolders.workspaceId, input.workspaceId),
20862
+ eq(schema.sandboxLeaseHolders.leaseId, input.leaseId),
20863
+ eq(schema.sandboxLeaseHolders.kind, "process"),
20864
+ inArray(
20865
+ schema.sandboxLeaseHolders.holderId,
20866
+ lostProcesses.map((process) => process.holderId),
20867
+ ),
20868
+ ),
20869
+ )
20870
+ .returning({ holderId: schema.sandboxLeaseHolders.holderId });
20871
+
20872
+ await tx.execute(sql`
20873
+ update sandbox_leases as lease set
20874
+ refcount = counts.total,
20875
+ turn_holders = counts.turns,
20876
+ viewer_holders = counts.viewers,
20877
+ updated_at = now()
20878
+ from (
20879
+ select count(*)::int as total,
20880
+ count(*) filter (where kind = 'turn')::int as turns,
20881
+ count(*) filter (where kind = 'viewer')::int as viewers
20882
+ from sandbox_lease_holders
20883
+ where lease_id = ${input.leaseId}
20884
+ ) as counts
20885
+ where lease.id = ${input.leaseId}
20886
+ `);
20887
+
20888
+ return {
20889
+ processesLost: lostProcesses.length,
20890
+ admissionsRejected: rejectedAdmissions.length,
20891
+ ptysClosed: closedPtys.length,
20892
+ processHoldersDeleted: deletedHolders.length,
20893
+ };
20894
+ }
20895
+
20082
20896
  /**
20083
20897
  * Atomically retire one exact warm provider instance after a resume-only caller
20084
20898
  * receives a provider NotFound. The epoch + instance predicates are the
@@ -20086,10 +20900,11 @@ export type MarkWarmLeaseInstanceLostResult =
20086
20900
  * box, but only the first one transitions the lease to cold and advances its
20087
20901
  * epoch. The next ordinary acquire elects one cold->warming spawner.
20088
20902
  *
20089
- * Holders remain intact because their logical work/viewer interest still exists.
20090
- * Only live provider identity is cleared. A persisted workspace archive is
20091
- * reduced to the same minimal cold envelope used by the drain/failure paths, so
20092
- * the elected replacement can hydrate it without carrying the dead box id.
20903
+ * Logical turn/viewer interest remains intact, while holders for active
20904
+ * processes that physically belonged to the lost provider are removed with
20905
+ * their terminal loss proof. A persisted workspace archive is reduced to the
20906
+ * same minimal cold envelope used by the drain/failure paths, so the elected
20907
+ * replacement can hydrate it without carrying the dead box id.
20093
20908
  */
20094
20909
  export async function markWarmLeaseInstanceLost(
20095
20910
  db: Database,
@@ -20109,6 +20924,33 @@ export async function markWarmLeaseInstanceLost(
20109
20924
  async (scopedDb) =>
20110
20925
  await scopedDb.transaction(async (txRaw) => {
20111
20926
  const tx = txRaw as unknown as Database;
20927
+ const observedRows = await tx.execute<LeaseRow>(sql`
20928
+ select * from sandbox_leases
20929
+ where workspace_id = ${input.workspaceId}
20930
+ and sandbox_group_id = ${input.sandboxGroupId}
20931
+ `);
20932
+ const observed = observedRows[0];
20933
+ if (
20934
+ !observed ||
20935
+ observed.liveness !== "warm" ||
20936
+ Number(observed.lease_epoch) !== input.expectedEpoch ||
20937
+ observed.instance_id !== input.expectedInstanceId
20938
+ ) {
20939
+ return {
20940
+ status: "stale" as const,
20941
+ lease: observed ? mapLeaseRow(observed) : null,
20942
+ };
20943
+ }
20944
+
20945
+ const blockerScope = {
20946
+ accountId: input.accountId,
20947
+ workspaceId: input.workspaceId,
20948
+ leaseId: observed.id,
20949
+ sandboxGroupId: input.sandboxGroupId,
20950
+ lostEpoch: input.expectedEpoch,
20951
+ lostInstanceId: input.expectedInstanceId,
20952
+ };
20953
+ await lockExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
20112
20954
  const currentRows = await tx.execute<LeaseRow>(sql`
20113
20955
  select * from sandbox_leases
20114
20956
  where workspace_id = ${input.workspaceId}
@@ -20118,6 +20960,7 @@ export async function markWarmLeaseInstanceLost(
20118
20960
  const current = currentRows[0];
20119
20961
  if (
20120
20962
  !current ||
20963
+ current.id !== observed.id ||
20121
20964
  current.liveness !== "warm" ||
20122
20965
  Number(current.lease_epoch) !== input.expectedEpoch ||
20123
20966
  current.instance_id !== input.expectedInstanceId
@@ -20128,6 +20971,8 @@ export async function markWarmLeaseInstanceLost(
20128
20971
  };
20129
20972
  }
20130
20973
 
20974
+ const settlement = await settleExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
20975
+
20131
20976
  const observedAt = new Date().toISOString();
20132
20977
  const before = recoveryStateFromLeaseRow(current);
20133
20978
  const archiveStatus = before.archive.status;
@@ -20198,7 +21043,126 @@ export async function markWarmLeaseInstanceLost(
20198
21043
  if (!updated) {
20199
21044
  throw new Error(`Warm sandbox lease vanished while retiring instance ${current.id}`);
20200
21045
  }
20201
- return { status: "marked" as const, lease: mapLeaseRow(updated) };
21046
+ return { status: "marked" as const, lease: mapLeaseRow(updated), settlement };
21047
+ }),
21048
+ );
21049
+ }
21050
+
21051
+ export type ReconcileColdLostLeaseInstanceBlockersResult =
21052
+ | {
21053
+ status: "reconciled";
21054
+ lease: LeaseSnapshot;
21055
+ settlement: LostProviderWorkspaceSettlement;
21056
+ }
21057
+ | { status: "stale"; lease: LeaseSnapshot | null };
21058
+
21059
+ /**
21060
+ * Settle blockers left by a provider-loss transition that predates exact loss
21061
+ * cleanup. This is deliberately narrower than markWarmLeaseInstanceLost: it
21062
+ * requires the already-cold lease to match the full operator-observed
21063
+ * generation/archive tuple, the current epoch to be exactly lostEpoch + 1, and
21064
+ * recovery truth to name the same missing provider. It never advances an epoch,
21065
+ * rematerializes a provider, or changes archive/recovery completeness.
21066
+ */
21067
+ export async function reconcileColdLostLeaseInstanceBlockers(
21068
+ db: Database,
21069
+ input: {
21070
+ accountId: string;
21071
+ workspaceId: string;
21072
+ sandboxGroupId: string;
21073
+ expectedCurrentEpoch: number;
21074
+ expectedLostEpoch: number;
21075
+ expectedLostInstanceId: string;
21076
+ expectedWorkspaceGeneration: number;
21077
+ expectedArchiveGeneration: number | null;
21078
+ expectedArchiveComplete: boolean;
21079
+ },
21080
+ ): Promise<ReconcileColdLostLeaseInstanceBlockersResult> {
21081
+ if (
21082
+ !Number.isSafeInteger(input.expectedCurrentEpoch) ||
21083
+ !Number.isSafeInteger(input.expectedLostEpoch) ||
21084
+ input.expectedCurrentEpoch !== input.expectedLostEpoch + 1
21085
+ ) {
21086
+ throw new Error("Cold lost-provider reconciliation requires currentEpoch = lostEpoch + 1");
21087
+ }
21088
+ return await withRlsContext(
21089
+ db,
21090
+ { accountId: input.accountId, workspaceId: input.workspaceId },
21091
+ async (scopedDb) =>
21092
+ await scopedDb.transaction(async (txRaw) => {
21093
+ const tx = txRaw as unknown as Database;
21094
+ const observedRows = await tx.execute<LeaseRow>(sql`
21095
+ select * from sandbox_leases
21096
+ where workspace_id = ${input.workspaceId}
21097
+ and sandbox_group_id = ${input.sandboxGroupId}
21098
+ `);
21099
+ const observed = observedRows[0];
21100
+ const observedRecovery = observed ? recoveryStateFromLeaseRow(observed) : null;
21101
+ if (
21102
+ !observed ||
21103
+ observed.liveness !== "cold" ||
21104
+ observed.instance_id !== null ||
21105
+ Number(observed.lease_epoch) !== input.expectedCurrentEpoch ||
21106
+ Number(observed.workspace_generation) !== input.expectedWorkspaceGeneration ||
21107
+ (observed.archive_generation === null ? null : Number(observed.archive_generation)) !==
21108
+ input.expectedArchiveGeneration ||
21109
+ hasCompleteWorkspaceArchive(observed) !== input.expectedArchiveComplete ||
21110
+ observedRecovery?.provider.status !== "missing" ||
21111
+ observedRecovery.provider.instanceId !== input.expectedLostInstanceId
21112
+ ) {
21113
+ return {
21114
+ status: "stale" as const,
21115
+ lease: observed ? mapLeaseRow(observed) : null,
21116
+ };
21117
+ }
21118
+
21119
+ const blockerScope = {
21120
+ accountId: input.accountId,
21121
+ workspaceId: input.workspaceId,
21122
+ leaseId: observed.id,
21123
+ sandboxGroupId: input.sandboxGroupId,
21124
+ lostEpoch: input.expectedLostEpoch,
21125
+ lostInstanceId: input.expectedLostInstanceId,
21126
+ };
21127
+ await lockExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
21128
+ const currentRows = await tx.execute<LeaseRow>(sql`
21129
+ select * from sandbox_leases
21130
+ where workspace_id = ${input.workspaceId}
21131
+ and sandbox_group_id = ${input.sandboxGroupId}
21132
+ for update
21133
+ `);
21134
+ const current = currentRows[0];
21135
+ const recovery = current ? recoveryStateFromLeaseRow(current) : null;
21136
+ if (
21137
+ !current ||
21138
+ current.id !== observed.id ||
21139
+ current.liveness !== "cold" ||
21140
+ current.instance_id !== null ||
21141
+ Number(current.lease_epoch) !== input.expectedCurrentEpoch ||
21142
+ Number(current.workspace_generation) !== input.expectedWorkspaceGeneration ||
21143
+ (current.archive_generation === null ? null : Number(current.archive_generation)) !==
21144
+ input.expectedArchiveGeneration ||
21145
+ hasCompleteWorkspaceArchive(current) !== input.expectedArchiveComplete ||
21146
+ recovery?.provider.status !== "missing" ||
21147
+ recovery.provider.instanceId !== input.expectedLostInstanceId
21148
+ ) {
21149
+ return {
21150
+ status: "stale" as const,
21151
+ lease: current ? mapLeaseRow(current) : null,
21152
+ };
21153
+ }
21154
+
21155
+ const settlement = await settleExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
21156
+ const refreshedRows = await tx.execute<LeaseRow>(sql`
21157
+ select * from sandbox_leases where id = ${current.id}
21158
+ `);
21159
+ const refreshed = refreshedRows[0];
21160
+ if (!refreshed) throw new Error("Cold sandbox lease vanished during reconciliation");
21161
+ return {
21162
+ status: "reconciled" as const,
21163
+ lease: mapLeaseRow(refreshed),
21164
+ settlement,
21165
+ };
20202
21166
  }),
20203
21167
  );
20204
21168
  }
@@ -21066,7 +22030,7 @@ export async function confirmDrainCold(
21066
22030
  // The fence is the split-brain guard: a stale-epoch reaper writes ZERO rows and
21067
22031
  // is told not to terminate.
21068
22032
  export class SandboxWorkspaceMutationFencedError extends Error {
21069
- readonly name = "SandboxWorkspaceMutationFencedError";
22033
+ readonly name: string = "SandboxWorkspaceMutationFencedError";
21070
22034
 
21071
22035
  constructor(
21072
22036
  public readonly code:
@@ -21132,8 +22096,131 @@ export type SandboxRetainedProcess = {
21132
22096
  settlementReason: string | null;
21133
22097
  startedAt: string;
21134
22098
  settledAt: string | null;
22099
+ reconcileAfter: string;
22100
+ reconcileClaimId: string | null;
22101
+ reconcileClaimedAt: string | null;
22102
+ reconcileAttempts: number;
22103
+ lastReconcileOutcome: string | null;
22104
+ reconcileProofOutcome: "exited" | "lost" | null;
22105
+ reconcileProofExitCode: number | null;
22106
+ reconcileProofReason:
22107
+ | "provider_exit_banner"
22108
+ | "provider_session_lost_banner"
22109
+ | "provider_instance_not_found"
22110
+ | null;
22111
+ reconcileProofObservedAt: string | null;
21135
22112
  };
21136
22113
 
22114
+ export type RetainedProcessProviderProof =
22115
+ | { outcome: "exited"; exitCode: number; reason: "provider_exit_banner" }
22116
+ | {
22117
+ outcome: "lost";
22118
+ exitCode: null;
22119
+ reason: "provider_session_lost_banner" | "provider_instance_not_found";
22120
+ };
22121
+
22122
+ export type SandboxRetainedProcessIdentity = Pick<
22123
+ SandboxRetainedProcess,
22124
+ | "leaseId"
22125
+ | "sandboxGroupId"
22126
+ | "parentAdmissionId"
22127
+ | "holderId"
22128
+ | "leaseEpoch"
22129
+ | "providerBackend"
22130
+ | "providerInstanceId"
22131
+ | "routeKind"
22132
+ | "routeTargetId"
22133
+ | "routeEpoch"
22134
+ | "providerSessionId"
22135
+ >;
22136
+
22137
+ export type SandboxRetainedProcessReconciliationClaim = {
22138
+ process: SandboxRetainedProcess;
22139
+ claimId: string;
22140
+ ownerState: string;
22141
+ ownerAttemptOutcome: string | null;
22142
+ };
22143
+
22144
+ export type ActiveRetainedProcessOwnerCount = {
22145
+ ownerState: string;
22146
+ activeCount: number;
22147
+ terminalOwnerCount: number;
22148
+ };
22149
+
22150
+ export type ExpiredDrainingSandboxLeaseCount = {
22151
+ backend: string;
22152
+ ageBucket: "lt_5m" | "5m_1h" | "1h_1d" | "gte_1d";
22153
+ count: number;
22154
+ };
22155
+
22156
+ export function retainedProcessSettlementIdentity(
22157
+ process: SandboxRetainedProcess,
22158
+ ): SandboxRetainedProcessIdentity {
22159
+ return {
22160
+ leaseId: process.leaseId,
22161
+ sandboxGroupId: process.sandboxGroupId,
22162
+ parentAdmissionId: process.parentAdmissionId,
22163
+ holderId: process.holderId,
22164
+ leaseEpoch: process.leaseEpoch,
22165
+ providerBackend: process.providerBackend,
22166
+ providerInstanceId: process.providerInstanceId,
22167
+ routeKind: process.routeKind,
22168
+ routeTargetId: process.routeTargetId,
22169
+ routeEpoch: process.routeEpoch,
22170
+ providerSessionId: process.providerSessionId,
22171
+ };
22172
+ }
22173
+
22174
+ export function retainedProcessReconciliationProof(
22175
+ process: SandboxRetainedProcess,
22176
+ ): RetainedProcessProviderProof | null {
22177
+ if (
22178
+ process.reconcileProofOutcome === "exited" &&
22179
+ process.reconcileProofReason === "provider_exit_banner" &&
22180
+ process.reconcileProofExitCode !== null
22181
+ ) {
22182
+ return {
22183
+ outcome: "exited",
22184
+ exitCode: process.reconcileProofExitCode,
22185
+ reason: "provider_exit_banner",
22186
+ };
22187
+ }
22188
+ if (
22189
+ process.reconcileProofOutcome === "lost" &&
22190
+ (process.reconcileProofReason === "provider_session_lost_banner" ||
22191
+ process.reconcileProofReason === "provider_instance_not_found")
22192
+ ) {
22193
+ return {
22194
+ outcome: "lost",
22195
+ exitCode: null,
22196
+ reason: process.reconcileProofReason,
22197
+ };
22198
+ }
22199
+ return null;
22200
+ }
22201
+
22202
+ /** Durable promotion committed before a mutable route/turn authority check
22203
+ * rejected the provider output. The process identity is safe to hand to the
22204
+ * exact-backend cancellation path; callers must still reject the output and
22205
+ * must never replay the provider mutation. */
22206
+ export class SandboxRetainedProcessPromotionFencedError extends SandboxWorkspaceMutationFencedError {
22207
+ readonly name = "SandboxRetainedProcessPromotionFencedError";
22208
+
22209
+ constructor(
22210
+ code:
22211
+ | "attempt_fenced"
22212
+ | "holder_fenced"
22213
+ | "lease_fenced"
22214
+ | "route_fenced"
22215
+ | "process_fenced"
22216
+ | "admission_fenced",
22217
+ message: string,
22218
+ public readonly process: SandboxRetainedProcess,
22219
+ ) {
22220
+ super(code, message);
22221
+ }
22222
+ }
22223
+
21137
22224
  type SandboxWorkspaceMutationSettlementResult =
21138
22225
  | { failure: null }
21139
22226
  | {
@@ -21246,6 +22333,18 @@ function normalizeRetainedProcessSettlementReason(reason: string): string {
21246
22333
  return bounded;
21247
22334
  }
21248
22335
 
22336
+ function normalizeRetainedProcessReconciliationOutcome(outcome: string): string {
22337
+ const normalized = outcome.trim();
22338
+ const byteLength = Buffer.byteLength(normalized, "utf8");
22339
+ if (byteLength < 1 || byteLength > 64) {
22340
+ throw new SandboxWorkspaceMutationFencedError(
22341
+ "process_fenced",
22342
+ "Retained process reconciliation outcome must contain between 1 and 64 UTF-8 bytes",
22343
+ );
22344
+ }
22345
+ return normalized;
22346
+ }
22347
+
21249
22348
  function mapWorkspaceMutationAdmission(row: {
21250
22349
  id: string;
21251
22350
  lease_id: string;
@@ -21311,9 +22410,38 @@ function mapRetainedProcess(
21311
22410
  settlementReason: row.settlementReason ?? null,
21312
22411
  startedAt: row.startedAt.toISOString(),
21313
22412
  settledAt: row.settledAt?.toISOString() ?? null,
22413
+ reconcileAfter: row.reconcileAfter.toISOString(),
22414
+ reconcileClaimId: row.reconcileClaimId ?? null,
22415
+ reconcileClaimedAt: row.reconcileClaimedAt?.toISOString() ?? null,
22416
+ reconcileAttempts: row.reconcileAttempts,
22417
+ lastReconcileOutcome: row.lastReconcileOutcome ?? null,
22418
+ reconcileProofOutcome: row.reconcileProofOutcome ?? null,
22419
+ reconcileProofExitCode: row.reconcileProofExitCode ?? null,
22420
+ reconcileProofReason:
22421
+ (row.reconcileProofReason as SandboxRetainedProcess["reconcileProofReason"]) ?? null,
22422
+ reconcileProofObservedAt: row.reconcileProofObservedAt?.toISOString() ?? null,
21314
22423
  };
21315
22424
  }
21316
22425
 
22426
+ function retainedProcessMatchesSettlementIdentity(
22427
+ process: typeof schema.sandboxRetainedProcesses.$inferSelect,
22428
+ expected: SandboxRetainedProcessIdentity,
22429
+ ): boolean {
22430
+ return (
22431
+ process.leaseId === expected.leaseId &&
22432
+ process.sandboxGroupId === expected.sandboxGroupId &&
22433
+ process.parentAdmissionId === expected.parentAdmissionId &&
22434
+ process.holderId === expected.holderId &&
22435
+ process.leaseEpoch === expected.leaseEpoch &&
22436
+ process.providerBackend === expected.providerBackend &&
22437
+ process.providerInstanceId === expected.providerInstanceId &&
22438
+ process.routeKind === expected.routeKind &&
22439
+ (process.routeTargetId ?? null) === expected.routeTargetId &&
22440
+ process.routeEpoch === expected.routeEpoch &&
22441
+ process.providerSessionId === expected.providerSessionId
22442
+ );
22443
+ }
22444
+
21317
22445
  async function lockWorkspaceMutationSessionTx(
21318
22446
  tx: Database,
21319
22447
  workspaceId: string,
@@ -22333,7 +23461,11 @@ export async function retainWorkspaceMutationProcess(
22333
23461
  }),
22334
23462
  );
22335
23463
  if (result.failure.failure !== null) {
22336
- throw new SandboxWorkspaceMutationFencedError(result.failure.failure, result.failure.detail);
23464
+ throw new SandboxRetainedProcessPromotionFencedError(
23465
+ result.failure.failure,
23466
+ result.failure.detail,
23467
+ result.process,
23468
+ );
22337
23469
  }
22338
23470
  return result.process;
22339
23471
  }
@@ -22360,6 +23492,293 @@ export async function getRetainedProcess(
22360
23492
  });
22361
23493
  }
22362
23494
 
23495
+ /** Claim a bounded, oldest-due batch of active retained processes whose exact
23496
+ * owner attempt is closed (or whose direct request already returned). Claim
23497
+ * expiry only recovers coordination after worker death; it is never provider
23498
+ * exit proof. Rows that settle between the global claim and scoped read are
23499
+ * deliberately omitted. */
23500
+ export async function claimTerminalRetainedProcesses(
23501
+ db: Database,
23502
+ input: { claimId: string; limit: number; claimTtlMs: number },
23503
+ ): Promise<SandboxRetainedProcessReconciliationClaim[]> {
23504
+ if (!Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) {
23505
+ throw new Error("Retained process reconciliation limit must be between 1 and 100");
23506
+ }
23507
+ if (
23508
+ !Number.isSafeInteger(input.claimTtlMs) ||
23509
+ input.claimTtlMs < 0 ||
23510
+ input.claimTtlMs > 3_600_000
23511
+ ) {
23512
+ throw new Error("Retained process reconciliation claim TTL is invalid");
23513
+ }
23514
+ const rows = await rawRows<{
23515
+ account_id: string;
23516
+ workspace_id: string;
23517
+ session_id: string;
23518
+ process_id: string;
23519
+ claim_id: string;
23520
+ owner_state: string;
23521
+ owner_attempt_outcome: string | null;
23522
+ }>(
23523
+ db,
23524
+ sql`
23525
+ select account_id, workspace_id, session_id, process_id, claim_id,
23526
+ owner_state, owner_attempt_outcome
23527
+ from opengeni_private.claim_terminal_retained_processes(
23528
+ ${input.claimId}::uuid, ${input.limit}::integer, ${input.claimTtlMs}::bigint
23529
+ )
23530
+ `,
23531
+ );
23532
+ const claims: SandboxRetainedProcessReconciliationClaim[] = [];
23533
+ for (const row of rows) {
23534
+ const process = await getRetainedProcess(db, {
23535
+ workspaceId: row.workspace_id,
23536
+ sessionId: row.session_id,
23537
+ processId: row.process_id,
23538
+ });
23539
+ if (
23540
+ !process ||
23541
+ process.accountId !== row.account_id ||
23542
+ process.state !== "active" ||
23543
+ process.reconcileClaimId !== row.claim_id
23544
+ ) {
23545
+ continue;
23546
+ }
23547
+ claims.push({
23548
+ process,
23549
+ claimId: row.claim_id,
23550
+ ownerState: row.owner_state,
23551
+ ownerAttemptOutcome: row.owner_attempt_outcome,
23552
+ });
23553
+ }
23554
+ return claims;
23555
+ }
23556
+
23557
+ /** Durably checkpoint exact provider exit/loss proof before canonical
23558
+ * settlement. This closes the worker-death window between provider observation
23559
+ * and settlement without converting owner state, age, timeout, or claim expiry
23560
+ * into physical proof. */
23561
+ export async function recordRetainedProcessReconciliationProof(
23562
+ db: Database,
23563
+ input: {
23564
+ accountId: string;
23565
+ workspaceId: string;
23566
+ sessionId: string;
23567
+ processId: string;
23568
+ expected: SandboxRetainedProcessIdentity;
23569
+ claimId: string;
23570
+ proof: RetainedProcessProviderProof;
23571
+ },
23572
+ ): Promise<SandboxRetainedProcess> {
23573
+ if (
23574
+ input.proof.outcome === "exited" &&
23575
+ (!Number.isSafeInteger(input.proof.exitCode) || input.proof.exitCode === null)
23576
+ ) {
23577
+ throw new SandboxWorkspaceMutationFencedError(
23578
+ "process_fenced",
23579
+ "Retained process reconciliation exit proof requires a safe integer exit code",
23580
+ );
23581
+ }
23582
+ return await withRlsContext(
23583
+ db,
23584
+ { accountId: input.accountId, workspaceId: input.workspaceId },
23585
+ async (scopedDb) =>
23586
+ await scopedDb.transaction(async (txRaw) => {
23587
+ const tx = txRaw as unknown as Database;
23588
+ const [process] = await tx
23589
+ .select()
23590
+ .from(schema.sandboxRetainedProcesses)
23591
+ .where(
23592
+ and(
23593
+ eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
23594
+ eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
23595
+ eq(schema.sandboxRetainedProcesses.sessionId, input.sessionId),
23596
+ eq(schema.sandboxRetainedProcesses.id, input.processId),
23597
+ ),
23598
+ )
23599
+ .for("update")
23600
+ .limit(1);
23601
+ if (
23602
+ !process ||
23603
+ process.state !== "active" ||
23604
+ !retainedProcessMatchesSettlementIdentity(process, input.expected)
23605
+ ) {
23606
+ throw new SandboxWorkspaceMutationFencedError(
23607
+ "process_fenced",
23608
+ "Retained process proof did not match an active copied durable identity",
23609
+ );
23610
+ }
23611
+ if (process.reconcileClaimId !== input.claimId) {
23612
+ throw new SandboxWorkspaceMutationFencedError(
23613
+ "process_fenced",
23614
+ "Retained process proof claim was lost or superseded",
23615
+ );
23616
+ }
23617
+ const existing = mapRetainedProcess(process);
23618
+ const existingProof = retainedProcessReconciliationProof(existing);
23619
+ if (existingProof) {
23620
+ if (
23621
+ existingProof.outcome !== input.proof.outcome ||
23622
+ existingProof.exitCode !== input.proof.exitCode ||
23623
+ existingProof.reason !== input.proof.reason
23624
+ ) {
23625
+ throw new SandboxWorkspaceMutationFencedError(
23626
+ "process_fenced",
23627
+ "Retained process already carries different provider proof",
23628
+ );
23629
+ }
23630
+ return existing;
23631
+ }
23632
+ const [updated] = await tx
23633
+ .update(schema.sandboxRetainedProcesses)
23634
+ .set({
23635
+ reconcileProofOutcome: input.proof.outcome,
23636
+ reconcileProofExitCode: input.proof.exitCode,
23637
+ reconcileProofReason: input.proof.reason,
23638
+ reconcileProofObservedAt: new Date(),
23639
+ lastReconcileOutcome: `proof_${input.proof.outcome}`,
23640
+ })
23641
+ .where(
23642
+ and(
23643
+ eq(schema.sandboxRetainedProcesses.id, process.id),
23644
+ eq(schema.sandboxRetainedProcesses.state, "active"),
23645
+ eq(schema.sandboxRetainedProcesses.reconcileClaimId, input.claimId),
23646
+ ),
23647
+ )
23648
+ .returning();
23649
+ if (!updated) {
23650
+ throw new SandboxWorkspaceMutationFencedError(
23651
+ "process_fenced",
23652
+ "Retained process changed while checkpointing provider proof",
23653
+ );
23654
+ }
23655
+ return mapRetainedProcess(updated);
23656
+ }),
23657
+ );
23658
+ }
23659
+
23660
+ /** Release one exact reconciliation claim after an ambiguous/running provider
23661
+ * observation. This schedules a bounded retry but never changes process,
23662
+ * admission, holder, lease, archive, snapshot, or workspace state. */
23663
+ export async function deferRetainedProcessReconciliation(
23664
+ db: Database,
23665
+ input: {
23666
+ accountId: string;
23667
+ workspaceId: string;
23668
+ sessionId: string;
23669
+ processId: string;
23670
+ expected: SandboxRetainedProcessIdentity;
23671
+ claimId: string;
23672
+ outcome: string;
23673
+ retryAfterMs: number;
23674
+ },
23675
+ ): Promise<boolean> {
23676
+ const outcome = normalizeRetainedProcessReconciliationOutcome(input.outcome);
23677
+ if (
23678
+ !Number.isSafeInteger(input.retryAfterMs) ||
23679
+ input.retryAfterMs < 0 ||
23680
+ input.retryAfterMs > 86_400_000
23681
+ ) {
23682
+ throw new SandboxWorkspaceMutationFencedError(
23683
+ "process_fenced",
23684
+ "Retained process reconciliation retry delay is invalid",
23685
+ );
23686
+ }
23687
+ return await withRlsContext(
23688
+ db,
23689
+ { accountId: input.accountId, workspaceId: input.workspaceId },
23690
+ async (scopedDb) =>
23691
+ await scopedDb.transaction(async (txRaw) => {
23692
+ const tx = txRaw as unknown as Database;
23693
+ const [process] = await tx
23694
+ .select()
23695
+ .from(schema.sandboxRetainedProcesses)
23696
+ .where(
23697
+ and(
23698
+ eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
23699
+ eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
23700
+ eq(schema.sandboxRetainedProcesses.sessionId, input.sessionId),
23701
+ eq(schema.sandboxRetainedProcesses.id, input.processId),
23702
+ ),
23703
+ )
23704
+ .for("update")
23705
+ .limit(1);
23706
+ if (!process || !retainedProcessMatchesSettlementIdentity(process, input.expected)) {
23707
+ throw new SandboxWorkspaceMutationFencedError(
23708
+ "process_fenced",
23709
+ "Retained process reconciliation did not match the copied durable identity",
23710
+ );
23711
+ }
23712
+ if (process.state !== "active") return false;
23713
+ if (process.reconcileClaimId !== input.claimId) {
23714
+ throw new SandboxWorkspaceMutationFencedError(
23715
+ "process_fenced",
23716
+ "Retained process reconciliation claim was lost or superseded",
23717
+ );
23718
+ }
23719
+ const [updated] = await tx
23720
+ .update(schema.sandboxRetainedProcesses)
23721
+ .set({
23722
+ reconcileAfter: new Date(Date.now() + input.retryAfterMs),
23723
+ reconcileClaimId: null,
23724
+ reconcileClaimedAt: null,
23725
+ lastReconcileOutcome: outcome,
23726
+ })
23727
+ .where(
23728
+ and(
23729
+ eq(schema.sandboxRetainedProcesses.id, process.id),
23730
+ eq(schema.sandboxRetainedProcesses.state, "active"),
23731
+ eq(schema.sandboxRetainedProcesses.reconcileClaimId, input.claimId),
23732
+ ),
23733
+ )
23734
+ .returning({ id: schema.sandboxRetainedProcesses.id });
23735
+ return Boolean(updated);
23736
+ }),
23737
+ );
23738
+ }
23739
+
23740
+ export async function countActiveRetainedProcessesByOwnerState(
23741
+ db: Database,
23742
+ ): Promise<ActiveRetainedProcessOwnerCount[]> {
23743
+ const rows = await rawRows<{
23744
+ owner_state: string;
23745
+ active_count: number | string;
23746
+ terminal_owner_count: number | string;
23747
+ }>(
23748
+ db,
23749
+ sql`
23750
+ select owner_state, active_count, terminal_owner_count
23751
+ from opengeni_private.count_active_retained_processes_by_owner_state()
23752
+ `,
23753
+ );
23754
+ return rows.map((row) => ({
23755
+ ownerState: row.owner_state,
23756
+ activeCount: Number(row.active_count),
23757
+ terminalOwnerCount: Number(row.terminal_owner_count),
23758
+ }));
23759
+ }
23760
+
23761
+ export async function countExpiredDrainingSandboxLeases(
23762
+ db: Database,
23763
+ ): Promise<ExpiredDrainingSandboxLeaseCount[]> {
23764
+ const rows = await rawRows<{
23765
+ backend: string;
23766
+ age_bucket: ExpiredDrainingSandboxLeaseCount["ageBucket"];
23767
+ count: number | string;
23768
+ }>(
23769
+ db,
23770
+ sql`
23771
+ select backend, age_bucket, count
23772
+ from opengeni_private.count_expired_draining_sandbox_leases()
23773
+ `,
23774
+ );
23775
+ return rows.map((row) => ({
23776
+ backend: row.backend,
23777
+ ageBucket: row.age_bucket,
23778
+ count: Number(row.count),
23779
+ }));
23780
+ }
23781
+
22363
23782
  /** Settle an exact retained process only after exit or definitive loss proof.
22364
23783
  * The process row, parent admission, non-TTL holder, and lease count transition
22365
23784
  * commit atomically. Duplicate identical proof is idempotent; conflicting proof
@@ -22371,6 +23790,8 @@ export async function settleRetainedProcess(
22371
23790
  workspaceId: string;
22372
23791
  sessionId: string;
22373
23792
  processId: string;
23793
+ expected: SandboxRetainedProcessIdentity;
23794
+ reconciliationClaimId?: string;
22374
23795
  outcome: "exited" | "lost";
22375
23796
  exitCode?: number | null;
22376
23797
  reason: string;
@@ -22411,6 +23832,12 @@ export async function settleRetainedProcess(
22411
23832
  "Retained process settlement did not match a durable process",
22412
23833
  );
22413
23834
  }
23835
+ if (!retainedProcessMatchesSettlementIdentity(process, input.expected)) {
23836
+ throw new SandboxWorkspaceMutationFencedError(
23837
+ "process_fenced",
23838
+ "Retained process settlement did not match the copied durable identity",
23839
+ );
23840
+ }
22414
23841
  if (process.state !== "active") {
22415
23842
  if (
22416
23843
  process.state !== input.outcome ||
@@ -22433,6 +23860,27 @@ export async function settleRetainedProcess(
22433
23860
  );
22434
23861
  return { settled: false, process: mapRetainedProcess(process) };
22435
23862
  }
23863
+ if (
23864
+ input.reconciliationClaimId !== undefined &&
23865
+ process.reconcileClaimId !== input.reconciliationClaimId
23866
+ ) {
23867
+ throw new SandboxWorkspaceMutationFencedError(
23868
+ "process_fenced",
23869
+ "Retained process settlement reconciliation claim was lost or superseded",
23870
+ );
23871
+ }
23872
+ const durableProof = retainedProcessReconciliationProof(mapRetainedProcess(process));
23873
+ if (
23874
+ durableProof &&
23875
+ (durableProof.outcome !== input.outcome ||
23876
+ durableProof.exitCode !== exitCode ||
23877
+ durableProof.reason !== reason)
23878
+ ) {
23879
+ throw new SandboxWorkspaceMutationFencedError(
23880
+ "process_fenced",
23881
+ "Retained process settlement conflicts with checkpointed provider proof",
23882
+ );
23883
+ }
22436
23884
  const admissions = await tx.execute<AdmissionIdentityRow>(sql`
22437
23885
  select * from sandbox_workspace_mutation_admissions
22438
23886
  where id = ${process.parentAdmissionId}
@@ -22440,6 +23888,13 @@ export async function settleRetainedProcess(
22440
23888
  and workspace_id = ${input.workspaceId}
22441
23889
  and session_id = ${input.sessionId}
22442
23890
  and lease_id = ${process.leaseId}
23891
+ and sandbox_group_id = ${process.sandboxGroupId}
23892
+ and lease_epoch = ${process.leaseEpoch}
23893
+ and provider_backend = ${process.providerBackend}
23894
+ and provider_instance_id = ${process.providerInstanceId}
23895
+ and route_kind = ${process.routeKind}
23896
+ and route_target_id is not distinct from ${process.routeTargetId}
23897
+ and route_epoch = ${process.routeEpoch}
22443
23898
  and provider_outcome = 'retained'
22444
23899
  and settled_at is null
22445
23900
  for update
@@ -22450,6 +23905,23 @@ export async function settleRetainedProcess(
22450
23905
  "Retained process parent admission is not open",
22451
23906
  );
22452
23907
  }
23908
+ const leases = await tx.execute<LeaseRow>(sql`
23909
+ select * from sandbox_leases
23910
+ where id = ${process.leaseId}
23911
+ and account_id = ${input.accountId}
23912
+ and workspace_id = ${input.workspaceId}
23913
+ and sandbox_group_id = ${process.sandboxGroupId}
23914
+ and lease_epoch = ${process.leaseEpoch}
23915
+ and backend = ${process.providerBackend}
23916
+ and instance_id = ${process.providerInstanceId}
23917
+ for update
23918
+ `);
23919
+ if (!leases[0]) {
23920
+ throw new SandboxWorkspaceMutationFencedError(
23921
+ "lease_fenced",
23922
+ "Retained process settlement cannot mutate a successor lease identity",
23923
+ );
23924
+ }
22453
23925
  await tx
22454
23926
  .update(schema.sandboxPtySessions)
22455
23927
  .set({ status: "closed", closedAt: new Date() })
@@ -22466,6 +23938,12 @@ export async function settleRetainedProcess(
22466
23938
  exitCode,
22467
23939
  settlementReason: reason,
22468
23940
  settledAt: new Date(),
23941
+ reconcileClaimId: null,
23942
+ reconcileClaimedAt: null,
23943
+ lastReconcileOutcome:
23944
+ input.reconciliationClaimId === undefined
23945
+ ? `owner_settled_${input.outcome}`
23946
+ : `reconciled_${input.outcome}`,
22469
23947
  })
22470
23948
  .where(
22471
23949
  and(
@@ -22490,6 +23968,8 @@ export async function settleRetainedProcess(
22490
23968
  await tx.execute(sql`
22491
23969
  delete from sandbox_lease_holders
22492
23970
  where lease_id = ${process.leaseId}
23971
+ and account_id = ${input.accountId}
23972
+ and workspace_id = ${input.workspaceId}
22493
23973
  and kind = 'process' and holder_id = ${process.holderId}
22494
23974
  `);
22495
23975
  const [counts] = await tx.execute<{
@@ -22518,6 +23998,10 @@ export async function settleRetainedProcess(
22518
23998
  }
22519
23999
  updated_at = now()
22520
24000
  where id = ${process.leaseId}
24001
+ and sandbox_group_id = ${process.sandboxGroupId}
24002
+ and lease_epoch = ${process.leaseEpoch}
24003
+ and backend = ${process.providerBackend}
24004
+ and instance_id = ${process.providerInstanceId}
22521
24005
  `);
22522
24006
  return { settled: true, process: mapRetainedProcess(updated) };
22523
24007
  }),
@@ -27521,7 +29005,7 @@ export async function initializeSessionStartAtomically(
27521
29005
  if (!goal) throw new Error("Failed to create initial session goal");
27522
29006
  }
27523
29007
 
27524
- let [userEvent] = await tx
29008
+ const existingUserEvents = await tx
27525
29009
  .select()
27526
29010
  .from(schema.sessionEvents)
27527
29011
  .where(
@@ -27533,6 +29017,7 @@ export async function initializeSessionStartAtomically(
27533
29017
  )
27534
29018
  .orderBy(asc(schema.sessionEvents.sequence))
27535
29019
  .limit(1);
29020
+ let userEvent: typeof schema.sessionEvents.$inferSelect | undefined = existingUserEvents[0];
27536
29021
  let sequence = session.lastSequence;
27537
29022
  const insertedEvents: Array<typeof schema.sessionEvents.$inferSelect> = [];
27538
29023
  const runnable = effectiveControl.state === "active";
@@ -27732,7 +29217,7 @@ export async function initializeSessionStartAtomically(
27732
29217
  tx as unknown as Database,
27733
29218
  input.consumeNewSessionDraft.subjectId,
27734
29219
  );
27735
- await consumeNewSessionDraftInTransaction(tx as unknown as Database, {
29220
+ await seedNewSessionDraftInTransaction(tx as unknown as Database, {
27736
29221
  workspaceId: input.workspaceId,
27737
29222
  subjectId: input.consumeNewSessionDraft.subjectId,
27738
29223
  expectedRevision: input.consumeNewSessionDraft.expectedRevision,
@@ -29568,7 +31053,7 @@ export async function peekSessionWork(
29568
31053
  if (
29569
31054
  latestInterruption &&
29570
31055
  latestInterruption.quiescedAt === null &&
29571
- ["settled", "rejected_stale"].includes(latestInterruption.interruptionState)
31056
+ latestInterruption.interruptionState === "settled"
29572
31057
  ) {
29573
31058
  return {
29574
31059
  kind: "cancellation-wait",
@@ -32894,6 +34379,9 @@ function sessionEventTypesAdvanceActivity(inputs: ReadonlyArray<{ type: string }
32894
34379
  function sessionMutationAdvancesActivity(update: {
32895
34380
  resources?: ResourceRef[];
32896
34381
  tools?: ToolRef[];
34382
+ toolPolicy?: SessionToolPolicy;
34383
+ toolPolicyVersion?: number;
34384
+ expectedToolPolicyVersion?: number;
32897
34385
  model?: string;
32898
34386
  metadata?: Record<string, unknown>;
32899
34387
  status?: SessionStatus;
@@ -33482,6 +34970,7 @@ type LockedSessionUpdateContext = {
33482
34970
  requireApproval: SessionMcpApprovalPolicy,
33483
34971
  ) => Promise<UpdateSessionMcpApprovalPolicyResult>;
33484
34972
  listPendingSessionTurns: () => Promise<SessionTurn[]>;
34973
+ getLockedSession: (sessionId: string) => Promise<Session | null>;
33485
34974
  };
33486
34975
 
33487
34976
  type LockedSessionUpdateResult = {
@@ -33489,6 +34978,9 @@ type LockedSessionUpdateResult = {
33489
34978
  update?: {
33490
34979
  resources?: ResourceRef[];
33491
34980
  tools?: ToolRef[];
34981
+ toolPolicy?: SessionToolPolicy;
34982
+ toolPolicyVersion?: number;
34983
+ expectedToolPolicyVersion?: number;
33492
34984
  model?: string;
33493
34985
  metadata?: Record<string, unknown>;
33494
34986
  status?: SessionStatus;
@@ -33504,18 +34996,39 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33504
34996
  session: Session,
33505
34997
  context: LockedSessionUpdateContext,
33506
34998
  ) => LockedSessionUpdateResult | Promise<LockedSessionUpdateResult>,
34999
+ options: { lockParentSession?: boolean } = {},
33507
35000
  ): Promise<SessionEvent[]> {
33508
35001
  return await withWorkspaceRls(
33509
35002
  db,
33510
35003
  workspaceId,
33511
35004
  async (scopedDb) =>
33512
35005
  await scopedDb.transaction(async (tx) => {
33513
- const locks = await lockSessionEventWriteRows(tx as unknown as Database, {
35006
+ const firstLocks = await lockSessionEventWriteRows(tx as unknown as Database, {
33514
35007
  workspaceId,
33515
35008
  controlLock: "share",
33516
35009
  sessionIds: [sessionId],
33517
35010
  });
33518
- const sessionRow = locks.sessions[0];
35011
+ const firstSessionRow = firstLocks.sessions.find((row) => row.id === sessionId);
35012
+ if (!firstSessionRow) {
35013
+ throw new Error(`Session not found: ${sessionId}`);
35014
+ }
35015
+ // Child-policy updates must serialize with a concurrent parent-policy
35016
+ // update. The target lock establishes the parent id, then the second
35017
+ // call acquires the complete UUID-ordered set under the already-held
35018
+ // workspace/control prefix.
35019
+ const lockSessionIds = options.lockParentSession
35020
+ ? [sessionId, firstSessionRow.parentSessionId].filter((id): id is string => Boolean(id))
35021
+ : [sessionId];
35022
+ const locks =
35023
+ lockSessionIds.length === 1
35024
+ ? firstLocks
35025
+ : await lockSessionEventWriteRows(tx as unknown as Database, {
35026
+ workspaceId,
35027
+ controlLock: "already_locked",
35028
+ workspaceLock: "already_locked",
35029
+ sessionIds: lockSessionIds,
35030
+ });
35031
+ const sessionRow = locks.sessions.find((row) => row.id === sessionId);
33519
35032
  if (!sessionRow) {
33520
35033
  throw new Error(`Session not found: ${sessionId}`);
33521
35034
  }
@@ -33554,6 +35067,18 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33554
35067
  .orderBy(asc(schema.sessionTurns.position), asc(schema.sessionTurns.createdAt));
33555
35068
  return rows.map(mapSessionTurn);
33556
35069
  },
35070
+ getLockedSession: async (lockedSessionId) => {
35071
+ const row = locks.sessions.find((candidate) => candidate.id === lockedSessionId);
35072
+ return row
35073
+ ? await mapSessionWithControl(
35074
+ tx as unknown as Database,
35075
+ row,
35076
+ [],
35077
+ undefined,
35078
+ locks.control ?? undefined,
35079
+ )
35080
+ : null;
35081
+ },
33557
35082
  });
33558
35083
  if (built.events.length === 0) {
33559
35084
  return [];
@@ -33591,12 +35116,16 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33591
35116
  const update = built.update ?? {};
33592
35117
  const advancesActivity =
33593
35118
  sessionMutationAdvancesActivity(update) || sessionEventTypesAdvanceActivity(values);
33594
- await tx
35119
+ const updated = await tx
33595
35120
  .update(schema.sessions)
33596
35121
  .set({
33597
35122
  lastSequence: sequence,
33598
35123
  ...(update.resources !== undefined ? { resources: update.resources } : {}),
33599
35124
  ...(update.tools !== undefined ? { tools: update.tools } : {}),
35125
+ ...(update.toolPolicy !== undefined ? { toolPolicy: update.toolPolicy } : {}),
35126
+ ...(update.toolPolicyVersion !== undefined
35127
+ ? { toolPolicyVersion: update.toolPolicyVersion }
35128
+ : {}),
33600
35129
  ...(update.model !== undefined ? { model: update.model } : {}),
33601
35130
  ...(update.metadata !== undefined ? { metadata: update.metadata } : {}),
33602
35131
  ...(update.status !== undefined ? { status: update.status } : {}),
@@ -33604,8 +35133,27 @@ export async function appendSessionEventsWithLockedSessionUpdate(
33604
35133
  ...(advancesActivity ? { updatedAt: now } : {}),
33605
35134
  })
33606
35135
  .where(
33607
- and(eq(schema.sessions.workspaceId, workspaceId), eq(schema.sessions.id, sessionId)),
35136
+ and(
35137
+ eq(schema.sessions.workspaceId, workspaceId),
35138
+ eq(schema.sessions.id, sessionId),
35139
+ ...(update.expectedToolPolicyVersion !== undefined
35140
+ ? [eq(schema.sessions.toolPolicyVersion, update.expectedToolPolicyVersion)]
35141
+ : []),
35142
+ ),
35143
+ )
35144
+ .returning({ id: schema.sessions.id });
35145
+ if (updated.length === 0) {
35146
+ const [current] = await tx
35147
+ .select({ toolPolicyVersion: schema.sessions.toolPolicyVersion })
35148
+ .from(schema.sessions)
35149
+ .where(
35150
+ and(eq(schema.sessions.workspaceId, workspaceId), eq(schema.sessions.id, sessionId)),
35151
+ )
35152
+ .limit(1);
35153
+ throw new SessionToolPolicyVersionConflictError(
35154
+ Number(current?.toolPolicyVersion ?? update.expectedToolPolicyVersion ?? 1),
33608
35155
  );
35156
+ }
33609
35157
  return inserted.map(mapEvent);
33610
35158
  }),
33611
35159
  );
@@ -33666,6 +35214,7 @@ function mapSession(
33666
35214
  mode: "legacy",
33667
35215
  inheritedFromSessionId: null,
33668
35216
  },
35217
+ toolPolicyVersion: Number(row.toolPolicyVersion ?? 1),
33669
35218
  metadata: row.metadata,
33670
35219
  createdBy: initiatorFromStorage(
33671
35220
  row.createdByKind,
@@ -34173,12 +35722,14 @@ function mapConnectionMetadata(row: {
34173
35722
  lastUsedAt: Date | null;
34174
35723
  lastError: string | null;
34175
35724
  version: number;
35725
+ verifiedInstallAt: Date | null;
35726
+ verifiedInstallVersion: number | null;
34176
35727
  metadata: Record<string, unknown>;
34177
35728
  createdBySubjectId: string | null;
34178
35729
  updatedBySubjectId: string | null;
34179
35730
  createdAt: Date;
34180
35731
  updatedAt: Date;
34181
- }): ConnectionMetadata {
35732
+ }): ConnectionMetadataWithVerification {
34182
35733
  return {
34183
35734
  id: row.id,
34184
35735
  accountId: row.accountId,
@@ -34193,6 +35744,8 @@ function mapConnectionMetadata(row: {
34193
35744
  lastUsedAt: row.lastUsedAt?.toISOString() ?? null,
34194
35745
  lastError: row.lastError,
34195
35746
  version: row.version,
35747
+ verifiedInstallAt: row.verifiedInstallAt?.toISOString() ?? null,
35748
+ verifiedInstallVersion: row.verifiedInstallVersion,
34196
35749
  metadata: row.metadata,
34197
35750
  createdBySubjectId: row.createdBySubjectId,
34198
35751
  updatedBySubjectId: row.updatedBySubjectId,
@@ -34201,6 +35754,32 @@ function mapConnectionMetadata(row: {
34201
35754
  };
34202
35755
  }
34203
35756
 
35757
+ function mapSlackBotPostOperation(
35758
+ row: typeof schema.slackBotPostOperations.$inferSelect,
35759
+ ): SlackBotPostOperation {
35760
+ return {
35761
+ id: row.id,
35762
+ accountId: row.accountId,
35763
+ workspaceId: row.workspaceId,
35764
+ connectionId: row.connectionId,
35765
+ operationId: row.operationId,
35766
+ clientMessageId: row.clientMessageId,
35767
+ targetKind: row.targetKind,
35768
+ targetId: row.targetId,
35769
+ requestDigest: row.requestDigest,
35770
+ status: row.status,
35771
+ claimHolderId: row.claimHolderId,
35772
+ claimExpiresAt: row.claimExpiresAt,
35773
+ attemptCount: row.attemptCount,
35774
+ lastFailureCode: row.lastFailureCode,
35775
+ slackChannelId: row.slackChannelId,
35776
+ slackMessageTimestamp: row.slackMessageTimestamp,
35777
+ completedAt: row.completedAt,
35778
+ createdAt: row.createdAt,
35779
+ updatedAt: row.updatedAt,
35780
+ };
35781
+ }
35782
+
34204
35783
  function mapKnowledgeMemory(row: typeof schema.knowledgeMemories.$inferSelect): KnowledgeMemory {
34205
35784
  return {
34206
35785
  id: row.id,
@@ -34270,10 +35849,17 @@ function mapGitHubInstallation(
34270
35849
  accountId: row.accountId,
34271
35850
  workspaceId: row.workspaceId,
34272
35851
  installationId: row.installationId,
35852
+ githubAccountId: row.githubAccountId,
34273
35853
  accountLogin: row.accountLogin,
34274
35854
  accountType: row.accountType,
34275
35855
  repositoryScope: row.repositoryScope as GitHubRepositoryScope,
34276
35856
  linkedBySubjectId: row.linkedBySubjectId,
35857
+ githubActorId: row.githubActorId,
35858
+ githubActorLogin: row.githubActorLogin,
35859
+ authorityKind: row.authorityKind as GitHubInstallationAuthorityKind | null,
35860
+ authorityCheckedAt: row.authorityCheckedAt?.toISOString() ?? null,
35861
+ authorityExpiresAt: row.authorityExpiresAt?.toISOString() ?? null,
35862
+ authorityNonce: row.authorityNonce,
34277
35863
  createdAt: row.createdAt.toISOString(),
34278
35864
  updatedAt: row.updatedAt.toISOString(),
34279
35865
  };