@opengeni/db 0.12.0 → 0.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-VUKRIBO5.js → chunk-CYA6BHV6.js} +150 -7
- package/dist/chunk-CYA6BHV6.js.map +1 -0
- package/dist/chunk-XRVNI2GB.js +896 -0
- package/dist/chunk-XRVNI2GB.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1166 -265
- package/dist/index.js.map +1 -1
- package/dist/provision-roles.d.ts +324 -38
- package/dist/provision-roles.js +1 -1
- package/dist/{schema-CnpD6BcX.d.ts → schema-CeXWlYcG.d.ts} +441 -12
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +1 -1
- package/drizzle/0122_codex_capacity_same_turn.sql +59 -0
- package/drizzle/0123_session_tool_policy_version.sql +14 -0
- package/drizzle/0124_session_event_duplicate_lookup.sql +4 -0
- package/drizzle/0125_document_drops_visibility.sql +29 -0
- package/drizzle/0126_document_access_constraints.sql +138 -0
- package/drizzle/0127_document_default_base_index.sql +5 -0
- package/drizzle/0128_github_installation_authority.sql +69 -0
- package/drizzle/0129_retained_process_reconciliation.sql +356 -0
- package/package.json +4 -3
- package/src/event-payload-sanitizer.ts +167 -55
- package/src/index.ts +1494 -258
- package/src/new-session-drafts.ts +127 -6
- package/src/provision-roles.ts +184 -2
- package/src/runtime-posture-cli.ts +57 -0
- package/src/runtime-posture.ts +760 -0
- package/src/schema.ts +156 -3
- package/src/session-control.ts +1 -0
- package/dist/chunk-BMFDXFPA.js +0 -155
- package/dist/chunk-BMFDXFPA.js.map +0 -1
- package/dist/chunk-VUKRIBO5.js.map +0 -1
package/src/index.ts
CHANGED
|
@@ -23,6 +23,7 @@ import type {
|
|
|
23
23
|
KnowledgeMemoryKind,
|
|
24
24
|
KnowledgeMemoryStatus,
|
|
25
25
|
KnowledgeSourceRef,
|
|
26
|
+
GitHubInstallationAuthorityKind,
|
|
26
27
|
GitHubRepositoryScope,
|
|
27
28
|
HostEventExport,
|
|
28
29
|
HostEventExportBatch,
|
|
@@ -159,7 +160,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
|
|
|
159
160
|
import postgres from "postgres";
|
|
160
161
|
import { decryptEnvironmentValue } from "./environment-crypto";
|
|
161
162
|
import { sanitizeEventPayload, sanitizeModelPayload } from "./event-payload-sanitizer";
|
|
162
|
-
import {
|
|
163
|
+
import { seedNewSessionDraftInTransaction } from "./new-session-drafts";
|
|
163
164
|
import {
|
|
164
165
|
runIdempotentPersistenceTransaction,
|
|
165
166
|
type IdempotentPersistenceTransactionOptions,
|
|
@@ -225,6 +226,7 @@ export {
|
|
|
225
226
|
sanitizeModelPayload,
|
|
226
227
|
} from "./event-payload-sanitizer";
|
|
227
228
|
export * from "./persistence-errors";
|
|
229
|
+
export * from "./runtime-posture";
|
|
228
230
|
export { sanitizeMemoryText } from "./memory-domain";
|
|
229
231
|
// Re-exported so external consumers can `import { migrate } from "@opengeni/db"`.
|
|
230
232
|
// The `@opengeni/db/migrate` subpath stays available too (internal callers + the
|
|
@@ -260,6 +262,16 @@ export * from "./memory-domain";
|
|
|
260
262
|
// unaffected.
|
|
261
263
|
export type Database = PgDatabase<any, typeof schema>;
|
|
262
264
|
|
|
265
|
+
/** Raised when a durable session tool-policy write lost its version fence. */
|
|
266
|
+
export class SessionToolPolicyVersionConflictError extends Error {
|
|
267
|
+
readonly code = "SESSION_TOOL_POLICY_CONFLICT";
|
|
268
|
+
|
|
269
|
+
constructor(readonly currentVersion: number) {
|
|
270
|
+
super("The session tool policy changed in another client");
|
|
271
|
+
this.name = "SessionToolPolicyVersionConflictError";
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
263
275
|
export type DbClient = {
|
|
264
276
|
db: Database;
|
|
265
277
|
close: () => Promise<void>;
|
|
@@ -999,15 +1011,27 @@ export async function withRlsContext<T>(
|
|
|
999
1011
|
// manufacturing a phantom "no active subscription" from a credential that is
|
|
1000
1012
|
// in fact active. Convert that silent false into a loud, root-cause-bearing
|
|
1001
1013
|
// error so the caller can retry rather than permanently mis-decide.
|
|
1002
|
-
const applied = await tx.execute<{
|
|
1003
|
-
|
|
1014
|
+
const applied = await tx.execute<{
|
|
1015
|
+
account_id: string | null;
|
|
1016
|
+
workspace_id: string | null;
|
|
1017
|
+
}>(
|
|
1018
|
+
sql`select
|
|
1019
|
+
current_setting('opengeni.account_id', true) as account_id,
|
|
1020
|
+
current_setting('opengeni.workspace_id', true) as workspace_id`,
|
|
1004
1021
|
);
|
|
1005
1022
|
const appliedAccountId = applied[0]?.account_id ?? "";
|
|
1023
|
+
const expectedWorkspaceId = context.workspaceId ?? "";
|
|
1024
|
+
const appliedWorkspaceId = applied[0]?.workspace_id ?? "";
|
|
1006
1025
|
if (appliedAccountId !== context.accountId) {
|
|
1007
1026
|
throw new Error(
|
|
1008
1027
|
`RLS context not applied on the active backend: expected account ${context.accountId}, got "${appliedAccountId}"`,
|
|
1009
1028
|
);
|
|
1010
1029
|
}
|
|
1030
|
+
if (appliedWorkspaceId !== expectedWorkspaceId) {
|
|
1031
|
+
throw new Error(
|
|
1032
|
+
`RLS context not applied on the active backend: expected workspace "${expectedWorkspaceId}", got "${appliedWorkspaceId}"`,
|
|
1033
|
+
);
|
|
1034
|
+
}
|
|
1011
1035
|
return await fn(scoped);
|
|
1012
1036
|
}, transactionConfig);
|
|
1013
1037
|
}
|
|
@@ -2044,10 +2068,17 @@ export type GitHubInstallation = {
|
|
|
2044
2068
|
accountId: string;
|
|
2045
2069
|
workspaceId: string;
|
|
2046
2070
|
installationId: number;
|
|
2071
|
+
githubAccountId: number | null;
|
|
2047
2072
|
accountLogin: string | null;
|
|
2048
2073
|
accountType: string | null;
|
|
2049
2074
|
repositoryScope: GitHubRepositoryScope;
|
|
2050
2075
|
linkedBySubjectId: string | null;
|
|
2076
|
+
githubActorId: number | null;
|
|
2077
|
+
githubActorLogin: string | null;
|
|
2078
|
+
authorityKind: GitHubInstallationAuthorityKind | null;
|
|
2079
|
+
authorityCheckedAt: string | null;
|
|
2080
|
+
authorityExpiresAt: string | null;
|
|
2081
|
+
authorityNonce: string | null;
|
|
2051
2082
|
createdAt: string;
|
|
2052
2083
|
updatedAt: string;
|
|
2053
2084
|
};
|
|
@@ -2056,6 +2087,59 @@ export type GitHubInstallationAccess = GitHubInstallation & {
|
|
|
2056
2087
|
repositoryIds: number[];
|
|
2057
2088
|
};
|
|
2058
2089
|
|
|
2090
|
+
export class GitHubInstallationAuthorityCommitError extends Error {
|
|
2091
|
+
constructor() {
|
|
2092
|
+
super("GitHub installation authority expired before the binding transaction completed");
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
const githubInstallationAuthorityMaxAgeMs = 10 * 60_000;
|
|
2097
|
+
|
|
2098
|
+
/**
|
|
2099
|
+
* True only for a binding created by the owner-authority transaction. Legacy
|
|
2100
|
+
* installation rows remain visible for audit/unlink, but can never make a
|
|
2101
|
+
* workspace healthy, enumerate repositories, or authorize a token mint.
|
|
2102
|
+
*
|
|
2103
|
+
* `authorityExpiresAt` is the expiry of the consumed proof, not of the durable
|
|
2104
|
+
* delegation. A binding remains delegated after that instant; GitHub's live
|
|
2105
|
+
* installation/repository checks govern its ongoing usability.
|
|
2106
|
+
*/
|
|
2107
|
+
export function hasAuditableGitHubInstallationAuthority(
|
|
2108
|
+
installation: GitHubInstallationAccess,
|
|
2109
|
+
): boolean {
|
|
2110
|
+
const checkedAt = installation.authorityCheckedAt
|
|
2111
|
+
? Date.parse(installation.authorityCheckedAt)
|
|
2112
|
+
: Number.NaN;
|
|
2113
|
+
const expiresAt = installation.authorityExpiresAt
|
|
2114
|
+
? Date.parse(installation.authorityExpiresAt)
|
|
2115
|
+
: Number.NaN;
|
|
2116
|
+
const common =
|
|
2117
|
+
installation.repositoryScope === "selected" &&
|
|
2118
|
+
installation.repositoryIds.length > 0 &&
|
|
2119
|
+
installation.repositoryIds.every((id) => Number.isSafeInteger(id) && id > 0) &&
|
|
2120
|
+
installation.githubAccountId !== null &&
|
|
2121
|
+
Number.isSafeInteger(installation.githubAccountId) &&
|
|
2122
|
+
installation.githubAccountId > 0 &&
|
|
2123
|
+
Boolean(installation.accountLogin) &&
|
|
2124
|
+
installation.githubActorId !== null &&
|
|
2125
|
+
Number.isSafeInteger(installation.githubActorId) &&
|
|
2126
|
+
installation.githubActorId > 0 &&
|
|
2127
|
+
Boolean(installation.githubActorLogin) &&
|
|
2128
|
+
Boolean(installation.linkedBySubjectId) &&
|
|
2129
|
+
Boolean(installation.authorityNonce) &&
|
|
2130
|
+
Number.isFinite(checkedAt) &&
|
|
2131
|
+
Number.isFinite(expiresAt) &&
|
|
2132
|
+
checkedAt < expiresAt;
|
|
2133
|
+
if (!common) {
|
|
2134
|
+
return false;
|
|
2135
|
+
}
|
|
2136
|
+
return installation.authorityKind === "personal_owner"
|
|
2137
|
+
? installation.accountType === "User" &&
|
|
2138
|
+
installation.githubActorId === installation.githubAccountId
|
|
2139
|
+
: installation.authorityKind === "organization_owner" &&
|
|
2140
|
+
installation.accountType === "Organization";
|
|
2141
|
+
}
|
|
2142
|
+
|
|
2059
2143
|
export async function upsertGitHubInstallation(
|
|
2060
2144
|
db: Database,
|
|
2061
2145
|
input: {
|
|
@@ -2095,6 +2179,9 @@ export async function upsertGitHubInstallation(
|
|
|
2095
2179
|
: {}),
|
|
2096
2180
|
updatedAt: new Date(),
|
|
2097
2181
|
},
|
|
2182
|
+
// This legacy metadata helper cannot mutate an owner-authorized row.
|
|
2183
|
+
// New bindings use bindAuthorizedGitHubInstallationRepositories.
|
|
2184
|
+
setWhere: isNull(schema.githubInstallations.authorityNonce),
|
|
2098
2185
|
})
|
|
2099
2186
|
.returning();
|
|
2100
2187
|
if (!row) {
|
|
@@ -2156,6 +2243,164 @@ export async function bindGitHubInstallationRepositories(
|
|
|
2156
2243
|
linkedBySubjectId: input.linkedBySubjectId,
|
|
2157
2244
|
updatedAt: new Date(),
|
|
2158
2245
|
},
|
|
2246
|
+
// Preserve the immutable authority + allowlist audit boundary.
|
|
2247
|
+
setWhere: isNull(schema.githubInstallations.authorityNonce),
|
|
2248
|
+
})
|
|
2249
|
+
.returning();
|
|
2250
|
+
if (!row) {
|
|
2251
|
+
throw new Error("Failed to bind GitHub installation");
|
|
2252
|
+
}
|
|
2253
|
+
await tx
|
|
2254
|
+
.delete(schema.githubInstallationRepositories)
|
|
2255
|
+
.where(
|
|
2256
|
+
and(
|
|
2257
|
+
eq(schema.githubInstallationRepositories.workspaceId, input.workspaceId),
|
|
2258
|
+
eq(schema.githubInstallationRepositories.installationId, input.installationId),
|
|
2259
|
+
),
|
|
2260
|
+
);
|
|
2261
|
+
for (let offset = 0; offset < repositoryIds.length; offset += 1_000) {
|
|
2262
|
+
await tx.insert(schema.githubInstallationRepositories).values(
|
|
2263
|
+
repositoryIds.slice(offset, offset + 1_000).map((repositoryId) => ({
|
|
2264
|
+
accountId: input.accountId,
|
|
2265
|
+
workspaceId: input.workspaceId,
|
|
2266
|
+
installationId: input.installationId,
|
|
2267
|
+
repositoryId,
|
|
2268
|
+
})),
|
|
2269
|
+
);
|
|
2270
|
+
}
|
|
2271
|
+
return { ...mapGitHubInstallation(row), repositoryIds };
|
|
2272
|
+
}),
|
|
2273
|
+
);
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
export async function bindAuthorizedGitHubInstallationRepositories(
|
|
2277
|
+
db: Database,
|
|
2278
|
+
input: {
|
|
2279
|
+
accountId: string;
|
|
2280
|
+
workspaceId: string;
|
|
2281
|
+
installationId: number;
|
|
2282
|
+
githubAccountId: number;
|
|
2283
|
+
accountLogin: string | null;
|
|
2284
|
+
accountType: string | null;
|
|
2285
|
+
linkedBySubjectId: string;
|
|
2286
|
+
githubActorId: number;
|
|
2287
|
+
githubActorLogin: string;
|
|
2288
|
+
authorityKind: GitHubInstallationAuthorityKind;
|
|
2289
|
+
authorityCheckedAt: Date;
|
|
2290
|
+
authorityExpiresAt: Date;
|
|
2291
|
+
authorityNonce: string;
|
|
2292
|
+
repositoryIds: number[];
|
|
2293
|
+
},
|
|
2294
|
+
): Promise<GitHubInstallationAccess | null> {
|
|
2295
|
+
if (!Number.isSafeInteger(input.installationId) || input.installationId <= 0) {
|
|
2296
|
+
throw new Error("GitHub installation id must be a positive safe integer");
|
|
2297
|
+
}
|
|
2298
|
+
if (!Number.isSafeInteger(input.githubAccountId) || input.githubAccountId <= 0) {
|
|
2299
|
+
throw new Error("GitHub account id must be a positive safe integer");
|
|
2300
|
+
}
|
|
2301
|
+
if (!Number.isSafeInteger(input.githubActorId) || input.githubActorId <= 0) {
|
|
2302
|
+
throw new Error("GitHub actor id must be a positive safe integer");
|
|
2303
|
+
}
|
|
2304
|
+
const authorityCheckedAtMs = input.authorityCheckedAt.getTime();
|
|
2305
|
+
const authorityExpiresAtMs = input.authorityExpiresAt.getTime();
|
|
2306
|
+
if (
|
|
2307
|
+
!Number.isFinite(authorityCheckedAtMs) ||
|
|
2308
|
+
!Number.isFinite(authorityExpiresAtMs) ||
|
|
2309
|
+
authorityCheckedAtMs >= authorityExpiresAtMs ||
|
|
2310
|
+
authorityExpiresAtMs - authorityCheckedAtMs > githubInstallationAuthorityMaxAgeMs ||
|
|
2311
|
+
!input.authorityNonce ||
|
|
2312
|
+
!input.githubActorLogin.trim() ||
|
|
2313
|
+
!input.accountLogin?.trim() ||
|
|
2314
|
+
!input.linkedBySubjectId.trim() ||
|
|
2315
|
+
(input.authorityKind !== "personal_owner" && input.authorityKind !== "organization_owner") ||
|
|
2316
|
+
(input.authorityKind === "personal_owner" &&
|
|
2317
|
+
(input.accountType !== "User" || input.githubActorId !== input.githubAccountId)) ||
|
|
2318
|
+
(input.authorityKind === "organization_owner" && input.accountType !== "Organization")
|
|
2319
|
+
) {
|
|
2320
|
+
throw new Error("GitHub installation authority proof is invalid or expired");
|
|
2321
|
+
}
|
|
2322
|
+
const repositoryIds = [...new Set(input.repositoryIds)];
|
|
2323
|
+
if (
|
|
2324
|
+
repositoryIds.length === 0 ||
|
|
2325
|
+
repositoryIds.length !== input.repositoryIds.length ||
|
|
2326
|
+
repositoryIds.some((id) => !Number.isSafeInteger(id) || id <= 0)
|
|
2327
|
+
) {
|
|
2328
|
+
throw new Error(
|
|
2329
|
+
"GitHub repository ids must be a nonempty, unique list of positive safe integers",
|
|
2330
|
+
);
|
|
2331
|
+
}
|
|
2332
|
+
return await withRlsContext(
|
|
2333
|
+
db,
|
|
2334
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
2335
|
+
async (scopedDb) =>
|
|
2336
|
+
await scopedDb.transaction(async (tx) => {
|
|
2337
|
+
await assertGitHubAuthorityWindowOpen(
|
|
2338
|
+
tx,
|
|
2339
|
+
input.authorityCheckedAt,
|
|
2340
|
+
input.authorityExpiresAt,
|
|
2341
|
+
);
|
|
2342
|
+
await tx
|
|
2343
|
+
.delete(schema.integrationOauthStateNonces)
|
|
2344
|
+
.where(
|
|
2345
|
+
and(
|
|
2346
|
+
eq(schema.integrationOauthStateNonces.workspaceId, input.workspaceId),
|
|
2347
|
+
lt(schema.integrationOauthStateNonces.expiresAt, input.authorityCheckedAt),
|
|
2348
|
+
),
|
|
2349
|
+
);
|
|
2350
|
+
const consumed = await tx
|
|
2351
|
+
.insert(schema.integrationOauthStateNonces)
|
|
2352
|
+
.values({
|
|
2353
|
+
accountId: input.accountId,
|
|
2354
|
+
workspaceId: input.workspaceId,
|
|
2355
|
+
subjectId: input.linkedBySubjectId,
|
|
2356
|
+
nonce: input.authorityNonce,
|
|
2357
|
+
expiresAt: input.authorityExpiresAt,
|
|
2358
|
+
usedAt: input.authorityCheckedAt,
|
|
2359
|
+
})
|
|
2360
|
+
.onConflictDoNothing({ target: schema.integrationOauthStateNonces.nonce })
|
|
2361
|
+
.returning({ nonce: schema.integrationOauthStateNonces.nonce });
|
|
2362
|
+
if (consumed.length === 0) {
|
|
2363
|
+
return null;
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
const [row] = await tx
|
|
2367
|
+
.insert(schema.githubInstallations)
|
|
2368
|
+
.values({
|
|
2369
|
+
accountId: input.accountId,
|
|
2370
|
+
workspaceId: input.workspaceId,
|
|
2371
|
+
installationId: input.installationId,
|
|
2372
|
+
githubAccountId: input.githubAccountId,
|
|
2373
|
+
accountLogin: input.accountLogin,
|
|
2374
|
+
accountType: input.accountType,
|
|
2375
|
+
repositoryScope: "selected",
|
|
2376
|
+
linkedBySubjectId: input.linkedBySubjectId,
|
|
2377
|
+
githubActorId: input.githubActorId,
|
|
2378
|
+
githubActorLogin: input.githubActorLogin,
|
|
2379
|
+
authorityKind: input.authorityKind,
|
|
2380
|
+
authorityCheckedAt: input.authorityCheckedAt,
|
|
2381
|
+
authorityExpiresAt: input.authorityExpiresAt,
|
|
2382
|
+
authorityNonce: input.authorityNonce,
|
|
2383
|
+
})
|
|
2384
|
+
.onConflictDoUpdate({
|
|
2385
|
+
target: [
|
|
2386
|
+
schema.githubInstallations.workspaceId,
|
|
2387
|
+
schema.githubInstallations.installationId,
|
|
2388
|
+
],
|
|
2389
|
+
set: {
|
|
2390
|
+
accountId: input.accountId,
|
|
2391
|
+
githubAccountId: input.githubAccountId,
|
|
2392
|
+
accountLogin: input.accountLogin,
|
|
2393
|
+
accountType: input.accountType,
|
|
2394
|
+
repositoryScope: "selected",
|
|
2395
|
+
linkedBySubjectId: input.linkedBySubjectId,
|
|
2396
|
+
githubActorId: input.githubActorId,
|
|
2397
|
+
githubActorLogin: input.githubActorLogin,
|
|
2398
|
+
authorityKind: input.authorityKind,
|
|
2399
|
+
authorityCheckedAt: input.authorityCheckedAt,
|
|
2400
|
+
authorityExpiresAt: input.authorityExpiresAt,
|
|
2401
|
+
authorityNonce: input.authorityNonce,
|
|
2402
|
+
updatedAt: input.authorityCheckedAt,
|
|
2403
|
+
},
|
|
2159
2404
|
})
|
|
2160
2405
|
.returning();
|
|
2161
2406
|
if (!row) {
|
|
@@ -2179,11 +2424,41 @@ export async function bindGitHubInstallationRepositories(
|
|
|
2179
2424
|
})),
|
|
2180
2425
|
);
|
|
2181
2426
|
}
|
|
2427
|
+
// Recheck against the database clock after all writes. Throwing here
|
|
2428
|
+
// rolls the transaction back if the proof expired while it was being
|
|
2429
|
+
// committed, so no partial nonce or binding can survive.
|
|
2430
|
+
await assertGitHubAuthorityWindowOpen(
|
|
2431
|
+
tx,
|
|
2432
|
+
input.authorityCheckedAt,
|
|
2433
|
+
input.authorityExpiresAt,
|
|
2434
|
+
);
|
|
2182
2435
|
return { ...mapGitHubInstallation(row), repositoryIds };
|
|
2183
2436
|
}),
|
|
2184
2437
|
);
|
|
2185
2438
|
}
|
|
2186
2439
|
|
|
2440
|
+
async function assertGitHubAuthorityWindowOpen(
|
|
2441
|
+
tx: Database,
|
|
2442
|
+
checkedAt: Date,
|
|
2443
|
+
expiresAt: Date,
|
|
2444
|
+
): Promise<void> {
|
|
2445
|
+
// postgres.js does not serialize JavaScript Date instances passed through
|
|
2446
|
+
// raw Drizzle SQL. Bind the already-validated ISO timestamp representation
|
|
2447
|
+
// and retain PostgreSQL's timestamptz/database-clock authority checks.
|
|
2448
|
+
const checkedAtIso = checkedAt.toISOString();
|
|
2449
|
+
const expiresAtIso = expiresAt.toISOString();
|
|
2450
|
+
const result = await tx.execute<{ valid: boolean }>(sql`
|
|
2451
|
+
select (
|
|
2452
|
+
${checkedAtIso}::timestamptz <= clock_timestamp()
|
|
2453
|
+
and clock_timestamp() < ${expiresAtIso}::timestamptz
|
|
2454
|
+
and ${expiresAtIso}::timestamptz <= ${checkedAtIso}::timestamptz + interval '10 minutes'
|
|
2455
|
+
) as valid
|
|
2456
|
+
`);
|
|
2457
|
+
if (result[0]?.valid !== true) {
|
|
2458
|
+
throw new GitHubInstallationAuthorityCommitError();
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
|
|
2187
2462
|
export async function listGitHubInstallationsForWorkspace(
|
|
2188
2463
|
db: Database,
|
|
2189
2464
|
workspaceId: string,
|
|
@@ -2256,7 +2531,7 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
|
|
|
2256
2531
|
}
|
|
2257
2532
|
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
2258
2533
|
const [installation] = await scopedDb
|
|
2259
|
-
.select(
|
|
2534
|
+
.select()
|
|
2260
2535
|
.from(schema.githubInstallations)
|
|
2261
2536
|
.where(
|
|
2262
2537
|
and(
|
|
@@ -2268,9 +2543,6 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
|
|
|
2268
2543
|
if (!installation) {
|
|
2269
2544
|
return false;
|
|
2270
2545
|
}
|
|
2271
|
-
if (installation.repositoryScope === "all") {
|
|
2272
|
-
return true;
|
|
2273
|
-
}
|
|
2274
2546
|
const allowed = await scopedDb
|
|
2275
2547
|
.select({
|
|
2276
2548
|
repositoryId: schema.githubInstallationRepositories.repositoryId,
|
|
@@ -2283,7 +2555,13 @@ export async function areGitHubRepositoriesAllowedForWorkspace(
|
|
|
2283
2555
|
inArray(schema.githubInstallationRepositories.repositoryId, requestedIds),
|
|
2284
2556
|
),
|
|
2285
2557
|
);
|
|
2286
|
-
|
|
2558
|
+
if (allowed.length !== requestedIds.length) {
|
|
2559
|
+
return false;
|
|
2560
|
+
}
|
|
2561
|
+
return hasAuditableGitHubInstallationAuthority({
|
|
2562
|
+
...mapGitHubInstallation(installation),
|
|
2563
|
+
repositoryIds: allowed.map((row) => row.repositoryId),
|
|
2564
|
+
});
|
|
2287
2565
|
});
|
|
2288
2566
|
}
|
|
2289
2567
|
|
|
@@ -9516,12 +9794,13 @@ export type CodexCapacityWait = {
|
|
|
9516
9794
|
accountId: string;
|
|
9517
9795
|
workspaceId: string;
|
|
9518
9796
|
sessionId: string;
|
|
9519
|
-
goalId: string;
|
|
9797
|
+
goalId: string | null;
|
|
9520
9798
|
blockedTurnId: string;
|
|
9799
|
+
blockedTurnGeneration: number;
|
|
9521
9800
|
workflowId: string;
|
|
9522
9801
|
generation: number;
|
|
9523
9802
|
status: CodexCapacityWaitStatus;
|
|
9524
|
-
goalVersion: number;
|
|
9803
|
+
goalVersion: number | null;
|
|
9525
9804
|
policyHash: string | null;
|
|
9526
9805
|
earliestResetAt: Date | null;
|
|
9527
9806
|
nextCheckAt: Date;
|
|
@@ -9588,9 +9867,9 @@ export type ReconcileCodexCapacityWaitResult =
|
|
|
9588
9867
|
| {
|
|
9589
9868
|
action: "resumed";
|
|
9590
9869
|
waiter: CodexCapacityWait;
|
|
9591
|
-
update: SessionSystemUpdate;
|
|
9592
9870
|
events: SessionEvent[];
|
|
9593
9871
|
}
|
|
9872
|
+
| { action: "paused"; waiter: CodexCapacityWait; events: SessionEvent[] }
|
|
9594
9873
|
| { action: "superseded"; waiter: CodexCapacityWait; events: SessionEvent[] }
|
|
9595
9874
|
| {
|
|
9596
9875
|
action: "stale";
|
|
@@ -9621,6 +9900,7 @@ function mapCodexCapacityWaiter(
|
|
|
9621
9900
|
sessionId: row.sessionId,
|
|
9622
9901
|
goalId: row.goalId,
|
|
9623
9902
|
blockedTurnId: row.blockedTurnId,
|
|
9903
|
+
blockedTurnGeneration: row.blockedTurnGeneration,
|
|
9624
9904
|
workflowId: row.workflowId,
|
|
9625
9905
|
generation: row.generation,
|
|
9626
9906
|
status: row.status as CodexCapacityWaitStatus,
|
|
@@ -9698,14 +9978,13 @@ function nextCodexCapacityCheckAt(
|
|
|
9698
9978
|
}
|
|
9699
9979
|
|
|
9700
9980
|
/**
|
|
9701
|
-
* Atomically
|
|
9702
|
-
* Lock order is allocator rotation row -> workspace
|
|
9703
|
-
* -> session -> exact turn -> exact attempt ->
|
|
9704
|
-
* reactive failure owns one) -> waiter.
|
|
9705
|
-
*
|
|
9706
|
-
*
|
|
9707
|
-
*
|
|
9708
|
-
* leave only half of the boundary visible.
|
|
9981
|
+
* Atomically close one all-unavailable attempt and arm exactly one durable wait
|
|
9982
|
+
* for the same logical turn. Lock order is allocator rotation row -> workspace
|
|
9983
|
+
* control -> actual workspace -> session -> exact turn -> exact attempt ->
|
|
9984
|
+
* optional goal -> live lease (when a reactive failure owns one) -> waiter.
|
|
9985
|
+
* The waiting turn/session pointer, durable events, exact lease release, and
|
|
9986
|
+
* waiter generation commit together; a crash cannot leave only half of the
|
|
9987
|
+
* boundary visible.
|
|
9709
9988
|
*/
|
|
9710
9989
|
export async function armCodexCapacityWait(
|
|
9711
9990
|
db: Database,
|
|
@@ -9716,8 +9995,8 @@ export async function armCodexCapacityWait(
|
|
|
9716
9995
|
turnId: string;
|
|
9717
9996
|
attemptId: string;
|
|
9718
9997
|
workflowId: string;
|
|
9719
|
-
goalId
|
|
9720
|
-
goalVersion
|
|
9998
|
+
goalId?: string | null;
|
|
9999
|
+
goalVersion?: number | null;
|
|
9721
10000
|
policyHash?: string | null;
|
|
9722
10001
|
earliestResetAt: Date | null;
|
|
9723
10002
|
resetKind: CodexCapacityResetKind;
|
|
@@ -9730,6 +10009,14 @@ export async function armCodexCapacityWait(
|
|
|
9730
10009
|
},
|
|
9731
10010
|
): Promise<ArmCodexCapacityWaitResult> {
|
|
9732
10011
|
const now = input.now ?? new Date();
|
|
10012
|
+
const goalId = input.goalId ?? null;
|
|
10013
|
+
const goalVersion = input.goalVersion ?? null;
|
|
10014
|
+
if (
|
|
10015
|
+
(goalId === null) !== (goalVersion === null) ||
|
|
10016
|
+
(goalVersion !== null && (!Number.isSafeInteger(goalVersion) || goalVersion < 1))
|
|
10017
|
+
) {
|
|
10018
|
+
throw new Error("Codex capacity goal fence must be absent or contain a positive version");
|
|
10019
|
+
}
|
|
9733
10020
|
return await withRlsContext(
|
|
9734
10021
|
db,
|
|
9735
10022
|
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
@@ -9755,18 +10042,20 @@ export async function armCodexCapacityWait(
|
|
|
9755
10042
|
workspaceControl: locks.control ?? undefined,
|
|
9756
10043
|
})
|
|
9757
10044
|
: null;
|
|
9758
|
-
const [goal] =
|
|
9759
|
-
|
|
9760
|
-
|
|
9761
|
-
|
|
9762
|
-
|
|
9763
|
-
|
|
9764
|
-
|
|
9765
|
-
|
|
9766
|
-
|
|
9767
|
-
|
|
9768
|
-
|
|
9769
|
-
|
|
10045
|
+
const [goal] = goalId
|
|
10046
|
+
? await tx
|
|
10047
|
+
.select()
|
|
10048
|
+
.from(schema.sessionGoals)
|
|
10049
|
+
.where(
|
|
10050
|
+
and(
|
|
10051
|
+
eq(schema.sessionGoals.workspaceId, input.workspaceId),
|
|
10052
|
+
eq(schema.sessionGoals.id, goalId),
|
|
10053
|
+
eq(schema.sessionGoals.sessionId, input.sessionId),
|
|
10054
|
+
),
|
|
10055
|
+
)
|
|
10056
|
+
.for("update")
|
|
10057
|
+
.limit(1)
|
|
10058
|
+
: [];
|
|
9770
10059
|
const leaseRows = input.leaseFence
|
|
9771
10060
|
? await tx.execute(sql<{ holder_id: string; generation: number }>`
|
|
9772
10061
|
select holder_id, generation
|
|
@@ -9807,7 +10096,10 @@ export async function armCodexCapacityWait(
|
|
|
9807
10096
|
if (
|
|
9808
10097
|
existing?.status === "waiting" &&
|
|
9809
10098
|
existing.blockedTurnId === input.turnId &&
|
|
9810
|
-
turn?.
|
|
10099
|
+
existing.blockedTurnGeneration === turn?.executionGeneration &&
|
|
10100
|
+
turn?.status === "waiting_capacity" &&
|
|
10101
|
+
session?.status === "waiting_capacity" &&
|
|
10102
|
+
session.activeTurnId === input.turnId
|
|
9811
10103
|
) {
|
|
9812
10104
|
return {
|
|
9813
10105
|
action: "waiting",
|
|
@@ -9825,13 +10117,14 @@ export async function armCodexCapacityWait(
|
|
|
9825
10117
|
Number(lease.generation) === input.leaseFence.generation &&
|
|
9826
10118
|
currentRedispatches === (input.expectedRedispatches ?? currentRedispatches));
|
|
9827
10119
|
if (
|
|
9828
|
-
!
|
|
10120
|
+
!session ||
|
|
10121
|
+
!turn ||
|
|
9829
10122
|
effectiveControl?.state !== "active" ||
|
|
9830
10123
|
effectiveControl.settlement !== null ||
|
|
9831
10124
|
session.activeTurnId !== input.turnId ||
|
|
9832
10125
|
session.status !== "running" ||
|
|
9833
|
-
|
|
9834
|
-
|
|
10126
|
+
(goalId !== null &&
|
|
10127
|
+
(!goal || goal.status !== "active" || goal.version !== goalVersion)) ||
|
|
9835
10128
|
turn.status !== "running" ||
|
|
9836
10129
|
turn.activeAttemptId !== input.attemptId ||
|
|
9837
10130
|
!leaseFenceValid ||
|
|
@@ -9851,7 +10144,7 @@ export async function armCodexCapacityWait(
|
|
|
9851
10144
|
sessionId: input.sessionId,
|
|
9852
10145
|
turnId: input.turnId,
|
|
9853
10146
|
executionGeneration: turn.executionGeneration,
|
|
9854
|
-
outcome: "
|
|
10147
|
+
outcome: "waiting_capacity",
|
|
9855
10148
|
closedAt: now,
|
|
9856
10149
|
});
|
|
9857
10150
|
|
|
@@ -9867,12 +10160,13 @@ export async function armCodexCapacityWait(
|
|
|
9867
10160
|
accountId: input.accountId,
|
|
9868
10161
|
workspaceId: input.workspaceId,
|
|
9869
10162
|
sessionId: input.sessionId,
|
|
9870
|
-
goalId
|
|
10163
|
+
goalId,
|
|
9871
10164
|
blockedTurnId: input.turnId,
|
|
10165
|
+
blockedTurnGeneration: turn.executionGeneration,
|
|
9872
10166
|
workflowId: input.workflowId,
|
|
9873
10167
|
generation,
|
|
9874
10168
|
status: "waiting",
|
|
9875
|
-
goalVersion
|
|
10169
|
+
goalVersion,
|
|
9876
10170
|
policyHash,
|
|
9877
10171
|
earliestResetAt: input.earliestResetAt,
|
|
9878
10172
|
nextCheckAt,
|
|
@@ -9917,32 +10211,17 @@ export async function armCodexCapacityWait(
|
|
|
9917
10211
|
workspaceId: input.workspaceId,
|
|
9918
10212
|
sessionId: input.sessionId,
|
|
9919
10213
|
sequence: ++sequence,
|
|
9920
|
-
type: "
|
|
10214
|
+
type: "codex.capacity.waiting",
|
|
9921
10215
|
payload: sanitizeEventPayload({
|
|
9922
10216
|
...input.failurePayload,
|
|
9923
10217
|
recovery: "codex_capacity",
|
|
9924
|
-
retryable:
|
|
10218
|
+
retryable: true,
|
|
9925
10219
|
rotated: true,
|
|
9926
|
-
capacityWaiterId: waiterRow.id,
|
|
9927
|
-
capacityWaitGeneration: waiterRow.generation,
|
|
9928
|
-
}),
|
|
9929
|
-
turnId: input.turnId,
|
|
9930
|
-
turnGeneration: turn.executionGeneration,
|
|
9931
|
-
turnAttemptId: input.attemptId,
|
|
9932
|
-
turnAssociation: "current",
|
|
9933
|
-
occurredAt: now,
|
|
9934
|
-
},
|
|
9935
|
-
{
|
|
9936
|
-
accountId: input.accountId,
|
|
9937
|
-
workspaceId: input.workspaceId,
|
|
9938
|
-
sessionId: input.sessionId,
|
|
9939
|
-
sequence: ++sequence,
|
|
9940
|
-
type: "codex.capacity.waiting",
|
|
9941
|
-
payload: sanitizeEventPayload({
|
|
9942
10220
|
waiterId: waiterRow.id,
|
|
9943
10221
|
generation: waiterRow.generation,
|
|
9944
|
-
goalId
|
|
9945
|
-
goalVersion
|
|
10222
|
+
goalId,
|
|
10223
|
+
goalVersion,
|
|
10224
|
+
blockedTurnGeneration: turn.executionGeneration,
|
|
9946
10225
|
policyHash,
|
|
9947
10226
|
resetKind: input.resetKind,
|
|
9948
10227
|
earliestResetAt: input.earliestResetAt?.toISOString() ?? null,
|
|
@@ -9960,7 +10239,7 @@ export async function armCodexCapacityWait(
|
|
|
9960
10239
|
sessionId: input.sessionId,
|
|
9961
10240
|
sequence: ++sequence,
|
|
9962
10241
|
type: "session.status.changed",
|
|
9963
|
-
payload: { status: "
|
|
10242
|
+
payload: { status: "waiting_capacity", reason: "codex_capacity" },
|
|
9964
10243
|
turnId: input.turnId,
|
|
9965
10244
|
turnGeneration: turn.executionGeneration,
|
|
9966
10245
|
turnAttemptId: input.attemptId,
|
|
@@ -9969,13 +10248,14 @@ export async function armCodexCapacityWait(
|
|
|
9969
10248
|
},
|
|
9970
10249
|
])
|
|
9971
10250
|
.returning();
|
|
9972
|
-
await tx
|
|
10251
|
+
const [waitingTurn] = await tx
|
|
9973
10252
|
.update(schema.sessionTurns)
|
|
9974
10253
|
.set({
|
|
9975
|
-
status: "
|
|
10254
|
+
status: "waiting_capacity",
|
|
9976
10255
|
activeAttemptId: null,
|
|
10256
|
+
metadata: metadataWithoutTurnDispatchAttempt(turn.metadata),
|
|
9977
10257
|
version: turn.version + 1,
|
|
9978
|
-
finishedAt:
|
|
10258
|
+
finishedAt: null,
|
|
9979
10259
|
updatedAt: now,
|
|
9980
10260
|
})
|
|
9981
10261
|
.where(
|
|
@@ -9983,13 +10263,18 @@ export async function armCodexCapacityWait(
|
|
|
9983
10263
|
eq(schema.sessionTurns.workspaceId, input.workspaceId),
|
|
9984
10264
|
eq(schema.sessionTurns.id, input.turnId),
|
|
9985
10265
|
eq(schema.sessionTurns.status, "running"),
|
|
10266
|
+
eq(schema.sessionTurns.activeAttemptId, input.attemptId),
|
|
9986
10267
|
),
|
|
9987
|
-
)
|
|
9988
|
-
|
|
10268
|
+
)
|
|
10269
|
+
.returning({ id: schema.sessionTurns.id });
|
|
10270
|
+
if (!waitingTurn) {
|
|
10271
|
+
throw new Error("Codex capacity blocked turn changed during atomic arm");
|
|
10272
|
+
}
|
|
10273
|
+
const [waitingSession] = await tx
|
|
9989
10274
|
.update(schema.sessions)
|
|
9990
10275
|
.set({
|
|
9991
|
-
status: "
|
|
9992
|
-
activeTurnId:
|
|
10276
|
+
status: "waiting_capacity",
|
|
10277
|
+
activeTurnId: input.turnId,
|
|
9993
10278
|
lastSequence: sequence,
|
|
9994
10279
|
updatedAt: now,
|
|
9995
10280
|
})
|
|
@@ -9997,15 +10282,24 @@ export async function armCodexCapacityWait(
|
|
|
9997
10282
|
and(
|
|
9998
10283
|
eq(schema.sessions.workspaceId, input.workspaceId),
|
|
9999
10284
|
eq(schema.sessions.id, input.sessionId),
|
|
10285
|
+
eq(schema.sessions.status, "running"),
|
|
10000
10286
|
eq(schema.sessions.activeTurnId, input.turnId),
|
|
10001
10287
|
),
|
|
10002
|
-
)
|
|
10003
|
-
|
|
10004
|
-
|
|
10005
|
-
|
|
10006
|
-
|
|
10007
|
-
|
|
10008
|
-
|
|
10288
|
+
)
|
|
10289
|
+
.returning({ id: schema.sessions.id });
|
|
10290
|
+
if (!waitingSession) {
|
|
10291
|
+
throw new Error("Codex capacity session changed during atomic arm");
|
|
10292
|
+
}
|
|
10293
|
+
if (input.leaseFence) {
|
|
10294
|
+
await tx.execute(sql`
|
|
10295
|
+
delete from codex_credential_leases
|
|
10296
|
+
where account_id = ${input.accountId}
|
|
10297
|
+
and workspace_id = ${input.workspaceId}
|
|
10298
|
+
and turn_id = ${input.turnId}
|
|
10299
|
+
and holder_id = ${input.leaseFence.holderId}
|
|
10300
|
+
and generation = ${input.leaseFence.generation}
|
|
10301
|
+
`);
|
|
10302
|
+
}
|
|
10009
10303
|
return {
|
|
10010
10304
|
action: "waiting",
|
|
10011
10305
|
waiter: mapCodexCapacityWaiter(waiterRow),
|
|
@@ -10037,8 +10331,8 @@ export async function getCodexCapacityWaitForSession(
|
|
|
10037
10331
|
}
|
|
10038
10332
|
|
|
10039
10333
|
/**
|
|
10040
|
-
* Same-transaction capacity-mutation/outbox seam for
|
|
10041
|
-
*
|
|
10334
|
+
* Same-transaction capacity-mutation/outbox seam for eligibility and
|
|
10335
|
+
* membership/default changes. The allocator rotation row is always the
|
|
10042
10336
|
* first lock. Mutations report whether capacity truth changed; only then are
|
|
10043
10337
|
* matching waiter wake revisions advanced and returned for best-effort signal.
|
|
10044
10338
|
*/
|
|
@@ -10153,6 +10447,7 @@ async function supersedeCodexCapacityWaitInTransaction(
|
|
|
10153
10447
|
tx: Database,
|
|
10154
10448
|
input: {
|
|
10155
10449
|
session: typeof schema.sessions.$inferSelect;
|
|
10450
|
+
blockedTurn: typeof schema.sessionTurns.$inferSelect;
|
|
10156
10451
|
waiter: typeof schema.codexCapacityWaiters.$inferSelect;
|
|
10157
10452
|
reason: string;
|
|
10158
10453
|
now: Date;
|
|
@@ -10176,9 +10471,52 @@ async function supersedeCodexCapacityWaitInTransaction(
|
|
|
10176
10471
|
if (!updated) {
|
|
10177
10472
|
return { waiter: mapCodexCapacityWaiter(input.waiter), events: [] };
|
|
10178
10473
|
}
|
|
10179
|
-
const
|
|
10180
|
-
|
|
10181
|
-
|
|
10474
|
+
const turnWasCurrent = input.session.activeTurnId === input.blockedTurn.id;
|
|
10475
|
+
const turnStillWaiting = input.blockedTurn.status === "waiting_capacity";
|
|
10476
|
+
const terminalTurnStatus = input.session.status === "cancelled" ? "cancelled" : "superseded";
|
|
10477
|
+
if (turnStillWaiting) {
|
|
10478
|
+
const [supersededTurn] = await tx
|
|
10479
|
+
.update(schema.sessionTurns)
|
|
10480
|
+
.set({
|
|
10481
|
+
status: terminalTurnStatus,
|
|
10482
|
+
activeAttemptId: null,
|
|
10483
|
+
cancelledBy: "codex_capacity_reconcile",
|
|
10484
|
+
cancelReason: input.reason,
|
|
10485
|
+
version: input.blockedTurn.version + 1,
|
|
10486
|
+
finishedAt: input.now,
|
|
10487
|
+
updatedAt: input.now,
|
|
10488
|
+
})
|
|
10489
|
+
.where(
|
|
10490
|
+
and(
|
|
10491
|
+
eq(schema.sessionTurns.workspaceId, input.session.workspaceId),
|
|
10492
|
+
eq(schema.sessionTurns.id, input.blockedTurn.id),
|
|
10493
|
+
eq(schema.sessionTurns.status, "waiting_capacity"),
|
|
10494
|
+
isNull(schema.sessionTurns.activeAttemptId),
|
|
10495
|
+
eq(schema.sessionTurns.executionGeneration, input.waiter.blockedTurnGeneration),
|
|
10496
|
+
),
|
|
10497
|
+
)
|
|
10498
|
+
.returning({ id: schema.sessionTurns.id });
|
|
10499
|
+
if (!supersededTurn) {
|
|
10500
|
+
throw new Error("Codex capacity blocked turn changed during atomic supersession");
|
|
10501
|
+
}
|
|
10502
|
+
}
|
|
10503
|
+
const [queued] = turnWasCurrent
|
|
10504
|
+
? await tx
|
|
10505
|
+
.select({ id: schema.sessionTurns.id })
|
|
10506
|
+
.from(schema.sessionTurns)
|
|
10507
|
+
.where(
|
|
10508
|
+
and(
|
|
10509
|
+
eq(schema.sessionTurns.workspaceId, input.session.workspaceId),
|
|
10510
|
+
eq(schema.sessionTurns.sessionId, input.session.id),
|
|
10511
|
+
eq(schema.sessionTurns.status, "queued"),
|
|
10512
|
+
),
|
|
10513
|
+
)
|
|
10514
|
+
.limit(1)
|
|
10515
|
+
: [];
|
|
10516
|
+
const nextSessionStatus =
|
|
10517
|
+
input.session.status === "cancelled" ? "cancelled" : queued ? "queued" : "idle";
|
|
10518
|
+
const eventValues: Array<typeof schema.sessionEvents.$inferInsert> = [
|
|
10519
|
+
{
|
|
10182
10520
|
accountId: input.session.accountId,
|
|
10183
10521
|
workspaceId: input.session.workspaceId,
|
|
10184
10522
|
sessionId: input.session.id,
|
|
@@ -10190,18 +10528,45 @@ async function supersedeCodexCapacityWaitInTransaction(
|
|
|
10190
10528
|
reason: input.reason,
|
|
10191
10529
|
}),
|
|
10192
10530
|
turnId: updated.blockedTurnId,
|
|
10531
|
+
turnGeneration: input.blockedTurn.executionGeneration,
|
|
10532
|
+
...(turnWasCurrent ? { turnAssociation: "current" } : {}),
|
|
10193
10533
|
occurredAt: input.now,
|
|
10194
|
-
}
|
|
10195
|
-
|
|
10196
|
-
|
|
10534
|
+
},
|
|
10535
|
+
];
|
|
10536
|
+
if (turnWasCurrent && input.session.status !== nextSessionStatus) {
|
|
10537
|
+
eventValues.push({
|
|
10538
|
+
accountId: input.session.accountId,
|
|
10539
|
+
workspaceId: input.session.workspaceId,
|
|
10540
|
+
sessionId: input.session.id,
|
|
10541
|
+
sequence: input.session.lastSequence + 2,
|
|
10542
|
+
type: "session.status.changed",
|
|
10543
|
+
payload: { status: nextSessionStatus, reason: input.reason },
|
|
10544
|
+
turnId: updated.blockedTurnId,
|
|
10545
|
+
turnGeneration: input.blockedTurn.executionGeneration,
|
|
10546
|
+
turnAssociation: "current",
|
|
10547
|
+
occurredAt: input.now,
|
|
10548
|
+
});
|
|
10549
|
+
}
|
|
10550
|
+
const inserted = await tx.insert(schema.sessionEvents).values(eventValues).returning();
|
|
10551
|
+
const lastSequence = input.session.lastSequence + inserted.length;
|
|
10552
|
+
const [updatedSession] = await tx
|
|
10197
10553
|
.update(schema.sessions)
|
|
10198
|
-
.set({
|
|
10554
|
+
.set({
|
|
10555
|
+
...(turnWasCurrent ? { status: nextSessionStatus, activeTurnId: null } : {}),
|
|
10556
|
+
lastSequence,
|
|
10557
|
+
updatedAt: input.now,
|
|
10558
|
+
})
|
|
10199
10559
|
.where(
|
|
10200
10560
|
and(
|
|
10201
10561
|
eq(schema.sessions.workspaceId, input.session.workspaceId),
|
|
10202
10562
|
eq(schema.sessions.id, input.session.id),
|
|
10563
|
+
...(turnWasCurrent ? [eq(schema.sessions.activeTurnId, input.blockedTurn.id)] : []),
|
|
10203
10564
|
),
|
|
10204
|
-
)
|
|
10565
|
+
)
|
|
10566
|
+
.returning({ id: schema.sessions.id });
|
|
10567
|
+
if (!updatedSession) {
|
|
10568
|
+
throw new Error("Codex capacity session changed during atomic supersession");
|
|
10569
|
+
}
|
|
10205
10570
|
return {
|
|
10206
10571
|
waiter: mapCodexCapacityWaiter(updated),
|
|
10207
10572
|
events: inserted.map(mapEvent),
|
|
@@ -10211,10 +10576,11 @@ async function supersedeCodexCapacityWaitInTransaction(
|
|
|
10211
10576
|
/**
|
|
10212
10577
|
* Row-lock and re-evaluate one waiter. Availability is decided by a pure
|
|
10213
10578
|
* caller supplied policy over the same rotation-row transaction as normal
|
|
10214
|
-
* acquisition. If available,
|
|
10215
|
-
*
|
|
10216
|
-
*
|
|
10217
|
-
*
|
|
10579
|
+
* acquisition. If available, the exact blocked turn becomes `recovering`;
|
|
10580
|
+
* duplicate timers/signals observe status=resumed and do no work. Effective
|
|
10581
|
+
* Pause leaves the waiter untouched, ordinary queued prompts remain behind the
|
|
10582
|
+
* current inference, and only an explicit semantic fence change supersedes the
|
|
10583
|
+
* waiter/blocked turn without inference.
|
|
10218
10584
|
*/
|
|
10219
10585
|
export async function reconcileCodexCapacityWait<
|
|
10220
10586
|
TPolicyScope = never,
|
|
@@ -10287,18 +10653,20 @@ export async function reconcileCodexCapacityWait<
|
|
|
10287
10653
|
workspaceControl: prefix.control ?? undefined,
|
|
10288
10654
|
})
|
|
10289
10655
|
: null;
|
|
10290
|
-
const [goal] =
|
|
10291
|
-
|
|
10292
|
-
|
|
10293
|
-
|
|
10294
|
-
|
|
10295
|
-
|
|
10296
|
-
|
|
10297
|
-
|
|
10298
|
-
|
|
10299
|
-
|
|
10300
|
-
|
|
10301
|
-
|
|
10656
|
+
const [goal] = waiterRead.goalId
|
|
10657
|
+
? await tx
|
|
10658
|
+
.select()
|
|
10659
|
+
.from(schema.sessionGoals)
|
|
10660
|
+
.where(
|
|
10661
|
+
and(
|
|
10662
|
+
eq(schema.sessionGoals.workspaceId, input.workspaceId),
|
|
10663
|
+
eq(schema.sessionGoals.id, waiterRead.goalId),
|
|
10664
|
+
eq(schema.sessionGoals.sessionId, input.sessionId),
|
|
10665
|
+
),
|
|
10666
|
+
)
|
|
10667
|
+
.for("update")
|
|
10668
|
+
.limit(1)
|
|
10669
|
+
: [];
|
|
10302
10670
|
const [waiter] = await tx
|
|
10303
10671
|
.select()
|
|
10304
10672
|
.from(schema.codexCapacityWaiters)
|
|
@@ -10307,7 +10675,6 @@ export async function reconcileCodexCapacityWait<
|
|
|
10307
10675
|
.limit(1);
|
|
10308
10676
|
if (
|
|
10309
10677
|
!session ||
|
|
10310
|
-
!goal ||
|
|
10311
10678
|
!blockedTurn ||
|
|
10312
10679
|
!waiter ||
|
|
10313
10680
|
session.accountId !== input.accountId ||
|
|
@@ -10326,55 +10693,40 @@ export async function reconcileCodexCapacityWait<
|
|
|
10326
10693
|
events: [],
|
|
10327
10694
|
} as const;
|
|
10328
10695
|
}
|
|
10696
|
+
if (effectiveControl?.state !== "active" || effectiveControl.settlement !== null) {
|
|
10697
|
+
return {
|
|
10698
|
+
action: "paused",
|
|
10699
|
+
waiter: mapCodexCapacityWaiter(waiter),
|
|
10700
|
+
events: [],
|
|
10701
|
+
} as const;
|
|
10702
|
+
}
|
|
10329
10703
|
|
|
10330
|
-
const [pending] = await tx
|
|
10331
|
-
.select({ id: schema.sessionTurns.id })
|
|
10332
|
-
.from(schema.sessionTurns)
|
|
10333
|
-
.where(
|
|
10334
|
-
and(
|
|
10335
|
-
eq(schema.sessionTurns.workspaceId, input.workspaceId),
|
|
10336
|
-
eq(schema.sessionTurns.sessionId, input.sessionId),
|
|
10337
|
-
inArray(schema.sessionTurns.status, [
|
|
10338
|
-
"queued",
|
|
10339
|
-
"running",
|
|
10340
|
-
"requires_action",
|
|
10341
|
-
"recovering",
|
|
10342
|
-
"waiting_capacity",
|
|
10343
|
-
]),
|
|
10344
|
-
),
|
|
10345
|
-
)
|
|
10346
|
-
.limit(1);
|
|
10347
|
-
const [laterTurn] = await tx
|
|
10348
|
-
.select({ id: schema.sessionTurns.id })
|
|
10349
|
-
.from(schema.sessionTurns)
|
|
10350
|
-
.where(
|
|
10351
|
-
and(
|
|
10352
|
-
eq(schema.sessionTurns.workspaceId, input.workspaceId),
|
|
10353
|
-
eq(schema.sessionTurns.sessionId, input.sessionId),
|
|
10354
|
-
gt(schema.sessionTurns.position, blockedTurn.position),
|
|
10355
|
-
),
|
|
10356
|
-
)
|
|
10357
|
-
.limit(1);
|
|
10358
10704
|
const currentPolicyHash = codexCapacityPolicyHashFromTurnMetadata(blockedTurn.metadata);
|
|
10359
10705
|
let supersedeReason: string | null = null;
|
|
10360
|
-
if (
|
|
10361
|
-
supersedeReason = "
|
|
10362
|
-
} else if (
|
|
10706
|
+
if (session.status === "cancelled") {
|
|
10707
|
+
supersedeReason = "session_cancelled";
|
|
10708
|
+
} else if (
|
|
10709
|
+
waiter.goalId !== null &&
|
|
10710
|
+
(!goal || goal.status !== "active" || goal.version !== waiter.goalVersion)
|
|
10711
|
+
) {
|
|
10363
10712
|
supersedeReason = "goal_changed";
|
|
10364
10713
|
} else if (currentPolicyHash !== waiter.policyHash) {
|
|
10365
10714
|
supersedeReason = "credential_policy_changed";
|
|
10366
|
-
} else if (session.
|
|
10367
|
-
supersedeReason = "
|
|
10368
|
-
} else if (
|
|
10715
|
+
} else if (session.activeTurnId !== blockedTurn.id) {
|
|
10716
|
+
supersedeReason = "active_turn_changed";
|
|
10717
|
+
} else if (session.status !== "waiting_capacity") {
|
|
10718
|
+
supersedeReason = "session_not_waiting_capacity";
|
|
10719
|
+
} else if (
|
|
10720
|
+
blockedTurn.status !== "waiting_capacity" ||
|
|
10721
|
+
blockedTurn.activeAttemptId !== null ||
|
|
10722
|
+
blockedTurn.executionGeneration !== waiter.blockedTurnGeneration
|
|
10723
|
+
) {
|
|
10369
10724
|
supersedeReason = "blocked_turn_changed";
|
|
10370
|
-
} else if (pending) {
|
|
10371
|
-
supersedeReason = "pending_work_exists";
|
|
10372
|
-
} else if (laterTurn) {
|
|
10373
|
-
supersedeReason = "newer_turn_exists";
|
|
10374
10725
|
}
|
|
10375
10726
|
if (supersedeReason) {
|
|
10376
10727
|
const superseded = await supersedeCodexCapacityWaitInTransaction(tx, {
|
|
10377
10728
|
session,
|
|
10729
|
+
blockedTurn,
|
|
10378
10730
|
waiter,
|
|
10379
10731
|
reason: supersedeReason,
|
|
10380
10732
|
now,
|
|
@@ -10446,69 +10798,6 @@ export async function reconcileCodexCapacityWait<
|
|
|
10446
10798
|
} as const;
|
|
10447
10799
|
}
|
|
10448
10800
|
|
|
10449
|
-
const prompt = [
|
|
10450
|
-
"[CODEX CAPACITY RESUME] Codex subscription capacity is available again.",
|
|
10451
|
-
`Continue the existing active goal from durable conversation history: ${goal.text}`,
|
|
10452
|
-
`Success criteria: ${goal.successCriteria ?? "none specified"}.`,
|
|
10453
|
-
"Do not replay completed tool side effects; verify any ambiguous in-flight effect before repeating it.",
|
|
10454
|
-
"If the goal is complete, call opengeni__goal_complete. If blocked for another reason, call opengeni__goal_pause.",
|
|
10455
|
-
].join("\n");
|
|
10456
|
-
const [update] = await tx
|
|
10457
|
-
.insert(schema.sessionSystemUpdates)
|
|
10458
|
-
.values({
|
|
10459
|
-
accountId: input.accountId,
|
|
10460
|
-
workspaceId: input.workspaceId,
|
|
10461
|
-
sessionId: input.sessionId,
|
|
10462
|
-
kind: "goal_continuation",
|
|
10463
|
-
classification: "info",
|
|
10464
|
-
sourceId: goal.id,
|
|
10465
|
-
dedupeKey: `codex-capacity-resume:${waiter.id}:${waiter.generation}`,
|
|
10466
|
-
summary: prompt,
|
|
10467
|
-
payload: {
|
|
10468
|
-
type: "goal_continuation",
|
|
10469
|
-
goalId: goal.id,
|
|
10470
|
-
goalVersion: goal.version,
|
|
10471
|
-
prompt,
|
|
10472
|
-
reason: "codex_capacity",
|
|
10473
|
-
capacityWaiterId: waiter.id,
|
|
10474
|
-
capacityWaitGeneration: waiter.generation,
|
|
10475
|
-
policy: {
|
|
10476
|
-
model: blockedTurn.model,
|
|
10477
|
-
reasoningEffort: blockedTurn.reasoningEffort,
|
|
10478
|
-
tools: blockedTurn.tools,
|
|
10479
|
-
sandboxBackend: blockedTurn.sandboxBackend,
|
|
10480
|
-
},
|
|
10481
|
-
},
|
|
10482
|
-
lineage: {
|
|
10483
|
-
goalId: goal.id,
|
|
10484
|
-
blockedTurnId: blockedTurn.id,
|
|
10485
|
-
capacityWaiterId: waiter.id,
|
|
10486
|
-
},
|
|
10487
|
-
state: "pending",
|
|
10488
|
-
})
|
|
10489
|
-
.returning();
|
|
10490
|
-
if (!update) {
|
|
10491
|
-
throw new Error("Codex capacity resume did not create an internal update");
|
|
10492
|
-
}
|
|
10493
|
-
await tx
|
|
10494
|
-
.insert(schema.usageEvents)
|
|
10495
|
-
.values({
|
|
10496
|
-
accountId: input.accountId,
|
|
10497
|
-
workspaceId: input.workspaceId,
|
|
10498
|
-
eventType: "agent_run.created",
|
|
10499
|
-
quantity: 1,
|
|
10500
|
-
unit: "run",
|
|
10501
|
-
sourceResourceType: "session_system_update",
|
|
10502
|
-
sourceResourceId: update.id,
|
|
10503
|
-
sessionId: input.sessionId,
|
|
10504
|
-
initiatorKind: "service",
|
|
10505
|
-
initiatorSubjectId: "goal-continuation",
|
|
10506
|
-
initiatorContext: { goalId: goal.id, reason: "codex_capacity" },
|
|
10507
|
-
origin: "goal",
|
|
10508
|
-
idempotencyKey: `agent_run.created:codex-capacity:${input.workspaceId}:${update.id}`,
|
|
10509
|
-
occurredAt: now,
|
|
10510
|
-
})
|
|
10511
|
-
.onConflictDoNothing({ target: schema.usageEvents.idempotencyKey });
|
|
10512
10801
|
const events = await tx
|
|
10513
10802
|
.insert(schema.sessionEvents)
|
|
10514
10803
|
.values([
|
|
@@ -10517,14 +10806,20 @@ export async function reconcileCodexCapacityWait<
|
|
|
10517
10806
|
workspaceId: input.workspaceId,
|
|
10518
10807
|
sessionId: input.sessionId,
|
|
10519
10808
|
sequence: session.lastSequence + 1,
|
|
10520
|
-
type: "
|
|
10809
|
+
type: "codex.capacity.resumed",
|
|
10521
10810
|
payload: sanitizeEventPayload({
|
|
10522
|
-
|
|
10523
|
-
|
|
10524
|
-
|
|
10525
|
-
|
|
10526
|
-
|
|
10811
|
+
waiterId: waiter.id,
|
|
10812
|
+
generation: waiter.generation,
|
|
10813
|
+
wakeRevision: waiter.wakeRevision,
|
|
10814
|
+
goalId: waiter.goalId,
|
|
10815
|
+
goalVersion: waiter.goalVersion,
|
|
10816
|
+
blockedTurnGeneration: waiter.blockedTurnGeneration,
|
|
10817
|
+
policyHash: waiter.policyHash,
|
|
10818
|
+
diagnostic: decision.diagnostic ?? null,
|
|
10527
10819
|
}),
|
|
10820
|
+
turnId: blockedTurn.id,
|
|
10821
|
+
turnGeneration: blockedTurn.executionGeneration,
|
|
10822
|
+
turnAssociation: "current",
|
|
10528
10823
|
occurredAt: now,
|
|
10529
10824
|
},
|
|
10530
10825
|
{
|
|
@@ -10532,18 +10827,11 @@ export async function reconcileCodexCapacityWait<
|
|
|
10532
10827
|
workspaceId: input.workspaceId,
|
|
10533
10828
|
sessionId: input.sessionId,
|
|
10534
10829
|
sequence: session.lastSequence + 2,
|
|
10535
|
-
type: "
|
|
10536
|
-
payload:
|
|
10537
|
-
waiterId: waiter.id,
|
|
10538
|
-
generation: waiter.generation,
|
|
10539
|
-
wakeRevision: waiter.wakeRevision,
|
|
10540
|
-
goalId: goal.id,
|
|
10541
|
-
goalVersion: goal.version,
|
|
10542
|
-
policyHash: waiter.policyHash,
|
|
10543
|
-
diagnostic: decision.diagnostic ?? null,
|
|
10544
|
-
updateId: update.id,
|
|
10545
|
-
}),
|
|
10830
|
+
type: "session.status.changed",
|
|
10831
|
+
payload: { status: "recovering", reason: "codex_capacity" },
|
|
10546
10832
|
turnId: blockedTurn.id,
|
|
10833
|
+
turnGeneration: blockedTurn.executionGeneration,
|
|
10834
|
+
turnAssociation: "current",
|
|
10547
10835
|
occurredAt: now,
|
|
10548
10836
|
},
|
|
10549
10837
|
])
|
|
@@ -10552,7 +10840,7 @@ export async function reconcileCodexCapacityWait<
|
|
|
10552
10840
|
.update(schema.codexCapacityWaiters)
|
|
10553
10841
|
.set({
|
|
10554
10842
|
status: "resumed",
|
|
10555
|
-
resumedUpdateId:
|
|
10843
|
+
resumedUpdateId: null,
|
|
10556
10844
|
observedWakeRevision: waiter.wakeRevision,
|
|
10557
10845
|
lastWakeReason: "capacity_available",
|
|
10558
10846
|
updatedAt: now,
|
|
@@ -10568,11 +10856,34 @@ export async function reconcileCodexCapacityWait<
|
|
|
10568
10856
|
if (!updatedWaiter) {
|
|
10569
10857
|
throw new Error("Codex capacity waiter changed during atomic resume");
|
|
10570
10858
|
}
|
|
10571
|
-
await tx
|
|
10859
|
+
const [recoveringTurn] = await tx
|
|
10860
|
+
.update(schema.sessionTurns)
|
|
10861
|
+
.set({
|
|
10862
|
+
status: "recovering",
|
|
10863
|
+
activeAttemptId: null,
|
|
10864
|
+
metadata: metadataWithoutTurnDispatchAttempt(blockedTurn.metadata),
|
|
10865
|
+
version: blockedTurn.version + 1,
|
|
10866
|
+
finishedAt: null,
|
|
10867
|
+
updatedAt: now,
|
|
10868
|
+
})
|
|
10869
|
+
.where(
|
|
10870
|
+
and(
|
|
10871
|
+
eq(schema.sessionTurns.workspaceId, input.workspaceId),
|
|
10872
|
+
eq(schema.sessionTurns.id, blockedTurn.id),
|
|
10873
|
+
eq(schema.sessionTurns.status, "waiting_capacity"),
|
|
10874
|
+
isNull(schema.sessionTurns.activeAttemptId),
|
|
10875
|
+
eq(schema.sessionTurns.executionGeneration, waiter.blockedTurnGeneration),
|
|
10876
|
+
),
|
|
10877
|
+
)
|
|
10878
|
+
.returning({ id: schema.sessionTurns.id });
|
|
10879
|
+
if (!recoveringTurn) {
|
|
10880
|
+
throw new Error("Codex capacity blocked turn changed during atomic resume");
|
|
10881
|
+
}
|
|
10882
|
+
const [recoveringSession] = await tx
|
|
10572
10883
|
.update(schema.sessions)
|
|
10573
10884
|
.set({
|
|
10574
|
-
status: "
|
|
10575
|
-
activeTurnId:
|
|
10885
|
+
status: "recovering",
|
|
10886
|
+
activeTurnId: blockedTurn.id,
|
|
10576
10887
|
lastSequence: session.lastSequence + 2,
|
|
10577
10888
|
updatedAt: now,
|
|
10578
10889
|
})
|
|
@@ -10580,13 +10891,17 @@ export async function reconcileCodexCapacityWait<
|
|
|
10580
10891
|
and(
|
|
10581
10892
|
eq(schema.sessions.workspaceId, input.workspaceId),
|
|
10582
10893
|
eq(schema.sessions.id, input.sessionId),
|
|
10583
|
-
|
|
10894
|
+
eq(schema.sessions.status, "waiting_capacity"),
|
|
10895
|
+
eq(schema.sessions.activeTurnId, blockedTurn.id),
|
|
10584
10896
|
),
|
|
10585
|
-
)
|
|
10897
|
+
)
|
|
10898
|
+
.returning({ id: schema.sessions.id });
|
|
10899
|
+
if (!recoveringSession) {
|
|
10900
|
+
throw new Error("Codex capacity session changed during atomic resume");
|
|
10901
|
+
}
|
|
10586
10902
|
return {
|
|
10587
10903
|
action: "resumed",
|
|
10588
10904
|
waiter: mapCodexCapacityWaiter(updatedWaiter),
|
|
10589
|
-
update: mapSessionSystemUpdate(update),
|
|
10590
10905
|
events: events.map(mapEvent),
|
|
10591
10906
|
} as const;
|
|
10592
10907
|
}),
|
|
@@ -14463,21 +14778,24 @@ export async function listSessionsForSubject(
|
|
|
14463
14778
|
if (ordinaryIds.length > limit) {
|
|
14464
14779
|
let snapshot = reusableSnapshot;
|
|
14465
14780
|
if (!snapshot) {
|
|
14466
|
-
|
|
14467
|
-
|
|
14468
|
-
|
|
14469
|
-
|
|
14470
|
-
|
|
14471
|
-
|
|
14472
|
-
|
|
14473
|
-
|
|
14474
|
-
|
|
14475
|
-
|
|
14476
|
-
|
|
14477
|
-
|
|
14478
|
-
|
|
14479
|
-
|
|
14480
|
-
|
|
14781
|
+
// Creation is serialized by the subject advisory lock above.
|
|
14782
|
+
// Retain only the newest N-1 before inserting so polling and
|
|
14783
|
+
// search identity churn cannot turn this bounded cache into a
|
|
14784
|
+
// user-visible 429. A continuation for an evicted row retains
|
|
14785
|
+
// the existing typed expiry/410 + client rebase contract.
|
|
14786
|
+
await tx.execute(sql`
|
|
14787
|
+
delete from ${schema.sessionListSnapshots} snapshot
|
|
14788
|
+
where snapshot.workspace_id = ${workspaceId}
|
|
14789
|
+
and snapshot.subject_id = ${options.subjectId}
|
|
14790
|
+
and snapshot.id in (
|
|
14791
|
+
select evicted.id
|
|
14792
|
+
from ${schema.sessionListSnapshots} evicted
|
|
14793
|
+
where evicted.workspace_id = ${workspaceId}
|
|
14794
|
+
and evicted.subject_id = ${options.subjectId}
|
|
14795
|
+
order by evicted.created_at desc, evicted.id desc
|
|
14796
|
+
offset ${SESSION_LIST_SNAPSHOT_MAX_ACTIVE_PER_SUBJECT - 1}
|
|
14797
|
+
)
|
|
14798
|
+
`);
|
|
14481
14799
|
const [workspace] = await tx
|
|
14482
14800
|
.select({ accountId: schema.workspaces.accountId })
|
|
14483
14801
|
.from(schema.workspaces)
|
|
@@ -19374,7 +19692,7 @@ export async function acquireLease(
|
|
|
19374
19692
|
if (liveness === "cold") {
|
|
19375
19693
|
const recovery = recoveryStateFromLeaseRow(row);
|
|
19376
19694
|
if (
|
|
19377
|
-
recovery.restore.status === "degraded" ||
|
|
19695
|
+
(recovery.restore.status === "degraded" && recovery.restore.retryable !== true) ||
|
|
19378
19696
|
recovery.restore.status === "unrecoverable"
|
|
19379
19697
|
) {
|
|
19380
19698
|
return {
|
|
@@ -20075,10 +20393,186 @@ export async function recordWarmingSandboxCreated(
|
|
|
20075
20393
|
);
|
|
20076
20394
|
}
|
|
20077
20395
|
|
|
20396
|
+
export type LostProviderWorkspaceSettlement = {
|
|
20397
|
+
processesLost: number;
|
|
20398
|
+
admissionsRejected: number;
|
|
20399
|
+
ptysClosed: number;
|
|
20400
|
+
processHoldersDeleted: number;
|
|
20401
|
+
};
|
|
20402
|
+
|
|
20078
20403
|
export type MarkWarmLeaseInstanceLostResult =
|
|
20079
|
-
| {
|
|
20404
|
+
| {
|
|
20405
|
+
status: "marked";
|
|
20406
|
+
lease: LeaseSnapshot;
|
|
20407
|
+
settlement: LostProviderWorkspaceSettlement;
|
|
20408
|
+
}
|
|
20080
20409
|
| { status: "stale"; lease: LeaseSnapshot | null };
|
|
20081
20410
|
|
|
20411
|
+
const LOST_PROVIDER_PROCESS_REASON = "provider_instance_lost";
|
|
20412
|
+
|
|
20413
|
+
/** Lock the exact provider-owned blocker set before taking the lease row. Normal
|
|
20414
|
+
* retained-process settlement locks process -> admission -> lease, so provider
|
|
20415
|
+
* loss must not hold the lease while waiting for one of those rows. The lease
|
|
20416
|
+
* tuple is revalidated under FOR UPDATE after these locks are acquired. */
|
|
20417
|
+
async function lockExactLostProviderWorkspaceBlockersTx(
|
|
20418
|
+
tx: Database,
|
|
20419
|
+
input: {
|
|
20420
|
+
accountId: string;
|
|
20421
|
+
workspaceId: string;
|
|
20422
|
+
leaseId: string;
|
|
20423
|
+
sandboxGroupId: string;
|
|
20424
|
+
lostEpoch: number;
|
|
20425
|
+
lostInstanceId: string;
|
|
20426
|
+
},
|
|
20427
|
+
): Promise<void> {
|
|
20428
|
+
await tx.execute(sql`
|
|
20429
|
+
select id from sandbox_retained_processes
|
|
20430
|
+
where account_id = ${input.accountId}
|
|
20431
|
+
and workspace_id = ${input.workspaceId}
|
|
20432
|
+
and lease_id = ${input.leaseId}
|
|
20433
|
+
and sandbox_group_id = ${input.sandboxGroupId}
|
|
20434
|
+
and lease_epoch = ${input.lostEpoch}
|
|
20435
|
+
and provider_instance_id = ${input.lostInstanceId}
|
|
20436
|
+
and state = 'active'
|
|
20437
|
+
order by id
|
|
20438
|
+
for update
|
|
20439
|
+
`);
|
|
20440
|
+
await tx.execute(sql`
|
|
20441
|
+
select id from sandbox_workspace_mutation_admissions
|
|
20442
|
+
where account_id = ${input.accountId}
|
|
20443
|
+
and workspace_id = ${input.workspaceId}
|
|
20444
|
+
and lease_id = ${input.leaseId}
|
|
20445
|
+
and sandbox_group_id = ${input.sandboxGroupId}
|
|
20446
|
+
and lease_epoch = ${input.lostEpoch}
|
|
20447
|
+
and provider_instance_id = ${input.lostInstanceId}
|
|
20448
|
+
and settled_at is null
|
|
20449
|
+
order by id
|
|
20450
|
+
for update
|
|
20451
|
+
`);
|
|
20452
|
+
await tx.execute(sql`
|
|
20453
|
+
select id from sandbox_pty_sessions
|
|
20454
|
+
where account_id = ${input.accountId}
|
|
20455
|
+
and workspace_id = ${input.workspaceId}
|
|
20456
|
+
and lease_id = ${input.leaseId}
|
|
20457
|
+
and sandbox_group_id = ${input.sandboxGroupId}
|
|
20458
|
+
and lease_epoch = ${input.lostEpoch}
|
|
20459
|
+
and provider_instance_id = ${input.lostInstanceId}
|
|
20460
|
+
and status = 'open'
|
|
20461
|
+
order by id
|
|
20462
|
+
for update
|
|
20463
|
+
`);
|
|
20464
|
+
}
|
|
20465
|
+
|
|
20466
|
+
async function settleExactLostProviderWorkspaceBlockersTx(
|
|
20467
|
+
tx: Database,
|
|
20468
|
+
input: {
|
|
20469
|
+
accountId: string;
|
|
20470
|
+
workspaceId: string;
|
|
20471
|
+
leaseId: string;
|
|
20472
|
+
sandboxGroupId: string;
|
|
20473
|
+
lostEpoch: number;
|
|
20474
|
+
lostInstanceId: string;
|
|
20475
|
+
},
|
|
20476
|
+
): Promise<LostProviderWorkspaceSettlement> {
|
|
20477
|
+
const lostProcesses = await tx
|
|
20478
|
+
.update(schema.sandboxRetainedProcesses)
|
|
20479
|
+
.set({
|
|
20480
|
+
state: "lost",
|
|
20481
|
+
exitCode: null,
|
|
20482
|
+
settlementReason: LOST_PROVIDER_PROCESS_REASON,
|
|
20483
|
+
settledAt: new Date(),
|
|
20484
|
+
})
|
|
20485
|
+
.where(
|
|
20486
|
+
and(
|
|
20487
|
+
eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
|
|
20488
|
+
eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
|
|
20489
|
+
eq(schema.sandboxRetainedProcesses.leaseId, input.leaseId),
|
|
20490
|
+
eq(schema.sandboxRetainedProcesses.sandboxGroupId, input.sandboxGroupId),
|
|
20491
|
+
eq(schema.sandboxRetainedProcesses.leaseEpoch, input.lostEpoch),
|
|
20492
|
+
eq(schema.sandboxRetainedProcesses.providerInstanceId, input.lostInstanceId),
|
|
20493
|
+
eq(schema.sandboxRetainedProcesses.state, "active"),
|
|
20494
|
+
),
|
|
20495
|
+
)
|
|
20496
|
+
.returning({
|
|
20497
|
+
id: schema.sandboxRetainedProcesses.id,
|
|
20498
|
+
holderId: schema.sandboxRetainedProcesses.holderId,
|
|
20499
|
+
});
|
|
20500
|
+
|
|
20501
|
+
const rejectedAdmissions = await tx
|
|
20502
|
+
.update(schema.sandboxWorkspaceMutationAdmissions)
|
|
20503
|
+
.set({ providerOutcome: "rejected", settledAt: new Date() })
|
|
20504
|
+
.where(
|
|
20505
|
+
and(
|
|
20506
|
+
eq(schema.sandboxWorkspaceMutationAdmissions.accountId, input.accountId),
|
|
20507
|
+
eq(schema.sandboxWorkspaceMutationAdmissions.workspaceId, input.workspaceId),
|
|
20508
|
+
eq(schema.sandboxWorkspaceMutationAdmissions.leaseId, input.leaseId),
|
|
20509
|
+
eq(schema.sandboxWorkspaceMutationAdmissions.sandboxGroupId, input.sandboxGroupId),
|
|
20510
|
+
eq(schema.sandboxWorkspaceMutationAdmissions.leaseEpoch, input.lostEpoch),
|
|
20511
|
+
eq(schema.sandboxWorkspaceMutationAdmissions.providerInstanceId, input.lostInstanceId),
|
|
20512
|
+
isNull(schema.sandboxWorkspaceMutationAdmissions.settledAt),
|
|
20513
|
+
),
|
|
20514
|
+
)
|
|
20515
|
+
.returning({ id: schema.sandboxWorkspaceMutationAdmissions.id });
|
|
20516
|
+
|
|
20517
|
+
const closedPtys = await tx
|
|
20518
|
+
.update(schema.sandboxPtySessions)
|
|
20519
|
+
.set({ status: "closed", closedAt: new Date() })
|
|
20520
|
+
.where(
|
|
20521
|
+
and(
|
|
20522
|
+
eq(schema.sandboxPtySessions.accountId, input.accountId),
|
|
20523
|
+
eq(schema.sandboxPtySessions.workspaceId, input.workspaceId),
|
|
20524
|
+
eq(schema.sandboxPtySessions.leaseId, input.leaseId),
|
|
20525
|
+
eq(schema.sandboxPtySessions.sandboxGroupId, input.sandboxGroupId),
|
|
20526
|
+
eq(schema.sandboxPtySessions.leaseEpoch, input.lostEpoch),
|
|
20527
|
+
eq(schema.sandboxPtySessions.providerInstanceId, input.lostInstanceId),
|
|
20528
|
+
eq(schema.sandboxPtySessions.status, "open"),
|
|
20529
|
+
),
|
|
20530
|
+
)
|
|
20531
|
+
.returning({ id: schema.sandboxPtySessions.id });
|
|
20532
|
+
|
|
20533
|
+
const deletedHolders =
|
|
20534
|
+
lostProcesses.length === 0
|
|
20535
|
+
? []
|
|
20536
|
+
: await tx
|
|
20537
|
+
.delete(schema.sandboxLeaseHolders)
|
|
20538
|
+
.where(
|
|
20539
|
+
and(
|
|
20540
|
+
eq(schema.sandboxLeaseHolders.accountId, input.accountId),
|
|
20541
|
+
eq(schema.sandboxLeaseHolders.workspaceId, input.workspaceId),
|
|
20542
|
+
eq(schema.sandboxLeaseHolders.leaseId, input.leaseId),
|
|
20543
|
+
eq(schema.sandboxLeaseHolders.kind, "process"),
|
|
20544
|
+
inArray(
|
|
20545
|
+
schema.sandboxLeaseHolders.holderId,
|
|
20546
|
+
lostProcesses.map((process) => process.holderId),
|
|
20547
|
+
),
|
|
20548
|
+
),
|
|
20549
|
+
)
|
|
20550
|
+
.returning({ holderId: schema.sandboxLeaseHolders.holderId });
|
|
20551
|
+
|
|
20552
|
+
await tx.execute(sql`
|
|
20553
|
+
update sandbox_leases as lease set
|
|
20554
|
+
refcount = counts.total,
|
|
20555
|
+
turn_holders = counts.turns,
|
|
20556
|
+
viewer_holders = counts.viewers,
|
|
20557
|
+
updated_at = now()
|
|
20558
|
+
from (
|
|
20559
|
+
select count(*)::int as total,
|
|
20560
|
+
count(*) filter (where kind = 'turn')::int as turns,
|
|
20561
|
+
count(*) filter (where kind = 'viewer')::int as viewers
|
|
20562
|
+
from sandbox_lease_holders
|
|
20563
|
+
where lease_id = ${input.leaseId}
|
|
20564
|
+
) as counts
|
|
20565
|
+
where lease.id = ${input.leaseId}
|
|
20566
|
+
`);
|
|
20567
|
+
|
|
20568
|
+
return {
|
|
20569
|
+
processesLost: lostProcesses.length,
|
|
20570
|
+
admissionsRejected: rejectedAdmissions.length,
|
|
20571
|
+
ptysClosed: closedPtys.length,
|
|
20572
|
+
processHoldersDeleted: deletedHolders.length,
|
|
20573
|
+
};
|
|
20574
|
+
}
|
|
20575
|
+
|
|
20082
20576
|
/**
|
|
20083
20577
|
* Atomically retire one exact warm provider instance after a resume-only caller
|
|
20084
20578
|
* receives a provider NotFound. The epoch + instance predicates are the
|
|
@@ -20086,10 +20580,11 @@ export type MarkWarmLeaseInstanceLostResult =
|
|
|
20086
20580
|
* box, but only the first one transitions the lease to cold and advances its
|
|
20087
20581
|
* epoch. The next ordinary acquire elects one cold->warming spawner.
|
|
20088
20582
|
*
|
|
20089
|
-
*
|
|
20090
|
-
*
|
|
20091
|
-
*
|
|
20092
|
-
*
|
|
20583
|
+
* Logical turn/viewer interest remains intact, while holders for active
|
|
20584
|
+
* processes that physically belonged to the lost provider are removed with
|
|
20585
|
+
* their terminal loss proof. A persisted workspace archive is reduced to the
|
|
20586
|
+
* same minimal cold envelope used by the drain/failure paths, so the elected
|
|
20587
|
+
* replacement can hydrate it without carrying the dead box id.
|
|
20093
20588
|
*/
|
|
20094
20589
|
export async function markWarmLeaseInstanceLost(
|
|
20095
20590
|
db: Database,
|
|
@@ -20109,6 +20604,33 @@ export async function markWarmLeaseInstanceLost(
|
|
|
20109
20604
|
async (scopedDb) =>
|
|
20110
20605
|
await scopedDb.transaction(async (txRaw) => {
|
|
20111
20606
|
const tx = txRaw as unknown as Database;
|
|
20607
|
+
const observedRows = await tx.execute<LeaseRow>(sql`
|
|
20608
|
+
select * from sandbox_leases
|
|
20609
|
+
where workspace_id = ${input.workspaceId}
|
|
20610
|
+
and sandbox_group_id = ${input.sandboxGroupId}
|
|
20611
|
+
`);
|
|
20612
|
+
const observed = observedRows[0];
|
|
20613
|
+
if (
|
|
20614
|
+
!observed ||
|
|
20615
|
+
observed.liveness !== "warm" ||
|
|
20616
|
+
Number(observed.lease_epoch) !== input.expectedEpoch ||
|
|
20617
|
+
observed.instance_id !== input.expectedInstanceId
|
|
20618
|
+
) {
|
|
20619
|
+
return {
|
|
20620
|
+
status: "stale" as const,
|
|
20621
|
+
lease: observed ? mapLeaseRow(observed) : null,
|
|
20622
|
+
};
|
|
20623
|
+
}
|
|
20624
|
+
|
|
20625
|
+
const blockerScope = {
|
|
20626
|
+
accountId: input.accountId,
|
|
20627
|
+
workspaceId: input.workspaceId,
|
|
20628
|
+
leaseId: observed.id,
|
|
20629
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
20630
|
+
lostEpoch: input.expectedEpoch,
|
|
20631
|
+
lostInstanceId: input.expectedInstanceId,
|
|
20632
|
+
};
|
|
20633
|
+
await lockExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
|
|
20112
20634
|
const currentRows = await tx.execute<LeaseRow>(sql`
|
|
20113
20635
|
select * from sandbox_leases
|
|
20114
20636
|
where workspace_id = ${input.workspaceId}
|
|
@@ -20118,6 +20640,7 @@ export async function markWarmLeaseInstanceLost(
|
|
|
20118
20640
|
const current = currentRows[0];
|
|
20119
20641
|
if (
|
|
20120
20642
|
!current ||
|
|
20643
|
+
current.id !== observed.id ||
|
|
20121
20644
|
current.liveness !== "warm" ||
|
|
20122
20645
|
Number(current.lease_epoch) !== input.expectedEpoch ||
|
|
20123
20646
|
current.instance_id !== input.expectedInstanceId
|
|
@@ -20128,6 +20651,8 @@ export async function markWarmLeaseInstanceLost(
|
|
|
20128
20651
|
};
|
|
20129
20652
|
}
|
|
20130
20653
|
|
|
20654
|
+
const settlement = await settleExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
|
|
20655
|
+
|
|
20131
20656
|
const observedAt = new Date().toISOString();
|
|
20132
20657
|
const before = recoveryStateFromLeaseRow(current);
|
|
20133
20658
|
const archiveStatus = before.archive.status;
|
|
@@ -20198,7 +20723,126 @@ export async function markWarmLeaseInstanceLost(
|
|
|
20198
20723
|
if (!updated) {
|
|
20199
20724
|
throw new Error(`Warm sandbox lease vanished while retiring instance ${current.id}`);
|
|
20200
20725
|
}
|
|
20201
|
-
return { status: "marked" as const, lease: mapLeaseRow(updated) };
|
|
20726
|
+
return { status: "marked" as const, lease: mapLeaseRow(updated), settlement };
|
|
20727
|
+
}),
|
|
20728
|
+
);
|
|
20729
|
+
}
|
|
20730
|
+
|
|
20731
|
+
export type ReconcileColdLostLeaseInstanceBlockersResult =
|
|
20732
|
+
| {
|
|
20733
|
+
status: "reconciled";
|
|
20734
|
+
lease: LeaseSnapshot;
|
|
20735
|
+
settlement: LostProviderWorkspaceSettlement;
|
|
20736
|
+
}
|
|
20737
|
+
| { status: "stale"; lease: LeaseSnapshot | null };
|
|
20738
|
+
|
|
20739
|
+
/**
|
|
20740
|
+
* Settle blockers left by a provider-loss transition that predates exact loss
|
|
20741
|
+
* cleanup. This is deliberately narrower than markWarmLeaseInstanceLost: it
|
|
20742
|
+
* requires the already-cold lease to match the full operator-observed
|
|
20743
|
+
* generation/archive tuple, the current epoch to be exactly lostEpoch + 1, and
|
|
20744
|
+
* recovery truth to name the same missing provider. It never advances an epoch,
|
|
20745
|
+
* rematerializes a provider, or changes archive/recovery completeness.
|
|
20746
|
+
*/
|
|
20747
|
+
export async function reconcileColdLostLeaseInstanceBlockers(
|
|
20748
|
+
db: Database,
|
|
20749
|
+
input: {
|
|
20750
|
+
accountId: string;
|
|
20751
|
+
workspaceId: string;
|
|
20752
|
+
sandboxGroupId: string;
|
|
20753
|
+
expectedCurrentEpoch: number;
|
|
20754
|
+
expectedLostEpoch: number;
|
|
20755
|
+
expectedLostInstanceId: string;
|
|
20756
|
+
expectedWorkspaceGeneration: number;
|
|
20757
|
+
expectedArchiveGeneration: number | null;
|
|
20758
|
+
expectedArchiveComplete: boolean;
|
|
20759
|
+
},
|
|
20760
|
+
): Promise<ReconcileColdLostLeaseInstanceBlockersResult> {
|
|
20761
|
+
if (
|
|
20762
|
+
!Number.isSafeInteger(input.expectedCurrentEpoch) ||
|
|
20763
|
+
!Number.isSafeInteger(input.expectedLostEpoch) ||
|
|
20764
|
+
input.expectedCurrentEpoch !== input.expectedLostEpoch + 1
|
|
20765
|
+
) {
|
|
20766
|
+
throw new Error("Cold lost-provider reconciliation requires currentEpoch = lostEpoch + 1");
|
|
20767
|
+
}
|
|
20768
|
+
return await withRlsContext(
|
|
20769
|
+
db,
|
|
20770
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
20771
|
+
async (scopedDb) =>
|
|
20772
|
+
await scopedDb.transaction(async (txRaw) => {
|
|
20773
|
+
const tx = txRaw as unknown as Database;
|
|
20774
|
+
const observedRows = await tx.execute<LeaseRow>(sql`
|
|
20775
|
+
select * from sandbox_leases
|
|
20776
|
+
where workspace_id = ${input.workspaceId}
|
|
20777
|
+
and sandbox_group_id = ${input.sandboxGroupId}
|
|
20778
|
+
`);
|
|
20779
|
+
const observed = observedRows[0];
|
|
20780
|
+
const observedRecovery = observed ? recoveryStateFromLeaseRow(observed) : null;
|
|
20781
|
+
if (
|
|
20782
|
+
!observed ||
|
|
20783
|
+
observed.liveness !== "cold" ||
|
|
20784
|
+
observed.instance_id !== null ||
|
|
20785
|
+
Number(observed.lease_epoch) !== input.expectedCurrentEpoch ||
|
|
20786
|
+
Number(observed.workspace_generation) !== input.expectedWorkspaceGeneration ||
|
|
20787
|
+
(observed.archive_generation === null ? null : Number(observed.archive_generation)) !==
|
|
20788
|
+
input.expectedArchiveGeneration ||
|
|
20789
|
+
hasCompleteWorkspaceArchive(observed) !== input.expectedArchiveComplete ||
|
|
20790
|
+
observedRecovery?.provider.status !== "missing" ||
|
|
20791
|
+
observedRecovery.provider.instanceId !== input.expectedLostInstanceId
|
|
20792
|
+
) {
|
|
20793
|
+
return {
|
|
20794
|
+
status: "stale" as const,
|
|
20795
|
+
lease: observed ? mapLeaseRow(observed) : null,
|
|
20796
|
+
};
|
|
20797
|
+
}
|
|
20798
|
+
|
|
20799
|
+
const blockerScope = {
|
|
20800
|
+
accountId: input.accountId,
|
|
20801
|
+
workspaceId: input.workspaceId,
|
|
20802
|
+
leaseId: observed.id,
|
|
20803
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
20804
|
+
lostEpoch: input.expectedLostEpoch,
|
|
20805
|
+
lostInstanceId: input.expectedLostInstanceId,
|
|
20806
|
+
};
|
|
20807
|
+
await lockExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
|
|
20808
|
+
const currentRows = await tx.execute<LeaseRow>(sql`
|
|
20809
|
+
select * from sandbox_leases
|
|
20810
|
+
where workspace_id = ${input.workspaceId}
|
|
20811
|
+
and sandbox_group_id = ${input.sandboxGroupId}
|
|
20812
|
+
for update
|
|
20813
|
+
`);
|
|
20814
|
+
const current = currentRows[0];
|
|
20815
|
+
const recovery = current ? recoveryStateFromLeaseRow(current) : null;
|
|
20816
|
+
if (
|
|
20817
|
+
!current ||
|
|
20818
|
+
current.id !== observed.id ||
|
|
20819
|
+
current.liveness !== "cold" ||
|
|
20820
|
+
current.instance_id !== null ||
|
|
20821
|
+
Number(current.lease_epoch) !== input.expectedCurrentEpoch ||
|
|
20822
|
+
Number(current.workspace_generation) !== input.expectedWorkspaceGeneration ||
|
|
20823
|
+
(current.archive_generation === null ? null : Number(current.archive_generation)) !==
|
|
20824
|
+
input.expectedArchiveGeneration ||
|
|
20825
|
+
hasCompleteWorkspaceArchive(current) !== input.expectedArchiveComplete ||
|
|
20826
|
+
recovery?.provider.status !== "missing" ||
|
|
20827
|
+
recovery.provider.instanceId !== input.expectedLostInstanceId
|
|
20828
|
+
) {
|
|
20829
|
+
return {
|
|
20830
|
+
status: "stale" as const,
|
|
20831
|
+
lease: current ? mapLeaseRow(current) : null,
|
|
20832
|
+
};
|
|
20833
|
+
}
|
|
20834
|
+
|
|
20835
|
+
const settlement = await settleExactLostProviderWorkspaceBlockersTx(tx, blockerScope);
|
|
20836
|
+
const refreshedRows = await tx.execute<LeaseRow>(sql`
|
|
20837
|
+
select * from sandbox_leases where id = ${current.id}
|
|
20838
|
+
`);
|
|
20839
|
+
const refreshed = refreshedRows[0];
|
|
20840
|
+
if (!refreshed) throw new Error("Cold sandbox lease vanished during reconciliation");
|
|
20841
|
+
return {
|
|
20842
|
+
status: "reconciled" as const,
|
|
20843
|
+
lease: mapLeaseRow(refreshed),
|
|
20844
|
+
settlement,
|
|
20845
|
+
};
|
|
20202
20846
|
}),
|
|
20203
20847
|
);
|
|
20204
20848
|
}
|
|
@@ -21066,7 +21710,7 @@ export async function confirmDrainCold(
|
|
|
21066
21710
|
// The fence is the split-brain guard: a stale-epoch reaper writes ZERO rows and
|
|
21067
21711
|
// is told not to terminate.
|
|
21068
21712
|
export class SandboxWorkspaceMutationFencedError extends Error {
|
|
21069
|
-
readonly name = "SandboxWorkspaceMutationFencedError";
|
|
21713
|
+
readonly name: string = "SandboxWorkspaceMutationFencedError";
|
|
21070
21714
|
|
|
21071
21715
|
constructor(
|
|
21072
21716
|
public readonly code:
|
|
@@ -21132,8 +21776,131 @@ export type SandboxRetainedProcess = {
|
|
|
21132
21776
|
settlementReason: string | null;
|
|
21133
21777
|
startedAt: string;
|
|
21134
21778
|
settledAt: string | null;
|
|
21779
|
+
reconcileAfter: string;
|
|
21780
|
+
reconcileClaimId: string | null;
|
|
21781
|
+
reconcileClaimedAt: string | null;
|
|
21782
|
+
reconcileAttempts: number;
|
|
21783
|
+
lastReconcileOutcome: string | null;
|
|
21784
|
+
reconcileProofOutcome: "exited" | "lost" | null;
|
|
21785
|
+
reconcileProofExitCode: number | null;
|
|
21786
|
+
reconcileProofReason:
|
|
21787
|
+
| "provider_exit_banner"
|
|
21788
|
+
| "provider_session_lost_banner"
|
|
21789
|
+
| "provider_instance_not_found"
|
|
21790
|
+
| null;
|
|
21791
|
+
reconcileProofObservedAt: string | null;
|
|
21135
21792
|
};
|
|
21136
21793
|
|
|
21794
|
+
export type RetainedProcessProviderProof =
|
|
21795
|
+
| { outcome: "exited"; exitCode: number; reason: "provider_exit_banner" }
|
|
21796
|
+
| {
|
|
21797
|
+
outcome: "lost";
|
|
21798
|
+
exitCode: null;
|
|
21799
|
+
reason: "provider_session_lost_banner" | "provider_instance_not_found";
|
|
21800
|
+
};
|
|
21801
|
+
|
|
21802
|
+
export type SandboxRetainedProcessIdentity = Pick<
|
|
21803
|
+
SandboxRetainedProcess,
|
|
21804
|
+
| "leaseId"
|
|
21805
|
+
| "sandboxGroupId"
|
|
21806
|
+
| "parentAdmissionId"
|
|
21807
|
+
| "holderId"
|
|
21808
|
+
| "leaseEpoch"
|
|
21809
|
+
| "providerBackend"
|
|
21810
|
+
| "providerInstanceId"
|
|
21811
|
+
| "routeKind"
|
|
21812
|
+
| "routeTargetId"
|
|
21813
|
+
| "routeEpoch"
|
|
21814
|
+
| "providerSessionId"
|
|
21815
|
+
>;
|
|
21816
|
+
|
|
21817
|
+
export type SandboxRetainedProcessReconciliationClaim = {
|
|
21818
|
+
process: SandboxRetainedProcess;
|
|
21819
|
+
claimId: string;
|
|
21820
|
+
ownerState: string;
|
|
21821
|
+
ownerAttemptOutcome: string | null;
|
|
21822
|
+
};
|
|
21823
|
+
|
|
21824
|
+
export type ActiveRetainedProcessOwnerCount = {
|
|
21825
|
+
ownerState: string;
|
|
21826
|
+
activeCount: number;
|
|
21827
|
+
terminalOwnerCount: number;
|
|
21828
|
+
};
|
|
21829
|
+
|
|
21830
|
+
export type ExpiredDrainingSandboxLeaseCount = {
|
|
21831
|
+
backend: string;
|
|
21832
|
+
ageBucket: "lt_5m" | "5m_1h" | "1h_1d" | "gte_1d";
|
|
21833
|
+
count: number;
|
|
21834
|
+
};
|
|
21835
|
+
|
|
21836
|
+
export function retainedProcessSettlementIdentity(
|
|
21837
|
+
process: SandboxRetainedProcess,
|
|
21838
|
+
): SandboxRetainedProcessIdentity {
|
|
21839
|
+
return {
|
|
21840
|
+
leaseId: process.leaseId,
|
|
21841
|
+
sandboxGroupId: process.sandboxGroupId,
|
|
21842
|
+
parentAdmissionId: process.parentAdmissionId,
|
|
21843
|
+
holderId: process.holderId,
|
|
21844
|
+
leaseEpoch: process.leaseEpoch,
|
|
21845
|
+
providerBackend: process.providerBackend,
|
|
21846
|
+
providerInstanceId: process.providerInstanceId,
|
|
21847
|
+
routeKind: process.routeKind,
|
|
21848
|
+
routeTargetId: process.routeTargetId,
|
|
21849
|
+
routeEpoch: process.routeEpoch,
|
|
21850
|
+
providerSessionId: process.providerSessionId,
|
|
21851
|
+
};
|
|
21852
|
+
}
|
|
21853
|
+
|
|
21854
|
+
export function retainedProcessReconciliationProof(
|
|
21855
|
+
process: SandboxRetainedProcess,
|
|
21856
|
+
): RetainedProcessProviderProof | null {
|
|
21857
|
+
if (
|
|
21858
|
+
process.reconcileProofOutcome === "exited" &&
|
|
21859
|
+
process.reconcileProofReason === "provider_exit_banner" &&
|
|
21860
|
+
process.reconcileProofExitCode !== null
|
|
21861
|
+
) {
|
|
21862
|
+
return {
|
|
21863
|
+
outcome: "exited",
|
|
21864
|
+
exitCode: process.reconcileProofExitCode,
|
|
21865
|
+
reason: "provider_exit_banner",
|
|
21866
|
+
};
|
|
21867
|
+
}
|
|
21868
|
+
if (
|
|
21869
|
+
process.reconcileProofOutcome === "lost" &&
|
|
21870
|
+
(process.reconcileProofReason === "provider_session_lost_banner" ||
|
|
21871
|
+
process.reconcileProofReason === "provider_instance_not_found")
|
|
21872
|
+
) {
|
|
21873
|
+
return {
|
|
21874
|
+
outcome: "lost",
|
|
21875
|
+
exitCode: null,
|
|
21876
|
+
reason: process.reconcileProofReason,
|
|
21877
|
+
};
|
|
21878
|
+
}
|
|
21879
|
+
return null;
|
|
21880
|
+
}
|
|
21881
|
+
|
|
21882
|
+
/** Durable promotion committed before a mutable route/turn authority check
|
|
21883
|
+
* rejected the provider output. The process identity is safe to hand to the
|
|
21884
|
+
* exact-backend cancellation path; callers must still reject the output and
|
|
21885
|
+
* must never replay the provider mutation. */
|
|
21886
|
+
export class SandboxRetainedProcessPromotionFencedError extends SandboxWorkspaceMutationFencedError {
|
|
21887
|
+
readonly name = "SandboxRetainedProcessPromotionFencedError";
|
|
21888
|
+
|
|
21889
|
+
constructor(
|
|
21890
|
+
code:
|
|
21891
|
+
| "attempt_fenced"
|
|
21892
|
+
| "holder_fenced"
|
|
21893
|
+
| "lease_fenced"
|
|
21894
|
+
| "route_fenced"
|
|
21895
|
+
| "process_fenced"
|
|
21896
|
+
| "admission_fenced",
|
|
21897
|
+
message: string,
|
|
21898
|
+
public readonly process: SandboxRetainedProcess,
|
|
21899
|
+
) {
|
|
21900
|
+
super(code, message);
|
|
21901
|
+
}
|
|
21902
|
+
}
|
|
21903
|
+
|
|
21137
21904
|
type SandboxWorkspaceMutationSettlementResult =
|
|
21138
21905
|
| { failure: null }
|
|
21139
21906
|
| {
|
|
@@ -21246,6 +22013,18 @@ function normalizeRetainedProcessSettlementReason(reason: string): string {
|
|
|
21246
22013
|
return bounded;
|
|
21247
22014
|
}
|
|
21248
22015
|
|
|
22016
|
+
function normalizeRetainedProcessReconciliationOutcome(outcome: string): string {
|
|
22017
|
+
const normalized = outcome.trim();
|
|
22018
|
+
const byteLength = Buffer.byteLength(normalized, "utf8");
|
|
22019
|
+
if (byteLength < 1 || byteLength > 64) {
|
|
22020
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
22021
|
+
"process_fenced",
|
|
22022
|
+
"Retained process reconciliation outcome must contain between 1 and 64 UTF-8 bytes",
|
|
22023
|
+
);
|
|
22024
|
+
}
|
|
22025
|
+
return normalized;
|
|
22026
|
+
}
|
|
22027
|
+
|
|
21249
22028
|
function mapWorkspaceMutationAdmission(row: {
|
|
21250
22029
|
id: string;
|
|
21251
22030
|
lease_id: string;
|
|
@@ -21311,9 +22090,38 @@ function mapRetainedProcess(
|
|
|
21311
22090
|
settlementReason: row.settlementReason ?? null,
|
|
21312
22091
|
startedAt: row.startedAt.toISOString(),
|
|
21313
22092
|
settledAt: row.settledAt?.toISOString() ?? null,
|
|
22093
|
+
reconcileAfter: row.reconcileAfter.toISOString(),
|
|
22094
|
+
reconcileClaimId: row.reconcileClaimId ?? null,
|
|
22095
|
+
reconcileClaimedAt: row.reconcileClaimedAt?.toISOString() ?? null,
|
|
22096
|
+
reconcileAttempts: row.reconcileAttempts,
|
|
22097
|
+
lastReconcileOutcome: row.lastReconcileOutcome ?? null,
|
|
22098
|
+
reconcileProofOutcome: row.reconcileProofOutcome ?? null,
|
|
22099
|
+
reconcileProofExitCode: row.reconcileProofExitCode ?? null,
|
|
22100
|
+
reconcileProofReason:
|
|
22101
|
+
(row.reconcileProofReason as SandboxRetainedProcess["reconcileProofReason"]) ?? null,
|
|
22102
|
+
reconcileProofObservedAt: row.reconcileProofObservedAt?.toISOString() ?? null,
|
|
21314
22103
|
};
|
|
21315
22104
|
}
|
|
21316
22105
|
|
|
22106
|
+
function retainedProcessMatchesSettlementIdentity(
|
|
22107
|
+
process: typeof schema.sandboxRetainedProcesses.$inferSelect,
|
|
22108
|
+
expected: SandboxRetainedProcessIdentity,
|
|
22109
|
+
): boolean {
|
|
22110
|
+
return (
|
|
22111
|
+
process.leaseId === expected.leaseId &&
|
|
22112
|
+
process.sandboxGroupId === expected.sandboxGroupId &&
|
|
22113
|
+
process.parentAdmissionId === expected.parentAdmissionId &&
|
|
22114
|
+
process.holderId === expected.holderId &&
|
|
22115
|
+
process.leaseEpoch === expected.leaseEpoch &&
|
|
22116
|
+
process.providerBackend === expected.providerBackend &&
|
|
22117
|
+
process.providerInstanceId === expected.providerInstanceId &&
|
|
22118
|
+
process.routeKind === expected.routeKind &&
|
|
22119
|
+
(process.routeTargetId ?? null) === expected.routeTargetId &&
|
|
22120
|
+
process.routeEpoch === expected.routeEpoch &&
|
|
22121
|
+
process.providerSessionId === expected.providerSessionId
|
|
22122
|
+
);
|
|
22123
|
+
}
|
|
22124
|
+
|
|
21317
22125
|
async function lockWorkspaceMutationSessionTx(
|
|
21318
22126
|
tx: Database,
|
|
21319
22127
|
workspaceId: string,
|
|
@@ -22333,7 +23141,11 @@ export async function retainWorkspaceMutationProcess(
|
|
|
22333
23141
|
}),
|
|
22334
23142
|
);
|
|
22335
23143
|
if (result.failure.failure !== null) {
|
|
22336
|
-
throw new
|
|
23144
|
+
throw new SandboxRetainedProcessPromotionFencedError(
|
|
23145
|
+
result.failure.failure,
|
|
23146
|
+
result.failure.detail,
|
|
23147
|
+
result.process,
|
|
23148
|
+
);
|
|
22337
23149
|
}
|
|
22338
23150
|
return result.process;
|
|
22339
23151
|
}
|
|
@@ -22360,6 +23172,293 @@ export async function getRetainedProcess(
|
|
|
22360
23172
|
});
|
|
22361
23173
|
}
|
|
22362
23174
|
|
|
23175
|
+
/** Claim a bounded, oldest-due batch of active retained processes whose exact
|
|
23176
|
+
* owner attempt is closed (or whose direct request already returned). Claim
|
|
23177
|
+
* expiry only recovers coordination after worker death; it is never provider
|
|
23178
|
+
* exit proof. Rows that settle between the global claim and scoped read are
|
|
23179
|
+
* deliberately omitted. */
|
|
23180
|
+
export async function claimTerminalRetainedProcesses(
|
|
23181
|
+
db: Database,
|
|
23182
|
+
input: { claimId: string; limit: number; claimTtlMs: number },
|
|
23183
|
+
): Promise<SandboxRetainedProcessReconciliationClaim[]> {
|
|
23184
|
+
if (!Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 100) {
|
|
23185
|
+
throw new Error("Retained process reconciliation limit must be between 1 and 100");
|
|
23186
|
+
}
|
|
23187
|
+
if (
|
|
23188
|
+
!Number.isSafeInteger(input.claimTtlMs) ||
|
|
23189
|
+
input.claimTtlMs < 0 ||
|
|
23190
|
+
input.claimTtlMs > 3_600_000
|
|
23191
|
+
) {
|
|
23192
|
+
throw new Error("Retained process reconciliation claim TTL is invalid");
|
|
23193
|
+
}
|
|
23194
|
+
const rows = await rawRows<{
|
|
23195
|
+
account_id: string;
|
|
23196
|
+
workspace_id: string;
|
|
23197
|
+
session_id: string;
|
|
23198
|
+
process_id: string;
|
|
23199
|
+
claim_id: string;
|
|
23200
|
+
owner_state: string;
|
|
23201
|
+
owner_attempt_outcome: string | null;
|
|
23202
|
+
}>(
|
|
23203
|
+
db,
|
|
23204
|
+
sql`
|
|
23205
|
+
select account_id, workspace_id, session_id, process_id, claim_id,
|
|
23206
|
+
owner_state, owner_attempt_outcome
|
|
23207
|
+
from opengeni_private.claim_terminal_retained_processes(
|
|
23208
|
+
${input.claimId}::uuid, ${input.limit}::integer, ${input.claimTtlMs}::bigint
|
|
23209
|
+
)
|
|
23210
|
+
`,
|
|
23211
|
+
);
|
|
23212
|
+
const claims: SandboxRetainedProcessReconciliationClaim[] = [];
|
|
23213
|
+
for (const row of rows) {
|
|
23214
|
+
const process = await getRetainedProcess(db, {
|
|
23215
|
+
workspaceId: row.workspace_id,
|
|
23216
|
+
sessionId: row.session_id,
|
|
23217
|
+
processId: row.process_id,
|
|
23218
|
+
});
|
|
23219
|
+
if (
|
|
23220
|
+
!process ||
|
|
23221
|
+
process.accountId !== row.account_id ||
|
|
23222
|
+
process.state !== "active" ||
|
|
23223
|
+
process.reconcileClaimId !== row.claim_id
|
|
23224
|
+
) {
|
|
23225
|
+
continue;
|
|
23226
|
+
}
|
|
23227
|
+
claims.push({
|
|
23228
|
+
process,
|
|
23229
|
+
claimId: row.claim_id,
|
|
23230
|
+
ownerState: row.owner_state,
|
|
23231
|
+
ownerAttemptOutcome: row.owner_attempt_outcome,
|
|
23232
|
+
});
|
|
23233
|
+
}
|
|
23234
|
+
return claims;
|
|
23235
|
+
}
|
|
23236
|
+
|
|
23237
|
+
/** Durably checkpoint exact provider exit/loss proof before canonical
|
|
23238
|
+
* settlement. This closes the worker-death window between provider observation
|
|
23239
|
+
* and settlement without converting owner state, age, timeout, or claim expiry
|
|
23240
|
+
* into physical proof. */
|
|
23241
|
+
export async function recordRetainedProcessReconciliationProof(
|
|
23242
|
+
db: Database,
|
|
23243
|
+
input: {
|
|
23244
|
+
accountId: string;
|
|
23245
|
+
workspaceId: string;
|
|
23246
|
+
sessionId: string;
|
|
23247
|
+
processId: string;
|
|
23248
|
+
expected: SandboxRetainedProcessIdentity;
|
|
23249
|
+
claimId: string;
|
|
23250
|
+
proof: RetainedProcessProviderProof;
|
|
23251
|
+
},
|
|
23252
|
+
): Promise<SandboxRetainedProcess> {
|
|
23253
|
+
if (
|
|
23254
|
+
input.proof.outcome === "exited" &&
|
|
23255
|
+
(!Number.isSafeInteger(input.proof.exitCode) || input.proof.exitCode === null)
|
|
23256
|
+
) {
|
|
23257
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23258
|
+
"process_fenced",
|
|
23259
|
+
"Retained process reconciliation exit proof requires a safe integer exit code",
|
|
23260
|
+
);
|
|
23261
|
+
}
|
|
23262
|
+
return await withRlsContext(
|
|
23263
|
+
db,
|
|
23264
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
23265
|
+
async (scopedDb) =>
|
|
23266
|
+
await scopedDb.transaction(async (txRaw) => {
|
|
23267
|
+
const tx = txRaw as unknown as Database;
|
|
23268
|
+
const [process] = await tx
|
|
23269
|
+
.select()
|
|
23270
|
+
.from(schema.sandboxRetainedProcesses)
|
|
23271
|
+
.where(
|
|
23272
|
+
and(
|
|
23273
|
+
eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
|
|
23274
|
+
eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
|
|
23275
|
+
eq(schema.sandboxRetainedProcesses.sessionId, input.sessionId),
|
|
23276
|
+
eq(schema.sandboxRetainedProcesses.id, input.processId),
|
|
23277
|
+
),
|
|
23278
|
+
)
|
|
23279
|
+
.for("update")
|
|
23280
|
+
.limit(1);
|
|
23281
|
+
if (
|
|
23282
|
+
!process ||
|
|
23283
|
+
process.state !== "active" ||
|
|
23284
|
+
!retainedProcessMatchesSettlementIdentity(process, input.expected)
|
|
23285
|
+
) {
|
|
23286
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23287
|
+
"process_fenced",
|
|
23288
|
+
"Retained process proof did not match an active copied durable identity",
|
|
23289
|
+
);
|
|
23290
|
+
}
|
|
23291
|
+
if (process.reconcileClaimId !== input.claimId) {
|
|
23292
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23293
|
+
"process_fenced",
|
|
23294
|
+
"Retained process proof claim was lost or superseded",
|
|
23295
|
+
);
|
|
23296
|
+
}
|
|
23297
|
+
const existing = mapRetainedProcess(process);
|
|
23298
|
+
const existingProof = retainedProcessReconciliationProof(existing);
|
|
23299
|
+
if (existingProof) {
|
|
23300
|
+
if (
|
|
23301
|
+
existingProof.outcome !== input.proof.outcome ||
|
|
23302
|
+
existingProof.exitCode !== input.proof.exitCode ||
|
|
23303
|
+
existingProof.reason !== input.proof.reason
|
|
23304
|
+
) {
|
|
23305
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23306
|
+
"process_fenced",
|
|
23307
|
+
"Retained process already carries different provider proof",
|
|
23308
|
+
);
|
|
23309
|
+
}
|
|
23310
|
+
return existing;
|
|
23311
|
+
}
|
|
23312
|
+
const [updated] = await tx
|
|
23313
|
+
.update(schema.sandboxRetainedProcesses)
|
|
23314
|
+
.set({
|
|
23315
|
+
reconcileProofOutcome: input.proof.outcome,
|
|
23316
|
+
reconcileProofExitCode: input.proof.exitCode,
|
|
23317
|
+
reconcileProofReason: input.proof.reason,
|
|
23318
|
+
reconcileProofObservedAt: new Date(),
|
|
23319
|
+
lastReconcileOutcome: `proof_${input.proof.outcome}`,
|
|
23320
|
+
})
|
|
23321
|
+
.where(
|
|
23322
|
+
and(
|
|
23323
|
+
eq(schema.sandboxRetainedProcesses.id, process.id),
|
|
23324
|
+
eq(schema.sandboxRetainedProcesses.state, "active"),
|
|
23325
|
+
eq(schema.sandboxRetainedProcesses.reconcileClaimId, input.claimId),
|
|
23326
|
+
),
|
|
23327
|
+
)
|
|
23328
|
+
.returning();
|
|
23329
|
+
if (!updated) {
|
|
23330
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23331
|
+
"process_fenced",
|
|
23332
|
+
"Retained process changed while checkpointing provider proof",
|
|
23333
|
+
);
|
|
23334
|
+
}
|
|
23335
|
+
return mapRetainedProcess(updated);
|
|
23336
|
+
}),
|
|
23337
|
+
);
|
|
23338
|
+
}
|
|
23339
|
+
|
|
23340
|
+
/** Release one exact reconciliation claim after an ambiguous/running provider
|
|
23341
|
+
* observation. This schedules a bounded retry but never changes process,
|
|
23342
|
+
* admission, holder, lease, archive, snapshot, or workspace state. */
|
|
23343
|
+
export async function deferRetainedProcessReconciliation(
|
|
23344
|
+
db: Database,
|
|
23345
|
+
input: {
|
|
23346
|
+
accountId: string;
|
|
23347
|
+
workspaceId: string;
|
|
23348
|
+
sessionId: string;
|
|
23349
|
+
processId: string;
|
|
23350
|
+
expected: SandboxRetainedProcessIdentity;
|
|
23351
|
+
claimId: string;
|
|
23352
|
+
outcome: string;
|
|
23353
|
+
retryAfterMs: number;
|
|
23354
|
+
},
|
|
23355
|
+
): Promise<boolean> {
|
|
23356
|
+
const outcome = normalizeRetainedProcessReconciliationOutcome(input.outcome);
|
|
23357
|
+
if (
|
|
23358
|
+
!Number.isSafeInteger(input.retryAfterMs) ||
|
|
23359
|
+
input.retryAfterMs < 0 ||
|
|
23360
|
+
input.retryAfterMs > 86_400_000
|
|
23361
|
+
) {
|
|
23362
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23363
|
+
"process_fenced",
|
|
23364
|
+
"Retained process reconciliation retry delay is invalid",
|
|
23365
|
+
);
|
|
23366
|
+
}
|
|
23367
|
+
return await withRlsContext(
|
|
23368
|
+
db,
|
|
23369
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId },
|
|
23370
|
+
async (scopedDb) =>
|
|
23371
|
+
await scopedDb.transaction(async (txRaw) => {
|
|
23372
|
+
const tx = txRaw as unknown as Database;
|
|
23373
|
+
const [process] = await tx
|
|
23374
|
+
.select()
|
|
23375
|
+
.from(schema.sandboxRetainedProcesses)
|
|
23376
|
+
.where(
|
|
23377
|
+
and(
|
|
23378
|
+
eq(schema.sandboxRetainedProcesses.accountId, input.accountId),
|
|
23379
|
+
eq(schema.sandboxRetainedProcesses.workspaceId, input.workspaceId),
|
|
23380
|
+
eq(schema.sandboxRetainedProcesses.sessionId, input.sessionId),
|
|
23381
|
+
eq(schema.sandboxRetainedProcesses.id, input.processId),
|
|
23382
|
+
),
|
|
23383
|
+
)
|
|
23384
|
+
.for("update")
|
|
23385
|
+
.limit(1);
|
|
23386
|
+
if (!process || !retainedProcessMatchesSettlementIdentity(process, input.expected)) {
|
|
23387
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23388
|
+
"process_fenced",
|
|
23389
|
+
"Retained process reconciliation did not match the copied durable identity",
|
|
23390
|
+
);
|
|
23391
|
+
}
|
|
23392
|
+
if (process.state !== "active") return false;
|
|
23393
|
+
if (process.reconcileClaimId !== input.claimId) {
|
|
23394
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23395
|
+
"process_fenced",
|
|
23396
|
+
"Retained process reconciliation claim was lost or superseded",
|
|
23397
|
+
);
|
|
23398
|
+
}
|
|
23399
|
+
const [updated] = await tx
|
|
23400
|
+
.update(schema.sandboxRetainedProcesses)
|
|
23401
|
+
.set({
|
|
23402
|
+
reconcileAfter: new Date(Date.now() + input.retryAfterMs),
|
|
23403
|
+
reconcileClaimId: null,
|
|
23404
|
+
reconcileClaimedAt: null,
|
|
23405
|
+
lastReconcileOutcome: outcome,
|
|
23406
|
+
})
|
|
23407
|
+
.where(
|
|
23408
|
+
and(
|
|
23409
|
+
eq(schema.sandboxRetainedProcesses.id, process.id),
|
|
23410
|
+
eq(schema.sandboxRetainedProcesses.state, "active"),
|
|
23411
|
+
eq(schema.sandboxRetainedProcesses.reconcileClaimId, input.claimId),
|
|
23412
|
+
),
|
|
23413
|
+
)
|
|
23414
|
+
.returning({ id: schema.sandboxRetainedProcesses.id });
|
|
23415
|
+
return Boolean(updated);
|
|
23416
|
+
}),
|
|
23417
|
+
);
|
|
23418
|
+
}
|
|
23419
|
+
|
|
23420
|
+
export async function countActiveRetainedProcessesByOwnerState(
|
|
23421
|
+
db: Database,
|
|
23422
|
+
): Promise<ActiveRetainedProcessOwnerCount[]> {
|
|
23423
|
+
const rows = await rawRows<{
|
|
23424
|
+
owner_state: string;
|
|
23425
|
+
active_count: number | string;
|
|
23426
|
+
terminal_owner_count: number | string;
|
|
23427
|
+
}>(
|
|
23428
|
+
db,
|
|
23429
|
+
sql`
|
|
23430
|
+
select owner_state, active_count, terminal_owner_count
|
|
23431
|
+
from opengeni_private.count_active_retained_processes_by_owner_state()
|
|
23432
|
+
`,
|
|
23433
|
+
);
|
|
23434
|
+
return rows.map((row) => ({
|
|
23435
|
+
ownerState: row.owner_state,
|
|
23436
|
+
activeCount: Number(row.active_count),
|
|
23437
|
+
terminalOwnerCount: Number(row.terminal_owner_count),
|
|
23438
|
+
}));
|
|
23439
|
+
}
|
|
23440
|
+
|
|
23441
|
+
export async function countExpiredDrainingSandboxLeases(
|
|
23442
|
+
db: Database,
|
|
23443
|
+
): Promise<ExpiredDrainingSandboxLeaseCount[]> {
|
|
23444
|
+
const rows = await rawRows<{
|
|
23445
|
+
backend: string;
|
|
23446
|
+
age_bucket: ExpiredDrainingSandboxLeaseCount["ageBucket"];
|
|
23447
|
+
count: number | string;
|
|
23448
|
+
}>(
|
|
23449
|
+
db,
|
|
23450
|
+
sql`
|
|
23451
|
+
select backend, age_bucket, count
|
|
23452
|
+
from opengeni_private.count_expired_draining_sandbox_leases()
|
|
23453
|
+
`,
|
|
23454
|
+
);
|
|
23455
|
+
return rows.map((row) => ({
|
|
23456
|
+
backend: row.backend,
|
|
23457
|
+
ageBucket: row.age_bucket,
|
|
23458
|
+
count: Number(row.count),
|
|
23459
|
+
}));
|
|
23460
|
+
}
|
|
23461
|
+
|
|
22363
23462
|
/** Settle an exact retained process only after exit or definitive loss proof.
|
|
22364
23463
|
* The process row, parent admission, non-TTL holder, and lease count transition
|
|
22365
23464
|
* commit atomically. Duplicate identical proof is idempotent; conflicting proof
|
|
@@ -22371,6 +23470,8 @@ export async function settleRetainedProcess(
|
|
|
22371
23470
|
workspaceId: string;
|
|
22372
23471
|
sessionId: string;
|
|
22373
23472
|
processId: string;
|
|
23473
|
+
expected: SandboxRetainedProcessIdentity;
|
|
23474
|
+
reconciliationClaimId?: string;
|
|
22374
23475
|
outcome: "exited" | "lost";
|
|
22375
23476
|
exitCode?: number | null;
|
|
22376
23477
|
reason: string;
|
|
@@ -22411,6 +23512,12 @@ export async function settleRetainedProcess(
|
|
|
22411
23512
|
"Retained process settlement did not match a durable process",
|
|
22412
23513
|
);
|
|
22413
23514
|
}
|
|
23515
|
+
if (!retainedProcessMatchesSettlementIdentity(process, input.expected)) {
|
|
23516
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23517
|
+
"process_fenced",
|
|
23518
|
+
"Retained process settlement did not match the copied durable identity",
|
|
23519
|
+
);
|
|
23520
|
+
}
|
|
22414
23521
|
if (process.state !== "active") {
|
|
22415
23522
|
if (
|
|
22416
23523
|
process.state !== input.outcome ||
|
|
@@ -22433,6 +23540,27 @@ export async function settleRetainedProcess(
|
|
|
22433
23540
|
);
|
|
22434
23541
|
return { settled: false, process: mapRetainedProcess(process) };
|
|
22435
23542
|
}
|
|
23543
|
+
if (
|
|
23544
|
+
input.reconciliationClaimId !== undefined &&
|
|
23545
|
+
process.reconcileClaimId !== input.reconciliationClaimId
|
|
23546
|
+
) {
|
|
23547
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23548
|
+
"process_fenced",
|
|
23549
|
+
"Retained process settlement reconciliation claim was lost or superseded",
|
|
23550
|
+
);
|
|
23551
|
+
}
|
|
23552
|
+
const durableProof = retainedProcessReconciliationProof(mapRetainedProcess(process));
|
|
23553
|
+
if (
|
|
23554
|
+
durableProof &&
|
|
23555
|
+
(durableProof.outcome !== input.outcome ||
|
|
23556
|
+
durableProof.exitCode !== exitCode ||
|
|
23557
|
+
durableProof.reason !== reason)
|
|
23558
|
+
) {
|
|
23559
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23560
|
+
"process_fenced",
|
|
23561
|
+
"Retained process settlement conflicts with checkpointed provider proof",
|
|
23562
|
+
);
|
|
23563
|
+
}
|
|
22436
23564
|
const admissions = await tx.execute<AdmissionIdentityRow>(sql`
|
|
22437
23565
|
select * from sandbox_workspace_mutation_admissions
|
|
22438
23566
|
where id = ${process.parentAdmissionId}
|
|
@@ -22440,6 +23568,13 @@ export async function settleRetainedProcess(
|
|
|
22440
23568
|
and workspace_id = ${input.workspaceId}
|
|
22441
23569
|
and session_id = ${input.sessionId}
|
|
22442
23570
|
and lease_id = ${process.leaseId}
|
|
23571
|
+
and sandbox_group_id = ${process.sandboxGroupId}
|
|
23572
|
+
and lease_epoch = ${process.leaseEpoch}
|
|
23573
|
+
and provider_backend = ${process.providerBackend}
|
|
23574
|
+
and provider_instance_id = ${process.providerInstanceId}
|
|
23575
|
+
and route_kind = ${process.routeKind}
|
|
23576
|
+
and route_target_id is not distinct from ${process.routeTargetId}
|
|
23577
|
+
and route_epoch = ${process.routeEpoch}
|
|
22443
23578
|
and provider_outcome = 'retained'
|
|
22444
23579
|
and settled_at is null
|
|
22445
23580
|
for update
|
|
@@ -22450,6 +23585,23 @@ export async function settleRetainedProcess(
|
|
|
22450
23585
|
"Retained process parent admission is not open",
|
|
22451
23586
|
);
|
|
22452
23587
|
}
|
|
23588
|
+
const leases = await tx.execute<LeaseRow>(sql`
|
|
23589
|
+
select * from sandbox_leases
|
|
23590
|
+
where id = ${process.leaseId}
|
|
23591
|
+
and account_id = ${input.accountId}
|
|
23592
|
+
and workspace_id = ${input.workspaceId}
|
|
23593
|
+
and sandbox_group_id = ${process.sandboxGroupId}
|
|
23594
|
+
and lease_epoch = ${process.leaseEpoch}
|
|
23595
|
+
and backend = ${process.providerBackend}
|
|
23596
|
+
and instance_id = ${process.providerInstanceId}
|
|
23597
|
+
for update
|
|
23598
|
+
`);
|
|
23599
|
+
if (!leases[0]) {
|
|
23600
|
+
throw new SandboxWorkspaceMutationFencedError(
|
|
23601
|
+
"lease_fenced",
|
|
23602
|
+
"Retained process settlement cannot mutate a successor lease identity",
|
|
23603
|
+
);
|
|
23604
|
+
}
|
|
22453
23605
|
await tx
|
|
22454
23606
|
.update(schema.sandboxPtySessions)
|
|
22455
23607
|
.set({ status: "closed", closedAt: new Date() })
|
|
@@ -22466,6 +23618,12 @@ export async function settleRetainedProcess(
|
|
|
22466
23618
|
exitCode,
|
|
22467
23619
|
settlementReason: reason,
|
|
22468
23620
|
settledAt: new Date(),
|
|
23621
|
+
reconcileClaimId: null,
|
|
23622
|
+
reconcileClaimedAt: null,
|
|
23623
|
+
lastReconcileOutcome:
|
|
23624
|
+
input.reconciliationClaimId === undefined
|
|
23625
|
+
? `owner_settled_${input.outcome}`
|
|
23626
|
+
: `reconciled_${input.outcome}`,
|
|
22469
23627
|
})
|
|
22470
23628
|
.where(
|
|
22471
23629
|
and(
|
|
@@ -22490,6 +23648,8 @@ export async function settleRetainedProcess(
|
|
|
22490
23648
|
await tx.execute(sql`
|
|
22491
23649
|
delete from sandbox_lease_holders
|
|
22492
23650
|
where lease_id = ${process.leaseId}
|
|
23651
|
+
and account_id = ${input.accountId}
|
|
23652
|
+
and workspace_id = ${input.workspaceId}
|
|
22493
23653
|
and kind = 'process' and holder_id = ${process.holderId}
|
|
22494
23654
|
`);
|
|
22495
23655
|
const [counts] = await tx.execute<{
|
|
@@ -22518,6 +23678,10 @@ export async function settleRetainedProcess(
|
|
|
22518
23678
|
}
|
|
22519
23679
|
updated_at = now()
|
|
22520
23680
|
where id = ${process.leaseId}
|
|
23681
|
+
and sandbox_group_id = ${process.sandboxGroupId}
|
|
23682
|
+
and lease_epoch = ${process.leaseEpoch}
|
|
23683
|
+
and backend = ${process.providerBackend}
|
|
23684
|
+
and instance_id = ${process.providerInstanceId}
|
|
22521
23685
|
`);
|
|
22522
23686
|
return { settled: true, process: mapRetainedProcess(updated) };
|
|
22523
23687
|
}),
|
|
@@ -27521,7 +28685,7 @@ export async function initializeSessionStartAtomically(
|
|
|
27521
28685
|
if (!goal) throw new Error("Failed to create initial session goal");
|
|
27522
28686
|
}
|
|
27523
28687
|
|
|
27524
|
-
|
|
28688
|
+
const existingUserEvents = await tx
|
|
27525
28689
|
.select()
|
|
27526
28690
|
.from(schema.sessionEvents)
|
|
27527
28691
|
.where(
|
|
@@ -27533,6 +28697,7 @@ export async function initializeSessionStartAtomically(
|
|
|
27533
28697
|
)
|
|
27534
28698
|
.orderBy(asc(schema.sessionEvents.sequence))
|
|
27535
28699
|
.limit(1);
|
|
28700
|
+
let userEvent: typeof schema.sessionEvents.$inferSelect | undefined = existingUserEvents[0];
|
|
27536
28701
|
let sequence = session.lastSequence;
|
|
27537
28702
|
const insertedEvents: Array<typeof schema.sessionEvents.$inferSelect> = [];
|
|
27538
28703
|
const runnable = effectiveControl.state === "active";
|
|
@@ -27732,7 +28897,7 @@ export async function initializeSessionStartAtomically(
|
|
|
27732
28897
|
tx as unknown as Database,
|
|
27733
28898
|
input.consumeNewSessionDraft.subjectId,
|
|
27734
28899
|
);
|
|
27735
|
-
await
|
|
28900
|
+
await seedNewSessionDraftInTransaction(tx as unknown as Database, {
|
|
27736
28901
|
workspaceId: input.workspaceId,
|
|
27737
28902
|
subjectId: input.consumeNewSessionDraft.subjectId,
|
|
27738
28903
|
expectedRevision: input.consumeNewSessionDraft.expectedRevision,
|
|
@@ -29568,7 +30733,7 @@ export async function peekSessionWork(
|
|
|
29568
30733
|
if (
|
|
29569
30734
|
latestInterruption &&
|
|
29570
30735
|
latestInterruption.quiescedAt === null &&
|
|
29571
|
-
|
|
30736
|
+
latestInterruption.interruptionState === "settled"
|
|
29572
30737
|
) {
|
|
29573
30738
|
return {
|
|
29574
30739
|
kind: "cancellation-wait",
|
|
@@ -32894,6 +34059,9 @@ function sessionEventTypesAdvanceActivity(inputs: ReadonlyArray<{ type: string }
|
|
|
32894
34059
|
function sessionMutationAdvancesActivity(update: {
|
|
32895
34060
|
resources?: ResourceRef[];
|
|
32896
34061
|
tools?: ToolRef[];
|
|
34062
|
+
toolPolicy?: SessionToolPolicy;
|
|
34063
|
+
toolPolicyVersion?: number;
|
|
34064
|
+
expectedToolPolicyVersion?: number;
|
|
32897
34065
|
model?: string;
|
|
32898
34066
|
metadata?: Record<string, unknown>;
|
|
32899
34067
|
status?: SessionStatus;
|
|
@@ -33482,6 +34650,7 @@ type LockedSessionUpdateContext = {
|
|
|
33482
34650
|
requireApproval: SessionMcpApprovalPolicy,
|
|
33483
34651
|
) => Promise<UpdateSessionMcpApprovalPolicyResult>;
|
|
33484
34652
|
listPendingSessionTurns: () => Promise<SessionTurn[]>;
|
|
34653
|
+
getLockedSession: (sessionId: string) => Promise<Session | null>;
|
|
33485
34654
|
};
|
|
33486
34655
|
|
|
33487
34656
|
type LockedSessionUpdateResult = {
|
|
@@ -33489,6 +34658,9 @@ type LockedSessionUpdateResult = {
|
|
|
33489
34658
|
update?: {
|
|
33490
34659
|
resources?: ResourceRef[];
|
|
33491
34660
|
tools?: ToolRef[];
|
|
34661
|
+
toolPolicy?: SessionToolPolicy;
|
|
34662
|
+
toolPolicyVersion?: number;
|
|
34663
|
+
expectedToolPolicyVersion?: number;
|
|
33492
34664
|
model?: string;
|
|
33493
34665
|
metadata?: Record<string, unknown>;
|
|
33494
34666
|
status?: SessionStatus;
|
|
@@ -33504,18 +34676,39 @@ export async function appendSessionEventsWithLockedSessionUpdate(
|
|
|
33504
34676
|
session: Session,
|
|
33505
34677
|
context: LockedSessionUpdateContext,
|
|
33506
34678
|
) => LockedSessionUpdateResult | Promise<LockedSessionUpdateResult>,
|
|
34679
|
+
options: { lockParentSession?: boolean } = {},
|
|
33507
34680
|
): Promise<SessionEvent[]> {
|
|
33508
34681
|
return await withWorkspaceRls(
|
|
33509
34682
|
db,
|
|
33510
34683
|
workspaceId,
|
|
33511
34684
|
async (scopedDb) =>
|
|
33512
34685
|
await scopedDb.transaction(async (tx) => {
|
|
33513
|
-
const
|
|
34686
|
+
const firstLocks = await lockSessionEventWriteRows(tx as unknown as Database, {
|
|
33514
34687
|
workspaceId,
|
|
33515
34688
|
controlLock: "share",
|
|
33516
34689
|
sessionIds: [sessionId],
|
|
33517
34690
|
});
|
|
33518
|
-
const
|
|
34691
|
+
const firstSessionRow = firstLocks.sessions.find((row) => row.id === sessionId);
|
|
34692
|
+
if (!firstSessionRow) {
|
|
34693
|
+
throw new Error(`Session not found: ${sessionId}`);
|
|
34694
|
+
}
|
|
34695
|
+
// Child-policy updates must serialize with a concurrent parent-policy
|
|
34696
|
+
// update. The target lock establishes the parent id, then the second
|
|
34697
|
+
// call acquires the complete UUID-ordered set under the already-held
|
|
34698
|
+
// workspace/control prefix.
|
|
34699
|
+
const lockSessionIds = options.lockParentSession
|
|
34700
|
+
? [sessionId, firstSessionRow.parentSessionId].filter((id): id is string => Boolean(id))
|
|
34701
|
+
: [sessionId];
|
|
34702
|
+
const locks =
|
|
34703
|
+
lockSessionIds.length === 1
|
|
34704
|
+
? firstLocks
|
|
34705
|
+
: await lockSessionEventWriteRows(tx as unknown as Database, {
|
|
34706
|
+
workspaceId,
|
|
34707
|
+
controlLock: "already_locked",
|
|
34708
|
+
workspaceLock: "already_locked",
|
|
34709
|
+
sessionIds: lockSessionIds,
|
|
34710
|
+
});
|
|
34711
|
+
const sessionRow = locks.sessions.find((row) => row.id === sessionId);
|
|
33519
34712
|
if (!sessionRow) {
|
|
33520
34713
|
throw new Error(`Session not found: ${sessionId}`);
|
|
33521
34714
|
}
|
|
@@ -33554,6 +34747,18 @@ export async function appendSessionEventsWithLockedSessionUpdate(
|
|
|
33554
34747
|
.orderBy(asc(schema.sessionTurns.position), asc(schema.sessionTurns.createdAt));
|
|
33555
34748
|
return rows.map(mapSessionTurn);
|
|
33556
34749
|
},
|
|
34750
|
+
getLockedSession: async (lockedSessionId) => {
|
|
34751
|
+
const row = locks.sessions.find((candidate) => candidate.id === lockedSessionId);
|
|
34752
|
+
return row
|
|
34753
|
+
? await mapSessionWithControl(
|
|
34754
|
+
tx as unknown as Database,
|
|
34755
|
+
row,
|
|
34756
|
+
[],
|
|
34757
|
+
undefined,
|
|
34758
|
+
locks.control ?? undefined,
|
|
34759
|
+
)
|
|
34760
|
+
: null;
|
|
34761
|
+
},
|
|
33557
34762
|
});
|
|
33558
34763
|
if (built.events.length === 0) {
|
|
33559
34764
|
return [];
|
|
@@ -33591,12 +34796,16 @@ export async function appendSessionEventsWithLockedSessionUpdate(
|
|
|
33591
34796
|
const update = built.update ?? {};
|
|
33592
34797
|
const advancesActivity =
|
|
33593
34798
|
sessionMutationAdvancesActivity(update) || sessionEventTypesAdvanceActivity(values);
|
|
33594
|
-
await tx
|
|
34799
|
+
const updated = await tx
|
|
33595
34800
|
.update(schema.sessions)
|
|
33596
34801
|
.set({
|
|
33597
34802
|
lastSequence: sequence,
|
|
33598
34803
|
...(update.resources !== undefined ? { resources: update.resources } : {}),
|
|
33599
34804
|
...(update.tools !== undefined ? { tools: update.tools } : {}),
|
|
34805
|
+
...(update.toolPolicy !== undefined ? { toolPolicy: update.toolPolicy } : {}),
|
|
34806
|
+
...(update.toolPolicyVersion !== undefined
|
|
34807
|
+
? { toolPolicyVersion: update.toolPolicyVersion }
|
|
34808
|
+
: {}),
|
|
33600
34809
|
...(update.model !== undefined ? { model: update.model } : {}),
|
|
33601
34810
|
...(update.metadata !== undefined ? { metadata: update.metadata } : {}),
|
|
33602
34811
|
...(update.status !== undefined ? { status: update.status } : {}),
|
|
@@ -33604,8 +34813,27 @@ export async function appendSessionEventsWithLockedSessionUpdate(
|
|
|
33604
34813
|
...(advancesActivity ? { updatedAt: now } : {}),
|
|
33605
34814
|
})
|
|
33606
34815
|
.where(
|
|
33607
|
-
and(
|
|
34816
|
+
and(
|
|
34817
|
+
eq(schema.sessions.workspaceId, workspaceId),
|
|
34818
|
+
eq(schema.sessions.id, sessionId),
|
|
34819
|
+
...(update.expectedToolPolicyVersion !== undefined
|
|
34820
|
+
? [eq(schema.sessions.toolPolicyVersion, update.expectedToolPolicyVersion)]
|
|
34821
|
+
: []),
|
|
34822
|
+
),
|
|
34823
|
+
)
|
|
34824
|
+
.returning({ id: schema.sessions.id });
|
|
34825
|
+
if (updated.length === 0) {
|
|
34826
|
+
const [current] = await tx
|
|
34827
|
+
.select({ toolPolicyVersion: schema.sessions.toolPolicyVersion })
|
|
34828
|
+
.from(schema.sessions)
|
|
34829
|
+
.where(
|
|
34830
|
+
and(eq(schema.sessions.workspaceId, workspaceId), eq(schema.sessions.id, sessionId)),
|
|
34831
|
+
)
|
|
34832
|
+
.limit(1);
|
|
34833
|
+
throw new SessionToolPolicyVersionConflictError(
|
|
34834
|
+
Number(current?.toolPolicyVersion ?? update.expectedToolPolicyVersion ?? 1),
|
|
33608
34835
|
);
|
|
34836
|
+
}
|
|
33609
34837
|
return inserted.map(mapEvent);
|
|
33610
34838
|
}),
|
|
33611
34839
|
);
|
|
@@ -33666,6 +34894,7 @@ function mapSession(
|
|
|
33666
34894
|
mode: "legacy",
|
|
33667
34895
|
inheritedFromSessionId: null,
|
|
33668
34896
|
},
|
|
34897
|
+
toolPolicyVersion: Number(row.toolPolicyVersion ?? 1),
|
|
33669
34898
|
metadata: row.metadata,
|
|
33670
34899
|
createdBy: initiatorFromStorage(
|
|
33671
34900
|
row.createdByKind,
|
|
@@ -34270,10 +35499,17 @@ function mapGitHubInstallation(
|
|
|
34270
35499
|
accountId: row.accountId,
|
|
34271
35500
|
workspaceId: row.workspaceId,
|
|
34272
35501
|
installationId: row.installationId,
|
|
35502
|
+
githubAccountId: row.githubAccountId,
|
|
34273
35503
|
accountLogin: row.accountLogin,
|
|
34274
35504
|
accountType: row.accountType,
|
|
34275
35505
|
repositoryScope: row.repositoryScope as GitHubRepositoryScope,
|
|
34276
35506
|
linkedBySubjectId: row.linkedBySubjectId,
|
|
35507
|
+
githubActorId: row.githubActorId,
|
|
35508
|
+
githubActorLogin: row.githubActorLogin,
|
|
35509
|
+
authorityKind: row.authorityKind as GitHubInstallationAuthorityKind | null,
|
|
35510
|
+
authorityCheckedAt: row.authorityCheckedAt?.toISOString() ?? null,
|
|
35511
|
+
authorityExpiresAt: row.authorityExpiresAt?.toISOString() ?? null,
|
|
35512
|
+
authorityNonce: row.authorityNonce,
|
|
34277
35513
|
createdAt: row.createdAt.toISOString(),
|
|
34278
35514
|
updatedAt: row.updatedAt.toISOString(),
|
|
34279
35515
|
};
|