@opengeni/db 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-O3D7DABC.js → chunk-B22X3IEZ.js} +369 -49
- package/dist/chunk-B22X3IEZ.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +15133 -13199
- package/dist/index.js.map +1 -1
- package/dist/provision-roles.d.ts +475 -211
- package/dist/{schema-Dp2MbBxx.d.ts → schema-BN5mB9xZ.d.ts} +1365 -256
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +13 -3
- package/drizzle/0063_session_control_mega_foundation.sql +1324 -0
- package/package.json +3 -7
- package/src/index.ts +1474 -2898
- package/src/schema.ts +391 -60
- package/src/session-control.ts +1759 -0
- package/src/session-queue-commands.ts +1753 -0
- package/src/session-tool-call-settlement.ts +269 -0
- package/dist/chunk-O3D7DABC.js.map +0 -1
- package/src/session-control-cutover-audit.ts +0 -1203
|
@@ -1,1203 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import type postgres from "postgres";
|
|
3
|
-
|
|
4
|
-
export const SESSION_CONTROL_CUTOVER_CONTRACT = "opengeni/session-control-cutover/v3" as const;
|
|
5
|
-
|
|
6
|
-
export type CutoverPhase = "baseline" | "migrated" | "final";
|
|
7
|
-
|
|
8
|
-
export type CutoverTurn = {
|
|
9
|
-
id: string;
|
|
10
|
-
triggerEventId: string;
|
|
11
|
-
temporalWorkflowId: string;
|
|
12
|
-
status: string;
|
|
13
|
-
source: string;
|
|
14
|
-
position: number;
|
|
15
|
-
version: number | null;
|
|
16
|
-
executionGeneration: number | null;
|
|
17
|
-
activeAttemptId: string | null;
|
|
18
|
-
startedAt: string | null;
|
|
19
|
-
finishedAt: string | null;
|
|
20
|
-
hasStartedEvidence: boolean;
|
|
21
|
-
};
|
|
22
|
-
|
|
23
|
-
export type CutoverGoal = {
|
|
24
|
-
id: string;
|
|
25
|
-
status: string;
|
|
26
|
-
version: number;
|
|
27
|
-
autoContinuations: number;
|
|
28
|
-
noProgressStreak: number;
|
|
29
|
-
lastContinuationTurnId: string | null;
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
export type CutoverRelationProof = {
|
|
33
|
-
count: number;
|
|
34
|
-
maxOrdinal: number | null;
|
|
35
|
-
stableSha256: string;
|
|
36
|
-
identitySha256: string;
|
|
37
|
-
preservedCount: number;
|
|
38
|
-
preservedThroughOrdinal: number | null;
|
|
39
|
-
preservedStableSha256: string;
|
|
40
|
-
preservedIdentitySha256: string;
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
export type CutoverRunState = {
|
|
44
|
-
id: string;
|
|
45
|
-
turnId: string | null;
|
|
46
|
-
stateVersion: number;
|
|
47
|
-
schemaVersion: string | null;
|
|
48
|
-
pendingApprovalCount: number;
|
|
49
|
-
frozenCodexCredentialId: string | null;
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
export type CutoverCapacityWaiter = {
|
|
53
|
-
id: string;
|
|
54
|
-
goalId: string;
|
|
55
|
-
blockedTurnId: string;
|
|
56
|
-
workflowId: string;
|
|
57
|
-
generation: number;
|
|
58
|
-
status: string;
|
|
59
|
-
goalVersion: number;
|
|
60
|
-
controlGeneration: number;
|
|
61
|
-
wakeRevision: number;
|
|
62
|
-
observedWakeRevision: number;
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
export type CutoverSandboxLease = {
|
|
66
|
-
id: string;
|
|
67
|
-
sandboxGroupId: string;
|
|
68
|
-
liveness: string;
|
|
69
|
-
instanceId: string | null;
|
|
70
|
-
backend: string;
|
|
71
|
-
leaseEpoch: number;
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
export type CutoverSystemUpdate = {
|
|
75
|
-
id: string;
|
|
76
|
-
kind: string;
|
|
77
|
-
sourceId: string;
|
|
78
|
-
dedupeKey: string;
|
|
79
|
-
state: string;
|
|
80
|
-
deliveredTurnId: string | null;
|
|
81
|
-
};
|
|
82
|
-
|
|
83
|
-
export type CutoverPendingToolCall = {
|
|
84
|
-
id: string;
|
|
85
|
-
turnId: string;
|
|
86
|
-
executionGeneration: number;
|
|
87
|
-
attemptId: string;
|
|
88
|
-
callId: string;
|
|
89
|
-
callType: string;
|
|
90
|
-
resultRecorded: boolean;
|
|
91
|
-
};
|
|
92
|
-
|
|
93
|
-
export type CutoverSession = {
|
|
94
|
-
id: string;
|
|
95
|
-
accountId: string;
|
|
96
|
-
workspaceId: string;
|
|
97
|
-
status: string;
|
|
98
|
-
temporalWorkflowId: string | null;
|
|
99
|
-
activeTurnId: string | null;
|
|
100
|
-
sandboxGroupId: string;
|
|
101
|
-
activeSandboxId: string | null;
|
|
102
|
-
activeEpoch: number;
|
|
103
|
-
controlState: string | null;
|
|
104
|
-
controlGeneration: number | null;
|
|
105
|
-
workspaceRunExceptionGeneration: number | null;
|
|
106
|
-
queueVersion: number | null;
|
|
107
|
-
queueHeadPosition: number | null;
|
|
108
|
-
queueTailPosition: number | null;
|
|
109
|
-
turns: CutoverTurn[];
|
|
110
|
-
goals: CutoverGoal[];
|
|
111
|
-
historyProof: CutoverRelationProof;
|
|
112
|
-
eventProof: CutoverRelationProof;
|
|
113
|
-
runStates: CutoverRunState[];
|
|
114
|
-
capacityWaiters: CutoverCapacityWaiter[];
|
|
115
|
-
sandboxLeases: CutoverSandboxLease[];
|
|
116
|
-
systemUpdates: CutoverSystemUpdate[];
|
|
117
|
-
pendingToolCalls: CutoverPendingToolCall[];
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
export type CutoverWorkspace = {
|
|
121
|
-
id: string;
|
|
122
|
-
accountId: string;
|
|
123
|
-
inferenceState: string | null;
|
|
124
|
-
inferenceGeneration: number | null;
|
|
125
|
-
};
|
|
126
|
-
|
|
127
|
-
export type CutoverInvariantCounts = {
|
|
128
|
-
multipleCurrentInferences: number;
|
|
129
|
-
queuedMachineTurns: number;
|
|
130
|
-
activeOpaqueCompactionItems: number;
|
|
131
|
-
invalidActiveTurnPointers: number;
|
|
132
|
-
runningTurnsWithoutAttempt: number;
|
|
133
|
-
attemptOwnedNonRunningTurns: number;
|
|
134
|
-
duplicateCurrentUsageSources: number;
|
|
135
|
-
invalidDuplicateUsageAssociations: number;
|
|
136
|
-
enrollableSessions: number;
|
|
137
|
-
workerDeathRedispatchExhausted: number;
|
|
138
|
-
};
|
|
139
|
-
|
|
140
|
-
export type SessionControlCutoverSnapshot = {
|
|
141
|
-
contractVersion: typeof SESSION_CONTROL_CUTOVER_CONTRACT;
|
|
142
|
-
phase: CutoverPhase;
|
|
143
|
-
capturedAt: string;
|
|
144
|
-
schemaMigrations: string[];
|
|
145
|
-
workspaces: CutoverWorkspace[];
|
|
146
|
-
sessions: CutoverSession[];
|
|
147
|
-
proofBaselineSha256: string | null;
|
|
148
|
-
invariants: CutoverInvariantCounts;
|
|
149
|
-
identityCount: number;
|
|
150
|
-
sha256: string;
|
|
151
|
-
};
|
|
152
|
-
|
|
153
|
-
export type CutoverReconciliation = {
|
|
154
|
-
contractVersion: typeof SESSION_CONTROL_CUTOVER_CONTRACT;
|
|
155
|
-
mode: "migration" | "final-fate";
|
|
156
|
-
baselineSha256: string;
|
|
157
|
-
observedSha256: string;
|
|
158
|
-
ok: boolean;
|
|
159
|
-
errors: string[];
|
|
160
|
-
fateCounts: Record<string, number>;
|
|
161
|
-
sha256: string;
|
|
162
|
-
};
|
|
163
|
-
|
|
164
|
-
type Row = Record<string, unknown>;
|
|
165
|
-
|
|
166
|
-
function stableValue(value: unknown): unknown {
|
|
167
|
-
if (Array.isArray(value)) return value.map(stableValue);
|
|
168
|
-
if (value !== null && typeof value === "object") {
|
|
169
|
-
return Object.fromEntries(
|
|
170
|
-
Object.entries(value as Record<string, unknown>)
|
|
171
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
|
172
|
-
.map(([key, entry]) => [key, stableValue(entry)]),
|
|
173
|
-
);
|
|
174
|
-
}
|
|
175
|
-
return value;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
export function stableJson(value: unknown): string {
|
|
179
|
-
return JSON.stringify(stableValue(value));
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
export function sha256(value: unknown): string {
|
|
183
|
-
return createHash("sha256").update(stableJson(value)).digest("hex");
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
function stringValue(value: unknown, name: string): string {
|
|
187
|
-
if (typeof value !== "string") throw new Error(`${name} is not a string`);
|
|
188
|
-
return value;
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
function nullableString(value: unknown, name: string): string | null {
|
|
192
|
-
if (value === null || value === undefined) return null;
|
|
193
|
-
return stringValue(value, name);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
function nullableTimestamp(value: unknown, name: string): string | null {
|
|
197
|
-
if (value === null || value === undefined) return null;
|
|
198
|
-
if (value instanceof Date) return value.toISOString();
|
|
199
|
-
return stringValue(value, name);
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
function numberValue(value: unknown, name: string): number {
|
|
203
|
-
const parsed = typeof value === "number" ? value : Number(value);
|
|
204
|
-
if (!Number.isFinite(parsed)) throw new Error(`${name} is not numeric`);
|
|
205
|
-
return parsed;
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
function nullableNumber(value: unknown, name: string): number | null {
|
|
209
|
-
if (value === null || value === undefined) return null;
|
|
210
|
-
return numberValue(value, name);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
function booleanValue(value: unknown): boolean {
|
|
214
|
-
return value === true || value === "true";
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
async function tableExists(sql: postgres.Sql, table: string): Promise<boolean> {
|
|
218
|
-
const rows = await sql<Row[]>`
|
|
219
|
-
select exists (
|
|
220
|
-
select 1 from information_schema.tables
|
|
221
|
-
where table_schema = current_schema() and table_name = ${table}
|
|
222
|
-
) as present`;
|
|
223
|
-
return booleanValue(rows[0]?.present);
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
async function columnExists(sql: postgres.Sql, table: string, column: string): Promise<boolean> {
|
|
227
|
-
const rows = await sql<Row[]>`
|
|
228
|
-
select exists (
|
|
229
|
-
select 1 from information_schema.columns
|
|
230
|
-
where table_schema = current_schema() and table_name = ${table} and column_name = ${column}
|
|
231
|
-
) as present`;
|
|
232
|
-
return booleanValue(rows[0]?.present);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
async function functionExists(sql: postgres.Sql, signature: string): Promise<boolean> {
|
|
236
|
-
const rows = await sql<Row[]>`select to_regprocedure(${signature}) is not null as present`;
|
|
237
|
-
return booleanValue(rows[0]?.present);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
async function unsafeRows(sql: postgres.Sql, query: string): Promise<Row[]> {
|
|
241
|
-
return (await sql.unsafe<Row[]>(query)) as unknown as Row[];
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
/**
|
|
245
|
-
* The cutover snapshot crosses every workspace on purpose. FORCE RLS makes a
|
|
246
|
-
* normal runtime role return an empty set rather than throwing, which could
|
|
247
|
-
* otherwise produce a vacuous green reconciliation. Require an explicitly
|
|
248
|
-
* privileged operator connection before reading any identity table.
|
|
249
|
-
*/
|
|
250
|
-
export async function assertSessionControlCutoverAuditAuthority(sql: postgres.Sql): Promise<void> {
|
|
251
|
-
const rows = await unsafeRows(
|
|
252
|
-
sql,
|
|
253
|
-
`select current_user as role_name, rolsuper, rolbypassrls
|
|
254
|
-
from pg_roles where rolname = current_user`,
|
|
255
|
-
);
|
|
256
|
-
const role = rows[0];
|
|
257
|
-
if (!role || (!booleanValue(role.rolsuper) && !booleanValue(role.rolbypassrls))) {
|
|
258
|
-
throw new Error(
|
|
259
|
-
`session-control cutover audit requires a superuser or BYPASSRLS role; ${String(role?.role_name ?? "unknown")} is not authorized`,
|
|
260
|
-
);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
function groupBySession<T extends { sessionId: string }>(rows: T[]): Map<string, T[]> {
|
|
265
|
-
const grouped = new Map<string, T[]>();
|
|
266
|
-
for (const row of rows) {
|
|
267
|
-
const values = grouped.get(row.sessionId) ?? [];
|
|
268
|
-
values.push(row);
|
|
269
|
-
grouped.set(row.sessionId, values);
|
|
270
|
-
}
|
|
271
|
-
return grouped;
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
type RelationProofAccumulator = {
|
|
275
|
-
count: number;
|
|
276
|
-
maxOrdinal: number | null;
|
|
277
|
-
stable: ReturnType<typeof createHash>;
|
|
278
|
-
identity: ReturnType<typeof createHash>;
|
|
279
|
-
preservedCount: number;
|
|
280
|
-
preservedStable: ReturnType<typeof createHash>;
|
|
281
|
-
preservedIdentity: ReturnType<typeof createHash>;
|
|
282
|
-
};
|
|
283
|
-
|
|
284
|
-
function collectProofGarbage(): void {
|
|
285
|
-
if (typeof Bun !== "undefined") Bun.gc(true);
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
function updateProofHash(hash: ReturnType<typeof createHash>, fields: readonly string[]): void {
|
|
289
|
-
hash.update(String(fields.length));
|
|
290
|
-
hash.update(":");
|
|
291
|
-
for (const field of fields) {
|
|
292
|
-
hash.update(String(Buffer.byteLength(field, "utf8")));
|
|
293
|
-
hash.update(":");
|
|
294
|
-
hash.update(field);
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
function newProofAccumulator(): RelationProofAccumulator {
|
|
299
|
-
return {
|
|
300
|
-
count: 0,
|
|
301
|
-
maxOrdinal: null,
|
|
302
|
-
stable: createHash("sha256"),
|
|
303
|
-
identity: createHash("sha256"),
|
|
304
|
-
preservedCount: 0,
|
|
305
|
-
preservedStable: createHash("sha256"),
|
|
306
|
-
preservedIdentity: createHash("sha256"),
|
|
307
|
-
};
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
async function captureRelationProofs(
|
|
311
|
-
sql: postgres.Sql,
|
|
312
|
-
cursorQuery: string,
|
|
313
|
-
expectedFieldCount: number,
|
|
314
|
-
stableFieldIndexes: readonly number[],
|
|
315
|
-
reference: SessionControlCutoverSnapshot | undefined,
|
|
316
|
-
referenceProof: (session: CutoverSession) => CutoverRelationProof,
|
|
317
|
-
): Promise<Map<string, CutoverRelationProof>> {
|
|
318
|
-
const referenceBoundaries = new Map(
|
|
319
|
-
(reference?.sessions ?? []).map((session) => [session.id, referenceProof(session).maxOrdinal]),
|
|
320
|
-
);
|
|
321
|
-
const accumulators = new Map<string, RelationProofAccumulator>();
|
|
322
|
-
let rowsSinceGarbageCollection = 0;
|
|
323
|
-
|
|
324
|
-
const consumeFields = (fields: unknown): void => {
|
|
325
|
-
if (!Array.isArray(fields) || fields.some((field) => typeof field !== "string")) {
|
|
326
|
-
throw new Error("relation proof cursor row contains invalid fields");
|
|
327
|
-
}
|
|
328
|
-
if (fields.length !== expectedFieldCount) {
|
|
329
|
-
throw new Error(
|
|
330
|
-
`relation proof cursor row has ${fields.length} fields; expected ${expectedFieldCount}`,
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
const sessionId = fields[0];
|
|
334
|
-
const id = fields[1];
|
|
335
|
-
const ordinalText = fields[2];
|
|
336
|
-
if (!sessionId || !id || !ordinalText) {
|
|
337
|
-
throw new Error("relation proof cursor row is missing its identity or ordinal");
|
|
338
|
-
}
|
|
339
|
-
const ordinal = numberValue(ordinalText, "relation.ordinal");
|
|
340
|
-
const stableFields = stableFieldIndexes.map((index) => {
|
|
341
|
-
const field = fields[index];
|
|
342
|
-
if (field === undefined) {
|
|
343
|
-
throw new Error(`relation proof cursor row is missing stable field ${index}`);
|
|
344
|
-
}
|
|
345
|
-
return field;
|
|
346
|
-
});
|
|
347
|
-
let accumulator = accumulators.get(sessionId);
|
|
348
|
-
if (!accumulator) {
|
|
349
|
-
accumulator = newProofAccumulator();
|
|
350
|
-
// Bun can represent a split substring as a view into the COPY chunk that
|
|
351
|
-
// produced it. A Map key then pins that entire network chunk for the life
|
|
352
|
-
// of the audit. Flatten the one durable key per session into independent
|
|
353
|
-
// storage; transient lookup strings remain collectable with their chunk.
|
|
354
|
-
const durableSessionId = Buffer.from(sessionId, "utf8").toString("utf8");
|
|
355
|
-
accumulators.set(durableSessionId, accumulator);
|
|
356
|
-
}
|
|
357
|
-
accumulator.count += 1;
|
|
358
|
-
accumulator.maxOrdinal = Math.max(accumulator.maxOrdinal ?? ordinal, ordinal);
|
|
359
|
-
updateProofHash(accumulator.stable, stableFields);
|
|
360
|
-
updateProofHash(accumulator.identity, [id]);
|
|
361
|
-
const boundary = referenceBoundaries.get(sessionId);
|
|
362
|
-
const preservesRow = reference
|
|
363
|
-
? boundary !== undefined && boundary !== null && ordinal <= boundary
|
|
364
|
-
: true;
|
|
365
|
-
if (preservesRow) {
|
|
366
|
-
accumulator.preservedCount += 1;
|
|
367
|
-
updateProofHash(accumulator.preservedStable, stableFields);
|
|
368
|
-
updateProofHash(accumulator.preservedIdentity, [id]);
|
|
369
|
-
}
|
|
370
|
-
};
|
|
371
|
-
|
|
372
|
-
const cursor = sql.unsafe<Array<{ fields: string[] }>>(cursorQuery).cursor(10_000);
|
|
373
|
-
for await (const rows of cursor) {
|
|
374
|
-
for (const row of rows) {
|
|
375
|
-
consumeFields(row.fields);
|
|
376
|
-
}
|
|
377
|
-
rowsSinceGarbageCollection += rows.length;
|
|
378
|
-
if (rowsSinceGarbageCollection >= 100_000) {
|
|
379
|
-
// The server cursor and fixed batch size bound live row materialization.
|
|
380
|
-
// Periodic collection bounds transient decoding allocations independently
|
|
381
|
-
// of production cardinality without depending on COPY stream termination.
|
|
382
|
-
collectProofGarbage();
|
|
383
|
-
rowsSinceGarbageCollection = 0;
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
collectProofGarbage();
|
|
387
|
-
|
|
388
|
-
const proofs = new Map<string, CutoverRelationProof>();
|
|
389
|
-
const sessionIds = new Set([
|
|
390
|
-
...accumulators.keys(),
|
|
391
|
-
...(reference?.sessions.map((session) => session.id) ?? []),
|
|
392
|
-
]);
|
|
393
|
-
for (const sessionId of sessionIds) {
|
|
394
|
-
const accumulator = accumulators.get(sessionId) ?? newProofAccumulator();
|
|
395
|
-
proofs.set(sessionId, {
|
|
396
|
-
count: accumulator.count,
|
|
397
|
-
maxOrdinal: accumulator.maxOrdinal,
|
|
398
|
-
stableSha256: accumulator.stable.digest("hex"),
|
|
399
|
-
identitySha256: accumulator.identity.digest("hex"),
|
|
400
|
-
preservedCount: accumulator.preservedCount,
|
|
401
|
-
preservedThroughOrdinal: reference
|
|
402
|
-
? (referenceBoundaries.get(sessionId) ?? null)
|
|
403
|
-
: accumulator.maxOrdinal,
|
|
404
|
-
preservedStableSha256: accumulator.preservedStable.digest("hex"),
|
|
405
|
-
preservedIdentitySha256: accumulator.preservedIdentity.digest("hex"),
|
|
406
|
-
});
|
|
407
|
-
}
|
|
408
|
-
return proofs;
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
function emptyRelationProof(): CutoverRelationProof {
|
|
412
|
-
const digest = createHash("sha256").digest("hex");
|
|
413
|
-
return {
|
|
414
|
-
count: 0,
|
|
415
|
-
maxOrdinal: null,
|
|
416
|
-
stableSha256: digest,
|
|
417
|
-
identitySha256: digest,
|
|
418
|
-
preservedCount: 0,
|
|
419
|
-
preservedThroughOrdinal: null,
|
|
420
|
-
preservedStableSha256: digest,
|
|
421
|
-
preservedIdentitySha256: digest,
|
|
422
|
-
};
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
function idsEqual<T extends { id: string }>(before: T[], after: T[]): boolean {
|
|
426
|
-
if (before.length !== after.length) return false;
|
|
427
|
-
const afterIds = new Set(after.map((entry) => entry.id));
|
|
428
|
-
return before.every((entry) => afterIds.has(entry.id));
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
function isSubsequence(needle: string[], haystack: string[]): boolean {
|
|
432
|
-
let index = 0;
|
|
433
|
-
for (const value of haystack) {
|
|
434
|
-
if (value === needle[index]) index += 1;
|
|
435
|
-
}
|
|
436
|
-
return index === needle.length;
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
function expectedMigratedTurnStatus(turn: CutoverTurn): string {
|
|
440
|
-
if (turn.status === "running" || (turn.status === "queued" && turn.hasStartedEvidence)) {
|
|
441
|
-
return "recovering";
|
|
442
|
-
}
|
|
443
|
-
if (
|
|
444
|
-
turn.status === "queued" &&
|
|
445
|
-
(turn.source === "scheduled_task" || turn.source === "system" || turn.source === "goal")
|
|
446
|
-
) {
|
|
447
|
-
return "superseded";
|
|
448
|
-
}
|
|
449
|
-
return turn.status;
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
function humanQueue(turns: CutoverTurn[]): string[] {
|
|
453
|
-
return turns
|
|
454
|
-
.filter(
|
|
455
|
-
(turn) =>
|
|
456
|
-
turn.status === "queued" &&
|
|
457
|
-
!turn.hasStartedEvidence &&
|
|
458
|
-
(turn.source === "user" || turn.source === "api"),
|
|
459
|
-
)
|
|
460
|
-
.sort((a, b) => a.position - b.position || a.id.localeCompare(b.id))
|
|
461
|
-
.map((turn) => turn.id);
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
function increment(counter: Record<string, number>, key: string): void {
|
|
465
|
-
counter[key] = (counter[key] ?? 0) + 1;
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
export async function captureSessionControlCutoverSnapshot(
|
|
469
|
-
sql: postgres.Sql,
|
|
470
|
-
phase: CutoverPhase,
|
|
471
|
-
capturedAt = new Date().toISOString(),
|
|
472
|
-
reference?: SessionControlCutoverSnapshot,
|
|
473
|
-
): Promise<SessionControlCutoverSnapshot> {
|
|
474
|
-
if (phase === "baseline" && reference) {
|
|
475
|
-
throw new Error("a baseline capture cannot reference an earlier baseline");
|
|
476
|
-
}
|
|
477
|
-
if (phase !== "baseline") {
|
|
478
|
-
if (!reference || reference.contractVersion !== SESSION_CONTROL_CUTOVER_CONTRACT) {
|
|
479
|
-
throw new Error(`${phase} capture requires a v3 baseline proof`);
|
|
480
|
-
}
|
|
481
|
-
if (reference.phase !== "baseline") {
|
|
482
|
-
throw new Error(`${phase} capture reference is not a baseline`);
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
await assertSessionControlCutoverAuditAuthority(sql);
|
|
486
|
-
const hasControl = await columnExists(sql, "sessions", "control_state");
|
|
487
|
-
const hasAttempts = await columnExists(sql, "session_turns", "execution_generation");
|
|
488
|
-
const hasWorkspaceControl = await columnExists(sql, "workspaces", "inference_state");
|
|
489
|
-
const hasUpdates = await tableExists(sql, "session_system_updates");
|
|
490
|
-
const hasPendingTools = await tableExists(sql, "session_pending_tool_calls");
|
|
491
|
-
const hasUsageDuplicateAssociation = await columnExists(
|
|
492
|
-
sql,
|
|
493
|
-
"session_events",
|
|
494
|
-
"duplicate_of_event_id",
|
|
495
|
-
);
|
|
496
|
-
const hasEnrollableScan = await functionExists(
|
|
497
|
-
sql,
|
|
498
|
-
"opengeni_private.list_enrollable_sessions(integer)",
|
|
499
|
-
);
|
|
500
|
-
const hasFrozenCredential = await columnExists(
|
|
501
|
-
sql,
|
|
502
|
-
"agent_run_states",
|
|
503
|
-
"frozen_codex_credential_id",
|
|
504
|
-
);
|
|
505
|
-
|
|
506
|
-
const migrationRows = await unsafeRows(sql, `select name from schema_migrations order by name`);
|
|
507
|
-
const workspaceRows = await unsafeRows(
|
|
508
|
-
sql,
|
|
509
|
-
`select id, account_id,
|
|
510
|
-
${hasWorkspaceControl ? "inference_state" : "null::text as inference_state"},
|
|
511
|
-
${hasWorkspaceControl ? "inference_generation" : "null::integer as inference_generation"}
|
|
512
|
-
from workspaces order by id`,
|
|
513
|
-
);
|
|
514
|
-
const sessionRows = await unsafeRows(
|
|
515
|
-
sql,
|
|
516
|
-
`select id, account_id, workspace_id, status, temporal_workflow_id, active_turn_id,
|
|
517
|
-
sandbox_group_id, active_sandbox_id, active_epoch,
|
|
518
|
-
${hasControl ? "control_state" : "null::text as control_state"},
|
|
519
|
-
${hasControl ? "control_generation" : "null::integer as control_generation"},
|
|
520
|
-
${hasControl ? "workspace_run_exception_generation" : "null::integer as workspace_run_exception_generation"},
|
|
521
|
-
${hasControl ? "queue_version" : "null::integer as queue_version"},
|
|
522
|
-
${hasControl ? "queue_head_position" : "null::bigint as queue_head_position"},
|
|
523
|
-
${hasControl ? "queue_tail_position" : "null::bigint as queue_tail_position"}
|
|
524
|
-
from sessions order by id`,
|
|
525
|
-
);
|
|
526
|
-
const turnRows = await unsafeRows(
|
|
527
|
-
sql,
|
|
528
|
-
`select t.session_id, t.id, t.trigger_event_id, t.temporal_workflow_id, t.status, t.source,
|
|
529
|
-
t.position, ${hasAttempts ? "t.version" : "null::integer as version"},
|
|
530
|
-
${hasAttempts ? "t.execution_generation" : "null::integer as execution_generation"},
|
|
531
|
-
${hasAttempts ? "t.active_attempt_id" : "null::uuid as active_attempt_id"},
|
|
532
|
-
t.started_at, t.finished_at,
|
|
533
|
-
exists (
|
|
534
|
-
select 1 from session_events e
|
|
535
|
-
where e.workspace_id = t.workspace_id and e.session_id = t.session_id
|
|
536
|
-
and e.turn_id = t.id and e.type = 'turn.started'
|
|
537
|
-
) as has_started_evidence
|
|
538
|
-
from session_turns t order by t.session_id, t.position, t.id`,
|
|
539
|
-
);
|
|
540
|
-
const goalRows = await unsafeRows(
|
|
541
|
-
sql,
|
|
542
|
-
`select session_id, id, status, version, auto_continuations, no_progress_streak,
|
|
543
|
-
last_continuation_turn_id from session_goals order by session_id, id`,
|
|
544
|
-
);
|
|
545
|
-
const historyProofs = await captureRelationProofs(
|
|
546
|
-
sql,
|
|
547
|
-
`select array[
|
|
548
|
-
session_id::text,
|
|
549
|
-
id::text,
|
|
550
|
-
position::text,
|
|
551
|
-
coalesce(turn_id::text, E'\\\\N'),
|
|
552
|
-
case when active then 't' else 'f' end,
|
|
553
|
-
coalesce(producer_codex_credential_id::text, E'\\\\N')
|
|
554
|
-
] as fields
|
|
555
|
-
from session_history_items order by session_id, position, id`,
|
|
556
|
-
6,
|
|
557
|
-
[1, 3, 2, 4, 5],
|
|
558
|
-
reference,
|
|
559
|
-
(session) => session.historyProof,
|
|
560
|
-
);
|
|
561
|
-
const eventProofs = await captureRelationProofs(
|
|
562
|
-
sql,
|
|
563
|
-
`select array[
|
|
564
|
-
session_id::text,
|
|
565
|
-
id::text,
|
|
566
|
-
sequence::text,
|
|
567
|
-
coalesce(turn_id::text, E'\\\\N')
|
|
568
|
-
] as fields
|
|
569
|
-
from session_events order by session_id, sequence, id`,
|
|
570
|
-
4,
|
|
571
|
-
[1, 3, 2],
|
|
572
|
-
reference,
|
|
573
|
-
(session) => session.eventProof,
|
|
574
|
-
);
|
|
575
|
-
const runStateRows = await unsafeRows(
|
|
576
|
-
sql,
|
|
577
|
-
`select session_id, id, turn_id, state_version,
|
|
578
|
-
case when serialized_run_state ~ '^\\s*\\{' then
|
|
579
|
-
serialized_run_state::jsonb ->> '$schemaVersion'
|
|
580
|
-
else null end as schema_version,
|
|
581
|
-
jsonb_array_length(pending_approvals) as pending_approval_count,
|
|
582
|
-
${hasFrozenCredential ? "frozen_codex_credential_id" : "null::uuid as frozen_codex_credential_id"}
|
|
583
|
-
from agent_run_states order by session_id, state_version, id`,
|
|
584
|
-
);
|
|
585
|
-
const waiterRows = await unsafeRows(
|
|
586
|
-
sql,
|
|
587
|
-
`select session_id, id, goal_id, blocked_turn_id, workflow_id, generation, status,
|
|
588
|
-
goal_version, control_generation, wake_revision, observed_wake_revision
|
|
589
|
-
from codex_capacity_waiters order by session_id, id`,
|
|
590
|
-
);
|
|
591
|
-
const leaseRows = await unsafeRows(
|
|
592
|
-
sql,
|
|
593
|
-
`select s.id as session_id, l.id, l.sandbox_group_id, l.liveness, l.instance_id,
|
|
594
|
-
l.backend, l.lease_epoch
|
|
595
|
-
from sessions s join sandbox_leases l on l.workspace_id = s.workspace_id
|
|
596
|
-
and l.sandbox_group_id = s.sandbox_group_id
|
|
597
|
-
order by s.id, l.id`,
|
|
598
|
-
);
|
|
599
|
-
const updateRows = hasUpdates
|
|
600
|
-
? await unsafeRows(
|
|
601
|
-
sql,
|
|
602
|
-
`select session_id, id, kind, source_id, dedupe_key, state, delivered_turn_id
|
|
603
|
-
from session_system_updates order by session_id, created_at, id`,
|
|
604
|
-
)
|
|
605
|
-
: [];
|
|
606
|
-
const pendingToolRows = hasPendingTools
|
|
607
|
-
? await unsafeRows(
|
|
608
|
-
sql,
|
|
609
|
-
`select session_id, id, turn_id, execution_generation, attempt_id, call_id, call_type,
|
|
610
|
-
result_recorded_at is not null as result_recorded
|
|
611
|
-
from session_pending_tool_calls order by session_id, turn_id, call_id`,
|
|
612
|
-
)
|
|
613
|
-
: [];
|
|
614
|
-
|
|
615
|
-
const turns = groupBySession(
|
|
616
|
-
turnRows.map((row) => ({
|
|
617
|
-
sessionId: stringValue(row.session_id, "turn.session_id"),
|
|
618
|
-
id: stringValue(row.id, "turn.id"),
|
|
619
|
-
triggerEventId: stringValue(row.trigger_event_id, "turn.trigger_event_id"),
|
|
620
|
-
temporalWorkflowId: stringValue(row.temporal_workflow_id, "turn.temporal_workflow_id"),
|
|
621
|
-
status: stringValue(row.status, "turn.status"),
|
|
622
|
-
source: stringValue(row.source, "turn.source"),
|
|
623
|
-
position: numberValue(row.position, "turn.position"),
|
|
624
|
-
version: nullableNumber(row.version, "turn.version"),
|
|
625
|
-
executionGeneration: nullableNumber(row.execution_generation, "turn.execution_generation"),
|
|
626
|
-
activeAttemptId: nullableString(row.active_attempt_id, "turn.active_attempt_id"),
|
|
627
|
-
startedAt: nullableTimestamp(row.started_at, "turn.started_at"),
|
|
628
|
-
finishedAt: nullableTimestamp(row.finished_at, "turn.finished_at"),
|
|
629
|
-
hasStartedEvidence: booleanValue(row.has_started_evidence),
|
|
630
|
-
})),
|
|
631
|
-
);
|
|
632
|
-
const goals = groupBySession(
|
|
633
|
-
goalRows.map((row) => ({
|
|
634
|
-
sessionId: stringValue(row.session_id, "goal.session_id"),
|
|
635
|
-
id: stringValue(row.id, "goal.id"),
|
|
636
|
-
status: stringValue(row.status, "goal.status"),
|
|
637
|
-
version: numberValue(row.version, "goal.version"),
|
|
638
|
-
autoContinuations: numberValue(row.auto_continuations, "goal.auto_continuations"),
|
|
639
|
-
noProgressStreak: numberValue(row.no_progress_streak, "goal.no_progress_streak"),
|
|
640
|
-
lastContinuationTurnId: nullableString(
|
|
641
|
-
row.last_continuation_turn_id,
|
|
642
|
-
"goal.last_continuation_turn_id",
|
|
643
|
-
),
|
|
644
|
-
})),
|
|
645
|
-
);
|
|
646
|
-
const runStates = groupBySession(
|
|
647
|
-
runStateRows.map((row) => ({
|
|
648
|
-
sessionId: stringValue(row.session_id, "run_state.session_id"),
|
|
649
|
-
id: stringValue(row.id, "run_state.id"),
|
|
650
|
-
turnId: nullableString(row.turn_id, "run_state.turn_id"),
|
|
651
|
-
stateVersion: numberValue(row.state_version, "run_state.state_version"),
|
|
652
|
-
schemaVersion: nullableString(row.schema_version, "run_state.schema_version"),
|
|
653
|
-
pendingApprovalCount: numberValue(
|
|
654
|
-
row.pending_approval_count,
|
|
655
|
-
"run_state.pending_approval_count",
|
|
656
|
-
),
|
|
657
|
-
frozenCodexCredentialId: nullableString(
|
|
658
|
-
row.frozen_codex_credential_id,
|
|
659
|
-
"run_state.frozen_codex_credential_id",
|
|
660
|
-
),
|
|
661
|
-
})),
|
|
662
|
-
);
|
|
663
|
-
const capacityWaiters = groupBySession(
|
|
664
|
-
waiterRows.map((row) => ({
|
|
665
|
-
sessionId: stringValue(row.session_id, "capacity_waiter.session_id"),
|
|
666
|
-
id: stringValue(row.id, "capacity_waiter.id"),
|
|
667
|
-
goalId: stringValue(row.goal_id, "capacity_waiter.goal_id"),
|
|
668
|
-
blockedTurnId: stringValue(row.blocked_turn_id, "capacity_waiter.blocked_turn_id"),
|
|
669
|
-
workflowId: stringValue(row.workflow_id, "capacity_waiter.workflow_id"),
|
|
670
|
-
generation: numberValue(row.generation, "capacity_waiter.generation"),
|
|
671
|
-
status: stringValue(row.status, "capacity_waiter.status"),
|
|
672
|
-
goalVersion: numberValue(row.goal_version, "capacity_waiter.goal_version"),
|
|
673
|
-
controlGeneration: numberValue(row.control_generation, "capacity_waiter.control_generation"),
|
|
674
|
-
wakeRevision: numberValue(row.wake_revision, "capacity_waiter.wake_revision"),
|
|
675
|
-
observedWakeRevision: numberValue(
|
|
676
|
-
row.observed_wake_revision,
|
|
677
|
-
"capacity_waiter.observed_wake_revision",
|
|
678
|
-
),
|
|
679
|
-
})),
|
|
680
|
-
);
|
|
681
|
-
const sandboxLeases = groupBySession(
|
|
682
|
-
leaseRows.map((row) => ({
|
|
683
|
-
sessionId: stringValue(row.session_id, "sandbox_lease.session_id"),
|
|
684
|
-
id: stringValue(row.id, "sandbox_lease.id"),
|
|
685
|
-
sandboxGroupId: stringValue(row.sandbox_group_id, "sandbox_lease.sandbox_group_id"),
|
|
686
|
-
liveness: stringValue(row.liveness, "sandbox_lease.liveness"),
|
|
687
|
-
instanceId: nullableString(row.instance_id, "sandbox_lease.instance_id"),
|
|
688
|
-
backend: stringValue(row.backend, "sandbox_lease.backend"),
|
|
689
|
-
leaseEpoch: numberValue(row.lease_epoch, "sandbox_lease.lease_epoch"),
|
|
690
|
-
})),
|
|
691
|
-
);
|
|
692
|
-
const systemUpdates = groupBySession(
|
|
693
|
-
updateRows.map((row) => ({
|
|
694
|
-
sessionId: stringValue(row.session_id, "system_update.session_id"),
|
|
695
|
-
id: stringValue(row.id, "system_update.id"),
|
|
696
|
-
kind: stringValue(row.kind, "system_update.kind"),
|
|
697
|
-
sourceId: stringValue(row.source_id, "system_update.source_id"),
|
|
698
|
-
dedupeKey: stringValue(row.dedupe_key, "system_update.dedupe_key"),
|
|
699
|
-
state: stringValue(row.state, "system_update.state"),
|
|
700
|
-
deliveredTurnId: nullableString(row.delivered_turn_id, "system_update.delivered_turn_id"),
|
|
701
|
-
})),
|
|
702
|
-
);
|
|
703
|
-
const pendingToolCalls = groupBySession(
|
|
704
|
-
pendingToolRows.map((row) => ({
|
|
705
|
-
sessionId: stringValue(row.session_id, "pending_tool.session_id"),
|
|
706
|
-
id: stringValue(row.id, "pending_tool.id"),
|
|
707
|
-
turnId: stringValue(row.turn_id, "pending_tool.turn_id"),
|
|
708
|
-
executionGeneration: numberValue(
|
|
709
|
-
row.execution_generation,
|
|
710
|
-
"pending_tool.execution_generation",
|
|
711
|
-
),
|
|
712
|
-
attemptId: stringValue(row.attempt_id, "pending_tool.attempt_id"),
|
|
713
|
-
callId: stringValue(row.call_id, "pending_tool.call_id"),
|
|
714
|
-
callType: stringValue(row.call_type, "pending_tool.call_type"),
|
|
715
|
-
resultRecorded: booleanValue(row.result_recorded),
|
|
716
|
-
})),
|
|
717
|
-
);
|
|
718
|
-
|
|
719
|
-
const sessions: CutoverSession[] = sessionRows.map((row) => {
|
|
720
|
-
const id = stringValue(row.id, "session.id");
|
|
721
|
-
return {
|
|
722
|
-
id,
|
|
723
|
-
accountId: stringValue(row.account_id, "session.account_id"),
|
|
724
|
-
workspaceId: stringValue(row.workspace_id, "session.workspace_id"),
|
|
725
|
-
status: stringValue(row.status, "session.status"),
|
|
726
|
-
temporalWorkflowId: nullableString(row.temporal_workflow_id, "session.temporal_workflow_id"),
|
|
727
|
-
activeTurnId: nullableString(row.active_turn_id, "session.active_turn_id"),
|
|
728
|
-
sandboxGroupId: stringValue(row.sandbox_group_id, "session.sandbox_group_id"),
|
|
729
|
-
activeSandboxId: nullableString(row.active_sandbox_id, "session.active_sandbox_id"),
|
|
730
|
-
activeEpoch: numberValue(row.active_epoch, "session.active_epoch"),
|
|
731
|
-
controlState: nullableString(row.control_state, "session.control_state"),
|
|
732
|
-
controlGeneration: nullableNumber(row.control_generation, "session.control_generation"),
|
|
733
|
-
workspaceRunExceptionGeneration: nullableNumber(
|
|
734
|
-
row.workspace_run_exception_generation,
|
|
735
|
-
"session.workspace_run_exception_generation",
|
|
736
|
-
),
|
|
737
|
-
queueVersion: nullableNumber(row.queue_version, "session.queue_version"),
|
|
738
|
-
queueHeadPosition: nullableNumber(row.queue_head_position, "session.queue_head_position"),
|
|
739
|
-
queueTailPosition: nullableNumber(row.queue_tail_position, "session.queue_tail_position"),
|
|
740
|
-
turns: turns.get(id) ?? [],
|
|
741
|
-
goals: goals.get(id) ?? [],
|
|
742
|
-
historyProof: historyProofs.get(id) ?? emptyRelationProof(),
|
|
743
|
-
eventProof: eventProofs.get(id) ?? emptyRelationProof(),
|
|
744
|
-
runStates: runStates.get(id) ?? [],
|
|
745
|
-
capacityWaiters: capacityWaiters.get(id) ?? [],
|
|
746
|
-
sandboxLeases: sandboxLeases.get(id) ?? [],
|
|
747
|
-
systemUpdates: systemUpdates.get(id) ?? [],
|
|
748
|
-
pendingToolCalls: pendingToolCalls.get(id) ?? [],
|
|
749
|
-
};
|
|
750
|
-
});
|
|
751
|
-
|
|
752
|
-
const workspaces: CutoverWorkspace[] = workspaceRows.map((row) => ({
|
|
753
|
-
id: stringValue(row.id, "workspace.id"),
|
|
754
|
-
accountId: stringValue(row.account_id, "workspace.account_id"),
|
|
755
|
-
inferenceState: nullableString(row.inference_state, "workspace.inference_state"),
|
|
756
|
-
inferenceGeneration: nullableNumber(row.inference_generation, "workspace.inference_generation"),
|
|
757
|
-
}));
|
|
758
|
-
|
|
759
|
-
const currentStatuses = new Set(["running", "requires_action", "recovering", "waiting_capacity"]);
|
|
760
|
-
let multipleCurrentInferences = 0;
|
|
761
|
-
let queuedMachineTurns = 0;
|
|
762
|
-
let invalidActiveTurnPointers = 0;
|
|
763
|
-
for (const session of sessions) {
|
|
764
|
-
const current = session.turns.filter((turn) => currentStatuses.has(turn.status));
|
|
765
|
-
if (current.length > 1) multipleCurrentInferences += 1;
|
|
766
|
-
queuedMachineTurns += session.turns.filter(
|
|
767
|
-
(turn) => turn.status === "queued" && turn.source !== "user" && turn.source !== "api",
|
|
768
|
-
).length;
|
|
769
|
-
const expected = current[0]?.id ?? null;
|
|
770
|
-
if (session.activeTurnId !== expected) invalidActiveTurnPointers += 1;
|
|
771
|
-
}
|
|
772
|
-
const opaqueRows = await unsafeRows(
|
|
773
|
-
sql,
|
|
774
|
-
`select count(*)::integer as count from session_history_items
|
|
775
|
-
where active = true and item ->> 'type' in ('compaction','compaction_summary')`,
|
|
776
|
-
);
|
|
777
|
-
const ownershipRows = hasAttempts
|
|
778
|
-
? await unsafeRows(
|
|
779
|
-
sql,
|
|
780
|
-
`select
|
|
781
|
-
count(*) filter (where status = 'running' and active_attempt_id is null)::integer
|
|
782
|
-
as running_without_attempt,
|
|
783
|
-
count(*) filter (where status <> 'running' and active_attempt_id is not null)::integer
|
|
784
|
-
as owned_non_running
|
|
785
|
-
from session_turns`,
|
|
786
|
-
)
|
|
787
|
-
: [{ running_without_attempt: 0, owned_non_running: 0 }];
|
|
788
|
-
const usageRows = hasUsageDuplicateAssociation
|
|
789
|
-
? await unsafeRows(
|
|
790
|
-
sql,
|
|
791
|
-
`select
|
|
792
|
-
(select count(*)::integer from (
|
|
793
|
-
select workspace_id, session_id, turn_id, payload ->> 'sourceKey'
|
|
794
|
-
from session_events
|
|
795
|
-
where type = 'agent.model.usage' and turn_association = 'current'
|
|
796
|
-
and turn_id is not null and nullif(payload ->> 'sourceKey', '') is not null
|
|
797
|
-
group by workspace_id, session_id, turn_id, payload ->> 'sourceKey'
|
|
798
|
-
having count(*) > 1
|
|
799
|
-
) duplicate_sources) as duplicate_current_sources,
|
|
800
|
-
(select count(*)::integer from session_events
|
|
801
|
-
where type = 'agent.model.usage' and turn_association = 'duplicate'
|
|
802
|
-
and (duplicate_of_event_id is null or duplicate_reason is null))
|
|
803
|
-
as invalid_duplicate_associations`,
|
|
804
|
-
)
|
|
805
|
-
: [{ duplicate_current_sources: 0, invalid_duplicate_associations: 0 }];
|
|
806
|
-
const enrollableRows = hasEnrollableScan
|
|
807
|
-
? await unsafeRows(
|
|
808
|
-
sql,
|
|
809
|
-
`select count(*)::integer as count
|
|
810
|
-
from opengeni_private.list_enrollable_sessions(10000)`,
|
|
811
|
-
)
|
|
812
|
-
: [{ count: 0 }];
|
|
813
|
-
const casualtyRows = await unsafeRows(
|
|
814
|
-
sql,
|
|
815
|
-
`select count(*)::integer as count from session_events
|
|
816
|
-
where type = 'turn.failed'
|
|
817
|
-
and payload ->> 'code' = 'worker_death_redispatch_exhausted'`,
|
|
818
|
-
);
|
|
819
|
-
|
|
820
|
-
const draft = {
|
|
821
|
-
contractVersion: SESSION_CONTROL_CUTOVER_CONTRACT,
|
|
822
|
-
phase,
|
|
823
|
-
capturedAt,
|
|
824
|
-
schemaMigrations: migrationRows.map((row) => stringValue(row.name, "schema_migration.name")),
|
|
825
|
-
workspaces,
|
|
826
|
-
sessions,
|
|
827
|
-
proofBaselineSha256: reference?.sha256 ?? null,
|
|
828
|
-
invariants: {
|
|
829
|
-
multipleCurrentInferences,
|
|
830
|
-
queuedMachineTurns,
|
|
831
|
-
activeOpaqueCompactionItems: numberValue(opaqueRows[0]?.count ?? 0, "opaque count"),
|
|
832
|
-
invalidActiveTurnPointers,
|
|
833
|
-
runningTurnsWithoutAttempt: numberValue(
|
|
834
|
-
ownershipRows[0]?.running_without_attempt ?? 0,
|
|
835
|
-
"running turns without attempt",
|
|
836
|
-
),
|
|
837
|
-
attemptOwnedNonRunningTurns: numberValue(
|
|
838
|
-
ownershipRows[0]?.owned_non_running ?? 0,
|
|
839
|
-
"attempt-owned non-running turns",
|
|
840
|
-
),
|
|
841
|
-
duplicateCurrentUsageSources: numberValue(
|
|
842
|
-
usageRows[0]?.duplicate_current_sources ?? 0,
|
|
843
|
-
"duplicate current usage sources",
|
|
844
|
-
),
|
|
845
|
-
invalidDuplicateUsageAssociations: numberValue(
|
|
846
|
-
usageRows[0]?.invalid_duplicate_associations ?? 0,
|
|
847
|
-
"invalid duplicate usage associations",
|
|
848
|
-
),
|
|
849
|
-
enrollableSessions: numberValue(enrollableRows[0]?.count ?? 0, "enrollable sessions"),
|
|
850
|
-
workerDeathRedispatchExhausted: numberValue(
|
|
851
|
-
casualtyRows[0]?.count ?? 0,
|
|
852
|
-
"worker death redispatch casualties",
|
|
853
|
-
),
|
|
854
|
-
},
|
|
855
|
-
identityCount: sessions.reduce(
|
|
856
|
-
(count, session) =>
|
|
857
|
-
count +
|
|
858
|
-
1 +
|
|
859
|
-
session.turns.length +
|
|
860
|
-
session.goals.length +
|
|
861
|
-
session.historyProof.count +
|
|
862
|
-
session.eventProof.count +
|
|
863
|
-
session.runStates.length +
|
|
864
|
-
session.capacityWaiters.length +
|
|
865
|
-
session.sandboxLeases.length +
|
|
866
|
-
session.systemUpdates.length +
|
|
867
|
-
session.pendingToolCalls.length,
|
|
868
|
-
workspaces.length,
|
|
869
|
-
),
|
|
870
|
-
};
|
|
871
|
-
return { ...draft, sha256: sha256(draft) };
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
function compareStableRows<T extends { id: string }>(
|
|
875
|
-
errors: string[],
|
|
876
|
-
sessionId: string,
|
|
877
|
-
name: string,
|
|
878
|
-
before: T[],
|
|
879
|
-
after: T[],
|
|
880
|
-
exact: boolean,
|
|
881
|
-
): void {
|
|
882
|
-
const afterById = new Map(after.map((row) => [row.id, row]));
|
|
883
|
-
for (const row of before) {
|
|
884
|
-
const observed = afterById.get(row.id);
|
|
885
|
-
if (!observed) {
|
|
886
|
-
errors.push(`${sessionId}: missing ${name} ${row.id}`);
|
|
887
|
-
continue;
|
|
888
|
-
}
|
|
889
|
-
if (exact && stableJson(row) !== stableJson(observed)) {
|
|
890
|
-
errors.push(`${sessionId}: ${name} ${row.id} changed during migration`);
|
|
891
|
-
}
|
|
892
|
-
}
|
|
893
|
-
if (exact && !idsEqual(before, after)) {
|
|
894
|
-
errors.push(`${sessionId}: ${name} identity set changed during migration`);
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
function compareRelationProof(
|
|
899
|
-
errors: string[],
|
|
900
|
-
sessionId: string,
|
|
901
|
-
name: string,
|
|
902
|
-
before: CutoverRelationProof,
|
|
903
|
-
after: CutoverRelationProof,
|
|
904
|
-
mode: "migration" | "final-fate",
|
|
905
|
-
): void {
|
|
906
|
-
if (after.preservedThroughOrdinal !== before.maxOrdinal) {
|
|
907
|
-
errors.push(`${sessionId}: ${name} proof used the wrong baseline boundary`);
|
|
908
|
-
return;
|
|
909
|
-
}
|
|
910
|
-
if (mode === "migration") {
|
|
911
|
-
if (
|
|
912
|
-
after.count < before.count ||
|
|
913
|
-
after.preservedCount !== before.count ||
|
|
914
|
-
after.preservedStableSha256 !== before.stableSha256 ||
|
|
915
|
-
after.preservedIdentitySha256 !== before.identitySha256
|
|
916
|
-
) {
|
|
917
|
-
errors.push(`${sessionId}: ${name} changed during migration`);
|
|
918
|
-
}
|
|
919
|
-
return;
|
|
920
|
-
}
|
|
921
|
-
if (
|
|
922
|
-
after.preservedCount !== before.count ||
|
|
923
|
-
after.preservedIdentitySha256 !== before.identitySha256
|
|
924
|
-
) {
|
|
925
|
-
errors.push(`${sessionId}: baseline ${name} identities were not preserved`);
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
function comparePendingToolCalls(
|
|
930
|
-
errors: string[],
|
|
931
|
-
sessionId: string,
|
|
932
|
-
before: CutoverPendingToolCall[],
|
|
933
|
-
after: CutoverPendingToolCall[],
|
|
934
|
-
beforeTurns: CutoverTurn[],
|
|
935
|
-
afterTurns: CutoverTurn[],
|
|
936
|
-
mode: "migration" | "final-fate",
|
|
937
|
-
): void {
|
|
938
|
-
const beforeById = new Map(before.map((call) => [call.id, call]));
|
|
939
|
-
const afterById = new Map(after.map((call) => [call.id, call]));
|
|
940
|
-
const beforeTurnsById = new Map(beforeTurns.map((turn) => [turn.id, turn]));
|
|
941
|
-
const afterTurnsById = new Map(afterTurns.map((turn) => [turn.id, turn]));
|
|
942
|
-
for (const call of before) {
|
|
943
|
-
const observed = afterById.get(call.id);
|
|
944
|
-
if (observed) {
|
|
945
|
-
if (mode === "migration" && stableJson(call) !== stableJson(observed)) {
|
|
946
|
-
errors.push(`${sessionId}: pending tool call ${call.id} changed during migration`);
|
|
947
|
-
}
|
|
948
|
-
continue;
|
|
949
|
-
}
|
|
950
|
-
const beforeTurn = beforeTurnsById.get(call.turnId);
|
|
951
|
-
const afterTurn = afterTurnsById.get(call.turnId);
|
|
952
|
-
const wasClosedByAttemptSettlement =
|
|
953
|
-
beforeTurn?.activeAttemptId !== null &&
|
|
954
|
-
beforeTurn?.activeAttemptId !== undefined &&
|
|
955
|
-
afterTurn?.activeAttemptId === null &&
|
|
956
|
-
afterTurn.status !== "running" &&
|
|
957
|
-
afterTurn.status !== "requires_action";
|
|
958
|
-
if (!wasClosedByAttemptSettlement) {
|
|
959
|
-
errors.push(`${sessionId}: pending tool call ${call.id} disappeared without settlement`);
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
|
-
for (const call of after) {
|
|
963
|
-
if (mode === "migration" && !beforeById.has(call.id)) {
|
|
964
|
-
errors.push(`${sessionId}: unexpected pending tool call ${call.id} appeared during cutover`);
|
|
965
|
-
}
|
|
966
|
-
}
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
function compareMigrationSystemUpdates(
|
|
970
|
-
errors: string[],
|
|
971
|
-
session: CutoverSession,
|
|
972
|
-
observed: CutoverSession,
|
|
973
|
-
): void {
|
|
974
|
-
const beforeById = new Map(session.systemUpdates.map((update) => [update.id, update]));
|
|
975
|
-
const expectedMigratedTurnIds = new Set(
|
|
976
|
-
session.turns
|
|
977
|
-
.filter(
|
|
978
|
-
(turn) =>
|
|
979
|
-
turn.status === "queued" &&
|
|
980
|
-
!turn.hasStartedEvidence &&
|
|
981
|
-
(turn.source === "scheduled_task" || turn.source === "system"),
|
|
982
|
-
)
|
|
983
|
-
.map((turn) => turn.id),
|
|
984
|
-
);
|
|
985
|
-
for (const update of session.systemUpdates) {
|
|
986
|
-
const next = observed.systemUpdates.find((candidate) => candidate.id === update.id);
|
|
987
|
-
if (!next) errors.push(`${session.id}: missing system update ${update.id}`);
|
|
988
|
-
else if (stableJson(update) !== stableJson(next)) {
|
|
989
|
-
errors.push(`${session.id}: system update ${update.id} changed during migration`);
|
|
990
|
-
}
|
|
991
|
-
}
|
|
992
|
-
for (const update of observed.systemUpdates) {
|
|
993
|
-
if (beforeById.has(update.id)) continue;
|
|
994
|
-
if (
|
|
995
|
-
!expectedMigratedTurnIds.has(update.sourceId) ||
|
|
996
|
-
update.dedupeKey !== `migrated-turn:${update.sourceId}`
|
|
997
|
-
) {
|
|
998
|
-
errors.push(`${session.id}: unexpected system update ${update.id} appeared during migration`);
|
|
999
|
-
}
|
|
1000
|
-
}
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
export function reconcileSessionControlCutover(
|
|
1004
|
-
baseline: SessionControlCutoverSnapshot,
|
|
1005
|
-
observed: SessionControlCutoverSnapshot,
|
|
1006
|
-
mode: "migration" | "final-fate",
|
|
1007
|
-
): CutoverReconciliation {
|
|
1008
|
-
const errors: string[] = [];
|
|
1009
|
-
const fateCounts: Record<string, number> = {};
|
|
1010
|
-
if (baseline.contractVersion !== SESSION_CONTROL_CUTOVER_CONTRACT) {
|
|
1011
|
-
errors.push(`unsupported baseline contract ${baseline.contractVersion}`);
|
|
1012
|
-
}
|
|
1013
|
-
if (observed.contractVersion !== SESSION_CONTROL_CUTOVER_CONTRACT) {
|
|
1014
|
-
errors.push(`unsupported observed contract ${observed.contractVersion}`);
|
|
1015
|
-
}
|
|
1016
|
-
if (baseline.phase !== "baseline") errors.push("the first snapshot is not a baseline");
|
|
1017
|
-
if (mode === "migration" && observed.phase !== "migrated") {
|
|
1018
|
-
errors.push("migration reconciliation requires a migrated snapshot");
|
|
1019
|
-
}
|
|
1020
|
-
if (mode === "final-fate" && observed.phase !== "final") {
|
|
1021
|
-
errors.push("final-fate reconciliation requires a final snapshot");
|
|
1022
|
-
}
|
|
1023
|
-
if (observed.proofBaselineSha256 !== baseline.sha256) {
|
|
1024
|
-
errors.push("observed relation proofs were not derived from this baseline");
|
|
1025
|
-
}
|
|
1026
|
-
if (observed.invariants.multipleCurrentInferences !== 0) {
|
|
1027
|
-
errors.push("observed snapshot has multiple current inferences for a session");
|
|
1028
|
-
}
|
|
1029
|
-
if (observed.invariants.queuedMachineTurns !== 0) {
|
|
1030
|
-
errors.push("observed snapshot still has machine work in the human prompt queue");
|
|
1031
|
-
}
|
|
1032
|
-
if (observed.invariants.activeOpaqueCompactionItems !== 0) {
|
|
1033
|
-
errors.push("observed snapshot has active opaque compaction history");
|
|
1034
|
-
}
|
|
1035
|
-
if (observed.invariants.invalidActiveTurnPointers !== 0) {
|
|
1036
|
-
errors.push("observed snapshot has invalid active-turn pointers");
|
|
1037
|
-
}
|
|
1038
|
-
if (observed.invariants.runningTurnsWithoutAttempt !== 0) {
|
|
1039
|
-
errors.push("observed snapshot has running turns without registered attempts");
|
|
1040
|
-
}
|
|
1041
|
-
if (observed.invariants.attemptOwnedNonRunningTurns !== 0) {
|
|
1042
|
-
errors.push("observed snapshot has non-running turns with attempt owners");
|
|
1043
|
-
}
|
|
1044
|
-
if (observed.invariants.duplicateCurrentUsageSources !== 0) {
|
|
1045
|
-
errors.push("observed snapshot has duplicate current usage sources");
|
|
1046
|
-
}
|
|
1047
|
-
if (observed.invariants.invalidDuplicateUsageAssociations !== 0) {
|
|
1048
|
-
errors.push("observed snapshot has invalid duplicate usage associations");
|
|
1049
|
-
}
|
|
1050
|
-
|
|
1051
|
-
const observedWorkspaces = new Map(
|
|
1052
|
-
observed.workspaces.map((workspace) => [workspace.id, workspace]),
|
|
1053
|
-
);
|
|
1054
|
-
for (const workspace of baseline.workspaces) {
|
|
1055
|
-
const next = observedWorkspaces.get(workspace.id);
|
|
1056
|
-
if (!next) errors.push(`missing workspace ${workspace.id}`);
|
|
1057
|
-
else if (next.accountId !== workspace.accountId) {
|
|
1058
|
-
errors.push(`workspace ${workspace.id} changed account identity`);
|
|
1059
|
-
}
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
const observedSessions = new Map(observed.sessions.map((session) => [session.id, session]));
|
|
1063
|
-
for (const session of baseline.sessions) {
|
|
1064
|
-
const next = observedSessions.get(session.id);
|
|
1065
|
-
if (!next) {
|
|
1066
|
-
errors.push(`missing session ${session.id}`);
|
|
1067
|
-
continue;
|
|
1068
|
-
}
|
|
1069
|
-
increment(fateCounts, next.status);
|
|
1070
|
-
if (next.accountId !== session.accountId || next.workspaceId !== session.workspaceId) {
|
|
1071
|
-
errors.push(`${session.id}: account/workspace identity changed`);
|
|
1072
|
-
}
|
|
1073
|
-
if (
|
|
1074
|
-
next.sandboxGroupId !== session.sandboxGroupId ||
|
|
1075
|
-
next.activeSandboxId !== session.activeSandboxId ||
|
|
1076
|
-
next.activeEpoch !== session.activeEpoch
|
|
1077
|
-
) {
|
|
1078
|
-
errors.push(`${session.id}: sandbox routing identity changed`);
|
|
1079
|
-
}
|
|
1080
|
-
|
|
1081
|
-
const nextTurns = new Map(next.turns.map((turn) => [turn.id, turn]));
|
|
1082
|
-
for (const turn of session.turns) {
|
|
1083
|
-
const nextTurn = nextTurns.get(turn.id);
|
|
1084
|
-
if (!nextTurn) {
|
|
1085
|
-
errors.push(`${session.id}: missing turn ${turn.id}`);
|
|
1086
|
-
continue;
|
|
1087
|
-
}
|
|
1088
|
-
if (
|
|
1089
|
-
nextTurn.triggerEventId !== turn.triggerEventId ||
|
|
1090
|
-
nextTurn.temporalWorkflowId !== turn.temporalWorkflowId ||
|
|
1091
|
-
nextTurn.source !== turn.source
|
|
1092
|
-
) {
|
|
1093
|
-
errors.push(`${session.id}: turn ${turn.id} changed causal identity`);
|
|
1094
|
-
}
|
|
1095
|
-
if (mode === "migration") {
|
|
1096
|
-
const expected = expectedMigratedTurnStatus(turn);
|
|
1097
|
-
if (nextTurn.status !== expected) {
|
|
1098
|
-
errors.push(
|
|
1099
|
-
`${session.id}: turn ${turn.id} expected ${expected}, got ${nextTurn.status}`,
|
|
1100
|
-
);
|
|
1101
|
-
}
|
|
1102
|
-
if (expected === "recovering" && nextTurn.activeAttemptId !== null) {
|
|
1103
|
-
errors.push(`${session.id}: recovering turn ${turn.id} retained an attempt owner`);
|
|
1104
|
-
}
|
|
1105
|
-
} else if (
|
|
1106
|
-
(turn.status === "running" || (turn.status === "queued" && turn.hasStartedEvidence)) &&
|
|
1107
|
-
nextTurn.status === "queued"
|
|
1108
|
-
) {
|
|
1109
|
-
errors.push(`${session.id}: current inference ${turn.id} was converted into queue work`);
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
const baselineQueue = humanQueue(session.turns);
|
|
1114
|
-
const observedQueue = next.turns
|
|
1115
|
-
.filter((turn) => turn.status === "queued")
|
|
1116
|
-
.sort((a, b) => a.position - b.position || a.id.localeCompare(b.id))
|
|
1117
|
-
.map((turn) => turn.id);
|
|
1118
|
-
if (mode === "migration") {
|
|
1119
|
-
if (stableJson(baselineQueue) !== stableJson(observedQueue)) {
|
|
1120
|
-
errors.push(`${session.id}: human prompt queue order changed during migration`);
|
|
1121
|
-
}
|
|
1122
|
-
} else {
|
|
1123
|
-
const remainingBaseline = observedQueue.filter((id) => baselineQueue.includes(id));
|
|
1124
|
-
if (!isSubsequence(remainingBaseline, baselineQueue)) {
|
|
1125
|
-
errors.push(`${session.id}: remaining baseline prompts changed relative order`);
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
compareStableRows(errors, session.id, "goal", session.goals, next.goals, mode === "migration");
|
|
1130
|
-
compareRelationProof(
|
|
1131
|
-
errors,
|
|
1132
|
-
session.id,
|
|
1133
|
-
"history",
|
|
1134
|
-
session.historyProof,
|
|
1135
|
-
next.historyProof,
|
|
1136
|
-
mode,
|
|
1137
|
-
);
|
|
1138
|
-
compareRelationProof(errors, session.id, "event", session.eventProof, next.eventProof, mode);
|
|
1139
|
-
compareStableRows(
|
|
1140
|
-
errors,
|
|
1141
|
-
session.id,
|
|
1142
|
-
"run state",
|
|
1143
|
-
session.runStates,
|
|
1144
|
-
next.runStates,
|
|
1145
|
-
mode === "migration",
|
|
1146
|
-
);
|
|
1147
|
-
compareStableRows(
|
|
1148
|
-
errors,
|
|
1149
|
-
session.id,
|
|
1150
|
-
"capacity waiter",
|
|
1151
|
-
session.capacityWaiters,
|
|
1152
|
-
next.capacityWaiters,
|
|
1153
|
-
mode === "migration",
|
|
1154
|
-
);
|
|
1155
|
-
compareStableRows(
|
|
1156
|
-
errors,
|
|
1157
|
-
session.id,
|
|
1158
|
-
"sandbox lease",
|
|
1159
|
-
session.sandboxLeases,
|
|
1160
|
-
next.sandboxLeases,
|
|
1161
|
-
mode === "migration",
|
|
1162
|
-
);
|
|
1163
|
-
if (mode === "migration") {
|
|
1164
|
-
compareMigrationSystemUpdates(errors, session, next);
|
|
1165
|
-
}
|
|
1166
|
-
comparePendingToolCalls(
|
|
1167
|
-
errors,
|
|
1168
|
-
session.id,
|
|
1169
|
-
session.pendingToolCalls,
|
|
1170
|
-
next.pendingToolCalls,
|
|
1171
|
-
session.turns,
|
|
1172
|
-
next.turns,
|
|
1173
|
-
mode,
|
|
1174
|
-
);
|
|
1175
|
-
|
|
1176
|
-
for (const turn of session.turns) {
|
|
1177
|
-
if (
|
|
1178
|
-
turn.status !== "queued" ||
|
|
1179
|
-
turn.hasStartedEvidence ||
|
|
1180
|
-
(turn.source !== "scheduled_task" && turn.source !== "system")
|
|
1181
|
-
) {
|
|
1182
|
-
continue;
|
|
1183
|
-
}
|
|
1184
|
-
const migrated = next.systemUpdates.find(
|
|
1185
|
-
(update) => update.sourceId === turn.id && update.dedupeKey === `migrated-turn:${turn.id}`,
|
|
1186
|
-
);
|
|
1187
|
-
if (!migrated) {
|
|
1188
|
-
errors.push(`${session.id}: machine turn ${turn.id} has no typed migrated update`);
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
}
|
|
1192
|
-
|
|
1193
|
-
const draft = {
|
|
1194
|
-
contractVersion: SESSION_CONTROL_CUTOVER_CONTRACT,
|
|
1195
|
-
mode,
|
|
1196
|
-
baselineSha256: baseline.sha256,
|
|
1197
|
-
observedSha256: observed.sha256,
|
|
1198
|
-
ok: errors.length === 0,
|
|
1199
|
-
errors,
|
|
1200
|
-
fateCounts,
|
|
1201
|
-
};
|
|
1202
|
-
return { ...draft, sha256: sha256(draft) };
|
|
1203
|
-
}
|