@opengeni/worker-bundle 0.16.17 → 0.16.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +23 -19
- package/dist/activities/file-upload-reaper.d.ts +6 -0
- package/dist/index.js +148 -63
- package/dist/index.js.map +1 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +77 -74
- package/src/activities/context-compaction.ts +0 -2
- package/src/activities/file-upload-reaper.ts +108 -7
- package/src/activities/sandbox-lease.ts +5 -2
- package/src/sandbox-routing.ts +5 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.21",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -41,16 +41,16 @@
|
|
|
41
41
|
"dependencies": {
|
|
42
42
|
"@opengeni/agent-proto": "^0.3.0",
|
|
43
43
|
"@opengeni/codex": "^0.2.11",
|
|
44
|
-
"@opengeni/config": "^0.
|
|
45
|
-
"@opengeni/contracts": "^0.
|
|
46
|
-
"@opengeni/core": "^0.
|
|
47
|
-
"@opengeni/db": "^0.
|
|
48
|
-
"@opengeni/documents": "^0.5.
|
|
49
|
-
"@opengeni/events": "^0.3.
|
|
50
|
-
"@opengeni/github": "^0.4.
|
|
51
|
-
"@opengeni/observability": "^0.
|
|
52
|
-
"@opengeni/runtime": "^0.18.
|
|
53
|
-
"@opengeni/storage": "^0.2.
|
|
44
|
+
"@opengeni/config": "^0.11.0",
|
|
45
|
+
"@opengeni/contracts": "^0.39.0",
|
|
46
|
+
"@opengeni/core": "^0.21.2",
|
|
47
|
+
"@opengeni/db": "^0.28.1",
|
|
48
|
+
"@opengeni/documents": "^0.5.10",
|
|
49
|
+
"@opengeni/events": "^0.3.80",
|
|
50
|
+
"@opengeni/github": "^0.4.31",
|
|
51
|
+
"@opengeni/observability": "^0.5.0",
|
|
52
|
+
"@opengeni/runtime": "^0.18.17",
|
|
53
|
+
"@opengeni/storage": "^0.2.68",
|
|
54
54
|
"@temporalio/activity": "^1.17.0",
|
|
55
55
|
"@temporalio/client": "^1.17.0",
|
|
56
56
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -89,7 +89,7 @@ import {
|
|
|
89
89
|
sandboxStateEntryFromRunState,
|
|
90
90
|
maxTurnsExceededRunState,
|
|
91
91
|
modelResponseServiceTierFromSdkEvent,
|
|
92
|
-
|
|
92
|
+
modelTerminalResponseFromSdkEvent,
|
|
93
93
|
normalizeModelCallUsage,
|
|
94
94
|
normalizeSdkEvent,
|
|
95
95
|
projectHistoryForProvider,
|
|
@@ -126,6 +126,7 @@ import {
|
|
|
126
126
|
runOwnedSandboxSetup,
|
|
127
127
|
RoutingMutationOutcomeUnknownError,
|
|
128
128
|
WorkspaceArchiveIntegrityError,
|
|
129
|
+
sdkBackendIdForSandboxBackend,
|
|
129
130
|
swapTargetEstablishability,
|
|
130
131
|
type SandboxFileDownload,
|
|
131
132
|
type SandboxFileDownloadFailure,
|
|
@@ -1123,10 +1124,9 @@ type TurnEventPublisher = (
|
|
|
1123
1124
|
immediate?: boolean,
|
|
1124
1125
|
) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
|
|
1125
1126
|
|
|
1126
|
-
export type
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
lastProviderContextTokensObserved: number | null;
|
|
1127
|
+
export type ModelResponseEventState = {
|
|
1128
|
+
responseCount: number;
|
|
1129
|
+
contextSignal: { revision: number; totalTokens: number } | null;
|
|
1130
1130
|
claimedSourceKeys: Set<string>;
|
|
1131
1131
|
};
|
|
1132
1132
|
|
|
@@ -1135,13 +1135,12 @@ export type CompactionModelUsageEventState = {
|
|
|
1135
1135
|
claimedSourceKeys: Set<string>;
|
|
1136
1136
|
};
|
|
1137
1137
|
|
|
1138
|
-
export function
|
|
1138
|
+
export function createModelResponseEventState(
|
|
1139
1139
|
claimedSourceKeys: Set<string> = new Set<string>(),
|
|
1140
|
-
):
|
|
1140
|
+
): ModelResponseEventState {
|
|
1141
1141
|
return {
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
lastProviderContextTokensObserved: null,
|
|
1142
|
+
responseCount: 0,
|
|
1143
|
+
contextSignal: null,
|
|
1145
1144
|
claimedSourceKeys,
|
|
1146
1145
|
};
|
|
1147
1146
|
}
|
|
@@ -1152,15 +1151,10 @@ export function createCompactionModelUsageEventState(
|
|
|
1152
1151
|
return { usageCount: 0, claimedSourceKeys };
|
|
1153
1152
|
}
|
|
1154
1153
|
|
|
1155
|
-
export function
|
|
1156
|
-
state:
|
|
1154
|
+
export function modelResponseContextSignal(
|
|
1155
|
+
state: ModelResponseEventState,
|
|
1157
1156
|
): { revision: number; totalTokens: number } | null {
|
|
1158
|
-
return state.
|
|
1159
|
-
? null
|
|
1160
|
-
: {
|
|
1161
|
-
revision: state.providerContextRevision,
|
|
1162
|
-
totalTokens: state.lastProviderContextTokensObserved,
|
|
1163
|
-
};
|
|
1157
|
+
return state.contextSignal;
|
|
1164
1158
|
}
|
|
1165
1159
|
|
|
1166
1160
|
export function assertModelResponseLatencyMode(input: {
|
|
@@ -1195,19 +1189,20 @@ export function assertModelResponseLatencyMode(input: {
|
|
|
1195
1189
|
}
|
|
1196
1190
|
|
|
1197
1191
|
/**
|
|
1198
|
-
* Process one SDK
|
|
1192
|
+
* Process one SDK terminal-response event through the production authority path.
|
|
1199
1193
|
*
|
|
1200
1194
|
* The pinned Responses SDK mirrors one provider terminal response as both a
|
|
1201
1195
|
* normalized `response_done` and a raw `model/response.completed` event. Claim
|
|
1202
|
-
* the stable response/source key before lease renewal or any
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1205
|
-
*
|
|
1206
|
-
*
|
|
1196
|
+
* the stable response/source key before lease renewal or any side effect, and
|
|
1197
|
+
* use that one positional ordinal for both response identity and same-run
|
|
1198
|
+
* context binding. A response without usage still clears attempt-owned token
|
|
1199
|
+
* state. When usage exists, the durable `agent.model.usage` source-key fence
|
|
1200
|
+
* remains the cross-restart authority: a replay may retry the idempotent billing
|
|
1201
|
+
* write, but it cannot advance metrics, context, or attempt-owned signals.
|
|
1207
1202
|
*/
|
|
1208
|
-
export async function
|
|
1209
|
-
event: Parameters<typeof
|
|
1210
|
-
state:
|
|
1203
|
+
export async function processModelResponseTerminalEvent(input: {
|
|
1204
|
+
event: Parameters<typeof modelTerminalResponseFromSdkEvent>[0];
|
|
1205
|
+
state: ModelResponseEventState;
|
|
1211
1206
|
dispatchId: string | null;
|
|
1212
1207
|
settings: Settings;
|
|
1213
1208
|
db: ActivityServices["db"];
|
|
@@ -1230,20 +1225,25 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1230
1225
|
renewLease: () => Promise<void>;
|
|
1231
1226
|
leaseLost: () => boolean;
|
|
1232
1227
|
leaseLostMessage: string;
|
|
1233
|
-
setLastInputTokens: (tokens: number) => Promise<void>;
|
|
1228
|
+
setLastInputTokens: (tokens: number | null) => Promise<void>;
|
|
1234
1229
|
}): Promise<
|
|
1235
|
-
| { status: "
|
|
1230
|
+
| { status: "not_response" }
|
|
1236
1231
|
| { status: "duplicate"; sourceKey: string }
|
|
1237
|
-
| {
|
|
1232
|
+
| {
|
|
1233
|
+
status: "processed";
|
|
1234
|
+
sourceKey: string;
|
|
1235
|
+
authoritative: boolean;
|
|
1236
|
+
usageReported: boolean;
|
|
1237
|
+
}
|
|
1238
1238
|
> {
|
|
1239
|
-
const
|
|
1240
|
-
if (!
|
|
1241
|
-
return { status: "
|
|
1239
|
+
const terminal = modelTerminalResponseFromSdkEvent(input.event);
|
|
1240
|
+
if (!terminal) {
|
|
1241
|
+
return { status: "not_response" };
|
|
1242
1242
|
}
|
|
1243
1243
|
|
|
1244
|
-
const responseOrdinal = input.state.
|
|
1244
|
+
const responseOrdinal = input.state.responseCount + 1;
|
|
1245
1245
|
const sourceKey = modelUsageSourceKey({
|
|
1246
|
-
responseId:
|
|
1246
|
+
responseId: terminal.responseId,
|
|
1247
1247
|
dispatchId: input.dispatchId,
|
|
1248
1248
|
positionalKey: `response-${responseOrdinal}`,
|
|
1249
1249
|
});
|
|
@@ -1251,20 +1251,25 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1251
1251
|
return { status: "duplicate", sourceKey };
|
|
1252
1252
|
}
|
|
1253
1253
|
input.state.claimedSourceKeys.add(sourceKey);
|
|
1254
|
-
input.state.
|
|
1254
|
+
input.state.responseCount = responseOrdinal;
|
|
1255
1255
|
|
|
1256
|
-
const
|
|
1256
|
+
const responseUsage = terminal.usage;
|
|
1257
|
+
|
|
1258
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage?.usage);
|
|
1257
1259
|
const accountContext = modelCallAccountContext({
|
|
1258
1260
|
servingCredentialId: input.servingCredentialId,
|
|
1259
1261
|
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
1260
1262
|
isFirstCallOfTurn: responseOrdinal === 1,
|
|
1261
1263
|
});
|
|
1262
|
-
|
|
1264
|
+
// A terminal response without usage still authoritatively replaces the
|
|
1265
|
+
// attempt-owned context signal. There is simply no billing event to claim.
|
|
1266
|
+
let authoritative = responseUsage === null;
|
|
1263
1267
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
1264
1268
|
renewLease: input.renewLease,
|
|
1265
1269
|
leaseLost: input.leaseLost,
|
|
1266
1270
|
leaseLostMessage: input.leaseLostMessage,
|
|
1267
1271
|
recordUsage: async () => {
|
|
1272
|
+
if (!responseUsage) return;
|
|
1268
1273
|
const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
1269
1274
|
accountId: input.accountId,
|
|
1270
1275
|
workspaceId: input.workspaceId,
|
|
@@ -1322,17 +1327,22 @@ export async function processModelResponseUsageEvent(input: {
|
|
|
1322
1327
|
recordAttemptSignals: async () => {
|
|
1323
1328
|
if (!authoritative) return;
|
|
1324
1329
|
const observedTotal = normalizedUsage.totalTokens;
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1330
|
+
input.state.contextSignal =
|
|
1331
|
+
observedTotal !== null && observedTotal > 0
|
|
1332
|
+
? { revision: responseOrdinal, totalTokens: observedTotal }
|
|
1333
|
+
: null;
|
|
1329
1334
|
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1335
|
+
await input.setLastInputTokens(
|
|
1336
|
+
observedInput !== null && observedInput > 0 ? observedInput : null,
|
|
1337
|
+
);
|
|
1333
1338
|
},
|
|
1334
1339
|
});
|
|
1335
|
-
return {
|
|
1340
|
+
return {
|
|
1341
|
+
status: "processed",
|
|
1342
|
+
sourceKey,
|
|
1343
|
+
authoritative,
|
|
1344
|
+
usageReported: responseUsage !== null,
|
|
1345
|
+
};
|
|
1336
1346
|
}
|
|
1337
1347
|
|
|
1338
1348
|
/**
|
|
@@ -2272,10 +2282,6 @@ export function createTurnSandboxProvisioner<T>(
|
|
|
2272
2282
|
};
|
|
2273
2283
|
}
|
|
2274
2284
|
|
|
2275
|
-
function sdkBackendIdForSandboxBackend(backend: Settings["sandboxBackend"]): string {
|
|
2276
|
-
return backend === "local" ? "unix_local" : backend;
|
|
2277
|
-
}
|
|
2278
|
-
|
|
2279
2285
|
/**
|
|
2280
2286
|
* Decide whether THIS turn may send OpenAI's `prompt_cache_key` request field.
|
|
2281
2287
|
*
|
|
@@ -2454,7 +2460,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2454
2460
|
// Still required by credential-loss/capacity settlements, whose own
|
|
2455
2461
|
// recovery transactions fence against worker-death redispatches.
|
|
2456
2462
|
let redispatchesAtDispatch = 0;
|
|
2457
|
-
const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
|
|
2463
|
+
const setLastInputTokensFenced = async (lastInputTokens: number | null): Promise<void> => {
|
|
2458
2464
|
if (!turnId || executionGeneration <= 0) {
|
|
2459
2465
|
throw new Error("Turn attempt was not initialized before token accounting");
|
|
2460
2466
|
}
|
|
@@ -6509,10 +6515,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6509
6515
|
attemptId: input.attemptId,
|
|
6510
6516
|
},
|
|
6511
6517
|
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
6512
|
-
// guards provide their exact current signal; provider
|
|
6513
|
-
//
|
|
6514
|
-
// recovery proves progress separately by comparing the
|
|
6515
|
-
// with the current active-history estimate
|
|
6518
|
+
// guards provide their exact current provider-based signal; provider
|
|
6519
|
+
// overflows do not, so their diagnostic signal stays unknown/zero.
|
|
6520
|
+
// Forced recovery proves progress separately by comparing the
|
|
6521
|
+
// replacement with the current active-history estimate.
|
|
6516
6522
|
recoverySignalTokens,
|
|
6517
6523
|
compactSummarizer,
|
|
6518
6524
|
{
|
|
@@ -6535,13 +6541,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6535
6541
|
return outcome;
|
|
6536
6542
|
};
|
|
6537
6543
|
|
|
6538
|
-
// Keep response
|
|
6544
|
+
// Keep response identity across every stream attempt in this
|
|
6539
6545
|
// activity. Context compaction retries the same logical dispatch by
|
|
6540
6546
|
// calling runStreamAttempt again; resetting this state there would reuse
|
|
6541
6547
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
6542
|
-
const
|
|
6543
|
-
claimedModelUsageSourceKeys,
|
|
6544
|
-
);
|
|
6548
|
+
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
6545
6549
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
6546
6550
|
if (!runInput) {
|
|
6547
6551
|
throw new Error("Run input was not prepared");
|
|
@@ -6555,6 +6559,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6555
6559
|
// schema or compatibility state.
|
|
6556
6560
|
let currentToolBatchCallIds = new Set<string>();
|
|
6557
6561
|
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
6562
|
+
let streamSawPerResponseUsage = false;
|
|
6558
6563
|
// Actual input tokens of the most recent model response this turn; the
|
|
6559
6564
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
6560
6565
|
throwIfWorkerShuttingDown();
|
|
@@ -6671,7 +6676,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6671
6676
|
: {}),
|
|
6672
6677
|
}
|
|
6673
6678
|
: {}),
|
|
6674
|
-
contextCompactionSignal: () =>
|
|
6679
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
6675
6680
|
contextCompactionRequested: () =>
|
|
6676
6681
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
6677
6682
|
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
@@ -6712,9 +6717,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6712
6717
|
}
|
|
6713
6718
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
6714
6719
|
let completedCurrentToolBatch = false;
|
|
6715
|
-
const
|
|
6720
|
+
const responseResult = await processModelResponseTerminalEvent({
|
|
6716
6721
|
event: next.value,
|
|
6717
|
-
state:
|
|
6722
|
+
state: modelResponseState,
|
|
6718
6723
|
dispatchId: modelUsageDispatchId,
|
|
6719
6724
|
settings,
|
|
6720
6725
|
db,
|
|
@@ -6745,7 +6750,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6745
6750
|
model: turn.model,
|
|
6746
6751
|
...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
|
|
6747
6752
|
});
|
|
6748
|
-
if (
|
|
6753
|
+
if (responseResult.status === "processed") {
|
|
6754
|
+
streamSawPerResponseUsage ||= responseResult.usageReported;
|
|
6749
6755
|
currentToolBatchCallIds = new Set<string>();
|
|
6750
6756
|
currentToolBatchCompletedCallIds = new Set<string>();
|
|
6751
6757
|
await reconcileConversationTruth();
|
|
@@ -6884,7 +6890,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6884
6890
|
stream.completed.catch(() => undefined),
|
|
6885
6891
|
cancellationSignal,
|
|
6886
6892
|
);
|
|
6887
|
-
if (
|
|
6893
|
+
if (!streamSawPerResponseUsage) {
|
|
6888
6894
|
const aggregateUsage = stream.state.usage;
|
|
6889
6895
|
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
6890
6896
|
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
@@ -6895,8 +6901,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6895
6901
|
});
|
|
6896
6902
|
if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
|
|
6897
6903
|
claimedModelUsageSourceKeys.add(aggregateSourceKey);
|
|
6898
|
-
// The
|
|
6899
|
-
//
|
|
6904
|
+
// The aggregate frame is only a billing fallback when no terminal
|
|
6905
|
+
// response exposed usage. It is not final-request context authority.
|
|
6900
6906
|
const aggregateAccountCtx = modelCallAccountContext({
|
|
6901
6907
|
servingCredentialId: effectiveCodexCredentialId,
|
|
6902
6908
|
priorSessionCredentialId: priorSessionCodexCredentialId,
|
|
@@ -6963,14 +6969,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6963
6969
|
},
|
|
6964
6970
|
recordAttemptSignals: async () => {
|
|
6965
6971
|
if (!aggregateAuthoritative) return;
|
|
6966
|
-
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
if (aggregateInput !== null && aggregateInput > 0) {
|
|
6972
|
-
await setLastInputTokensFenced(aggregateInput);
|
|
6973
|
-
}
|
|
6972
|
+
// Stream aggregates can cover multiple requests and therefore
|
|
6973
|
+
// cannot identify the final request that compaction must bind.
|
|
6974
|
+
// They remain billing/telemetry fallback data only.
|
|
6975
|
+
modelResponseState.contextSignal = null;
|
|
6976
|
+
await setLastInputTokensFenced(null);
|
|
6974
6977
|
},
|
|
6975
6978
|
});
|
|
6976
6979
|
}
|
|
@@ -339,7 +339,6 @@ async function compactContextRemoteV2(
|
|
|
339
339
|
expectedAttemptId: scope.attemptId,
|
|
340
340
|
replacementItems: replacementHistory.slice(0, -1),
|
|
341
341
|
summaryItem: tailItem as Record<string, unknown>,
|
|
342
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
343
342
|
...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
|
|
344
343
|
eventPayload: {
|
|
345
344
|
trigger: options.trigger ?? "auto",
|
|
@@ -415,7 +414,6 @@ async function compactContextPortable(
|
|
|
415
414
|
expectedAttemptId: scope.attemptId,
|
|
416
415
|
replacementItems: replacementHistory.slice(0, -1),
|
|
417
416
|
summaryItem: summaryItem as Record<string, unknown>,
|
|
418
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
419
417
|
...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
|
|
420
418
|
eventPayload: {
|
|
421
419
|
trigger: options.trigger ?? "auto",
|
|
@@ -1,10 +1,51 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
claimDueTranscriptionRecordingObjectCleanup,
|
|
3
|
+
claimExpiredFileUploadCleanup,
|
|
4
|
+
completeDueTranscriptionRecordingObjectCleanup,
|
|
5
|
+
completeExpiredFileUploadCleanup,
|
|
6
|
+
purgeExpiredTranscriptionRecordings,
|
|
7
|
+
} from "@opengeni/db";
|
|
2
8
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
3
9
|
import type { ActivityServices } from "./types";
|
|
4
10
|
|
|
5
11
|
export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
|
|
6
12
|
export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
|
|
7
13
|
export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
|
|
14
|
+
const SAFE_CLEANUP_ERROR_CATEGORIES = new Set([
|
|
15
|
+
"aborted",
|
|
16
|
+
"conflict",
|
|
17
|
+
"invalid_response",
|
|
18
|
+
"network",
|
|
19
|
+
"not_found",
|
|
20
|
+
"permission_denied",
|
|
21
|
+
"provider",
|
|
22
|
+
"timeout",
|
|
23
|
+
"unavailable",
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
function cleanupErrorAttributes(error: unknown): {
|
|
27
|
+
errorCategory: string;
|
|
28
|
+
errorStatus?: number;
|
|
29
|
+
} {
|
|
30
|
+
let errorCategory = "provider_error";
|
|
31
|
+
let errorStatus: number | undefined;
|
|
32
|
+
if (error && typeof error === "object") {
|
|
33
|
+
const rawCode = (error as { code?: unknown }).code;
|
|
34
|
+
if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
|
|
35
|
+
errorCategory = rawCode;
|
|
36
|
+
} else if (rawCode !== undefined) {
|
|
37
|
+
errorCategory = "unknown";
|
|
38
|
+
}
|
|
39
|
+
const rawStatus = Number(
|
|
40
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
41
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
42
|
+
);
|
|
43
|
+
if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
|
|
44
|
+
errorStatus = rawStatus;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return { errorCategory, ...(errorStatus === undefined ? {} : { errorStatus }) };
|
|
48
|
+
}
|
|
8
49
|
|
|
9
50
|
export type ReapExpiredFileUploadsResult = {
|
|
10
51
|
claimed: number;
|
|
@@ -18,6 +59,11 @@ export type FileUploadReaperActivityOptions = {
|
|
|
18
59
|
batchSize?: number;
|
|
19
60
|
/** Failure-injection seam; production uses the configured provider delete. */
|
|
20
61
|
deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
|
|
62
|
+
claimFileUploads?: typeof claimExpiredFileUploadCleanup;
|
|
63
|
+
completeFileUpload?: typeof completeExpiredFileUploadCleanup;
|
|
64
|
+
claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
|
|
65
|
+
completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
|
|
66
|
+
purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
|
|
21
67
|
};
|
|
22
68
|
|
|
23
69
|
/**
|
|
@@ -33,14 +79,29 @@ export function createFileUploadReaperActivities(
|
|
|
33
79
|
const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
|
|
34
80
|
const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
|
|
35
81
|
const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
|
|
82
|
+
const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
|
|
83
|
+
const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
|
|
84
|
+
const claimTranscriptionObjects =
|
|
85
|
+
options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
|
|
86
|
+
const completeTranscriptionObject =
|
|
87
|
+
options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
|
|
88
|
+
const purgeTranscriptionRecordings =
|
|
89
|
+
options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
|
|
36
90
|
|
|
37
91
|
async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
|
|
38
92
|
const { db, objectStorage, observability } = await services();
|
|
39
93
|
if (!objectStorage) {
|
|
94
|
+
try {
|
|
95
|
+
await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
|
|
96
|
+
} catch (error) {
|
|
97
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
98
|
+
...cleanupErrorAttributes(error),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
40
101
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
41
102
|
}
|
|
42
103
|
|
|
43
|
-
const claims = await
|
|
104
|
+
const claims = await claimFileUploads(db, {
|
|
44
105
|
graceMs,
|
|
45
106
|
claimTimeoutMs,
|
|
46
107
|
limit: batchSize,
|
|
@@ -50,7 +111,7 @@ export function createFileUploadReaperActivities(
|
|
|
50
111
|
for (const claim of claims) {
|
|
51
112
|
try {
|
|
52
113
|
await deleteObject(objectStorage, claim.objectKey);
|
|
53
|
-
const settled = await
|
|
114
|
+
const settled = await completeFileUpload(db, claim);
|
|
54
115
|
if (!settled) {
|
|
55
116
|
throw new Error("cleanup claim no longer owns a reclaimable upload");
|
|
56
117
|
}
|
|
@@ -61,18 +122,58 @@ export function createFileUploadReaperActivities(
|
|
|
61
122
|
workspaceId: claim.workspaceId,
|
|
62
123
|
uploadId: claim.uploadId,
|
|
63
124
|
fileId: claim.fileId,
|
|
64
|
-
|
|
125
|
+
...cleanupErrorAttributes(error),
|
|
65
126
|
});
|
|
66
127
|
}
|
|
67
128
|
}
|
|
68
|
-
|
|
129
|
+
const recordingClaims = await claimTranscriptionObjects(db, {
|
|
130
|
+
graceMs,
|
|
131
|
+
claimTimeoutMs,
|
|
132
|
+
limit: batchSize,
|
|
133
|
+
});
|
|
134
|
+
for (const claim of recordingClaims) {
|
|
135
|
+
try {
|
|
136
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
137
|
+
const settled = await completeTranscriptionObject(db, claim);
|
|
138
|
+
if (!settled) {
|
|
139
|
+
throw new Error("cleanup claim no longer owns a transcription recording object");
|
|
140
|
+
}
|
|
141
|
+
deleted += 1;
|
|
142
|
+
} catch (error) {
|
|
143
|
+
failed += 1;
|
|
144
|
+
observability.warn(
|
|
145
|
+
"expired transcription recording object cleanup failed; claim remains reclaimable",
|
|
146
|
+
{
|
|
147
|
+
workspaceId: claim.workspaceId,
|
|
148
|
+
recordingId: claim.recordingId,
|
|
149
|
+
...cleanupErrorAttributes(error),
|
|
150
|
+
},
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
let transcriptionRecordingsPurged = 0;
|
|
155
|
+
try {
|
|
156
|
+
transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
|
|
157
|
+
graceMs,
|
|
158
|
+
limit: batchSize,
|
|
159
|
+
});
|
|
160
|
+
} catch (error) {
|
|
161
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
162
|
+
...cleanupErrorAttributes(error),
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
const claimed = claims.length + recordingClaims.length;
|
|
166
|
+
if (claimed > 0 || transcriptionRecordingsPurged > 0) {
|
|
69
167
|
observability.info("expired file upload cleanup swept", {
|
|
70
|
-
claimed
|
|
168
|
+
claimed,
|
|
71
169
|
deleted,
|
|
72
170
|
failed,
|
|
171
|
+
fileUploads: claims.length,
|
|
172
|
+
transcriptionRecordingObjects: recordingClaims.length,
|
|
173
|
+
transcriptionRecordingsPurged,
|
|
73
174
|
});
|
|
74
175
|
}
|
|
75
|
-
return { claimed
|
|
176
|
+
return { claimed, deleted, failed };
|
|
76
177
|
}
|
|
77
178
|
|
|
78
179
|
return { reapExpiredFileUploads };
|
|
@@ -95,6 +95,7 @@ import {
|
|
|
95
95
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
96
96
|
resolveModalCheckpointProviderBinding,
|
|
97
97
|
resolveModalCheckpointProviderBindingForSession,
|
|
98
|
+
sandboxBackendForSdkBackendId,
|
|
98
99
|
sandboxCommandExitCode,
|
|
99
100
|
sandboxCommandStillRunning,
|
|
100
101
|
sweepModalOrphanSandboxes,
|
|
@@ -1428,7 +1429,8 @@ async function probeDrainableProviderReadiness(
|
|
|
1428
1429
|
if (!lease.instanceId || !lease.resumeState) {
|
|
1429
1430
|
throw new Error("Expired workspace capture has no resumable provider identity");
|
|
1430
1431
|
}
|
|
1431
|
-
const
|
|
1432
|
+
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
1433
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
1432
1434
|
let established: Awaited<ReturnType<typeof establishSandboxSessionFromEnvelope>>;
|
|
1433
1435
|
try {
|
|
1434
1436
|
established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {
|
|
@@ -1811,7 +1813,8 @@ export async function terminateProviderBox(
|
|
|
1811
1813
|
persistArchive: PersistArchiveFn,
|
|
1812
1814
|
createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
|
|
1813
1815
|
): Promise<ProviderTerminationOutcome> {
|
|
1814
|
-
const
|
|
1816
|
+
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
1817
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
1815
1818
|
// 'none' / no backend -> nothing to terminate.
|
|
1816
1819
|
if (!backend || backend === "none") {
|
|
1817
1820
|
return { terminated: true, providerMissingBeforeCapture: false };
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
RoutingBackendRecoveryRequiredError,
|
|
45
45
|
RoutingSandboxSession,
|
|
46
46
|
resolveModalCheckpointProviderBindingForSession,
|
|
47
|
+
sandboxBackendForSdkBackendId,
|
|
47
48
|
verifySandboxExecReadiness,
|
|
48
49
|
type ControlRpc,
|
|
49
50
|
type EstablishedSandboxSession,
|
|
@@ -322,7 +323,8 @@ async function resolveCurrentHomeBackend(
|
|
|
322
323
|
}
|
|
323
324
|
|
|
324
325
|
const resumeState = lease.resumeState;
|
|
325
|
-
const
|
|
326
|
+
const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
327
|
+
const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
|
|
326
328
|
if (
|
|
327
329
|
!resumeState ||
|
|
328
330
|
resumeBackend !== ids.backend ||
|
|
@@ -377,7 +379,8 @@ async function resolveCurrentHomeBackend(
|
|
|
377
379
|
// The provider identity must agree with the exact durable lease row before
|
|
378
380
|
// any caller can publish or route through this rebound handle. A mismatch is
|
|
379
381
|
// unverifiable local state, not permission to try the old handle.
|
|
380
|
-
|
|
382
|
+
const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
|
|
383
|
+
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
381
384
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
382
385
|
}
|
|
383
386
|
services.onHomeSandboxRebound?.({
|