@opengeni/worker-bundle 0.16.17 → 0.16.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.16.17",
3
+ "version": "0.16.21",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -41,16 +41,16 @@
41
41
  "dependencies": {
42
42
  "@opengeni/agent-proto": "^0.3.0",
43
43
  "@opengeni/codex": "^0.2.11",
44
- "@opengeni/config": "^0.10.14",
45
- "@opengeni/contracts": "^0.38.3",
46
- "@opengeni/core": "^0.20.16",
47
- "@opengeni/db": "^0.27.11",
48
- "@opengeni/documents": "^0.5.7",
49
- "@opengeni/events": "^0.3.77",
50
- "@opengeni/github": "^0.4.30",
51
- "@opengeni/observability": "^0.4.16",
52
- "@opengeni/runtime": "^0.18.15",
53
- "@opengeni/storage": "^0.2.67",
44
+ "@opengeni/config": "^0.11.0",
45
+ "@opengeni/contracts": "^0.39.0",
46
+ "@opengeni/core": "^0.21.2",
47
+ "@opengeni/db": "^0.28.1",
48
+ "@opengeni/documents": "^0.5.10",
49
+ "@opengeni/events": "^0.3.80",
50
+ "@opengeni/github": "^0.4.31",
51
+ "@opengeni/observability": "^0.5.0",
52
+ "@opengeni/runtime": "^0.18.17",
53
+ "@opengeni/storage": "^0.2.68",
54
54
  "@temporalio/activity": "^1.17.0",
55
55
  "@temporalio/client": "^1.17.0",
56
56
  "@temporalio/worker": "^1.17.0",
@@ -89,7 +89,7 @@ import {
89
89
  sandboxStateEntryFromRunState,
90
90
  maxTurnsExceededRunState,
91
91
  modelResponseServiceTierFromSdkEvent,
92
- modelResponseUsageFromSdkEvent,
92
+ modelTerminalResponseFromSdkEvent,
93
93
  normalizeModelCallUsage,
94
94
  normalizeSdkEvent,
95
95
  projectHistoryForProvider,
@@ -126,6 +126,7 @@ import {
126
126
  runOwnedSandboxSetup,
127
127
  RoutingMutationOutcomeUnknownError,
128
128
  WorkspaceArchiveIntegrityError,
129
+ sdkBackendIdForSandboxBackend,
129
130
  swapTargetEstablishability,
130
131
  type SandboxFileDownload,
131
132
  type SandboxFileDownloadFailure,
@@ -1123,10 +1124,9 @@ type TurnEventPublisher = (
1123
1124
  immediate?: boolean,
1124
1125
  ) => Promise<{ events: SessionEvent[]; accepted: boolean }>;
1125
1126
 
1126
- export type ModelResponseUsageEventState = {
1127
- responseUsageCount: number;
1128
- providerContextRevision: number;
1129
- lastProviderContextTokensObserved: number | null;
1127
+ export type ModelResponseEventState = {
1128
+ responseCount: number;
1129
+ contextSignal: { revision: number; totalTokens: number } | null;
1130
1130
  claimedSourceKeys: Set<string>;
1131
1131
  };
1132
1132
 
@@ -1135,13 +1135,12 @@ export type CompactionModelUsageEventState = {
1135
1135
  claimedSourceKeys: Set<string>;
1136
1136
  };
1137
1137
 
1138
- export function createModelResponseUsageEventState(
1138
+ export function createModelResponseEventState(
1139
1139
  claimedSourceKeys: Set<string> = new Set<string>(),
1140
- ): ModelResponseUsageEventState {
1140
+ ): ModelResponseEventState {
1141
1141
  return {
1142
- responseUsageCount: 0,
1143
- providerContextRevision: 0,
1144
- lastProviderContextTokensObserved: null,
1142
+ responseCount: 0,
1143
+ contextSignal: null,
1145
1144
  claimedSourceKeys,
1146
1145
  };
1147
1146
  }
@@ -1152,15 +1151,10 @@ export function createCompactionModelUsageEventState(
1152
1151
  return { usageCount: 0, claimedSourceKeys };
1153
1152
  }
1154
1153
 
1155
- export function modelResponseUsageContextSignal(
1156
- state: ModelResponseUsageEventState,
1154
+ export function modelResponseContextSignal(
1155
+ state: ModelResponseEventState,
1157
1156
  ): { revision: number; totalTokens: number } | null {
1158
- return state.lastProviderContextTokensObserved === null
1159
- ? null
1160
- : {
1161
- revision: state.providerContextRevision,
1162
- totalTokens: state.lastProviderContextTokensObserved,
1163
- };
1157
+ return state.contextSignal;
1164
1158
  }
1165
1159
 
1166
1160
  export function assertModelResponseLatencyMode(input: {
@@ -1195,19 +1189,20 @@ export function assertModelResponseLatencyMode(input: {
1195
1189
  }
1196
1190
 
1197
1191
  /**
1198
- * Process one SDK stream event through the exact production usage path.
1192
+ * Process one SDK terminal-response event through the production authority path.
1199
1193
  *
1200
1194
  * The pinned Responses SDK mirrors one provider terminal response as both a
1201
1195
  * normalized `response_done` and a raw `model/response.completed` event. Claim
1202
- * the stable response/source key before lease renewal or any usage side effect,
1203
- * and advance the positional ordinal only for a newly claimed response. The
1204
- * durable `agent.model.usage` source-key fence remains the cross-restart
1205
- * authority: a replay may retry the idempotent billing write, but it cannot
1206
- * advance process-local metrics, provider context, or attempt-owned signals.
1196
+ * the stable response/source key before lease renewal or any side effect, and
1197
+ * use that one positional ordinal for both response identity and same-run
1198
+ * context binding. A response without usage still clears attempt-owned token
1199
+ * state. When usage exists, the durable `agent.model.usage` source-key fence
1200
+ * remains the cross-restart authority: a replay may retry the idempotent billing
1201
+ * write, but it cannot advance metrics, context, or attempt-owned signals.
1207
1202
  */
1208
- export async function processModelResponseUsageEvent(input: {
1209
- event: Parameters<typeof modelResponseUsageFromSdkEvent>[0];
1210
- state: ModelResponseUsageEventState;
1203
+ export async function processModelResponseTerminalEvent(input: {
1204
+ event: Parameters<typeof modelTerminalResponseFromSdkEvent>[0];
1205
+ state: ModelResponseEventState;
1211
1206
  dispatchId: string | null;
1212
1207
  settings: Settings;
1213
1208
  db: ActivityServices["db"];
@@ -1230,20 +1225,25 @@ export async function processModelResponseUsageEvent(input: {
1230
1225
  renewLease: () => Promise<void>;
1231
1226
  leaseLost: () => boolean;
1232
1227
  leaseLostMessage: string;
1233
- setLastInputTokens: (tokens: number) => Promise<void>;
1228
+ setLastInputTokens: (tokens: number | null) => Promise<void>;
1234
1229
  }): Promise<
1235
- | { status: "not_usage" }
1230
+ | { status: "not_response" }
1236
1231
  | { status: "duplicate"; sourceKey: string }
1237
- | { status: "processed"; sourceKey: string; authoritative: boolean }
1232
+ | {
1233
+ status: "processed";
1234
+ sourceKey: string;
1235
+ authoritative: boolean;
1236
+ usageReported: boolean;
1237
+ }
1238
1238
  > {
1239
- const responseUsage = modelResponseUsageFromSdkEvent(input.event);
1240
- if (!responseUsage) {
1241
- return { status: "not_usage" };
1239
+ const terminal = modelTerminalResponseFromSdkEvent(input.event);
1240
+ if (!terminal) {
1241
+ return { status: "not_response" };
1242
1242
  }
1243
1243
 
1244
- const responseOrdinal = input.state.responseUsageCount + 1;
1244
+ const responseOrdinal = input.state.responseCount + 1;
1245
1245
  const sourceKey = modelUsageSourceKey({
1246
- responseId: responseUsage.responseId,
1246
+ responseId: terminal.responseId,
1247
1247
  dispatchId: input.dispatchId,
1248
1248
  positionalKey: `response-${responseOrdinal}`,
1249
1249
  });
@@ -1251,20 +1251,25 @@ export async function processModelResponseUsageEvent(input: {
1251
1251
  return { status: "duplicate", sourceKey };
1252
1252
  }
1253
1253
  input.state.claimedSourceKeys.add(sourceKey);
1254
- input.state.responseUsageCount = responseOrdinal;
1254
+ input.state.responseCount = responseOrdinal;
1255
1255
 
1256
- const normalizedUsage = normalizeModelCallUsage(responseUsage.usage);
1256
+ const responseUsage = terminal.usage;
1257
+
1258
+ const normalizedUsage = normalizeModelCallUsage(responseUsage?.usage);
1257
1259
  const accountContext = modelCallAccountContext({
1258
1260
  servingCredentialId: input.servingCredentialId,
1259
1261
  priorSessionCredentialId: input.priorSessionCredentialId,
1260
1262
  isFirstCallOfTurn: responseOrdinal === 1,
1261
1263
  });
1262
- let authoritative = false;
1264
+ // A terminal response without usage still authoritatively replaces the
1265
+ // attempt-owned context signal. There is simply no billing event to claim.
1266
+ let authoritative = responseUsage === null;
1263
1267
  await recordCompletedModelCallBeforeOwnershipFences({
1264
1268
  renewLease: input.renewLease,
1265
1269
  leaseLost: input.leaseLost,
1266
1270
  leaseLostMessage: input.leaseLostMessage,
1267
1271
  recordUsage: async () => {
1272
+ if (!responseUsage) return;
1268
1273
  const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
1269
1274
  accountId: input.accountId,
1270
1275
  workspaceId: input.workspaceId,
@@ -1322,17 +1327,22 @@ export async function processModelResponseUsageEvent(input: {
1322
1327
  recordAttemptSignals: async () => {
1323
1328
  if (!authoritative) return;
1324
1329
  const observedTotal = normalizedUsage.totalTokens;
1325
- if (observedTotal !== null) {
1326
- input.state.lastProviderContextTokensObserved = observedTotal;
1327
- input.state.providerContextRevision += 1;
1328
- }
1330
+ input.state.contextSignal =
1331
+ observedTotal !== null && observedTotal > 0
1332
+ ? { revision: responseOrdinal, totalTokens: observedTotal }
1333
+ : null;
1329
1334
  const observedInput = normalizedUsage.telemetry.inputTokens;
1330
- if (observedInput !== null && observedInput > 0) {
1331
- await input.setLastInputTokens(observedInput);
1332
- }
1335
+ await input.setLastInputTokens(
1336
+ observedInput !== null && observedInput > 0 ? observedInput : null,
1337
+ );
1333
1338
  },
1334
1339
  });
1335
- return { status: "processed", sourceKey, authoritative };
1340
+ return {
1341
+ status: "processed",
1342
+ sourceKey,
1343
+ authoritative,
1344
+ usageReported: responseUsage !== null,
1345
+ };
1336
1346
  }
1337
1347
 
1338
1348
  /**
@@ -2272,10 +2282,6 @@ export function createTurnSandboxProvisioner<T>(
2272
2282
  };
2273
2283
  }
2274
2284
 
2275
- function sdkBackendIdForSandboxBackend(backend: Settings["sandboxBackend"]): string {
2276
- return backend === "local" ? "unix_local" : backend;
2277
- }
2278
-
2279
2285
  /**
2280
2286
  * Decide whether THIS turn may send OpenAI's `prompt_cache_key` request field.
2281
2287
  *
@@ -2454,7 +2460,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2454
2460
  // Still required by credential-loss/capacity settlements, whose own
2455
2461
  // recovery transactions fence against worker-death redispatches.
2456
2462
  let redispatchesAtDispatch = 0;
2457
- const setLastInputTokensFenced = async (lastInputTokens: number): Promise<void> => {
2463
+ const setLastInputTokensFenced = async (lastInputTokens: number | null): Promise<void> => {
2458
2464
  if (!turnId || executionGeneration <= 0) {
2459
2465
  throw new Error("Turn attempt was not initialized before token accounting");
2460
2466
  }
@@ -6509,10 +6515,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6509
6515
  attemptId: input.attemptId,
6510
6516
  },
6511
6517
  // Never reuse the persisted prior-turn signal for recovery. Proactive
6512
- // guards provide their exact current signal; provider overflows do not,
6513
- // so null derives decision metadata from active history. Forced
6514
- // recovery proves progress separately by comparing the replacement
6515
- // with the current active-history estimate in maybeCompactContext.
6518
+ // guards provide their exact current provider-based signal; provider
6519
+ // overflows do not, so their diagnostic signal stays unknown/zero.
6520
+ // Forced recovery proves progress separately by comparing the
6521
+ // replacement with the current active-history estimate.
6516
6522
  recoverySignalTokens,
6517
6523
  compactSummarizer,
6518
6524
  {
@@ -6535,13 +6541,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6535
6541
  return outcome;
6536
6542
  };
6537
6543
 
6538
- // Keep response usage identity across every stream attempt in this
6544
+ // Keep response identity across every stream attempt in this
6539
6545
  // activity. Context compaction retries the same logical dispatch by
6540
6546
  // calling runStreamAttempt again; resetting this state there would reuse
6541
6547
  // the first no-response-ID fallback key and suppress a real model call.
6542
- const modelResponseUsageState = createModelResponseUsageEventState(
6543
- claimedModelUsageSourceKeys,
6544
- );
6548
+ const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
6545
6549
  const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
6546
6550
  if (!runInput) {
6547
6551
  throw new Error("Run input was not prepared");
@@ -6555,6 +6559,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6555
6559
  // schema or compatibility state.
6556
6560
  let currentToolBatchCallIds = new Set<string>();
6557
6561
  let currentToolBatchCompletedCallIds = new Set<string>();
6562
+ let streamSawPerResponseUsage = false;
6558
6563
  // Actual input tokens of the most recent model response this turn; the
6559
6564
  // pre-read trigger for the NEXT turn. Persisted at every turn-end path.
6560
6565
  throwIfWorkerShuttingDown();
@@ -6671,7 +6676,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6671
6676
  : {}),
6672
6677
  }
6673
6678
  : {}),
6674
- contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
6679
+ contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
6675
6680
  contextCompactionRequested: () =>
6676
6681
  isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
6677
6682
  callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
@@ -6712,9 +6717,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6712
6717
  }
6713
6718
  let stableToolCallIdsToClear: string[] | null = null;
6714
6719
  let completedCurrentToolBatch = false;
6715
- const responseUsageResult = await processModelResponseUsageEvent({
6720
+ const responseResult = await processModelResponseTerminalEvent({
6716
6721
  event: next.value,
6717
- state: modelResponseUsageState,
6722
+ state: modelResponseState,
6718
6723
  dispatchId: modelUsageDispatchId,
6719
6724
  settings,
6720
6725
  db,
@@ -6745,7 +6750,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6745
6750
  model: turn.model,
6746
6751
  ...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
6747
6752
  });
6748
- if (responseUsageResult.status === "processed") {
6753
+ if (responseResult.status === "processed") {
6754
+ streamSawPerResponseUsage ||= responseResult.usageReported;
6749
6755
  currentToolBatchCallIds = new Set<string>();
6750
6756
  currentToolBatchCompletedCallIds = new Set<string>();
6751
6757
  await reconcileConversationTruth();
@@ -6884,7 +6890,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6884
6890
  stream.completed.catch(() => undefined),
6885
6891
  cancellationSignal,
6886
6892
  );
6887
- if (modelResponseUsageState.responseUsageCount === 0) {
6893
+ if (!streamSawPerResponseUsage) {
6888
6894
  const aggregateUsage = stream.state.usage;
6889
6895
  const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
6890
6896
  const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
@@ -6895,8 +6901,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6895
6901
  });
6896
6902
  if (!claimedModelUsageSourceKeys.has(aggregateSourceKey)) {
6897
6903
  claimedModelUsageSourceKeys.add(aggregateSourceKey);
6898
- // The single aggregate frame is this turn's only model-usage record, so
6899
- // it is the first (account-switch surfaces here just like a first response).
6904
+ // The aggregate frame is only a billing fallback when no terminal
6905
+ // response exposed usage. It is not final-request context authority.
6900
6906
  const aggregateAccountCtx = modelCallAccountContext({
6901
6907
  servingCredentialId: effectiveCodexCredentialId,
6902
6908
  priorSessionCredentialId: priorSessionCodexCredentialId,
@@ -6963,14 +6969,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
6963
6969
  },
6964
6970
  recordAttemptSignals: async () => {
6965
6971
  if (!aggregateAuthoritative) return;
6966
- if (normalizedAggregateUsage.totalTokens !== null) {
6967
- modelResponseUsageState.lastProviderContextTokensObserved =
6968
- normalizedAggregateUsage.totalTokens;
6969
- modelResponseUsageState.providerContextRevision += 1;
6970
- }
6971
- if (aggregateInput !== null && aggregateInput > 0) {
6972
- await setLastInputTokensFenced(aggregateInput);
6973
- }
6972
+ // Stream aggregates can cover multiple requests and therefore
6973
+ // cannot identify the final request that compaction must bind.
6974
+ // They remain billing/telemetry fallback data only.
6975
+ modelResponseState.contextSignal = null;
6976
+ await setLastInputTokensFenced(null);
6974
6977
  },
6975
6978
  });
6976
6979
  }
@@ -339,7 +339,6 @@ async function compactContextRemoteV2(
339
339
  expectedAttemptId: scope.attemptId,
340
340
  replacementItems: replacementHistory.slice(0, -1),
341
341
  summaryItem: tailItem as Record<string, unknown>,
342
- replacementInputTokens: estimatedTokensAfter,
343
342
  ...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
344
343
  eventPayload: {
345
344
  trigger: options.trigger ?? "auto",
@@ -415,7 +414,6 @@ async function compactContextPortable(
415
414
  expectedAttemptId: scope.attemptId,
416
415
  replacementItems: replacementHistory.slice(0, -1),
417
416
  summaryItem: summaryItem as Record<string, unknown>,
418
- replacementInputTokens: estimatedTokensAfter,
419
417
  ...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
420
418
  eventPayload: {
421
419
  trigger: options.trigger ?? "auto",
@@ -1,10 +1,51 @@
1
- import { claimExpiredFileUploadCleanup, completeExpiredFileUploadCleanup } from "@opengeni/db";
1
+ import {
2
+ claimDueTranscriptionRecordingObjectCleanup,
3
+ claimExpiredFileUploadCleanup,
4
+ completeDueTranscriptionRecordingObjectCleanup,
5
+ completeExpiredFileUploadCleanup,
6
+ purgeExpiredTranscriptionRecordings,
7
+ } from "@opengeni/db";
2
8
  import type { ObjectStorage } from "@opengeni/storage";
3
9
  import type { ActivityServices } from "./types";
4
10
 
5
11
  export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
6
12
  export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
7
13
  export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
14
+ const SAFE_CLEANUP_ERROR_CATEGORIES = new Set([
15
+ "aborted",
16
+ "conflict",
17
+ "invalid_response",
18
+ "network",
19
+ "not_found",
20
+ "permission_denied",
21
+ "provider",
22
+ "timeout",
23
+ "unavailable",
24
+ ]);
25
+
26
+ function cleanupErrorAttributes(error: unknown): {
27
+ errorCategory: string;
28
+ errorStatus?: number;
29
+ } {
30
+ let errorCategory = "provider_error";
31
+ let errorStatus: number | undefined;
32
+ if (error && typeof error === "object") {
33
+ const rawCode = (error as { code?: unknown }).code;
34
+ if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
35
+ errorCategory = rawCode;
36
+ } else if (rawCode !== undefined) {
37
+ errorCategory = "unknown";
38
+ }
39
+ const rawStatus = Number(
40
+ (error as { status?: unknown; statusCode?: unknown }).status ??
41
+ (error as { statusCode?: unknown }).statusCode,
42
+ );
43
+ if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
44
+ errorStatus = rawStatus;
45
+ }
46
+ }
47
+ return { errorCategory, ...(errorStatus === undefined ? {} : { errorStatus }) };
48
+ }
8
49
 
9
50
  export type ReapExpiredFileUploadsResult = {
10
51
  claimed: number;
@@ -18,6 +59,11 @@ export type FileUploadReaperActivityOptions = {
18
59
  batchSize?: number;
19
60
  /** Failure-injection seam; production uses the configured provider delete. */
20
61
  deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
62
+ claimFileUploads?: typeof claimExpiredFileUploadCleanup;
63
+ completeFileUpload?: typeof completeExpiredFileUploadCleanup;
64
+ claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
65
+ completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
66
+ purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
21
67
  };
22
68
 
23
69
  /**
@@ -33,14 +79,29 @@ export function createFileUploadReaperActivities(
33
79
  const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
34
80
  const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
35
81
  const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
82
+ const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
83
+ const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
84
+ const claimTranscriptionObjects =
85
+ options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
86
+ const completeTranscriptionObject =
87
+ options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
88
+ const purgeTranscriptionRecordings =
89
+ options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
36
90
 
37
91
  async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
38
92
  const { db, objectStorage, observability } = await services();
39
93
  if (!objectStorage) {
94
+ try {
95
+ await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
96
+ } catch (error) {
97
+ observability.warn("expired transcription recording metadata purge failed", {
98
+ ...cleanupErrorAttributes(error),
99
+ });
100
+ }
40
101
  return { claimed: 0, deleted: 0, failed: 0 };
41
102
  }
42
103
 
43
- const claims = await claimExpiredFileUploadCleanup(db, {
104
+ const claims = await claimFileUploads(db, {
44
105
  graceMs,
45
106
  claimTimeoutMs,
46
107
  limit: batchSize,
@@ -50,7 +111,7 @@ export function createFileUploadReaperActivities(
50
111
  for (const claim of claims) {
51
112
  try {
52
113
  await deleteObject(objectStorage, claim.objectKey);
53
- const settled = await completeExpiredFileUploadCleanup(db, claim);
114
+ const settled = await completeFileUpload(db, claim);
54
115
  if (!settled) {
55
116
  throw new Error("cleanup claim no longer owns a reclaimable upload");
56
117
  }
@@ -61,18 +122,58 @@ export function createFileUploadReaperActivities(
61
122
  workspaceId: claim.workspaceId,
62
123
  uploadId: claim.uploadId,
63
124
  fileId: claim.fileId,
64
- error: error instanceof Error ? error.message : String(error),
125
+ ...cleanupErrorAttributes(error),
65
126
  });
66
127
  }
67
128
  }
68
- if (claims.length > 0) {
129
+ const recordingClaims = await claimTranscriptionObjects(db, {
130
+ graceMs,
131
+ claimTimeoutMs,
132
+ limit: batchSize,
133
+ });
134
+ for (const claim of recordingClaims) {
135
+ try {
136
+ await deleteObject(objectStorage, claim.objectKey);
137
+ const settled = await completeTranscriptionObject(db, claim);
138
+ if (!settled) {
139
+ throw new Error("cleanup claim no longer owns a transcription recording object");
140
+ }
141
+ deleted += 1;
142
+ } catch (error) {
143
+ failed += 1;
144
+ observability.warn(
145
+ "expired transcription recording object cleanup failed; claim remains reclaimable",
146
+ {
147
+ workspaceId: claim.workspaceId,
148
+ recordingId: claim.recordingId,
149
+ ...cleanupErrorAttributes(error),
150
+ },
151
+ );
152
+ }
153
+ }
154
+ let transcriptionRecordingsPurged = 0;
155
+ try {
156
+ transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
157
+ graceMs,
158
+ limit: batchSize,
159
+ });
160
+ } catch (error) {
161
+ observability.warn("expired transcription recording metadata purge failed", {
162
+ ...cleanupErrorAttributes(error),
163
+ });
164
+ }
165
+ const claimed = claims.length + recordingClaims.length;
166
+ if (claimed > 0 || transcriptionRecordingsPurged > 0) {
69
167
  observability.info("expired file upload cleanup swept", {
70
- claimed: claims.length,
168
+ claimed,
71
169
  deleted,
72
170
  failed,
171
+ fileUploads: claims.length,
172
+ transcriptionRecordingObjects: recordingClaims.length,
173
+ transcriptionRecordingsPurged,
73
174
  });
74
175
  }
75
- return { claimed: claims.length, deleted, failed };
176
+ return { claimed, deleted, failed };
76
177
  }
77
178
 
78
179
  return { reapExpiredFileUploads };
@@ -95,6 +95,7 @@ import {
95
95
  resolveModalCheckpointProviderBindingForLiveSandbox,
96
96
  resolveModalCheckpointProviderBinding,
97
97
  resolveModalCheckpointProviderBindingForSession,
98
+ sandboxBackendForSdkBackendId,
98
99
  sandboxCommandExitCode,
99
100
  sandboxCommandStillRunning,
100
101
  sweepModalOrphanSandboxes,
@@ -1428,7 +1429,8 @@ async function probeDrainableProviderReadiness(
1428
1429
  if (!lease.instanceId || !lease.resumeState) {
1429
1430
  throw new Error("Expired workspace capture has no resumable provider identity");
1430
1431
  }
1431
- const backend = (lease.resumeBackendId ?? lease.backend) as string;
1432
+ const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
1433
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
1432
1434
  let established: Awaited<ReturnType<typeof establishSandboxSessionFromEnvelope>>;
1433
1435
  try {
1434
1436
  established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {
@@ -1811,7 +1813,8 @@ export async function terminateProviderBox(
1811
1813
  persistArchive: PersistArchiveFn,
1812
1814
  createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,
1813
1815
  ): Promise<ProviderTerminationOutcome> {
1814
- const backend = (lease.resumeBackendId ?? lease.backend) as string;
1816
+ const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
1817
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
1815
1818
  // 'none' / no backend -> nothing to terminate.
1816
1819
  if (!backend || backend === "none") {
1817
1820
  return { terminated: true, providerMissingBeforeCapture: false };
@@ -44,6 +44,7 @@ import {
44
44
  RoutingBackendRecoveryRequiredError,
45
45
  RoutingSandboxSession,
46
46
  resolveModalCheckpointProviderBindingForSession,
47
+ sandboxBackendForSdkBackendId,
47
48
  verifySandboxExecReadiness,
48
49
  type ControlRpc,
49
50
  type EstablishedSandboxSession,
@@ -322,7 +323,8 @@ async function resolveCurrentHomeBackend(
322
323
  }
323
324
 
324
325
  const resumeState = lease.resumeState;
325
- const resumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
326
+ const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
327
+ const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
326
328
  if (
327
329
  !resumeState ||
328
330
  resumeBackend !== ids.backend ||
@@ -377,7 +379,8 @@ async function resolveCurrentHomeBackend(
377
379
  // The provider identity must agree with the exact durable lease row before
378
380
  // any caller can publish or route through this rebound handle. A mismatch is
379
381
  // unverifiable local state, not permission to try the old handle.
380
- if (rebound.instanceId !== lease.instanceId || rebound.backendId !== resumeBackend) {
382
+ const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
383
+ if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
381
384
  throw homeRouteRecoveryError(lease, lease.leaseEpoch);
382
385
  }
383
386
  services.onHomeSandboxRebound?.({