@opengeni/worker-bundle 0.16.14 → 0.16.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +33 -52
- package/dist/activities/codex-rotation.d.ts +5 -9
- package/dist/activities/context-compaction.d.ts +0 -4
- package/dist/activities/file-upload-reaper.d.ts +6 -0
- package/dist/activities/run-input.d.ts +16 -73
- package/dist/index.js +350 -278
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +44 -28
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +242 -257
- package/src/activities/codex-capacity.ts +0 -1
- package/src/activities/codex-rotation.ts +10 -83
- package/src/activities/context-compaction.ts +2 -14
- package/src/activities/file-upload-reaper.ts +108 -7
- package/src/activities/run-input.ts +90 -150
- package/src/activities/sandbox-lease.ts +5 -2
- package/src/sandbox-routing.ts +5 -2
- package/src/workflows/session.ts +55 -32
|
@@ -94,7 +94,6 @@ export function codexCapacityDecision(
|
|
|
94
94
|
sessionPinnedCredentialId: context.sessionPinnedCredentialId,
|
|
95
95
|
sessionPinSource: context.sessionPinSource,
|
|
96
96
|
sessionLastCredentialId: context.sessionLastCredentialId,
|
|
97
|
-
continuationCredentialId: null,
|
|
98
97
|
now,
|
|
99
98
|
});
|
|
100
99
|
if (selected.credentialId) {
|
|
@@ -88,10 +88,7 @@ export function classifyCodexPin(args: {
|
|
|
88
88
|
export type RotationDecision =
|
|
89
89
|
// The chosen account. `moved` ⇒ it differs from the current active pointer, so
|
|
90
90
|
// the caller must persist the pointer move (and the switch is a "rotation").
|
|
91
|
-
|
|
92
|
-
// chosen account does NOT cover — populated ONLY on a Tier-2/unknown pick that
|
|
93
|
-
// can't cover (prefer-not-require: failover still happens, but the pill warns).
|
|
94
|
-
| { kind: "active"; credentialId: string; moved: boolean; droppedConnectors?: string[] }
|
|
91
|
+
| { kind: "active"; credentialId: string; moved: boolean }
|
|
95
92
|
// Every eligible account is capped/cooling: idle until the soonest instant ANY
|
|
96
93
|
// account clears every blocking condition (the multi-account generalization of
|
|
97
94
|
// the single-account idle-until-reset).
|
|
@@ -157,46 +154,10 @@ function cooling(acct: CodexRotationAccount, now: Date): boolean {
|
|
|
157
154
|
}
|
|
158
155
|
|
|
159
156
|
/**
|
|
160
|
-
*
|
|
161
|
-
* cached connector set is a SUPERSET of the session's used set. An empty used set is
|
|
162
|
-
* trivially covered by everyone (→ Tier 1 == all eligibles == byte-identical to P3).
|
|
163
|
-
* A null (never-probed) set is UNKNOWN: never credited as covering (Tier 2 only),
|
|
164
|
-
* but — critically — never excluded, so failover is fully preserved.
|
|
165
|
-
*/
|
|
166
|
-
function covers(acct: CodexRotationAccount, usedConnectors: string[]): boolean {
|
|
167
|
-
if (usedConnectors.length === 0) {
|
|
168
|
-
return true;
|
|
169
|
-
}
|
|
170
|
-
const owned = acct.connectorNamespaces;
|
|
171
|
-
if (owned === null) {
|
|
172
|
-
return false; // unprobed ⇒ unknown ⇒ never Tier 1 (but still a Tier-2 candidate)
|
|
173
|
-
}
|
|
174
|
-
const ownedSet = new Set(owned);
|
|
175
|
-
return usedConnectors.every((connector) => ownedSet.has(connector));
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/**
|
|
179
|
-
* The used connectors the chosen account does NOT cover (the "switch dropped a
|
|
180
|
-
* connector" note). Empty when the account covers (superset) or nothing was used.
|
|
181
|
-
* A null (unknown) set surfaces ALL used connectors — we can't prove it covers them.
|
|
182
|
-
*/
|
|
183
|
-
function droppedConnectorsFor(acct: CodexRotationAccount, usedConnectors: string[]): string[] {
|
|
184
|
-
if (usedConnectors.length === 0) {
|
|
185
|
-
return [];
|
|
186
|
-
}
|
|
187
|
-
const owned = acct.connectorNamespaces;
|
|
188
|
-
if (owned === null) {
|
|
189
|
-
return [...usedConnectors];
|
|
190
|
-
}
|
|
191
|
-
const ownedSet = new Set(owned);
|
|
192
|
-
return usedConnectors.filter((connector) => !ownedSet.has(connector));
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Healthy for an already-frozen/in-flight turn = connected/usable (status
|
|
157
|
+
* Healthy for an already-leased/in-flight turn = connected/usable (status
|
|
197
158
|
* "active", excludes needs_relogin/error), not cooling, and not exhausted in
|
|
198
159
|
* either provider window. Allocator eligibility is
|
|
199
|
-
* deliberately separate so disabling NEW work cannot break
|
|
160
|
+
* deliberately separate so disabling NEW work cannot break an existing live lease.
|
|
200
161
|
*/
|
|
201
162
|
export function isCodexCredentialHealthy(acct: CodexRotationAccount, now: Date): boolean {
|
|
202
163
|
return (
|
|
@@ -433,8 +394,8 @@ export const REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5_000; // 5s
|
|
|
433
394
|
* Circuit-breaker slack (Finding 1b) added to the connected-account count to bound
|
|
434
395
|
* consecutive reactive failovers. Each legitimate failover cools one account, so
|
|
435
396
|
* after at most N failovers every account is capped and the reactive path takes the
|
|
436
|
-
* all-capped idle instead; the +margin absorbs benign re-ranks
|
|
437
|
-
*
|
|
397
|
+
* all-capped idle instead; the +margin absorbs benign capacity re-ranks without
|
|
398
|
+
* tripping the breaker in normal use.
|
|
438
399
|
*/
|
|
439
400
|
export const REACTIVE_ROTATION_MARGIN = 2;
|
|
440
401
|
/**
|
|
@@ -491,11 +452,6 @@ export function chooseRotationActive(args: {
|
|
|
491
452
|
priorCredentialId: string | null;
|
|
492
453
|
accounts: CodexRotationAccount[];
|
|
493
454
|
now: Date;
|
|
494
|
-
// P4 (Part B): the connectors the leaving session has access to (the leaving
|
|
495
|
-
// account's cached connector set, used as the "session needs these" proxy).
|
|
496
|
-
// Optional + defaults to [] ⇒ when absent/empty the ranker is BYTE-IDENTICAL to
|
|
497
|
-
// P3 (every account trivially covers → Tier 1 == all eligibles). Self-gating.
|
|
498
|
-
usedConnectors?: string[];
|
|
499
455
|
}): RotationDecision {
|
|
500
456
|
const { activeCredentialId, priorCredentialId, accounts, now } = args;
|
|
501
457
|
// Normalize at the entry point: rotation-enabled ALWAYS behaves as sharded
|
|
@@ -503,7 +459,6 @@ export function chooseRotationActive(args: {
|
|
|
503
459
|
// tests can still express stored legacy values.
|
|
504
460
|
const rotationStrategy = effectiveRotationStrategy(args.rotationStrategy);
|
|
505
461
|
const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
|
|
506
|
-
const usedConnectors = args.usedConnectors ?? [];
|
|
507
462
|
|
|
508
463
|
if (allocatableAccounts.length === 0) {
|
|
509
464
|
return { kind: "none" };
|
|
@@ -523,14 +478,10 @@ export function chooseRotationActive(args: {
|
|
|
523
478
|
earliestResetAt: earliestReset(allocatableAccounts, now),
|
|
524
479
|
};
|
|
525
480
|
}
|
|
526
|
-
// P4: surface the dropped-connector note when the chosen account doesn't cover
|
|
527
|
-
// the session's used connectors (a Tier-2/unknown failover pick). Empty ⇒ omit.
|
|
528
|
-
const dropped = droppedConnectorsFor(chosen, usedConnectors);
|
|
529
481
|
return {
|
|
530
482
|
kind: "active",
|
|
531
483
|
credentialId: chosen.id,
|
|
532
484
|
moved: chosen.id !== activeCredentialId,
|
|
533
|
-
...(dropped.length > 0 ? { droppedConnectors: dropped } : {}),
|
|
534
485
|
};
|
|
535
486
|
};
|
|
536
487
|
|
|
@@ -568,26 +519,20 @@ export function chooseRotationActive(args: {
|
|
|
568
519
|
return decide(eligibles[0]);
|
|
569
520
|
}
|
|
570
521
|
|
|
571
|
-
// most_remaining (default + the correctness path).
|
|
572
|
-
// the SAME eligible set (prefer-not-require):
|
|
573
|
-
// Tier 1 — eligibles whose connector set COVERS the session's used set.
|
|
574
|
-
// Tier 2 — ALL eligibles, only if Tier 1 is empty (failover fully preserved).
|
|
575
|
-
// Within the chosen tier:
|
|
522
|
+
// most_remaining (default + the correctness path). Across the eligible set:
|
|
576
523
|
// 1. least active leases (concurrent turns spread before quota metadata moves),
|
|
577
524
|
// 2. most remaining binding quota,
|
|
578
525
|
// 3. fewest historical selections,
|
|
579
526
|
// 4. least-recently selected, then stable created_at/id input order.
|
|
580
527
|
// Base CodexAccountStatus values (pure legacy tests/reactive rank) default the
|
|
581
528
|
// lease/cursor metadata to zero/null, preserving deterministic stable ties.
|
|
582
|
-
const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
|
|
583
|
-
const pool = covering.length > 0 ? covering : eligibles;
|
|
584
529
|
const activeLeases = (acct: CodexRotationAccount): number =>
|
|
585
530
|
"activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
|
|
586
531
|
const selections = (acct: CodexRotationAccount): number =>
|
|
587
532
|
"selectionCount" in acct ? acct.selectionCount : 0;
|
|
588
533
|
const lastSelected = (acct: CodexRotationAccount): number =>
|
|
589
534
|
"lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
|
|
590
|
-
const chosen =
|
|
535
|
+
const chosen = eligibles.reduce<CodexRotationAccount | undefined>((best, acct) => {
|
|
591
536
|
if (!best) {
|
|
592
537
|
return acct;
|
|
593
538
|
}
|
|
@@ -613,7 +558,7 @@ export function chooseRotationActive(args: {
|
|
|
613
558
|
* the same until BOTH the deployment flag and workspace cutover are enabled;
|
|
614
559
|
* otherwise a rolling fleet would run two allocators at once. When the active
|
|
615
560
|
* pointer is no longer eligible, zeroing the additive metadata delegates to the
|
|
616
|
-
* same capacity
|
|
561
|
+
* same capacity ranking that the old selector used.
|
|
617
562
|
*/
|
|
618
563
|
export function chooseLegacyRotationActive(
|
|
619
564
|
args: Parameters<typeof chooseRotationActive>[0],
|
|
@@ -650,8 +595,6 @@ export function selectCodexCredentialLeaseForTurn(args: {
|
|
|
650
595
|
sessionPinnedCredentialId: string | null;
|
|
651
596
|
sessionPinSource: CodexPinSource | null;
|
|
652
597
|
sessionLastCredentialId: string | null;
|
|
653
|
-
/** Same-turn checkpoint/frozen account; temporary disable cannot move it. */
|
|
654
|
-
continuationCredentialId?: string | null;
|
|
655
598
|
now: Date;
|
|
656
599
|
}): CodexTurnLeaseSelection {
|
|
657
600
|
const {
|
|
@@ -681,21 +624,6 @@ export function selectCodexCredentialLeaseForTurn(args: {
|
|
|
681
624
|
};
|
|
682
625
|
}
|
|
683
626
|
|
|
684
|
-
const continuation = args.continuationCredentialId
|
|
685
|
-
? accounts.find((account) => account.id === args.continuationCredentialId)
|
|
686
|
-
: undefined;
|
|
687
|
-
if (continuation && isCodexCredentialHealthy(continuation, args.now)) {
|
|
688
|
-
return {
|
|
689
|
-
credentialId: continuation.id,
|
|
690
|
-
advanceActivePointer: false,
|
|
691
|
-
decision: {
|
|
692
|
-
kind: "active",
|
|
693
|
-
credentialId: continuation.id,
|
|
694
|
-
moved: continuation.id !== activeCredentialId,
|
|
695
|
-
},
|
|
696
|
-
};
|
|
697
|
-
}
|
|
698
|
-
|
|
699
627
|
const connectedIds = new Set(
|
|
700
628
|
accounts.filter((account) => account.allocatorEnabled).map((account) => account.id),
|
|
701
629
|
);
|
|
@@ -708,8 +636,8 @@ export function selectCodexCredentialLeaseForTurn(args: {
|
|
|
708
636
|
});
|
|
709
637
|
|
|
710
638
|
// A manual pin is user intent, not a policy hint. Never silently fail it over.
|
|
711
|
-
// We still reject allocator-disabled rows for a NEW turn; healthy live
|
|
712
|
-
// same-turn
|
|
639
|
+
// We still reject allocator-disabled rows for a NEW turn; a healthy live
|
|
640
|
+
// same-turn lease already returned above before this admission filter.
|
|
713
641
|
if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
|
|
714
642
|
const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
|
|
715
643
|
if (!pinned?.allocatorEnabled) {
|
|
@@ -802,7 +730,6 @@ export function selectCodexCredentialLeaseForTurn(args: {
|
|
|
802
730
|
priorCredentialId: priorId,
|
|
803
731
|
accounts,
|
|
804
732
|
now: args.now,
|
|
805
|
-
usedConnectors: accounts.find((account) => account.id === priorId)?.connectorNamespaces ?? [],
|
|
806
733
|
});
|
|
807
734
|
return {
|
|
808
735
|
credentialId: decision.kind === "active" ? decision.credentialId : null,
|
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
} from "@opengeni/runtime";
|
|
23
23
|
import { contextInputBudgetTokens, type Settings } from "@opengeni/config";
|
|
24
24
|
import type { SessionEvent } from "@opengeni/contracts";
|
|
25
|
-
import {
|
|
25
|
+
import { projectRejectedProviderArtifacts } from "./run-input";
|
|
26
26
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
27
27
|
|
|
28
28
|
export type MaybeCompactResult =
|
|
@@ -85,15 +85,12 @@ export async function maybeCompactContext(
|
|
|
85
85
|
force?: boolean;
|
|
86
86
|
clearRequestedCompaction?: boolean;
|
|
87
87
|
trigger?: "auto" | "operator" | "proactive" | "overflow";
|
|
88
|
-
codexAccount?: TurnCodexAccount;
|
|
89
88
|
/** Frozen session mode; remote_v2 selects the Codex opaque path. */
|
|
90
89
|
codexCompactionMode?: "remote_v2" | "portable";
|
|
91
90
|
/** True when this turn's resolved provider is codex-subscription. */
|
|
92
91
|
isCodexSubscriptionTurn?: boolean;
|
|
93
92
|
/** Injected remote requester; required for the remote_v2 branch. */
|
|
94
93
|
requestRemoteCompactionV2?: RemoteCompactionV2Requester;
|
|
95
|
-
/** Tag persisted history with the Codex credential that produced it. */
|
|
96
|
-
producerCodexCredentialId?: string | null;
|
|
97
94
|
/**
|
|
98
95
|
* Live fanout for the attempt-fenced `compaction.started` event so the
|
|
99
96
|
* timeline can show progress before the provider call returns. Must never
|
|
@@ -144,10 +141,7 @@ export async function maybeCompactContext(
|
|
|
144
141
|
};
|
|
145
142
|
}
|
|
146
143
|
|
|
147
|
-
const canonicalItems =
|
|
148
|
-
active,
|
|
149
|
-
options.codexAccount ?? { currentCodexCredentialId: null },
|
|
150
|
-
) as CompactionItem[];
|
|
144
|
+
const canonicalItems = projectRejectedProviderArtifacts(active) as CompactionItem[];
|
|
151
145
|
const projectForWire = async (input: CompactionItem[]): Promise<CompactionItem[]> =>
|
|
152
146
|
sanitizeHistoryItemsForModel(
|
|
153
147
|
options.projectModelInput ? await options.projectModelInput(input) : input,
|
|
@@ -312,7 +306,6 @@ async function compactContextRemoteV2(
|
|
|
312
306
|
clearRequestedCompaction?: boolean;
|
|
313
307
|
trigger?: "auto" | "operator" | "proactive" | "overflow";
|
|
314
308
|
requestRemoteCompactionV2?: RemoteCompactionV2Requester;
|
|
315
|
-
producerCodexCredentialId?: string | null;
|
|
316
309
|
},
|
|
317
310
|
projectForWire: (items: CompactionItem[]) => Promise<CompactionItem[]>,
|
|
318
311
|
): Promise<MaybeCompactResult> {
|
|
@@ -346,11 +339,7 @@ async function compactContextRemoteV2(
|
|
|
346
339
|
expectedAttemptId: scope.attemptId,
|
|
347
340
|
replacementItems: replacementHistory.slice(0, -1),
|
|
348
341
|
summaryItem: tailItem as Record<string, unknown>,
|
|
349
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
350
342
|
...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
|
|
351
|
-
...(options.producerCodexCredentialId
|
|
352
|
-
? { producerCodexCredentialId: options.producerCodexCredentialId }
|
|
353
|
-
: {}),
|
|
354
343
|
eventPayload: {
|
|
355
344
|
trigger: options.trigger ?? "auto",
|
|
356
345
|
implementation: REMOTE_COMPACTION_V2_IMPLEMENTATION,
|
|
@@ -425,7 +414,6 @@ async function compactContextPortable(
|
|
|
425
414
|
expectedAttemptId: scope.attemptId,
|
|
426
415
|
replacementItems: replacementHistory.slice(0, -1),
|
|
427
416
|
summaryItem: summaryItem as Record<string, unknown>,
|
|
428
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
429
417
|
...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
|
|
430
418
|
eventPayload: {
|
|
431
419
|
trigger: options.trigger ?? "auto",
|
|
@@ -1,10 +1,51 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
claimDueTranscriptionRecordingObjectCleanup,
|
|
3
|
+
claimExpiredFileUploadCleanup,
|
|
4
|
+
completeDueTranscriptionRecordingObjectCleanup,
|
|
5
|
+
completeExpiredFileUploadCleanup,
|
|
6
|
+
purgeExpiredTranscriptionRecordings,
|
|
7
|
+
} from "@opengeni/db";
|
|
2
8
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
3
9
|
import type { ActivityServices } from "./types";
|
|
4
10
|
|
|
5
11
|
export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
|
|
6
12
|
export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
|
|
7
13
|
export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
|
|
14
|
+
const SAFE_CLEANUP_ERROR_CATEGORIES = new Set([
|
|
15
|
+
"aborted",
|
|
16
|
+
"conflict",
|
|
17
|
+
"invalid_response",
|
|
18
|
+
"network",
|
|
19
|
+
"not_found",
|
|
20
|
+
"permission_denied",
|
|
21
|
+
"provider",
|
|
22
|
+
"timeout",
|
|
23
|
+
"unavailable",
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
function cleanupErrorAttributes(error: unknown): {
|
|
27
|
+
errorCategory: string;
|
|
28
|
+
errorStatus?: number;
|
|
29
|
+
} {
|
|
30
|
+
let errorCategory = "provider_error";
|
|
31
|
+
let errorStatus: number | undefined;
|
|
32
|
+
if (error && typeof error === "object") {
|
|
33
|
+
const rawCode = (error as { code?: unknown }).code;
|
|
34
|
+
if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
|
|
35
|
+
errorCategory = rawCode;
|
|
36
|
+
} else if (rawCode !== undefined) {
|
|
37
|
+
errorCategory = "unknown";
|
|
38
|
+
}
|
|
39
|
+
const rawStatus = Number(
|
|
40
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
41
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
42
|
+
);
|
|
43
|
+
if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
|
|
44
|
+
errorStatus = rawStatus;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return { errorCategory, ...(errorStatus === undefined ? {} : { errorStatus }) };
|
|
48
|
+
}
|
|
8
49
|
|
|
9
50
|
export type ReapExpiredFileUploadsResult = {
|
|
10
51
|
claimed: number;
|
|
@@ -18,6 +59,11 @@ export type FileUploadReaperActivityOptions = {
|
|
|
18
59
|
batchSize?: number;
|
|
19
60
|
/** Failure-injection seam; production uses the configured provider delete. */
|
|
20
61
|
deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
|
|
62
|
+
claimFileUploads?: typeof claimExpiredFileUploadCleanup;
|
|
63
|
+
completeFileUpload?: typeof completeExpiredFileUploadCleanup;
|
|
64
|
+
claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
|
|
65
|
+
completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
|
|
66
|
+
purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
|
|
21
67
|
};
|
|
22
68
|
|
|
23
69
|
/**
|
|
@@ -33,14 +79,29 @@ export function createFileUploadReaperActivities(
|
|
|
33
79
|
const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
|
|
34
80
|
const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
|
|
35
81
|
const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
|
|
82
|
+
const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
|
|
83
|
+
const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
|
|
84
|
+
const claimTranscriptionObjects =
|
|
85
|
+
options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
|
|
86
|
+
const completeTranscriptionObject =
|
|
87
|
+
options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
|
|
88
|
+
const purgeTranscriptionRecordings =
|
|
89
|
+
options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
|
|
36
90
|
|
|
37
91
|
async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
|
|
38
92
|
const { db, objectStorage, observability } = await services();
|
|
39
93
|
if (!objectStorage) {
|
|
94
|
+
try {
|
|
95
|
+
await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
|
|
96
|
+
} catch (error) {
|
|
97
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
98
|
+
...cleanupErrorAttributes(error),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
40
101
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
41
102
|
}
|
|
42
103
|
|
|
43
|
-
const claims = await
|
|
104
|
+
const claims = await claimFileUploads(db, {
|
|
44
105
|
graceMs,
|
|
45
106
|
claimTimeoutMs,
|
|
46
107
|
limit: batchSize,
|
|
@@ -50,7 +111,7 @@ export function createFileUploadReaperActivities(
|
|
|
50
111
|
for (const claim of claims) {
|
|
51
112
|
try {
|
|
52
113
|
await deleteObject(objectStorage, claim.objectKey);
|
|
53
|
-
const settled = await
|
|
114
|
+
const settled = await completeFileUpload(db, claim);
|
|
54
115
|
if (!settled) {
|
|
55
116
|
throw new Error("cleanup claim no longer owns a reclaimable upload");
|
|
56
117
|
}
|
|
@@ -61,18 +122,58 @@ export function createFileUploadReaperActivities(
|
|
|
61
122
|
workspaceId: claim.workspaceId,
|
|
62
123
|
uploadId: claim.uploadId,
|
|
63
124
|
fileId: claim.fileId,
|
|
64
|
-
|
|
125
|
+
...cleanupErrorAttributes(error),
|
|
65
126
|
});
|
|
66
127
|
}
|
|
67
128
|
}
|
|
68
|
-
|
|
129
|
+
const recordingClaims = await claimTranscriptionObjects(db, {
|
|
130
|
+
graceMs,
|
|
131
|
+
claimTimeoutMs,
|
|
132
|
+
limit: batchSize,
|
|
133
|
+
});
|
|
134
|
+
for (const claim of recordingClaims) {
|
|
135
|
+
try {
|
|
136
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
137
|
+
const settled = await completeTranscriptionObject(db, claim);
|
|
138
|
+
if (!settled) {
|
|
139
|
+
throw new Error("cleanup claim no longer owns a transcription recording object");
|
|
140
|
+
}
|
|
141
|
+
deleted += 1;
|
|
142
|
+
} catch (error) {
|
|
143
|
+
failed += 1;
|
|
144
|
+
observability.warn(
|
|
145
|
+
"expired transcription recording object cleanup failed; claim remains reclaimable",
|
|
146
|
+
{
|
|
147
|
+
workspaceId: claim.workspaceId,
|
|
148
|
+
recordingId: claim.recordingId,
|
|
149
|
+
...cleanupErrorAttributes(error),
|
|
150
|
+
},
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
let transcriptionRecordingsPurged = 0;
|
|
155
|
+
try {
|
|
156
|
+
transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
|
|
157
|
+
graceMs,
|
|
158
|
+
limit: batchSize,
|
|
159
|
+
});
|
|
160
|
+
} catch (error) {
|
|
161
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
162
|
+
...cleanupErrorAttributes(error),
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
const claimed = claims.length + recordingClaims.length;
|
|
166
|
+
if (claimed > 0 || transcriptionRecordingsPurged > 0) {
|
|
69
167
|
observability.info("expired file upload cleanup swept", {
|
|
70
|
-
claimed
|
|
168
|
+
claimed,
|
|
71
169
|
deleted,
|
|
72
170
|
failed,
|
|
171
|
+
fileUploads: claims.length,
|
|
172
|
+
transcriptionRecordingObjects: recordingClaims.length,
|
|
173
|
+
transcriptionRecordingsPurged,
|
|
73
174
|
});
|
|
74
175
|
}
|
|
75
|
-
return { claimed
|
|
176
|
+
return { claimed, deleted, failed };
|
|
76
177
|
}
|
|
77
178
|
|
|
78
179
|
return { reapExpiredFileUploads };
|