@opengeni/worker-bundle 0.16.14 → 0.16.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -94,7 +94,6 @@ export function codexCapacityDecision(
94
94
  sessionPinnedCredentialId: context.sessionPinnedCredentialId,
95
95
  sessionPinSource: context.sessionPinSource,
96
96
  sessionLastCredentialId: context.sessionLastCredentialId,
97
- continuationCredentialId: null,
98
97
  now,
99
98
  });
100
99
  if (selected.credentialId) {
@@ -88,10 +88,7 @@ export function classifyCodexPin(args: {
88
88
  export type RotationDecision =
89
89
  // The chosen account. `moved` ⇒ it differs from the current active pointer, so
90
90
  // the caller must persist the pointer move (and the switch is a "rotation").
91
- // `droppedConnectors` (P4 Part B): the leaving session's used connectors that the
92
- // chosen account does NOT cover — populated ONLY on a Tier-2/unknown pick that
93
- // can't cover (prefer-not-require: failover still happens, but the pill warns).
94
- | { kind: "active"; credentialId: string; moved: boolean; droppedConnectors?: string[] }
91
+ | { kind: "active"; credentialId: string; moved: boolean }
95
92
  // Every eligible account is capped/cooling: idle until the soonest instant ANY
96
93
  // account clears every blocking condition (the multi-account generalization of
97
94
  // the single-account idle-until-reset).
@@ -157,46 +154,10 @@ function cooling(acct: CodexRotationAccount, now: Date): boolean {
157
154
  }
158
155
 
159
156
  /**
160
- * P4 connector coverage (prefer-not-require). `acct` COVERS `usedConnectors` iff its
161
- * cached connector set is a SUPERSET of the session's used set. An empty used set is
162
- * trivially covered by everyone (→ Tier 1 == all eligibles == byte-identical to P3).
163
- * A null (never-probed) set is UNKNOWN: never credited as covering (Tier 2 only),
164
- * but — critically — never excluded, so failover is fully preserved.
165
- */
166
- function covers(acct: CodexRotationAccount, usedConnectors: string[]): boolean {
167
- if (usedConnectors.length === 0) {
168
- return true;
169
- }
170
- const owned = acct.connectorNamespaces;
171
- if (owned === null) {
172
- return false; // unprobed ⇒ unknown ⇒ never Tier 1 (but still a Tier-2 candidate)
173
- }
174
- const ownedSet = new Set(owned);
175
- return usedConnectors.every((connector) => ownedSet.has(connector));
176
- }
177
-
178
- /**
179
- * The used connectors the chosen account does NOT cover (the "switch dropped a
180
- * connector" note). Empty when the account covers (superset) or nothing was used.
181
- * A null (unknown) set surfaces ALL used connectors — we can't prove it covers them.
182
- */
183
- function droppedConnectorsFor(acct: CodexRotationAccount, usedConnectors: string[]): string[] {
184
- if (usedConnectors.length === 0) {
185
- return [];
186
- }
187
- const owned = acct.connectorNamespaces;
188
- if (owned === null) {
189
- return [...usedConnectors];
190
- }
191
- const ownedSet = new Set(owned);
192
- return usedConnectors.filter((connector) => !ownedSet.has(connector));
193
- }
194
-
195
- /**
196
- * Healthy for an already-frozen/in-flight turn = connected/usable (status
157
+ * Healthy for an already-leased/in-flight turn = connected/usable (status
197
158
  * "active", excludes needs_relogin/error), not cooling, and not exhausted in
198
159
  * either provider window. Allocator eligibility is
199
- * deliberately separate so disabling NEW work cannot break refresh/resume.
160
+ * deliberately separate so disabling NEW work cannot break an existing live lease.
200
161
  */
201
162
  export function isCodexCredentialHealthy(acct: CodexRotationAccount, now: Date): boolean {
202
163
  return (
@@ -433,8 +394,8 @@ export const REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5_000; // 5s
433
394
  * Circuit-breaker slack (Finding 1b) added to the connected-account count to bound
434
395
  * consecutive reactive failovers. Each legitimate failover cools one account, so
435
396
  * after at most N failovers every account is capped and the reactive path takes the
436
- * all-capped idle instead; the +margin absorbs benign re-ranks (a connector-covering
437
- * pick order) without tripping the breaker in normal use.
397
+ * all-capped idle instead; the +margin absorbs benign capacity re-ranks without
398
+ * tripping the breaker in normal use.
438
399
  */
439
400
  export const REACTIVE_ROTATION_MARGIN = 2;
440
401
  /**
@@ -491,11 +452,6 @@ export function chooseRotationActive(args: {
491
452
  priorCredentialId: string | null;
492
453
  accounts: CodexRotationAccount[];
493
454
  now: Date;
494
- // P4 (Part B): the connectors the leaving session has access to (the leaving
495
- // account's cached connector set, used as the "session needs these" proxy).
496
- // Optional + defaults to [] ⇒ when absent/empty the ranker is BYTE-IDENTICAL to
497
- // P3 (every account trivially covers → Tier 1 == all eligibles). Self-gating.
498
- usedConnectors?: string[];
499
455
  }): RotationDecision {
500
456
  const { activeCredentialId, priorCredentialId, accounts, now } = args;
501
457
  // Normalize at the entry point: rotation-enabled ALWAYS behaves as sharded
@@ -503,7 +459,6 @@ export function chooseRotationActive(args: {
503
459
  // tests can still express stored legacy values.
504
460
  const rotationStrategy = effectiveRotationStrategy(args.rotationStrategy);
505
461
  const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
506
- const usedConnectors = args.usedConnectors ?? [];
507
462
 
508
463
  if (allocatableAccounts.length === 0) {
509
464
  return { kind: "none" };
@@ -523,14 +478,10 @@ export function chooseRotationActive(args: {
523
478
  earliestResetAt: earliestReset(allocatableAccounts, now),
524
479
  };
525
480
  }
526
- // P4: surface the dropped-connector note when the chosen account doesn't cover
527
- // the session's used connectors (a Tier-2/unknown failover pick). Empty ⇒ omit.
528
- const dropped = droppedConnectorsFor(chosen, usedConnectors);
529
481
  return {
530
482
  kind: "active",
531
483
  credentialId: chosen.id,
532
484
  moved: chosen.id !== activeCredentialId,
533
- ...(dropped.length > 0 ? { droppedConnectors: dropped } : {}),
534
485
  };
535
486
  };
536
487
 
@@ -568,26 +519,20 @@ export function chooseRotationActive(args: {
568
519
  return decide(eligibles[0]);
569
520
  }
570
521
 
571
- // most_remaining (default + the correctness path). Two-tier coverage pick over
572
- // the SAME eligible set (prefer-not-require):
573
- // Tier 1 — eligibles whose connector set COVERS the session's used set.
574
- // Tier 2 — ALL eligibles, only if Tier 1 is empty (failover fully preserved).
575
- // Within the chosen tier:
522
+ // most_remaining (default + the correctness path). Across the eligible set:
576
523
  // 1. least active leases (concurrent turns spread before quota metadata moves),
577
524
  // 2. most remaining binding quota,
578
525
  // 3. fewest historical selections,
579
526
  // 4. least-recently selected, then stable created_at/id input order.
580
527
  // Base CodexAccountStatus values (pure legacy tests/reactive rank) default the
581
528
  // lease/cursor metadata to zero/null, preserving deterministic stable ties.
582
- const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
583
- const pool = covering.length > 0 ? covering : eligibles;
584
529
  const activeLeases = (acct: CodexRotationAccount): number =>
585
530
  "activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
586
531
  const selections = (acct: CodexRotationAccount): number =>
587
532
  "selectionCount" in acct ? acct.selectionCount : 0;
588
533
  const lastSelected = (acct: CodexRotationAccount): number =>
589
534
  "lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
590
- const chosen = pool.reduce<CodexRotationAccount | undefined>((best, acct) => {
535
+ const chosen = eligibles.reduce<CodexRotationAccount | undefined>((best, acct) => {
591
536
  if (!best) {
592
537
  return acct;
593
538
  }
@@ -613,7 +558,7 @@ export function chooseRotationActive(args: {
613
558
  * the same until BOTH the deployment flag and workspace cutover are enabled;
614
559
  * otherwise a rolling fleet would run two allocators at once. When the active
615
560
  * pointer is no longer eligible, zeroing the additive metadata delegates to the
616
- * same capacity/connector ranking that the old selector used.
561
+ * same capacity ranking that the old selector used.
617
562
  */
618
563
  export function chooseLegacyRotationActive(
619
564
  args: Parameters<typeof chooseRotationActive>[0],
@@ -650,8 +595,6 @@ export function selectCodexCredentialLeaseForTurn(args: {
650
595
  sessionPinnedCredentialId: string | null;
651
596
  sessionPinSource: CodexPinSource | null;
652
597
  sessionLastCredentialId: string | null;
653
- /** Same-turn checkpoint/frozen account; temporary disable cannot move it. */
654
- continuationCredentialId?: string | null;
655
598
  now: Date;
656
599
  }): CodexTurnLeaseSelection {
657
600
  const {
@@ -681,21 +624,6 @@ export function selectCodexCredentialLeaseForTurn(args: {
681
624
  };
682
625
  }
683
626
 
684
- const continuation = args.continuationCredentialId
685
- ? accounts.find((account) => account.id === args.continuationCredentialId)
686
- : undefined;
687
- if (continuation && isCodexCredentialHealthy(continuation, args.now)) {
688
- return {
689
- credentialId: continuation.id,
690
- advanceActivePointer: false,
691
- decision: {
692
- kind: "active",
693
- credentialId: continuation.id,
694
- moved: continuation.id !== activeCredentialId,
695
- },
696
- };
697
- }
698
-
699
627
  const connectedIds = new Set(
700
628
  accounts.filter((account) => account.allocatorEnabled).map((account) => account.id),
701
629
  );
@@ -708,8 +636,8 @@ export function selectCodexCredentialLeaseForTurn(args: {
708
636
  });
709
637
 
710
638
  // A manual pin is user intent, not a policy hint. Never silently fail it over.
711
- // We still reject allocator-disabled rows for a NEW turn; healthy live/frozen
712
- // same-turn continuity already returned above before this admission filter.
639
+ // We still reject allocator-disabled rows for a NEW turn; a healthy live
640
+ // same-turn lease already returned above before this admission filter.
713
641
  if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
714
642
  const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
715
643
  if (!pinned?.allocatorEnabled) {
@@ -802,7 +730,6 @@ export function selectCodexCredentialLeaseForTurn(args: {
802
730
  priorCredentialId: priorId,
803
731
  accounts,
804
732
  now: args.now,
805
- usedConnectors: accounts.find((account) => account.id === priorId)?.connectorNamespaces ?? [],
806
733
  });
807
734
  return {
808
735
  credentialId: decision.kind === "active" ? decision.credentialId : null,
@@ -22,7 +22,7 @@ import {
22
22
  } from "@opengeni/runtime";
23
23
  import { contextInputBudgetTokens, type Settings } from "@opengeni/config";
24
24
  import type { SessionEvent } from "@opengeni/contracts";
25
- import { applyCodexHistoryStrip, type TurnCodexAccount } from "./run-input";
25
+ import { projectRejectedProviderArtifacts } from "./run-input";
26
26
  import { TurnAttemptFencedError } from "./turn-attempt-fenced";
27
27
 
28
28
  export type MaybeCompactResult =
@@ -85,15 +85,12 @@ export async function maybeCompactContext(
85
85
  force?: boolean;
86
86
  clearRequestedCompaction?: boolean;
87
87
  trigger?: "auto" | "operator" | "proactive" | "overflow";
88
- codexAccount?: TurnCodexAccount;
89
88
  /** Frozen session mode; remote_v2 selects the Codex opaque path. */
90
89
  codexCompactionMode?: "remote_v2" | "portable";
91
90
  /** True when this turn's resolved provider is codex-subscription. */
92
91
  isCodexSubscriptionTurn?: boolean;
93
92
  /** Injected remote requester; required for the remote_v2 branch. */
94
93
  requestRemoteCompactionV2?: RemoteCompactionV2Requester;
95
- /** Tag persisted history with the Codex credential that produced it. */
96
- producerCodexCredentialId?: string | null;
97
94
  /**
98
95
  * Live fanout for the attempt-fenced `compaction.started` event so the
99
96
  * timeline can show progress before the provider call returns. Must never
@@ -144,10 +141,7 @@ export async function maybeCompactContext(
144
141
  };
145
142
  }
146
143
 
147
- const canonicalItems = applyCodexHistoryStrip(
148
- active,
149
- options.codexAccount ?? { currentCodexCredentialId: null },
150
- ) as CompactionItem[];
144
+ const canonicalItems = projectRejectedProviderArtifacts(active) as CompactionItem[];
151
145
  const projectForWire = async (input: CompactionItem[]): Promise<CompactionItem[]> =>
152
146
  sanitizeHistoryItemsForModel(
153
147
  options.projectModelInput ? await options.projectModelInput(input) : input,
@@ -312,7 +306,6 @@ async function compactContextRemoteV2(
312
306
  clearRequestedCompaction?: boolean;
313
307
  trigger?: "auto" | "operator" | "proactive" | "overflow";
314
308
  requestRemoteCompactionV2?: RemoteCompactionV2Requester;
315
- producerCodexCredentialId?: string | null;
316
309
  },
317
310
  projectForWire: (items: CompactionItem[]) => Promise<CompactionItem[]>,
318
311
  ): Promise<MaybeCompactResult> {
@@ -346,11 +339,7 @@ async function compactContextRemoteV2(
346
339
  expectedAttemptId: scope.attemptId,
347
340
  replacementItems: replacementHistory.slice(0, -1),
348
341
  summaryItem: tailItem as Record<string, unknown>,
349
- replacementInputTokens: estimatedTokensAfter,
350
342
  ...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
351
- ...(options.producerCodexCredentialId
352
- ? { producerCodexCredentialId: options.producerCodexCredentialId }
353
- : {}),
354
343
  eventPayload: {
355
344
  trigger: options.trigger ?? "auto",
356
345
  implementation: REMOTE_COMPACTION_V2_IMPLEMENTATION,
@@ -425,7 +414,6 @@ async function compactContextPortable(
425
414
  expectedAttemptId: scope.attemptId,
426
415
  replacementItems: replacementHistory.slice(0, -1),
427
416
  summaryItem: summaryItem as Record<string, unknown>,
428
- replacementInputTokens: estimatedTokensAfter,
429
417
  ...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
430
418
  eventPayload: {
431
419
  trigger: options.trigger ?? "auto",
@@ -1,10 +1,51 @@
1
- import { claimExpiredFileUploadCleanup, completeExpiredFileUploadCleanup } from "@opengeni/db";
1
+ import {
2
+ claimDueTranscriptionRecordingObjectCleanup,
3
+ claimExpiredFileUploadCleanup,
4
+ completeDueTranscriptionRecordingObjectCleanup,
5
+ completeExpiredFileUploadCleanup,
6
+ purgeExpiredTranscriptionRecordings,
7
+ } from "@opengeni/db";
2
8
  import type { ObjectStorage } from "@opengeni/storage";
3
9
  import type { ActivityServices } from "./types";
4
10
 
5
11
  export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
6
12
  export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
7
13
  export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
14
+ const SAFE_CLEANUP_ERROR_CATEGORIES = new Set([
15
+ "aborted",
16
+ "conflict",
17
+ "invalid_response",
18
+ "network",
19
+ "not_found",
20
+ "permission_denied",
21
+ "provider",
22
+ "timeout",
23
+ "unavailable",
24
+ ]);
25
+
26
+ function cleanupErrorAttributes(error: unknown): {
27
+ errorCategory: string;
28
+ errorStatus?: number;
29
+ } {
30
+ let errorCategory = "provider_error";
31
+ let errorStatus: number | undefined;
32
+ if (error && typeof error === "object") {
33
+ const rawCode = (error as { code?: unknown }).code;
34
+ if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
35
+ errorCategory = rawCode;
36
+ } else if (rawCode !== undefined) {
37
+ errorCategory = "unknown";
38
+ }
39
+ const rawStatus = Number(
40
+ (error as { status?: unknown; statusCode?: unknown }).status ??
41
+ (error as { statusCode?: unknown }).statusCode,
42
+ );
43
+ if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
44
+ errorStatus = rawStatus;
45
+ }
46
+ }
47
+ return { errorCategory, ...(errorStatus === undefined ? {} : { errorStatus }) };
48
+ }
8
49
 
9
50
  export type ReapExpiredFileUploadsResult = {
10
51
  claimed: number;
@@ -18,6 +59,11 @@ export type FileUploadReaperActivityOptions = {
18
59
  batchSize?: number;
19
60
  /** Failure-injection seam; production uses the configured provider delete. */
20
61
  deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
62
+ claimFileUploads?: typeof claimExpiredFileUploadCleanup;
63
+ completeFileUpload?: typeof completeExpiredFileUploadCleanup;
64
+ claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
65
+ completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
66
+ purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
21
67
  };
22
68
 
23
69
  /**
@@ -33,14 +79,29 @@ export function createFileUploadReaperActivities(
33
79
  const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
34
80
  const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
35
81
  const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
82
+ const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
83
+ const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
84
+ const claimTranscriptionObjects =
85
+ options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
86
+ const completeTranscriptionObject =
87
+ options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
88
+ const purgeTranscriptionRecordings =
89
+ options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
36
90
 
37
91
  async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
38
92
  const { db, objectStorage, observability } = await services();
39
93
  if (!objectStorage) {
94
+ try {
95
+ await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
96
+ } catch (error) {
97
+ observability.warn("expired transcription recording metadata purge failed", {
98
+ ...cleanupErrorAttributes(error),
99
+ });
100
+ }
40
101
  return { claimed: 0, deleted: 0, failed: 0 };
41
102
  }
42
103
 
43
- const claims = await claimExpiredFileUploadCleanup(db, {
104
+ const claims = await claimFileUploads(db, {
44
105
  graceMs,
45
106
  claimTimeoutMs,
46
107
  limit: batchSize,
@@ -50,7 +111,7 @@ export function createFileUploadReaperActivities(
50
111
  for (const claim of claims) {
51
112
  try {
52
113
  await deleteObject(objectStorage, claim.objectKey);
53
- const settled = await completeExpiredFileUploadCleanup(db, claim);
114
+ const settled = await completeFileUpload(db, claim);
54
115
  if (!settled) {
55
116
  throw new Error("cleanup claim no longer owns a reclaimable upload");
56
117
  }
@@ -61,18 +122,58 @@ export function createFileUploadReaperActivities(
61
122
  workspaceId: claim.workspaceId,
62
123
  uploadId: claim.uploadId,
63
124
  fileId: claim.fileId,
64
- error: error instanceof Error ? error.message : String(error),
125
+ ...cleanupErrorAttributes(error),
65
126
  });
66
127
  }
67
128
  }
68
- if (claims.length > 0) {
129
+ const recordingClaims = await claimTranscriptionObjects(db, {
130
+ graceMs,
131
+ claimTimeoutMs,
132
+ limit: batchSize,
133
+ });
134
+ for (const claim of recordingClaims) {
135
+ try {
136
+ await deleteObject(objectStorage, claim.objectKey);
137
+ const settled = await completeTranscriptionObject(db, claim);
138
+ if (!settled) {
139
+ throw new Error("cleanup claim no longer owns a transcription recording object");
140
+ }
141
+ deleted += 1;
142
+ } catch (error) {
143
+ failed += 1;
144
+ observability.warn(
145
+ "expired transcription recording object cleanup failed; claim remains reclaimable",
146
+ {
147
+ workspaceId: claim.workspaceId,
148
+ recordingId: claim.recordingId,
149
+ ...cleanupErrorAttributes(error),
150
+ },
151
+ );
152
+ }
153
+ }
154
+ let transcriptionRecordingsPurged = 0;
155
+ try {
156
+ transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
157
+ graceMs,
158
+ limit: batchSize,
159
+ });
160
+ } catch (error) {
161
+ observability.warn("expired transcription recording metadata purge failed", {
162
+ ...cleanupErrorAttributes(error),
163
+ });
164
+ }
165
+ const claimed = claims.length + recordingClaims.length;
166
+ if (claimed > 0 || transcriptionRecordingsPurged > 0) {
69
167
  observability.info("expired file upload cleanup swept", {
70
- claimed: claims.length,
168
+ claimed,
71
169
  deleted,
72
170
  failed,
171
+ fileUploads: claims.length,
172
+ transcriptionRecordingObjects: recordingClaims.length,
173
+ transcriptionRecordingsPurged,
73
174
  });
74
175
  }
75
- return { claimed: claims.length, deleted, failed };
176
+ return { claimed, deleted, failed };
76
177
  }
77
178
 
78
179
  return { reapExpiredFileUploads };