@opengeni/worker-bundle 0.16.17 → 0.16.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/activities/agent-turn.d.ts +83 -32
  2. package/dist/activities/context-compaction.d.ts +2 -0
  3. package/dist/activities/environment.d.ts +1 -0
  4. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  5. package/dist/activities/file-upload-reaper.d.ts +6 -0
  6. package/dist/activities/git-credential-renewal.d.ts +1 -1
  7. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  8. package/dist/activities/retained-screenshots.d.ts +93 -0
  9. package/dist/activities/run-credential-renewal.d.ts +1 -1
  10. package/dist/activities/run-input.d.ts +2 -0
  11. package/dist/activities/toolspace-token-renewal.d.ts +1 -1
  12. package/dist/activities/workspace-capture.d.ts +1 -0
  13. package/dist/activities.d.ts +3 -0
  14. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  15. package/dist/chunk-RAMNYSAW.js.map +1 -0
  16. package/dist/host-export.js +1 -1
  17. package/dist/index.js +1465 -453
  18. package/dist/index.js.map +1 -1
  19. package/dist/sandbox-resume.d.ts +6 -0
  20. package/dist/sandbox-routing.d.ts +11 -6
  21. package/dist/workflow-bundle.js +2 -1
  22. package/package.json +12 -12
  23. package/src/activities/agent-turn.ts +559 -434
  24. package/src/activities/codex-capacity.ts +10 -1
  25. package/src/activities/codex-rotation.ts +19 -5
  26. package/src/activities/context-compaction.ts +9 -5
  27. package/src/activities/environment.ts +13 -24
  28. package/src/activities/expiring-material-renewal.ts +16 -7
  29. package/src/activities/file-upload-reaper.ts +108 -7
  30. package/src/activities/git-credential-renewal.ts +3 -3
  31. package/src/activities/recording.ts +4 -5
  32. package/src/activities/retained-screenshot-reaper.ts +202 -0
  33. package/src/activities/retained-screenshots.ts +892 -0
  34. package/src/activities/rig-verification.ts +13 -34
  35. package/src/activities/run-credential-renewal.ts +9 -2
  36. package/src/activities/run-input.ts +18 -4
  37. package/src/activities/sandbox-lease.ts +5 -2
  38. package/src/activities/scheduled-tasks.ts +19 -3
  39. package/src/activities/toolspace-token-renewal.ts +9 -2
  40. package/src/activities/workspace-capture.ts +33 -3
  41. package/src/activities.ts +10 -2
  42. package/src/host-export-pump.ts +28 -11
  43. package/src/http.ts +3 -6
  44. package/src/observability-metrics.ts +3 -3
  45. package/src/sandbox-resume.ts +45 -12
  46. package/src/sandbox-routing.ts +35 -17
  47. package/src/worker-service-lifecycle.ts +8 -4
  48. package/src/workflows/file-upload-reaper.ts +4 -1
  49. package/dist/activities/redaction.d.ts +0 -1
  50. package/dist/chunk-BJUW5LCM.js.map +0 -1
  51. package/src/activities/redaction.ts +0 -9
@@ -12,6 +12,7 @@ import {
12
12
  CODEX_USAGE_EXHAUSTED_PCT,
13
13
  authoritativeCodexCapacityResetAt,
14
14
  isCodexCredentialEligible,
15
+ isCodexCredentialHealthy,
15
16
  selectCodexCredentialLeaseForTurn,
16
17
  } from "./codex-rotation";
17
18
  import type {
@@ -96,7 +97,15 @@ export function codexCapacityDecision(
96
97
  sessionLastCredentialId: context.sessionLastCredentialId,
97
98
  now,
98
99
  });
99
- if (selected.credentialId) {
100
+ const selectedAccount = selected.credentialId
101
+ ? context.accounts.find((account) => account.id === selected.credentialId)
102
+ : undefined;
103
+ const selectedIsAvailable =
104
+ selectedAccount !== undefined &&
105
+ (selectedAccount.id === context.existingCredentialId
106
+ ? isCodexCredentialHealthy(selectedAccount, now)
107
+ : isCodexCredentialEligible(selectedAccount, now));
108
+ if (selected.credentialId && selectedIsAvailable) {
100
109
  return {
101
110
  kind: "available",
102
111
  credentialId: selected.credentialId,
@@ -694,13 +694,27 @@ export function selectCodexCredentialLeaseForTurn(args: {
694
694
 
695
695
  // A stale policy pin is intentionally ignored here and cleared by the caller
696
696
  // after the atomic selection. Rotation-off/unpinned behavior otherwise keeps
697
- // the pre-lease pin > active-pointer contract.
697
+ // the pre-lease pin > active-pointer contract. The pointer is preference, not
698
+ // proof of capacity: rotation-off must wait on a capped pointer rather than
699
+ // falsely admitting it or silently failing over to another subscription.
698
700
  if (!rotationEnabled) {
699
- const credentialId =
700
- activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
701
+ const active = activeCredentialId
702
+ ? accounts.find((account) => account.id === activeCredentialId)
703
+ : undefined;
704
+ if (!active?.allocatorEnabled) {
705
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
706
+ }
707
+ if (!isCodexCredentialHealthy(active, args.now)) {
708
+ return {
709
+ credentialId: null,
710
+ decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
711
+ advanceActivePointer: false,
712
+ };
713
+ }
701
714
  return {
702
- credentialId,
703
- decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" },
715
+ credentialId: active.id,
716
+ decision: { kind: "active", credentialId: active.id, moved: false },
717
+ advanceActivePointer: false,
704
718
  };
705
719
  }
706
720
 
@@ -97,6 +97,8 @@ export async function maybeCompactContext(
97
97
  * append again — the event is already durable.
98
98
  */
99
99
  publishLiveEvents?: (events: SessionEvent[]) => Promise<void>;
100
+ /** Materialize retained screenshot receipts only in the attempt-local model view. */
101
+ materializeHistory?: (items: CompactionItem[]) => Promise<CompactionItem[]>;
100
102
  /** Turn-scoped attachment/modality view; canonical persisted rows stay untouched. */
101
103
  projectModelInput?: (items: CompactionItem[]) => Promise<CompactionItem[]>;
102
104
  } = {},
@@ -142,11 +144,15 @@ export async function maybeCompactContext(
142
144
  }
143
145
 
144
146
  const canonicalItems = projectRejectedProviderArtifacts(active) as CompactionItem[];
145
- const projectForWire = async (input: CompactionItem[]): Promise<CompactionItem[]> =>
146
- sanitizeHistoryItemsForModel(
147
- options.projectModelInput ? await options.projectModelInput(input) : input,
147
+ const projectForWire = async (input: CompactionItem[]): Promise<CompactionItem[]> => {
148
+ const materialized = options.materializeHistory
149
+ ? await options.materializeHistory(input)
150
+ : input;
151
+ return sanitizeHistoryItemsForModel(
152
+ options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
148
153
  settings.modelToolOutputTruncationTokens,
149
154
  ) as CompactionItem[];
155
+ };
150
156
  const items = await projectForWire(canonicalItems);
151
157
  const decision = decideCompaction({
152
158
  items,
@@ -339,7 +345,6 @@ async function compactContextRemoteV2(
339
345
  expectedAttemptId: scope.attemptId,
340
346
  replacementItems: replacementHistory.slice(0, -1),
341
347
  summaryItem: tailItem as Record<string, unknown>,
342
- replacementInputTokens: estimatedTokensAfter,
343
348
  ...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
344
349
  eventPayload: {
345
350
  trigger: options.trigger ?? "auto",
@@ -415,7 +420,6 @@ async function compactContextPortable(
415
420
  expectedAttemptId: scope.attemptId,
416
421
  replacementItems: replacementHistory.slice(0, -1),
417
422
  summaryItem: summaryItem as Record<string, unknown>,
418
- replacementInputTokens: estimatedTokensAfter,
419
423
  ...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
420
424
  eventPayload: {
421
425
  trigger: options.trigger ?? "auto",
@@ -7,6 +7,7 @@ import {
7
7
  import {
8
8
  gitCredentialBindingIdForRepository,
9
9
  gitCredentialProviderForRepository,
10
+ gitRemoteIdentity,
10
11
  signDelegatedAccessToken,
11
12
  type ConnectionCredentialsPort,
12
13
  type GitCredentialProvider,
@@ -202,6 +203,7 @@ export async function sandboxEnvironmentForRun(
202
203
  // value later. `= {}` default so the non-optional reads below are safe.
203
204
  options: RunGitCredentialOptions & {
204
205
  skipGitHubToken?: boolean;
206
+ skipToolspace?: boolean;
205
207
  deferGitHubToken?: boolean;
206
208
  toolspaceAuthority?: SandboxToolspaceAuthority;
207
209
  } = {},
@@ -233,18 +235,18 @@ export async function sandboxEnvironmentForRun(
233
235
  // proactively renews every selected provider behind these stable pointers.
234
236
  const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
235
237
  const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
236
- // TOOLSPACE (selfhosted parity): the toolspace token is minted for EVERY
237
- // backend, including a connected machine. Unlike platform git provider tokens
238
- // (inert on selfhosted → skipped above), the toolspace token is the machine's
239
- // only path to programmatic tool calling, and it grants no more than the
240
- // machine owner's own authority (toolspace:call, own-session-bound,
241
- // short-lived and renewable, per-active-turn budgeted, approval-tools
242
- // excluded). Delivery mirrors the docker path: the caller threads it
243
- // OFF-MANIFEST as the seed the runtime writes to
244
- // $OPENGENI_TOOLSPACE_TOKEN_FILE over the box's exec channel.
238
+ // Connected Machines own their environment and credentials. They do not
239
+ // receive OpenGeni Toolspace pointers, package hints, or delegated tokens.
240
+ // Scrub the stable pointers as well as skipping the mint because a session
241
+ // whose home backend is managed may be actively routed to selfhosted.
242
+ if (options.skipToolspace) {
243
+ delete environment.OPENGENI_TOOLSPACE_URL;
244
+ delete environment.OPENGENI_TOOLSPACE_TOKEN_FILE;
245
+ delete environment.OPENGENI_OGTOOL_PACKAGE_SPEC;
246
+ }
245
247
  const toolspaceScope = options.scope;
246
248
  const toolspaceToken =
247
- toolspaceScope && options.toolspaceAuthority
249
+ !options.skipToolspace && toolspaceScope && options.toolspaceAuthority
248
250
  ? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority)
249
251
  : undefined;
250
252
  if (toolspaceToken && toolspaceScope) {
@@ -777,7 +779,7 @@ function gitCredentialSelections(resources: ResourceRef[]): GitCredentialSelecti
777
779
  );
778
780
  }
779
781
  bindingProviders.set(credentialBindingId, provider);
780
- const normalizedRemote = normalizedGitRemote(resource.uri);
782
+ const normalizedRemote = gitRemoteIdentity(resource.uri, provider);
781
783
  const claimedBinding = remoteBindings.get(normalizedRemote);
782
784
  if (claimedBinding && claimedBinding !== bindingKey) {
783
785
  throw new Error(
@@ -836,19 +838,6 @@ function normalizedGitHost(uri: string): string {
836
838
  }
837
839
  }
838
840
 
839
- function normalizedGitRemote(uri: string): string {
840
- try {
841
- const url = new URL(uri);
842
- return `${url.protocol.toLowerCase()}//${url.host.toLowerCase()}${url.pathname.replace(/\/+$/, "").replace(/\.git$/, "")}`;
843
- } catch {
844
- return uri
845
- .trim()
846
- .replace(/\/+$/, "")
847
- .replace(/\.git$/, "")
848
- .toLowerCase();
849
- }
850
- }
851
-
852
841
  function repositoryCredentialProvider(
853
842
  resource: Extract<ResourceRef, { kind: "repository" }>,
854
843
  ): GitCredentialProvider | null {
@@ -12,17 +12,25 @@ export type ExpiringMaterialRenewalController = {
12
12
  stop(): Promise<void>;
13
13
  };
14
14
 
15
- export type ExpiringMaterialRenewalOptions<T> = {
15
+ export type ExpiringMaterialRenewalPublicErrorClass =
16
+ | "RunCredentialRenewalOperationError"
17
+ | "ToolspaceTokenRenewalOperationError";
18
+
19
+ export type ExpiringMaterialRenewalOptions<
20
+ T,
21
+ ErrorClass extends ExpiringMaterialRenewalPublicErrorClass,
22
+ > = {
16
23
  initialExpiresAt: Date | null;
17
24
  resolve: () => Promise<T>;
18
25
  write: (material: T) => Promise<void>;
19
26
  expiresAt: (material: T) => Date | null;
27
+ publicErrorClass: ErrorClass;
20
28
  policy: ExpiringMaterialRenewalPolicy;
21
29
  now?: () => number;
22
30
  schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
23
31
  clearSchedule?: (timer: RenewalTimer) => void;
24
32
  onSuccess?: (result: { material: T; nextDelayMs: number }) => void;
25
- onFailure?: (failure: { retryDelayMs: number; errorClass: string }) => void;
33
+ onFailure?: (failure: { retryDelayMs: number; errorClass: ErrorClass }) => void;
26
34
  };
27
35
 
28
36
  export function nextExpiringMaterialRenewalDelay(
@@ -45,9 +53,10 @@ export function nextExpiringMaterialRenewalDelay(
45
53
  * A late resolver cannot write after stop. A physical sandbox write that already
46
54
  * started is drained so turn settlement cannot race a stale mutation.
47
55
  */
48
- export function startExpiringMaterialRenewalLoop<T>(
49
- options: ExpiringMaterialRenewalOptions<T>,
50
- ): ExpiringMaterialRenewalController {
56
+ export function startExpiringMaterialRenewalLoop<
57
+ T,
58
+ ErrorClass extends ExpiringMaterialRenewalPublicErrorClass,
59
+ >(options: ExpiringMaterialRenewalOptions<T, ErrorClass>): ExpiringMaterialRenewalController {
51
60
  const now = options.now ?? Date.now;
52
61
  const schedule =
53
62
  options.schedule ??
@@ -102,14 +111,14 @@ export function startExpiringMaterialRenewalLoop<T>(
102
111
  // Observability never owns credential liveness.
103
112
  }
104
113
  scheduleRefresh(nextDelayMs);
105
- } catch (error) {
114
+ } catch {
106
115
  if (stopped) return;
107
116
  const scheduledRetryMs = retryDelayMs;
108
117
  retryDelayMs = Math.min(retryDelayMs * 2, options.policy.maxRetryMs);
109
118
  try {
110
119
  options.onFailure?.({
111
120
  retryDelayMs: scheduledRetryMs,
112
- errorClass: error instanceof Error ? error.name : "UnknownError",
121
+ errorClass: options.publicErrorClass,
113
122
  });
114
123
  } catch {
115
124
  // Observability never owns credential liveness.
@@ -1,10 +1,51 @@
1
- import { claimExpiredFileUploadCleanup, completeExpiredFileUploadCleanup } from "@opengeni/db";
1
+ import {
2
+ claimDueTranscriptionRecordingObjectCleanup,
3
+ claimExpiredFileUploadCleanup,
4
+ completeDueTranscriptionRecordingObjectCleanup,
5
+ completeExpiredFileUploadCleanup,
6
+ purgeExpiredTranscriptionRecordings,
7
+ } from "@opengeni/db";
2
8
  import type { ObjectStorage } from "@opengeni/storage";
3
9
  import type { ActivityServices } from "./types";
4
10
 
5
11
  export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
6
12
  export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
7
13
  export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
14
+ const SAFE_CLEANUP_ERROR_CATEGORIES = new Set([
15
+ "aborted",
16
+ "conflict",
17
+ "invalid_response",
18
+ "network",
19
+ "not_found",
20
+ "permission_denied",
21
+ "provider",
22
+ "timeout",
23
+ "unavailable",
24
+ ]);
25
+
26
+ function cleanupErrorAttributes(error: unknown): {
27
+ errorCategory: string;
28
+ errorStatus?: number;
29
+ } {
30
+ let errorCategory = "provider_error";
31
+ let errorStatus: number | undefined;
32
+ if (error && typeof error === "object") {
33
+ const rawCode = (error as { code?: unknown }).code;
34
+ if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
35
+ errorCategory = rawCode;
36
+ } else if (rawCode !== undefined) {
37
+ errorCategory = "unknown";
38
+ }
39
+ const rawStatus = Number(
40
+ (error as { status?: unknown; statusCode?: unknown }).status ??
41
+ (error as { statusCode?: unknown }).statusCode,
42
+ );
43
+ if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
44
+ errorStatus = rawStatus;
45
+ }
46
+ }
47
+ return { errorCategory, ...(errorStatus === undefined ? {} : { errorStatus }) };
48
+ }
8
49
 
9
50
  export type ReapExpiredFileUploadsResult = {
10
51
  claimed: number;
@@ -18,6 +59,11 @@ export type FileUploadReaperActivityOptions = {
18
59
  batchSize?: number;
19
60
  /** Failure-injection seam; production uses the configured provider delete. */
20
61
  deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
62
+ claimFileUploads?: typeof claimExpiredFileUploadCleanup;
63
+ completeFileUpload?: typeof completeExpiredFileUploadCleanup;
64
+ claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
65
+ completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
66
+ purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
21
67
  };
22
68
 
23
69
  /**
@@ -33,14 +79,29 @@ export function createFileUploadReaperActivities(
33
79
  const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
34
80
  const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
35
81
  const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
82
+ const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
83
+ const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
84
+ const claimTranscriptionObjects =
85
+ options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
86
+ const completeTranscriptionObject =
87
+ options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
88
+ const purgeTranscriptionRecordings =
89
+ options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
36
90
 
37
91
  async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
38
92
  const { db, objectStorage, observability } = await services();
39
93
  if (!objectStorage) {
94
+ try {
95
+ await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
96
+ } catch (error) {
97
+ observability.warn("expired transcription recording metadata purge failed", {
98
+ ...cleanupErrorAttributes(error),
99
+ });
100
+ }
40
101
  return { claimed: 0, deleted: 0, failed: 0 };
41
102
  }
42
103
 
43
- const claims = await claimExpiredFileUploadCleanup(db, {
104
+ const claims = await claimFileUploads(db, {
44
105
  graceMs,
45
106
  claimTimeoutMs,
46
107
  limit: batchSize,
@@ -50,7 +111,7 @@ export function createFileUploadReaperActivities(
50
111
  for (const claim of claims) {
51
112
  try {
52
113
  await deleteObject(objectStorage, claim.objectKey);
53
- const settled = await completeExpiredFileUploadCleanup(db, claim);
114
+ const settled = await completeFileUpload(db, claim);
54
115
  if (!settled) {
55
116
  throw new Error("cleanup claim no longer owns a reclaimable upload");
56
117
  }
@@ -61,18 +122,58 @@ export function createFileUploadReaperActivities(
61
122
  workspaceId: claim.workspaceId,
62
123
  uploadId: claim.uploadId,
63
124
  fileId: claim.fileId,
64
- error: error instanceof Error ? error.message : String(error),
125
+ ...cleanupErrorAttributes(error),
65
126
  });
66
127
  }
67
128
  }
68
- if (claims.length > 0) {
129
+ const recordingClaims = await claimTranscriptionObjects(db, {
130
+ graceMs,
131
+ claimTimeoutMs,
132
+ limit: batchSize,
133
+ });
134
+ for (const claim of recordingClaims) {
135
+ try {
136
+ await deleteObject(objectStorage, claim.objectKey);
137
+ const settled = await completeTranscriptionObject(db, claim);
138
+ if (!settled) {
139
+ throw new Error("cleanup claim no longer owns a transcription recording object");
140
+ }
141
+ deleted += 1;
142
+ } catch (error) {
143
+ failed += 1;
144
+ observability.warn(
145
+ "expired transcription recording object cleanup failed; claim remains reclaimable",
146
+ {
147
+ workspaceId: claim.workspaceId,
148
+ recordingId: claim.recordingId,
149
+ ...cleanupErrorAttributes(error),
150
+ },
151
+ );
152
+ }
153
+ }
154
+ let transcriptionRecordingsPurged = 0;
155
+ try {
156
+ transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
157
+ graceMs,
158
+ limit: batchSize,
159
+ });
160
+ } catch (error) {
161
+ observability.warn("expired transcription recording metadata purge failed", {
162
+ ...cleanupErrorAttributes(error),
163
+ });
164
+ }
165
+ const claimed = claims.length + recordingClaims.length;
166
+ if (claimed > 0 || transcriptionRecordingsPurged > 0) {
69
167
  observability.info("expired file upload cleanup swept", {
70
- claimed: claims.length,
168
+ claimed,
71
169
  deleted,
72
170
  failed,
171
+ fileUploads: claims.length,
172
+ transcriptionRecordingObjects: recordingClaims.length,
173
+ transcriptionRecordingsPurged,
73
174
  });
74
175
  }
75
- return { claimed: claims.length, deleted, failed };
176
+ return { claimed, deleted, failed };
76
177
  }
77
178
 
78
179
  return { reapExpiredFileUploads };
@@ -16,7 +16,7 @@ export type GitCredentialRenewalResult = {
16
16
  export type GitCredentialRenewalFailure = {
17
17
  providers: readonly GitCredentialProvider[];
18
18
  retryDelayMs: number;
19
- errorClass: string;
19
+ errorClass: "GitCredentialRenewalOperationError";
20
20
  };
21
21
 
22
22
  export type GitCredentialRenewalController = {
@@ -133,7 +133,7 @@ export function startGitCredentialRenewalLoop(
133
133
  // Observability callbacks never own credential liveness.
134
134
  }
135
135
  scheduleRefresh(nextDelayMs);
136
- } catch (error) {
136
+ } catch {
137
137
  if (stopped) return;
138
138
  const scheduledRetryMs = retryDelayMs;
139
139
  retryDelayMs = Math.min(retryDelayMs * 2, GIT_CREDENTIAL_MAX_RETRY_MS);
@@ -141,7 +141,7 @@ export function startGitCredentialRenewalLoop(
141
141
  options.onFailure?.({
142
142
  providers,
143
143
  retryDelayMs: scheduledRetryMs,
144
- errorClass: error instanceof Error ? error.name : "UnknownError",
144
+ errorClass: "GitCredentialRenewalOperationError",
145
145
  });
146
146
  } catch {
147
147
  // Observability callbacks never own credential liveness.
@@ -270,11 +270,10 @@ export async function discardUnpublishedRecording(args: {
270
270
 
271
271
  export { DOWNLOAD_URL_TTL_SECONDS };
272
272
 
273
- // Agent/ffmpeg-controlled free text rides redact() like every payload, but we
274
- // also cap it here (defense in depth — a path/URL with creds shouldn't ride a
275
- // reason/detail field unbounded; the redactor scrubs known secret shapes).
273
+ // Recording diagnostics are canonical OpenGeni data. Preserve the exact text;
274
+ // transport-specific byte limits must reject or paginate rather than rewrite it.
276
275
  function scrubFreeText(value: string | null | undefined): string | null {
277
276
  if (value === null || value === undefined) return null;
278
- const trimmed = String(value).slice(0, 2_000);
279
- return trimmed.length === 0 ? null : trimmed;
277
+ const exact = String(value);
278
+ return exact.length === 0 ? null : exact;
280
279
  }
@@ -0,0 +1,202 @@
1
+ import type { FileAsset } from "@opengeni/contracts";
2
+ import {
3
+ claimRetainedScreenshotMaintenance,
4
+ completeRetainedScreenshotMaintenance,
5
+ promoteRetainedScreenshotMaintenanceCleanup,
6
+ type RetainedScreenshotMaintenanceClaim,
7
+ } from "@opengeni/db";
8
+ import type { ObjectHead, ObjectStorage } from "@opengeni/storage";
9
+ import type { ActivityServices } from "./types";
10
+
11
+ export const RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1_000;
12
+ export const RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
13
+ export const RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
14
+
15
+ export type MaintainRetainedScreenshotsResult = {
16
+ claimed: number;
17
+ ready: number;
18
+ deleted: number;
19
+ failed: number;
20
+ retryable: number;
21
+ };
22
+
23
+ export type RetainedScreenshotMaintenanceActivityOptions = {
24
+ pendingGraceMs?: number;
25
+ claimTimeoutMs?: number;
26
+ batchSize?: number;
27
+ /** Provider seams used by deterministic unit tests. */
28
+ fileExists?: (storage: ObjectStorage, file: FileAsset) => Promise<boolean>;
29
+ headFile?: (storage: ObjectStorage, file: FileAsset) => Promise<ObjectHead>;
30
+ deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
31
+ };
32
+
33
+ /**
34
+ * Reconcile interrupted screenshot writes and reap expired screenshot objects.
35
+ * Claims move rows out of the readable `ready`/writable `pending` states before
36
+ * provider access. Provider failures leave the durable claim reclaimable; DB
37
+ * completion happens only after exact metadata verification or an idempotent
38
+ * object delete.
39
+ */
40
+ export function createRetainedScreenshotMaintenanceActivities(
41
+ services: () => Promise<ActivityServices>,
42
+ options: RetainedScreenshotMaintenanceActivityOptions = {},
43
+ ) {
44
+ const pendingGraceMs = options.pendingGraceMs ?? RETAINED_SCREENSHOT_PENDING_GRACE_MS;
45
+ const claimTimeoutMs = options.claimTimeoutMs ?? RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS;
46
+ const batchSize = options.batchSize ?? RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE;
47
+ const fileExists = options.fileExists ?? ((storage, file) => storage.fileExists(file));
48
+ const headFile = options.headFile ?? ((storage, file) => storage.headFile(file));
49
+ const deleteObject = options.deleteObject ?? ((storage, key) => storage.deleteObject(key));
50
+
51
+ async function maintainRetainedScreenshots(): Promise<MaintainRetainedScreenshotsResult> {
52
+ const { db, objectStorage, observability } = await services();
53
+ if (!objectStorage) {
54
+ return { claimed: 0, ready: 0, deleted: 0, failed: 0, retryable: 0 };
55
+ }
56
+
57
+ const claims = await claimRetainedScreenshotMaintenance(db, {
58
+ pendingGraceMs,
59
+ claimTimeoutMs,
60
+ limit: batchSize,
61
+ });
62
+ const result: MaintainRetainedScreenshotsResult = {
63
+ claimed: claims.length,
64
+ ready: 0,
65
+ deleted: 0,
66
+ failed: 0,
67
+ retryable: 0,
68
+ };
69
+
70
+ for (const claim of claims) {
71
+ try {
72
+ if (claim.action === "reconcile") {
73
+ const outcome = await reconcileClaim(db, objectStorage, claim, {
74
+ fileExists,
75
+ headFile,
76
+ deleteObject,
77
+ });
78
+ if (outcome === "ready") {
79
+ result.ready += 1;
80
+ } else {
81
+ result.failed += 1;
82
+ }
83
+ continue;
84
+ }
85
+
86
+ await deleteObject(objectStorage, claim.objectKey);
87
+ const outcome = terminalDeleteOutcome(claim.cleanupReason);
88
+ const settled = await completeRetainedScreenshotMaintenance(db, {
89
+ accountId: claim.accountId,
90
+ workspaceId: claim.workspaceId,
91
+ artifactId: claim.artifactId,
92
+ claimId: claim.claimId,
93
+ outcome,
94
+ });
95
+ if (!settled) throw new Error("retained screenshot cleanup claim was superseded");
96
+ result.deleted += 1;
97
+ } catch (error) {
98
+ result.retryable += 1;
99
+ observability.warn("retained screenshot maintenance failed; claim remains reclaimable", {
100
+ workspaceId: claim.workspaceId,
101
+ sessionId: claim.sessionId,
102
+ artifactId: claim.artifactId,
103
+ action: claim.action,
104
+ error: error instanceof Error ? error.message : String(error),
105
+ });
106
+ }
107
+ }
108
+
109
+ if (claims.length > 0) {
110
+ observability.info("retained screenshot maintenance swept", result);
111
+ }
112
+ return result;
113
+ }
114
+
115
+ return { maintainRetainedScreenshots };
116
+ }
117
+
118
+ async function reconcileClaim(
119
+ db: Parameters<typeof completeRetainedScreenshotMaintenance>[0],
120
+ storage: ObjectStorage,
121
+ claim: RetainedScreenshotMaintenanceClaim,
122
+ provider: Required<
123
+ Pick<RetainedScreenshotMaintenanceActivityOptions, "fileExists" | "headFile" | "deleteObject">
124
+ >,
125
+ ): Promise<"ready" | "failed"> {
126
+ const observation = await observeClaimObject(storage, claim, provider);
127
+ if (observation === "matches") {
128
+ const settled = await completeRetainedScreenshotMaintenance(db, {
129
+ accountId: claim.accountId,
130
+ workspaceId: claim.workspaceId,
131
+ artifactId: claim.artifactId,
132
+ claimId: claim.claimId,
133
+ outcome: "ready",
134
+ });
135
+ if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
136
+ return "ready";
137
+ }
138
+
139
+ // A mismatch/missing observation cannot authorize provider deletion while
140
+ // the artifact is still eligible for a concurrent ready settlement. Promote
141
+ // the exact claim first; a ready winner clears the claim and makes this stale.
142
+ const promoted = await promoteRetainedScreenshotMaintenanceCleanup(db, {
143
+ accountId: claim.accountId,
144
+ workspaceId: claim.workspaceId,
145
+ artifactId: claim.artifactId,
146
+ claimId: claim.claimId,
147
+ cleanupReason: "failed",
148
+ });
149
+ if (!promoted) throw new Error("retained screenshot reconcile claim was superseded");
150
+
151
+ // Deletes are idempotent. Calling it for an authoritative missing observation
152
+ // also closes a race with a late writer that began before cleanup ownership.
153
+ await provider.deleteObject(storage, claim.objectKey);
154
+ const settled = await completeRetainedScreenshotMaintenance(db, {
155
+ accountId: claim.accountId,
156
+ workspaceId: claim.workspaceId,
157
+ artifactId: claim.artifactId,
158
+ claimId: claim.claimId,
159
+ outcome: "failed",
160
+ });
161
+ if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
162
+ return "failed";
163
+ }
164
+
165
+ async function observeClaimObject(
166
+ storage: ObjectStorage,
167
+ claim: RetainedScreenshotMaintenanceClaim,
168
+ provider: Required<Pick<RetainedScreenshotMaintenanceActivityOptions, "fileExists" | "headFile">>,
169
+ ): Promise<"matches" | "mismatch" | "missing"> {
170
+ const file = claimFile(storage, claim);
171
+ if (!(await provider.fileExists(storage, file))) return "missing";
172
+ const head = await provider.headFile(storage, file);
173
+ return head.ContentLength === claim.sizeBytes &&
174
+ head.ContentType === claim.mediaType &&
175
+ head.Metadata?.sha256 === claim.sha256
176
+ ? "matches"
177
+ : "mismatch";
178
+ }
179
+
180
+ function claimFile(storage: ObjectStorage, claim: RetainedScreenshotMaintenanceClaim): FileAsset {
181
+ const timestamp = new Date(0).toISOString();
182
+ return {
183
+ id: claim.artifactId,
184
+ workspaceId: claim.workspaceId,
185
+ status: "pending_upload",
186
+ filename: `computer-screenshot-${claim.artifactId}.png`,
187
+ safeFilename: `computer-screenshot-${claim.artifactId}.png`,
188
+ contentType: claim.mediaType,
189
+ sizeBytes: claim.sizeBytes,
190
+ sha256: claim.sha256,
191
+ bucket: storage.bucket,
192
+ objectKey: claim.objectKey,
193
+ createdAt: timestamp,
194
+ updatedAt: timestamp,
195
+ };
196
+ }
197
+
198
+ function terminalDeleteOutcome(reason: string | null): "failed" | "expired" | "deleted" {
199
+ if (reason === "expired") return "expired";
200
+ if (reason === "failed") return "failed";
201
+ return "deleted";
202
+ }