@opengeni/worker-bundle 0.16.17 → 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +83 -32
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/environment.d.ts +1 -0
- package/dist/activities/expiring-material-renewal.d.ts +5 -3
- package/dist/activities/file-upload-reaper.d.ts +6 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
- package/dist/activities/retained-screenshots.d.ts +93 -0
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +2 -0
- package/dist/activities/toolspace-token-renewal.d.ts +1 -1
- package/dist/activities/workspace-capture.d.ts +1 -0
- package/dist/activities.d.ts +3 -0
- package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
- package/dist/chunk-RAMNYSAW.js.map +1 -0
- package/dist/host-export.js +1 -1
- package/dist/index.js +1465 -453
- package/dist/index.js.map +1 -1
- package/dist/sandbox-resume.d.ts +6 -0
- package/dist/sandbox-routing.d.ts +11 -6
- package/dist/workflow-bundle.js +2 -1
- package/package.json +12 -12
- package/src/activities/agent-turn.ts +559 -434
- package/src/activities/codex-capacity.ts +10 -1
- package/src/activities/codex-rotation.ts +19 -5
- package/src/activities/context-compaction.ts +9 -5
- package/src/activities/environment.ts +13 -24
- package/src/activities/expiring-material-renewal.ts +16 -7
- package/src/activities/file-upload-reaper.ts +108 -7
- package/src/activities/git-credential-renewal.ts +3 -3
- package/src/activities/recording.ts +4 -5
- package/src/activities/retained-screenshot-reaper.ts +202 -0
- package/src/activities/retained-screenshots.ts +892 -0
- package/src/activities/rig-verification.ts +13 -34
- package/src/activities/run-credential-renewal.ts +9 -2
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +5 -2
- package/src/activities/scheduled-tasks.ts +19 -3
- package/src/activities/toolspace-token-renewal.ts +9 -2
- package/src/activities/workspace-capture.ts +33 -3
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +28 -11
- package/src/http.ts +3 -6
- package/src/observability-metrics.ts +3 -3
- package/src/sandbox-resume.ts +45 -12
- package/src/sandbox-routing.ts +35 -17
- package/src/worker-service-lifecycle.ts +8 -4
- package/src/workflows/file-upload-reaper.ts +4 -1
- package/dist/activities/redaction.d.ts +0 -1
- package/dist/chunk-BJUW5LCM.js.map +0 -1
- package/src/activities/redaction.ts +0 -9
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
CODEX_USAGE_EXHAUSTED_PCT,
|
|
13
13
|
authoritativeCodexCapacityResetAt,
|
|
14
14
|
isCodexCredentialEligible,
|
|
15
|
+
isCodexCredentialHealthy,
|
|
15
16
|
selectCodexCredentialLeaseForTurn,
|
|
16
17
|
} from "./codex-rotation";
|
|
17
18
|
import type {
|
|
@@ -96,7 +97,15 @@ export function codexCapacityDecision(
|
|
|
96
97
|
sessionLastCredentialId: context.sessionLastCredentialId,
|
|
97
98
|
now,
|
|
98
99
|
});
|
|
99
|
-
|
|
100
|
+
const selectedAccount = selected.credentialId
|
|
101
|
+
? context.accounts.find((account) => account.id === selected.credentialId)
|
|
102
|
+
: undefined;
|
|
103
|
+
const selectedIsAvailable =
|
|
104
|
+
selectedAccount !== undefined &&
|
|
105
|
+
(selectedAccount.id === context.existingCredentialId
|
|
106
|
+
? isCodexCredentialHealthy(selectedAccount, now)
|
|
107
|
+
: isCodexCredentialEligible(selectedAccount, now));
|
|
108
|
+
if (selected.credentialId && selectedIsAvailable) {
|
|
100
109
|
return {
|
|
101
110
|
kind: "available",
|
|
102
111
|
credentialId: selected.credentialId,
|
|
@@ -694,13 +694,27 @@ export function selectCodexCredentialLeaseForTurn(args: {
|
|
|
694
694
|
|
|
695
695
|
// A stale policy pin is intentionally ignored here and cleared by the caller
|
|
696
696
|
// after the atomic selection. Rotation-off/unpinned behavior otherwise keeps
|
|
697
|
-
// the pre-lease pin > active-pointer contract.
|
|
697
|
+
// the pre-lease pin > active-pointer contract. The pointer is preference, not
|
|
698
|
+
// proof of capacity: rotation-off must wait on a capped pointer rather than
|
|
699
|
+
// falsely admitting it or silently failing over to another subscription.
|
|
698
700
|
if (!rotationEnabled) {
|
|
699
|
-
const
|
|
700
|
-
|
|
701
|
+
const active = activeCredentialId
|
|
702
|
+
? accounts.find((account) => account.id === activeCredentialId)
|
|
703
|
+
: undefined;
|
|
704
|
+
if (!active?.allocatorEnabled) {
|
|
705
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
706
|
+
}
|
|
707
|
+
if (!isCodexCredentialHealthy(active, args.now)) {
|
|
708
|
+
return {
|
|
709
|
+
credentialId: null,
|
|
710
|
+
decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
|
|
711
|
+
advanceActivePointer: false,
|
|
712
|
+
};
|
|
713
|
+
}
|
|
701
714
|
return {
|
|
702
|
-
credentialId,
|
|
703
|
-
decision:
|
|
715
|
+
credentialId: active.id,
|
|
716
|
+
decision: { kind: "active", credentialId: active.id, moved: false },
|
|
717
|
+
advanceActivePointer: false,
|
|
704
718
|
};
|
|
705
719
|
}
|
|
706
720
|
|
|
@@ -97,6 +97,8 @@ export async function maybeCompactContext(
|
|
|
97
97
|
* append again — the event is already durable.
|
|
98
98
|
*/
|
|
99
99
|
publishLiveEvents?: (events: SessionEvent[]) => Promise<void>;
|
|
100
|
+
/** Materialize retained screenshot receipts only in the attempt-local model view. */
|
|
101
|
+
materializeHistory?: (items: CompactionItem[]) => Promise<CompactionItem[]>;
|
|
100
102
|
/** Turn-scoped attachment/modality view; canonical persisted rows stay untouched. */
|
|
101
103
|
projectModelInput?: (items: CompactionItem[]) => Promise<CompactionItem[]>;
|
|
102
104
|
} = {},
|
|
@@ -142,11 +144,15 @@ export async function maybeCompactContext(
|
|
|
142
144
|
}
|
|
143
145
|
|
|
144
146
|
const canonicalItems = projectRejectedProviderArtifacts(active) as CompactionItem[];
|
|
145
|
-
const projectForWire = async (input: CompactionItem[]): Promise<CompactionItem[]> =>
|
|
146
|
-
|
|
147
|
-
|
|
147
|
+
const projectForWire = async (input: CompactionItem[]): Promise<CompactionItem[]> => {
|
|
148
|
+
const materialized = options.materializeHistory
|
|
149
|
+
? await options.materializeHistory(input)
|
|
150
|
+
: input;
|
|
151
|
+
return sanitizeHistoryItemsForModel(
|
|
152
|
+
options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
|
|
148
153
|
settings.modelToolOutputTruncationTokens,
|
|
149
154
|
) as CompactionItem[];
|
|
155
|
+
};
|
|
150
156
|
const items = await projectForWire(canonicalItems);
|
|
151
157
|
const decision = decideCompaction({
|
|
152
158
|
items,
|
|
@@ -339,7 +345,6 @@ async function compactContextRemoteV2(
|
|
|
339
345
|
expectedAttemptId: scope.attemptId,
|
|
340
346
|
replacementItems: replacementHistory.slice(0, -1),
|
|
341
347
|
summaryItem: tailItem as Record<string, unknown>,
|
|
342
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
343
348
|
...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
|
|
344
349
|
eventPayload: {
|
|
345
350
|
trigger: options.trigger ?? "auto",
|
|
@@ -415,7 +420,6 @@ async function compactContextPortable(
|
|
|
415
420
|
expectedAttemptId: scope.attemptId,
|
|
416
421
|
replacementItems: replacementHistory.slice(0, -1),
|
|
417
422
|
summaryItem: summaryItem as Record<string, unknown>,
|
|
418
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
419
423
|
...(options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {}),
|
|
420
424
|
eventPayload: {
|
|
421
425
|
trigger: options.trigger ?? "auto",
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
import {
|
|
8
8
|
gitCredentialBindingIdForRepository,
|
|
9
9
|
gitCredentialProviderForRepository,
|
|
10
|
+
gitRemoteIdentity,
|
|
10
11
|
signDelegatedAccessToken,
|
|
11
12
|
type ConnectionCredentialsPort,
|
|
12
13
|
type GitCredentialProvider,
|
|
@@ -202,6 +203,7 @@ export async function sandboxEnvironmentForRun(
|
|
|
202
203
|
// value later. `= {}` default so the non-optional reads below are safe.
|
|
203
204
|
options: RunGitCredentialOptions & {
|
|
204
205
|
skipGitHubToken?: boolean;
|
|
206
|
+
skipToolspace?: boolean;
|
|
205
207
|
deferGitHubToken?: boolean;
|
|
206
208
|
toolspaceAuthority?: SandboxToolspaceAuthority;
|
|
207
209
|
} = {},
|
|
@@ -233,18 +235,18 @@ export async function sandboxEnvironmentForRun(
|
|
|
233
235
|
// proactively renews every selected provider behind these stable pointers.
|
|
234
236
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
235
237
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
238
|
+
// Connected Machines own their environment and credentials. They do not
|
|
239
|
+
// receive OpenGeni Toolspace pointers, package hints, or delegated tokens.
|
|
240
|
+
// Scrub the stable pointers as well as skipping the mint because a session
|
|
241
|
+
// whose home backend is managed may be actively routed to selfhosted.
|
|
242
|
+
if (options.skipToolspace) {
|
|
243
|
+
delete environment.OPENGENI_TOOLSPACE_URL;
|
|
244
|
+
delete environment.OPENGENI_TOOLSPACE_TOKEN_FILE;
|
|
245
|
+
delete environment.OPENGENI_OGTOOL_PACKAGE_SPEC;
|
|
246
|
+
}
|
|
245
247
|
const toolspaceScope = options.scope;
|
|
246
248
|
const toolspaceToken =
|
|
247
|
-
toolspaceScope && options.toolspaceAuthority
|
|
249
|
+
!options.skipToolspace && toolspaceScope && options.toolspaceAuthority
|
|
248
250
|
? await mintSandboxToolspaceToken(settings, toolspaceScope, options.toolspaceAuthority)
|
|
249
251
|
: undefined;
|
|
250
252
|
if (toolspaceToken && toolspaceScope) {
|
|
@@ -777,7 +779,7 @@ function gitCredentialSelections(resources: ResourceRef[]): GitCredentialSelecti
|
|
|
777
779
|
);
|
|
778
780
|
}
|
|
779
781
|
bindingProviders.set(credentialBindingId, provider);
|
|
780
|
-
const normalizedRemote =
|
|
782
|
+
const normalizedRemote = gitRemoteIdentity(resource.uri, provider);
|
|
781
783
|
const claimedBinding = remoteBindings.get(normalizedRemote);
|
|
782
784
|
if (claimedBinding && claimedBinding !== bindingKey) {
|
|
783
785
|
throw new Error(
|
|
@@ -836,19 +838,6 @@ function normalizedGitHost(uri: string): string {
|
|
|
836
838
|
}
|
|
837
839
|
}
|
|
838
840
|
|
|
839
|
-
function normalizedGitRemote(uri: string): string {
|
|
840
|
-
try {
|
|
841
|
-
const url = new URL(uri);
|
|
842
|
-
return `${url.protocol.toLowerCase()}//${url.host.toLowerCase()}${url.pathname.replace(/\/+$/, "").replace(/\.git$/, "")}`;
|
|
843
|
-
} catch {
|
|
844
|
-
return uri
|
|
845
|
-
.trim()
|
|
846
|
-
.replace(/\/+$/, "")
|
|
847
|
-
.replace(/\.git$/, "")
|
|
848
|
-
.toLowerCase();
|
|
849
|
-
}
|
|
850
|
-
}
|
|
851
|
-
|
|
852
841
|
function repositoryCredentialProvider(
|
|
853
842
|
resource: Extract<ResourceRef, { kind: "repository" }>,
|
|
854
843
|
): GitCredentialProvider | null {
|
|
@@ -12,17 +12,25 @@ export type ExpiringMaterialRenewalController = {
|
|
|
12
12
|
stop(): Promise<void>;
|
|
13
13
|
};
|
|
14
14
|
|
|
15
|
-
export type
|
|
15
|
+
export type ExpiringMaterialRenewalPublicErrorClass =
|
|
16
|
+
| "RunCredentialRenewalOperationError"
|
|
17
|
+
| "ToolspaceTokenRenewalOperationError";
|
|
18
|
+
|
|
19
|
+
export type ExpiringMaterialRenewalOptions<
|
|
20
|
+
T,
|
|
21
|
+
ErrorClass extends ExpiringMaterialRenewalPublicErrorClass,
|
|
22
|
+
> = {
|
|
16
23
|
initialExpiresAt: Date | null;
|
|
17
24
|
resolve: () => Promise<T>;
|
|
18
25
|
write: (material: T) => Promise<void>;
|
|
19
26
|
expiresAt: (material: T) => Date | null;
|
|
27
|
+
publicErrorClass: ErrorClass;
|
|
20
28
|
policy: ExpiringMaterialRenewalPolicy;
|
|
21
29
|
now?: () => number;
|
|
22
30
|
schedule?: (callback: () => void, delayMs: number) => RenewalTimer;
|
|
23
31
|
clearSchedule?: (timer: RenewalTimer) => void;
|
|
24
32
|
onSuccess?: (result: { material: T; nextDelayMs: number }) => void;
|
|
25
|
-
onFailure?: (failure: { retryDelayMs: number; errorClass:
|
|
33
|
+
onFailure?: (failure: { retryDelayMs: number; errorClass: ErrorClass }) => void;
|
|
26
34
|
};
|
|
27
35
|
|
|
28
36
|
export function nextExpiringMaterialRenewalDelay(
|
|
@@ -45,9 +53,10 @@ export function nextExpiringMaterialRenewalDelay(
|
|
|
45
53
|
* A late resolver cannot write after stop. A physical sandbox write that already
|
|
46
54
|
* started is drained so turn settlement cannot race a stale mutation.
|
|
47
55
|
*/
|
|
48
|
-
export function startExpiringMaterialRenewalLoop<
|
|
49
|
-
|
|
50
|
-
|
|
56
|
+
export function startExpiringMaterialRenewalLoop<
|
|
57
|
+
T,
|
|
58
|
+
ErrorClass extends ExpiringMaterialRenewalPublicErrorClass,
|
|
59
|
+
>(options: ExpiringMaterialRenewalOptions<T, ErrorClass>): ExpiringMaterialRenewalController {
|
|
51
60
|
const now = options.now ?? Date.now;
|
|
52
61
|
const schedule =
|
|
53
62
|
options.schedule ??
|
|
@@ -102,14 +111,14 @@ export function startExpiringMaterialRenewalLoop<T>(
|
|
|
102
111
|
// Observability never owns credential liveness.
|
|
103
112
|
}
|
|
104
113
|
scheduleRefresh(nextDelayMs);
|
|
105
|
-
} catch
|
|
114
|
+
} catch {
|
|
106
115
|
if (stopped) return;
|
|
107
116
|
const scheduledRetryMs = retryDelayMs;
|
|
108
117
|
retryDelayMs = Math.min(retryDelayMs * 2, options.policy.maxRetryMs);
|
|
109
118
|
try {
|
|
110
119
|
options.onFailure?.({
|
|
111
120
|
retryDelayMs: scheduledRetryMs,
|
|
112
|
-
errorClass:
|
|
121
|
+
errorClass: options.publicErrorClass,
|
|
113
122
|
});
|
|
114
123
|
} catch {
|
|
115
124
|
// Observability never owns credential liveness.
|
|
@@ -1,10 +1,51 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
claimDueTranscriptionRecordingObjectCleanup,
|
|
3
|
+
claimExpiredFileUploadCleanup,
|
|
4
|
+
completeDueTranscriptionRecordingObjectCleanup,
|
|
5
|
+
completeExpiredFileUploadCleanup,
|
|
6
|
+
purgeExpiredTranscriptionRecordings,
|
|
7
|
+
} from "@opengeni/db";
|
|
2
8
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
3
9
|
import type { ActivityServices } from "./types";
|
|
4
10
|
|
|
5
11
|
export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
|
|
6
12
|
export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
|
|
7
13
|
export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
|
|
14
|
+
const SAFE_CLEANUP_ERROR_CATEGORIES = new Set([
|
|
15
|
+
"aborted",
|
|
16
|
+
"conflict",
|
|
17
|
+
"invalid_response",
|
|
18
|
+
"network",
|
|
19
|
+
"not_found",
|
|
20
|
+
"permission_denied",
|
|
21
|
+
"provider",
|
|
22
|
+
"timeout",
|
|
23
|
+
"unavailable",
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
function cleanupErrorAttributes(error: unknown): {
|
|
27
|
+
errorCategory: string;
|
|
28
|
+
errorStatus?: number;
|
|
29
|
+
} {
|
|
30
|
+
let errorCategory = "provider_error";
|
|
31
|
+
let errorStatus: number | undefined;
|
|
32
|
+
if (error && typeof error === "object") {
|
|
33
|
+
const rawCode = (error as { code?: unknown }).code;
|
|
34
|
+
if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
|
|
35
|
+
errorCategory = rawCode;
|
|
36
|
+
} else if (rawCode !== undefined) {
|
|
37
|
+
errorCategory = "unknown";
|
|
38
|
+
}
|
|
39
|
+
const rawStatus = Number(
|
|
40
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
41
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
42
|
+
);
|
|
43
|
+
if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
|
|
44
|
+
errorStatus = rawStatus;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return { errorCategory, ...(errorStatus === undefined ? {} : { errorStatus }) };
|
|
48
|
+
}
|
|
8
49
|
|
|
9
50
|
export type ReapExpiredFileUploadsResult = {
|
|
10
51
|
claimed: number;
|
|
@@ -18,6 +59,11 @@ export type FileUploadReaperActivityOptions = {
|
|
|
18
59
|
batchSize?: number;
|
|
19
60
|
/** Failure-injection seam; production uses the configured provider delete. */
|
|
20
61
|
deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
|
|
62
|
+
claimFileUploads?: typeof claimExpiredFileUploadCleanup;
|
|
63
|
+
completeFileUpload?: typeof completeExpiredFileUploadCleanup;
|
|
64
|
+
claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
|
|
65
|
+
completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
|
|
66
|
+
purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
|
|
21
67
|
};
|
|
22
68
|
|
|
23
69
|
/**
|
|
@@ -33,14 +79,29 @@ export function createFileUploadReaperActivities(
|
|
|
33
79
|
const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
|
|
34
80
|
const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
|
|
35
81
|
const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
|
|
82
|
+
const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
|
|
83
|
+
const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
|
|
84
|
+
const claimTranscriptionObjects =
|
|
85
|
+
options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
|
|
86
|
+
const completeTranscriptionObject =
|
|
87
|
+
options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
|
|
88
|
+
const purgeTranscriptionRecordings =
|
|
89
|
+
options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
|
|
36
90
|
|
|
37
91
|
async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
|
|
38
92
|
const { db, objectStorage, observability } = await services();
|
|
39
93
|
if (!objectStorage) {
|
|
94
|
+
try {
|
|
95
|
+
await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
|
|
96
|
+
} catch (error) {
|
|
97
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
98
|
+
...cleanupErrorAttributes(error),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
40
101
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
41
102
|
}
|
|
42
103
|
|
|
43
|
-
const claims = await
|
|
104
|
+
const claims = await claimFileUploads(db, {
|
|
44
105
|
graceMs,
|
|
45
106
|
claimTimeoutMs,
|
|
46
107
|
limit: batchSize,
|
|
@@ -50,7 +111,7 @@ export function createFileUploadReaperActivities(
|
|
|
50
111
|
for (const claim of claims) {
|
|
51
112
|
try {
|
|
52
113
|
await deleteObject(objectStorage, claim.objectKey);
|
|
53
|
-
const settled = await
|
|
114
|
+
const settled = await completeFileUpload(db, claim);
|
|
54
115
|
if (!settled) {
|
|
55
116
|
throw new Error("cleanup claim no longer owns a reclaimable upload");
|
|
56
117
|
}
|
|
@@ -61,18 +122,58 @@ export function createFileUploadReaperActivities(
|
|
|
61
122
|
workspaceId: claim.workspaceId,
|
|
62
123
|
uploadId: claim.uploadId,
|
|
63
124
|
fileId: claim.fileId,
|
|
64
|
-
|
|
125
|
+
...cleanupErrorAttributes(error),
|
|
65
126
|
});
|
|
66
127
|
}
|
|
67
128
|
}
|
|
68
|
-
|
|
129
|
+
const recordingClaims = await claimTranscriptionObjects(db, {
|
|
130
|
+
graceMs,
|
|
131
|
+
claimTimeoutMs,
|
|
132
|
+
limit: batchSize,
|
|
133
|
+
});
|
|
134
|
+
for (const claim of recordingClaims) {
|
|
135
|
+
try {
|
|
136
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
137
|
+
const settled = await completeTranscriptionObject(db, claim);
|
|
138
|
+
if (!settled) {
|
|
139
|
+
throw new Error("cleanup claim no longer owns a transcription recording object");
|
|
140
|
+
}
|
|
141
|
+
deleted += 1;
|
|
142
|
+
} catch (error) {
|
|
143
|
+
failed += 1;
|
|
144
|
+
observability.warn(
|
|
145
|
+
"expired transcription recording object cleanup failed; claim remains reclaimable",
|
|
146
|
+
{
|
|
147
|
+
workspaceId: claim.workspaceId,
|
|
148
|
+
recordingId: claim.recordingId,
|
|
149
|
+
...cleanupErrorAttributes(error),
|
|
150
|
+
},
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
let transcriptionRecordingsPurged = 0;
|
|
155
|
+
try {
|
|
156
|
+
transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
|
|
157
|
+
graceMs,
|
|
158
|
+
limit: batchSize,
|
|
159
|
+
});
|
|
160
|
+
} catch (error) {
|
|
161
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
162
|
+
...cleanupErrorAttributes(error),
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
const claimed = claims.length + recordingClaims.length;
|
|
166
|
+
if (claimed > 0 || transcriptionRecordingsPurged > 0) {
|
|
69
167
|
observability.info("expired file upload cleanup swept", {
|
|
70
|
-
claimed
|
|
168
|
+
claimed,
|
|
71
169
|
deleted,
|
|
72
170
|
failed,
|
|
171
|
+
fileUploads: claims.length,
|
|
172
|
+
transcriptionRecordingObjects: recordingClaims.length,
|
|
173
|
+
transcriptionRecordingsPurged,
|
|
73
174
|
});
|
|
74
175
|
}
|
|
75
|
-
return { claimed
|
|
176
|
+
return { claimed, deleted, failed };
|
|
76
177
|
}
|
|
77
178
|
|
|
78
179
|
return { reapExpiredFileUploads };
|
|
@@ -16,7 +16,7 @@ export type GitCredentialRenewalResult = {
|
|
|
16
16
|
export type GitCredentialRenewalFailure = {
|
|
17
17
|
providers: readonly GitCredentialProvider[];
|
|
18
18
|
retryDelayMs: number;
|
|
19
|
-
errorClass:
|
|
19
|
+
errorClass: "GitCredentialRenewalOperationError";
|
|
20
20
|
};
|
|
21
21
|
|
|
22
22
|
export type GitCredentialRenewalController = {
|
|
@@ -133,7 +133,7 @@ export function startGitCredentialRenewalLoop(
|
|
|
133
133
|
// Observability callbacks never own credential liveness.
|
|
134
134
|
}
|
|
135
135
|
scheduleRefresh(nextDelayMs);
|
|
136
|
-
} catch
|
|
136
|
+
} catch {
|
|
137
137
|
if (stopped) return;
|
|
138
138
|
const scheduledRetryMs = retryDelayMs;
|
|
139
139
|
retryDelayMs = Math.min(retryDelayMs * 2, GIT_CREDENTIAL_MAX_RETRY_MS);
|
|
@@ -141,7 +141,7 @@ export function startGitCredentialRenewalLoop(
|
|
|
141
141
|
options.onFailure?.({
|
|
142
142
|
providers,
|
|
143
143
|
retryDelayMs: scheduledRetryMs,
|
|
144
|
-
errorClass:
|
|
144
|
+
errorClass: "GitCredentialRenewalOperationError",
|
|
145
145
|
});
|
|
146
146
|
} catch {
|
|
147
147
|
// Observability callbacks never own credential liveness.
|
|
@@ -270,11 +270,10 @@ export async function discardUnpublishedRecording(args: {
|
|
|
270
270
|
|
|
271
271
|
export { DOWNLOAD_URL_TTL_SECONDS };
|
|
272
272
|
|
|
273
|
-
//
|
|
274
|
-
//
|
|
275
|
-
// reason/detail field unbounded; the redactor scrubs known secret shapes).
|
|
273
|
+
// Recording diagnostics are canonical OpenGeni data. Preserve the exact text;
|
|
274
|
+
// transport-specific byte limits must reject or paginate rather than rewrite it.
|
|
276
275
|
function scrubFreeText(value: string | null | undefined): string | null {
|
|
277
276
|
if (value === null || value === undefined) return null;
|
|
278
|
-
const
|
|
279
|
-
return
|
|
277
|
+
const exact = String(value);
|
|
278
|
+
return exact.length === 0 ? null : exact;
|
|
280
279
|
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import type { FileAsset } from "@opengeni/contracts";
|
|
2
|
+
import {
|
|
3
|
+
claimRetainedScreenshotMaintenance,
|
|
4
|
+
completeRetainedScreenshotMaintenance,
|
|
5
|
+
promoteRetainedScreenshotMaintenanceCleanup,
|
|
6
|
+
type RetainedScreenshotMaintenanceClaim,
|
|
7
|
+
} from "@opengeni/db";
|
|
8
|
+
import type { ObjectHead, ObjectStorage } from "@opengeni/storage";
|
|
9
|
+
import type { ActivityServices } from "./types";
|
|
10
|
+
|
|
11
|
+
export const RETAINED_SCREENSHOT_PENDING_GRACE_MS = 15 * 60 * 1_000;
|
|
12
|
+
export const RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
|
|
13
|
+
export const RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE = 100;
|
|
14
|
+
|
|
15
|
+
export type MaintainRetainedScreenshotsResult = {
|
|
16
|
+
claimed: number;
|
|
17
|
+
ready: number;
|
|
18
|
+
deleted: number;
|
|
19
|
+
failed: number;
|
|
20
|
+
retryable: number;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type RetainedScreenshotMaintenanceActivityOptions = {
|
|
24
|
+
pendingGraceMs?: number;
|
|
25
|
+
claimTimeoutMs?: number;
|
|
26
|
+
batchSize?: number;
|
|
27
|
+
/** Provider seams used by deterministic unit tests. */
|
|
28
|
+
fileExists?: (storage: ObjectStorage, file: FileAsset) => Promise<boolean>;
|
|
29
|
+
headFile?: (storage: ObjectStorage, file: FileAsset) => Promise<ObjectHead>;
|
|
30
|
+
deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Reconcile interrupted screenshot writes and reap expired screenshot objects.
|
|
35
|
+
* Claims move rows out of the readable `ready`/writable `pending` states before
|
|
36
|
+
* provider access. Provider failures leave the durable claim reclaimable; DB
|
|
37
|
+
* completion happens only after exact metadata verification or an idempotent
|
|
38
|
+
* object delete.
|
|
39
|
+
*/
|
|
40
|
+
export function createRetainedScreenshotMaintenanceActivities(
|
|
41
|
+
services: () => Promise<ActivityServices>,
|
|
42
|
+
options: RetainedScreenshotMaintenanceActivityOptions = {},
|
|
43
|
+
) {
|
|
44
|
+
const pendingGraceMs = options.pendingGraceMs ?? RETAINED_SCREENSHOT_PENDING_GRACE_MS;
|
|
45
|
+
const claimTimeoutMs = options.claimTimeoutMs ?? RETAINED_SCREENSHOT_CLAIM_TIMEOUT_MS;
|
|
46
|
+
const batchSize = options.batchSize ?? RETAINED_SCREENSHOT_MAINTENANCE_BATCH_SIZE;
|
|
47
|
+
const fileExists = options.fileExists ?? ((storage, file) => storage.fileExists(file));
|
|
48
|
+
const headFile = options.headFile ?? ((storage, file) => storage.headFile(file));
|
|
49
|
+
const deleteObject = options.deleteObject ?? ((storage, key) => storage.deleteObject(key));
|
|
50
|
+
|
|
51
|
+
async function maintainRetainedScreenshots(): Promise<MaintainRetainedScreenshotsResult> {
|
|
52
|
+
const { db, objectStorage, observability } = await services();
|
|
53
|
+
if (!objectStorage) {
|
|
54
|
+
return { claimed: 0, ready: 0, deleted: 0, failed: 0, retryable: 0 };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const claims = await claimRetainedScreenshotMaintenance(db, {
|
|
58
|
+
pendingGraceMs,
|
|
59
|
+
claimTimeoutMs,
|
|
60
|
+
limit: batchSize,
|
|
61
|
+
});
|
|
62
|
+
const result: MaintainRetainedScreenshotsResult = {
|
|
63
|
+
claimed: claims.length,
|
|
64
|
+
ready: 0,
|
|
65
|
+
deleted: 0,
|
|
66
|
+
failed: 0,
|
|
67
|
+
retryable: 0,
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
for (const claim of claims) {
|
|
71
|
+
try {
|
|
72
|
+
if (claim.action === "reconcile") {
|
|
73
|
+
const outcome = await reconcileClaim(db, objectStorage, claim, {
|
|
74
|
+
fileExists,
|
|
75
|
+
headFile,
|
|
76
|
+
deleteObject,
|
|
77
|
+
});
|
|
78
|
+
if (outcome === "ready") {
|
|
79
|
+
result.ready += 1;
|
|
80
|
+
} else {
|
|
81
|
+
result.failed += 1;
|
|
82
|
+
}
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
87
|
+
const outcome = terminalDeleteOutcome(claim.cleanupReason);
|
|
88
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
89
|
+
accountId: claim.accountId,
|
|
90
|
+
workspaceId: claim.workspaceId,
|
|
91
|
+
artifactId: claim.artifactId,
|
|
92
|
+
claimId: claim.claimId,
|
|
93
|
+
outcome,
|
|
94
|
+
});
|
|
95
|
+
if (!settled) throw new Error("retained screenshot cleanup claim was superseded");
|
|
96
|
+
result.deleted += 1;
|
|
97
|
+
} catch (error) {
|
|
98
|
+
result.retryable += 1;
|
|
99
|
+
observability.warn("retained screenshot maintenance failed; claim remains reclaimable", {
|
|
100
|
+
workspaceId: claim.workspaceId,
|
|
101
|
+
sessionId: claim.sessionId,
|
|
102
|
+
artifactId: claim.artifactId,
|
|
103
|
+
action: claim.action,
|
|
104
|
+
error: error instanceof Error ? error.message : String(error),
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (claims.length > 0) {
|
|
110
|
+
observability.info("retained screenshot maintenance swept", result);
|
|
111
|
+
}
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return { maintainRetainedScreenshots };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function reconcileClaim(
|
|
119
|
+
db: Parameters<typeof completeRetainedScreenshotMaintenance>[0],
|
|
120
|
+
storage: ObjectStorage,
|
|
121
|
+
claim: RetainedScreenshotMaintenanceClaim,
|
|
122
|
+
provider: Required<
|
|
123
|
+
Pick<RetainedScreenshotMaintenanceActivityOptions, "fileExists" | "headFile" | "deleteObject">
|
|
124
|
+
>,
|
|
125
|
+
): Promise<"ready" | "failed"> {
|
|
126
|
+
const observation = await observeClaimObject(storage, claim, provider);
|
|
127
|
+
if (observation === "matches") {
|
|
128
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
129
|
+
accountId: claim.accountId,
|
|
130
|
+
workspaceId: claim.workspaceId,
|
|
131
|
+
artifactId: claim.artifactId,
|
|
132
|
+
claimId: claim.claimId,
|
|
133
|
+
outcome: "ready",
|
|
134
|
+
});
|
|
135
|
+
if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
|
|
136
|
+
return "ready";
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// A mismatch/missing observation cannot authorize provider deletion while
|
|
140
|
+
// the artifact is still eligible for a concurrent ready settlement. Promote
|
|
141
|
+
// the exact claim first; a ready winner clears the claim and makes this stale.
|
|
142
|
+
const promoted = await promoteRetainedScreenshotMaintenanceCleanup(db, {
|
|
143
|
+
accountId: claim.accountId,
|
|
144
|
+
workspaceId: claim.workspaceId,
|
|
145
|
+
artifactId: claim.artifactId,
|
|
146
|
+
claimId: claim.claimId,
|
|
147
|
+
cleanupReason: "failed",
|
|
148
|
+
});
|
|
149
|
+
if (!promoted) throw new Error("retained screenshot reconcile claim was superseded");
|
|
150
|
+
|
|
151
|
+
// Deletes are idempotent. Calling it for an authoritative missing observation
|
|
152
|
+
// also closes a race with a late writer that began before cleanup ownership.
|
|
153
|
+
await provider.deleteObject(storage, claim.objectKey);
|
|
154
|
+
const settled = await completeRetainedScreenshotMaintenance(db, {
|
|
155
|
+
accountId: claim.accountId,
|
|
156
|
+
workspaceId: claim.workspaceId,
|
|
157
|
+
artifactId: claim.artifactId,
|
|
158
|
+
claimId: claim.claimId,
|
|
159
|
+
outcome: "failed",
|
|
160
|
+
});
|
|
161
|
+
if (!settled) throw new Error("retained screenshot reconcile claim was superseded");
|
|
162
|
+
return "failed";
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async function observeClaimObject(
|
|
166
|
+
storage: ObjectStorage,
|
|
167
|
+
claim: RetainedScreenshotMaintenanceClaim,
|
|
168
|
+
provider: Required<Pick<RetainedScreenshotMaintenanceActivityOptions, "fileExists" | "headFile">>,
|
|
169
|
+
): Promise<"matches" | "mismatch" | "missing"> {
|
|
170
|
+
const file = claimFile(storage, claim);
|
|
171
|
+
if (!(await provider.fileExists(storage, file))) return "missing";
|
|
172
|
+
const head = await provider.headFile(storage, file);
|
|
173
|
+
return head.ContentLength === claim.sizeBytes &&
|
|
174
|
+
head.ContentType === claim.mediaType &&
|
|
175
|
+
head.Metadata?.sha256 === claim.sha256
|
|
176
|
+
? "matches"
|
|
177
|
+
: "mismatch";
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function claimFile(storage: ObjectStorage, claim: RetainedScreenshotMaintenanceClaim): FileAsset {
|
|
181
|
+
const timestamp = new Date(0).toISOString();
|
|
182
|
+
return {
|
|
183
|
+
id: claim.artifactId,
|
|
184
|
+
workspaceId: claim.workspaceId,
|
|
185
|
+
status: "pending_upload",
|
|
186
|
+
filename: `computer-screenshot-${claim.artifactId}.png`,
|
|
187
|
+
safeFilename: `computer-screenshot-${claim.artifactId}.png`,
|
|
188
|
+
contentType: claim.mediaType,
|
|
189
|
+
sizeBytes: claim.sizeBytes,
|
|
190
|
+
sha256: claim.sha256,
|
|
191
|
+
bucket: storage.bucket,
|
|
192
|
+
objectKey: claim.objectKey,
|
|
193
|
+
createdAt: timestamp,
|
|
194
|
+
updatedAt: timestamp,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function terminalDeleteOutcome(reason: string | null): "failed" | "expired" | "deleted" {
|
|
199
|
+
if (reason === "expired") return "expired";
|
|
200
|
+
if (reason === "failed") return "failed";
|
|
201
|
+
return "deleted";
|
|
202
|
+
}
|