@opengeni/worker-bundle 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7113 -1652
- package/dist/index.js.map +1 -1
- package/package.json +18 -18
- package/src/activities/agent-turn.ts +4337 -1052
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
|
@@ -7,19 +7,18 @@ import {
|
|
|
7
7
|
import {
|
|
8
8
|
signDelegatedAccessToken,
|
|
9
9
|
type ConnectionCredentialsPort,
|
|
10
|
+
type GitCredentialProvider,
|
|
11
|
+
type GitCredentialRepositoryRef,
|
|
10
12
|
type GitCredentials,
|
|
11
13
|
type ResourceRef,
|
|
12
14
|
type SandboxSecrets,
|
|
13
15
|
} from "@opengeni/contracts";
|
|
14
16
|
import {
|
|
15
|
-
|
|
17
|
+
loadVariableSetForRun as loadWorkspaceEnvironmentForRunFromDb,
|
|
16
18
|
type Database,
|
|
17
|
-
type WorkspaceEnvironmentForRun,
|
|
19
|
+
type VariableSetForRun as WorkspaceEnvironmentForRun,
|
|
18
20
|
} from "@opengeni/db";
|
|
19
|
-
import {
|
|
20
|
-
createGitHubAppInstallationToken,
|
|
21
|
-
githubAppBotIdentity,
|
|
22
|
-
} from "@opengeni/github";
|
|
21
|
+
import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
|
|
23
22
|
|
|
24
23
|
// Re-exported from the shared @opengeni/db leaf (moved there so the API-direct
|
|
25
24
|
// attach paths can load the SAME decrypted workspace environment the turn
|
|
@@ -38,6 +37,13 @@ export type ConnectionScope = {
|
|
|
38
37
|
workspaceId: string;
|
|
39
38
|
};
|
|
40
39
|
|
|
40
|
+
export type GitTokenSeeds = Partial<Record<GitCredentialProvider, string>>;
|
|
41
|
+
export type GitTokenExpiries = Partial<Record<GitCredentialProvider, string>>;
|
|
42
|
+
export type MintedRunGitCredentials = {
|
|
43
|
+
gitTokens: GitTokenSeeds;
|
|
44
|
+
expiresAt: GitTokenExpiries;
|
|
45
|
+
};
|
|
46
|
+
|
|
41
47
|
// §7.6 P4a — load the run's workspace environment, delegating the DECRYPT to a
|
|
42
48
|
// host `sandboxSecrets` provider when one is bound (the host owns the secret
|
|
43
49
|
// vault + encryption key in embedded/separate topologies) and otherwise running
|
|
@@ -64,7 +70,7 @@ export async function loadWorkspaceEnvironmentForRunWithCredentials(
|
|
|
64
70
|
const secrets: SandboxSecrets = await sandboxSecrets({
|
|
65
71
|
accountId: scope.accountId,
|
|
66
72
|
workspaceId: scope.workspaceId,
|
|
67
|
-
environmentId,
|
|
73
|
+
variableSetId: environmentId,
|
|
68
74
|
});
|
|
69
75
|
// FORK-7: the provider must echo THIS run's workspace before we apply its
|
|
70
76
|
// decrypted values into the sandbox.
|
|
@@ -98,38 +104,48 @@ export async function sandboxEnvironmentForRun(
|
|
|
98
104
|
settings: Settings,
|
|
99
105
|
resources: ResourceRef[],
|
|
100
106
|
workspaceEnvironment: Record<string, string> = {},
|
|
101
|
-
// §7.6 P4a
|
|
107
|
+
// §7.6 P4a - optional host git-credential provider + the run scope it needs
|
|
102
108
|
// (unset, the standalone default → self-mint from `settings` byte-for-byte).
|
|
103
109
|
// `skipGitHubToken` (Stage D): a connected-machine turn skips the inert platform
|
|
104
|
-
// token mint entirely
|
|
110
|
+
// token mint entirely and returns the stable base env unchanged. `deferGitHubToken`
|
|
111
|
+
// is the lazy CLOUD path: apply stable git-auth pointers now, mint only the token
|
|
112
|
+
// value later. `= {}` default so the non-optional reads below are safe.
|
|
105
113
|
options: {
|
|
106
114
|
skipGitHubToken?: boolean;
|
|
115
|
+
deferGitHubToken?: boolean;
|
|
107
116
|
scope?: ConnectionScope;
|
|
108
117
|
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
109
118
|
sessionId?: string;
|
|
110
119
|
runId?: string;
|
|
111
120
|
} = {},
|
|
112
|
-
): Promise<{
|
|
121
|
+
): Promise<{
|
|
122
|
+
environment: Record<string, string>;
|
|
123
|
+
gitToken?: string;
|
|
124
|
+
gitTokens?: GitTokenSeeds;
|
|
125
|
+
gitTokenExpiresAt?: GitTokenExpiries;
|
|
126
|
+
toolspaceToken?: string;
|
|
127
|
+
}> {
|
|
113
128
|
// Precedence: deployment allowlist < git identity < workspace environment
|
|
114
129
|
// < backend-aware HOME (the STABLE base, shared with the API-direct attach
|
|
115
|
-
// paths via stableSandboxEnvironmentForRun) < platform run-scoped
|
|
130
|
+
// paths via stableSandboxEnvironmentForRun) < platform run-scoped git auth
|
|
116
131
|
// (applied below, always last). Reserved name validation at write time prevents
|
|
117
132
|
// workspace values from colliding with the platform-managed entries.
|
|
118
133
|
//
|
|
119
|
-
// TOKEN-BROKER (B1):
|
|
120
|
-
//
|
|
121
|
-
// extraheader).
|
|
122
|
-
//
|
|
123
|
-
// clone
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
// OPENGENI_GIT_TOKEN_FILE), so
|
|
127
|
-
//
|
|
128
|
-
//
|
|
134
|
+
// TOKEN-BROKER (B1): run-scoped git provider tokens are NO LONGER layered into
|
|
135
|
+
// the box/agent MANIFEST env (no GH_TOKEN/GITHUB_TOKEN/GITLAB_TOKEN/
|
|
136
|
+
// AZURE_DEVOPS_EXT_PAT/GIT_CONFIG_* extraheader). They are minted once per turn
|
|
137
|
+
// and returned separately as provider token seeds; the caller threads them
|
|
138
|
+
// OFF-MANIFEST as clone-seed exec env vars so the clone hook writes stable token
|
|
139
|
+
// files. The manifest carries only stable pointers (GIT_ASKPASS,
|
|
140
|
+
// GIT_TERMINAL_PROMPT, identity, OPENGENI_GIT_CREDENTIALS_DIR, and
|
|
141
|
+
// OPENGENI_GIT_TOKEN_FILE), so token VALUES never ride the manifest and the SDK's
|
|
142
|
+
// per-turn provided-session env delta stays empty even though tokens rotate.
|
|
143
|
+
// GitHub keeps the legacy `gitToken`/OPENGENI_GIT_TOKEN_FILE alias. The worker
|
|
144
|
+
// proactively renews every selected provider behind these stable pointers.
|
|
129
145
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
130
146
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
131
147
|
// TOOLSPACE (selfhosted parity): the toolspace token is minted for EVERY
|
|
132
|
-
// backend, including a connected machine. Unlike
|
|
148
|
+
// backend, including a connected machine. Unlike platform git provider tokens
|
|
133
149
|
// (inert on selfhosted → skipped above), the toolspace token is the machine's
|
|
134
150
|
// only path to programmatic tool calling, and it grants no more than the
|
|
135
151
|
// machine owner's own authority (toolspace:call, own-session-bound, turn TTL,
|
|
@@ -138,11 +154,11 @@ export async function sandboxEnvironmentForRun(
|
|
|
138
154
|
// $OPENGENI_TOOLSPACE_TOKEN_FILE over the box's exec channel.
|
|
139
155
|
let toolspaceToken: string | undefined;
|
|
140
156
|
if (
|
|
141
|
-
settings.toolspaceEnabled
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
157
|
+
settings.toolspaceEnabled &&
|
|
158
|
+
settings.delegationSecret &&
|
|
159
|
+
options.scope &&
|
|
160
|
+
options.sessionId &&
|
|
161
|
+
options.runId
|
|
146
162
|
) {
|
|
147
163
|
toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
|
|
148
164
|
accountId: options.scope.accountId,
|
|
@@ -153,47 +169,36 @@ export async function sandboxEnvironmentForRun(
|
|
|
153
169
|
sessionId: options.sessionId,
|
|
154
170
|
exp: Math.floor(Date.now() / 1000) + 60 * 60,
|
|
155
171
|
});
|
|
156
|
-
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
|
|
172
|
+
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(
|
|
173
|
+
settings,
|
|
174
|
+
options.scope.workspaceId,
|
|
175
|
+
);
|
|
157
176
|
}
|
|
158
|
-
const
|
|
177
|
+
const selections = gitCredentialSelections(resources);
|
|
159
178
|
// NO-TOKEN SKIP (Stage D, change B): when the turn's EFFECTIVE compute backend is
|
|
160
|
-
// a connected machine (selfhosted),
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
179
|
+
// a connected machine (selfhosted), platform git provider tokens are INERT: exec
|
|
180
|
+
// routes over NATS to the user's machine, which uses ITS OWN git credentials, and
|
|
181
|
+
// the box those tokens would auth is never created. So skip the token mint entirely
|
|
182
|
+
// and return the STABLE base env (no gitToken/gitTokens). Env-
|
|
164
183
|
// parity holds: the SAME base object still feeds buildManifest + the SelfhostedSession
|
|
165
184
|
// manifest, so the SDK's per-turn provided-session env delta stays empty
|
|
166
185
|
// (validateNoEnvironmentDelta). The API-direct viewer attach path already drops the
|
|
167
186
|
// token under this exact contract — proof a box runs fine without it.
|
|
168
|
-
if (
|
|
187
|
+
if (selections.length === 0 || options.skipGitHubToken) {
|
|
169
188
|
return { environment, ...(toolspaceToken ? { toolspaceToken } : {}) };
|
|
170
189
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
let identity: { name: string; email: string } | null;
|
|
178
|
-
if (options?.gitCredentials && options.scope) {
|
|
179
|
-
const minted: GitCredentials = await options.gitCredentials({
|
|
180
|
-
accountId: options.scope.accountId,
|
|
181
|
-
workspaceId: options.scope.workspaceId,
|
|
182
|
-
installationId: selection.installationId,
|
|
183
|
-
repositoryIds: selection.repositoryIds,
|
|
184
|
-
});
|
|
185
|
-
// FORK-7: assert the provider scoped the token to THIS run's workspace
|
|
186
|
-
// before accepting the token for clone seeding.
|
|
187
|
-
assertWorkspaceEcho("gitCredentials", options.scope, minted.workspaceId);
|
|
188
|
-
token = minted.token;
|
|
189
|
-
identity = minted.identity ?? githubAppBotIdentity(settings);
|
|
190
|
-
} else {
|
|
191
|
-
token = await createGitHubAppInstallationToken(settings, {
|
|
192
|
-
installationId: selection.installationId,
|
|
193
|
-
repositoryIds: selection.repositoryIds,
|
|
194
|
-
});
|
|
195
|
-
identity = githubAppBotIdentity(settings);
|
|
190
|
+
if (options.deferGitHubToken) {
|
|
191
|
+
applyGitAuthPointerEnvironment(
|
|
192
|
+
environment,
|
|
193
|
+
await resolveRunGitIdentityWithSelections(settings, selections, options),
|
|
194
|
+
);
|
|
195
|
+
return { environment, ...(toolspaceToken ? { toolspaceToken } : {}) };
|
|
196
196
|
}
|
|
197
|
+
// Run-scoped sandbox preparation for repository resources. GitHub retains the
|
|
198
|
+
// legacy request shape and standalone self-mint path. Non-GitHub providers are
|
|
199
|
+
// host-brokered only: without a `gitCredentials` port there is no token value
|
|
200
|
+
// to seed, and the runtime wrappers degrade to passthrough.
|
|
201
|
+
const minted = await mintRunGitTokensWithIdentity(settings, selections, options);
|
|
197
202
|
// TOKEN-BROKER (B2): the askpass helper is PROVISIONED AT SETUP (runtime) into a
|
|
198
203
|
// per-box, user-writable path in the SAME dir as the token file, instead of a
|
|
199
204
|
// baked image script at /usr/local/bin/opengeni-git-askpass. The clone-hook seed
|
|
@@ -204,29 +209,267 @@ export async function sandboxEnvironmentForRun(
|
|
|
204
209
|
// surface (viewer attach, channel-A) declares the IDENTICAL env when it
|
|
205
210
|
// cold-creates the box for a repo-attached session — an attach-warmed box
|
|
206
211
|
// missing these keys kills the next repo turn on the SDK's manifest-env guard.
|
|
207
|
-
applyGitAuthPointerEnvironment(environment, identity);
|
|
208
|
-
return {
|
|
212
|
+
applyGitAuthPointerEnvironment(environment, minted.identity);
|
|
213
|
+
return {
|
|
214
|
+
environment,
|
|
215
|
+
...(minted.gitTokens.github ? { gitToken: minted.gitTokens.github } : {}),
|
|
216
|
+
...(Object.keys(minted.gitTokens).length > 0 ? { gitTokens: minted.gitTokens } : {}),
|
|
217
|
+
...(Object.keys(minted.expiresAt).length > 0 ? { gitTokenExpiresAt: minted.expiresAt } : {}),
|
|
218
|
+
...(toolspaceToken ? { toolspaceToken } : {}),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export async function mintRunGitCredentials(
|
|
223
|
+
settings: Settings,
|
|
224
|
+
resources: ResourceRef[],
|
|
225
|
+
options: {
|
|
226
|
+
scope?: ConnectionScope;
|
|
227
|
+
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
228
|
+
} = {},
|
|
229
|
+
): Promise<MintedRunGitCredentials | undefined> {
|
|
230
|
+
const selections = gitCredentialSelections(resources);
|
|
231
|
+
if (selections.length === 0) {
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
234
|
+
const minted = await mintRunGitTokensWithIdentity(settings, selections, options);
|
|
235
|
+
return Object.keys(minted.gitTokens).length > 0
|
|
236
|
+
? { gitTokens: minted.gitTokens, expiresAt: minted.expiresAt }
|
|
237
|
+
: undefined;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export async function mintRunGitTokens(
|
|
241
|
+
settings: Settings,
|
|
242
|
+
resources: ResourceRef[],
|
|
243
|
+
options: {
|
|
244
|
+
scope?: ConnectionScope;
|
|
245
|
+
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
246
|
+
} = {},
|
|
247
|
+
): Promise<GitTokenSeeds | undefined> {
|
|
248
|
+
return (await mintRunGitCredentials(settings, resources, options))?.gitTokens;
|
|
209
249
|
}
|
|
210
250
|
|
|
211
|
-
function
|
|
212
|
-
|
|
251
|
+
export async function mintRunGitToken(
|
|
252
|
+
settings: Settings,
|
|
253
|
+
resources: ResourceRef[],
|
|
254
|
+
options: {
|
|
255
|
+
scope?: ConnectionScope;
|
|
256
|
+
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
257
|
+
} = {},
|
|
258
|
+
): Promise<string | undefined> {
|
|
259
|
+
return (await mintRunGitTokens(settings, resources, options))?.github;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export async function resolveRunGitIdentity(
|
|
263
|
+
settings: Settings,
|
|
264
|
+
resources: ResourceRef[],
|
|
265
|
+
options: {
|
|
266
|
+
scope?: ConnectionScope;
|
|
267
|
+
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
268
|
+
} = {},
|
|
269
|
+
): Promise<{ name: string; email: string } | null> {
|
|
270
|
+
const selections = gitCredentialSelections(resources);
|
|
271
|
+
if (selections.length === 0) {
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
return await resolveRunGitIdentityWithSelections(settings, selections, options);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
async function mintRunGitTokensWithIdentity(
|
|
278
|
+
settings: Settings,
|
|
279
|
+
selections: GitCredentialSelection[],
|
|
280
|
+
options: {
|
|
281
|
+
scope?: ConnectionScope;
|
|
282
|
+
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
283
|
+
},
|
|
284
|
+
): Promise<{
|
|
285
|
+
gitTokens: GitTokenSeeds;
|
|
286
|
+
expiresAt: GitTokenExpiries;
|
|
287
|
+
identity: { name: string; email: string } | null;
|
|
288
|
+
}> {
|
|
289
|
+
const gitTokens: GitTokenSeeds = {};
|
|
290
|
+
const expiresAt: GitTokenExpiries = {};
|
|
291
|
+
let identity: { name: string; email: string } | null = null;
|
|
292
|
+
for (const selection of selections) {
|
|
293
|
+
let token: string | null = null;
|
|
294
|
+
if (options?.gitCredentials && options.scope) {
|
|
295
|
+
const request = gitCredentialsRequestForSelection(options.scope, selection, "token");
|
|
296
|
+
const minted: GitCredentials = await options.gitCredentials(request);
|
|
297
|
+
// FORK-7: assert the provider scoped the token to THIS run's workspace
|
|
298
|
+
// before accepting the token for clone seeding.
|
|
299
|
+
assertWorkspaceEcho("gitCredentials", options.scope, minted.workspaceId);
|
|
300
|
+
if (!minted.token) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
"connection-credential provider (gitCredentials) did not return a token for a token request",
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
token = minted.token;
|
|
306
|
+
if (minted.expiresAt) {
|
|
307
|
+
expiresAt[selection.provider] = validatedGitCredentialExpiry(
|
|
308
|
+
selection.provider,
|
|
309
|
+
minted.expiresAt,
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
if (minted.identity) {
|
|
313
|
+
identity = minted.identity;
|
|
314
|
+
} else if (selection.provider === "github") {
|
|
315
|
+
identity = githubAppBotIdentity(settings);
|
|
316
|
+
}
|
|
317
|
+
} else if (selection.provider === "github" && selection.installationId > 0) {
|
|
318
|
+
const minted = await createGitHubAppInstallationTokenWithExpiry(settings, {
|
|
319
|
+
installationId: selection.installationId,
|
|
320
|
+
repositoryIds: selection.repositoryIds,
|
|
321
|
+
});
|
|
322
|
+
token = minted.token;
|
|
323
|
+
if (minted.expiresAt) {
|
|
324
|
+
expiresAt.github = validatedGitCredentialExpiry("github", minted.expiresAt);
|
|
325
|
+
}
|
|
326
|
+
identity = githubAppBotIdentity(settings);
|
|
327
|
+
}
|
|
328
|
+
if (token) {
|
|
329
|
+
gitTokens[selection.provider] = token;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return { gitTokens, expiresAt, identity };
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function validatedGitCredentialExpiry(provider: GitCredentialProvider, value: string): string {
|
|
336
|
+
const timestamp = Date.parse(value);
|
|
337
|
+
if (!Number.isFinite(timestamp)) {
|
|
338
|
+
throw new Error(`connection-credential provider (${provider}) returned an invalid expiresAt`);
|
|
339
|
+
}
|
|
340
|
+
return new Date(timestamp).toISOString();
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
async function resolveRunGitIdentityWithSelections(
|
|
344
|
+
settings: Settings,
|
|
345
|
+
selections: GitCredentialSelection[],
|
|
346
|
+
options: {
|
|
347
|
+
scope?: ConnectionScope;
|
|
348
|
+
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
349
|
+
},
|
|
350
|
+
): Promise<{ name: string; email: string } | null> {
|
|
351
|
+
let identity: { name: string; email: string } | null = null;
|
|
352
|
+
for (const selection of selections) {
|
|
353
|
+
if (options.gitCredentials && options.scope) {
|
|
354
|
+
const resolved: GitCredentials = await options.gitCredentials(
|
|
355
|
+
gitCredentialsRequestForSelection(options.scope, selection, "identity"),
|
|
356
|
+
);
|
|
357
|
+
assertWorkspaceEcho("gitCredentials", options.scope, resolved.workspaceId);
|
|
358
|
+
if (resolved.identity) {
|
|
359
|
+
identity = resolved.identity;
|
|
360
|
+
} else if (selection.provider === "github") {
|
|
361
|
+
identity = githubAppBotIdentity(settings);
|
|
362
|
+
}
|
|
363
|
+
} else if (selection.provider === "github" && selection.installationId > 0) {
|
|
364
|
+
identity = githubAppBotIdentity(settings);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
return identity;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
type GitCredentialSelection = {
|
|
371
|
+
provider: GitCredentialProvider;
|
|
372
|
+
installationId: number;
|
|
373
|
+
repositoryIds: number[];
|
|
374
|
+
repositoryRefs: GitCredentialRepositoryRef[];
|
|
375
|
+
};
|
|
376
|
+
|
|
377
|
+
function gitCredentialsRequestForSelection(
|
|
378
|
+
scope: ConnectionScope,
|
|
379
|
+
selection: GitCredentialSelection,
|
|
380
|
+
purpose?: "token" | "identity",
|
|
381
|
+
): Parameters<NonNullable<ConnectionCredentialsPort["gitCredentials"]>>[0] {
|
|
382
|
+
const legacy = {
|
|
383
|
+
accountId: scope.accountId,
|
|
384
|
+
workspaceId: scope.workspaceId,
|
|
385
|
+
...(purpose ? { purpose } : {}),
|
|
386
|
+
installationId: selection.installationId,
|
|
387
|
+
repositoryIds: selection.repositoryIds,
|
|
388
|
+
};
|
|
389
|
+
if (selection.provider === "github") {
|
|
390
|
+
return {
|
|
391
|
+
...legacy,
|
|
392
|
+
repositoryRefs: selection.repositoryRefs,
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
return {
|
|
396
|
+
...legacy,
|
|
397
|
+
provider: selection.provider,
|
|
398
|
+
repositoryRefs: selection.repositoryRefs,
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function gitCredentialSelections(resources: ResourceRef[]): GitCredentialSelection[] {
|
|
403
|
+
const byProvider = new Map<GitCredentialProvider, GitCredentialSelection>();
|
|
404
|
+
for (const resource of resources) {
|
|
213
405
|
if (resource.kind !== "repository") {
|
|
214
|
-
|
|
406
|
+
continue;
|
|
215
407
|
}
|
|
216
|
-
const
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
408
|
+
const provider = repositoryCredentialProvider(resource);
|
|
409
|
+
if (!provider) {
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
const entry = byProvider.get(provider) ?? {
|
|
413
|
+
provider,
|
|
414
|
+
installationId: 0,
|
|
415
|
+
repositoryIds: [],
|
|
416
|
+
repositoryRefs: [],
|
|
417
|
+
};
|
|
418
|
+
const ref = gitCredentialRepositoryRef(resource, provider);
|
|
419
|
+
entry.repositoryRefs.push(ref);
|
|
420
|
+
if (provider === "github") {
|
|
421
|
+
const installationId = positiveInteger(
|
|
422
|
+
resource.githubInstallationId ?? resource.installationId,
|
|
423
|
+
);
|
|
424
|
+
const repositoryId = positiveInteger(resource.githubRepositoryId ?? resource.repositoryId);
|
|
425
|
+
if (installationId && repositoryId) {
|
|
426
|
+
if (entry.installationId > 0 && entry.installationId !== installationId) {
|
|
427
|
+
throw new Error("GitHub App repository resources must belong to one installation");
|
|
428
|
+
}
|
|
429
|
+
entry.installationId = installationId;
|
|
430
|
+
entry.repositoryIds.push(repositoryId);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
byProvider.set(provider, entry);
|
|
222
434
|
}
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
435
|
+
return [...byProvider.values()];
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function repositoryCredentialProvider(
|
|
439
|
+
resource: Extract<ResourceRef, { kind: "repository" }>,
|
|
440
|
+
): GitCredentialProvider | null {
|
|
441
|
+
if (resource.provider) {
|
|
442
|
+
return resource.provider;
|
|
226
443
|
}
|
|
444
|
+
if (
|
|
445
|
+
positiveInteger(resource.githubInstallationId) &&
|
|
446
|
+
positiveInteger(resource.githubRepositoryId)
|
|
447
|
+
) {
|
|
448
|
+
return "github";
|
|
449
|
+
}
|
|
450
|
+
return null;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
function gitCredentialRepositoryRef(
|
|
454
|
+
resource: Extract<ResourceRef, { kind: "repository" }>,
|
|
455
|
+
provider: GitCredentialProvider,
|
|
456
|
+
): GitCredentialRepositoryRef {
|
|
227
457
|
return {
|
|
228
|
-
|
|
229
|
-
|
|
458
|
+
provider,
|
|
459
|
+
uri: resource.uri,
|
|
460
|
+
ref: resource.ref,
|
|
461
|
+
...(resource.repositoryId !== undefined
|
|
462
|
+
? { repositoryId: resource.repositoryId }
|
|
463
|
+
: resource.githubRepositoryId !== undefined
|
|
464
|
+
? { repositoryId: resource.githubRepositoryId }
|
|
465
|
+
: {}),
|
|
466
|
+
...(resource.installationId !== undefined
|
|
467
|
+
? { installationId: resource.installationId }
|
|
468
|
+
: resource.githubInstallationId !== undefined
|
|
469
|
+
? { installationId: resource.githubInstallationId }
|
|
470
|
+
: {}),
|
|
471
|
+
...(resource.projectId !== undefined ? { projectId: resource.projectId } : {}),
|
|
472
|
+
...(resource.connectionId ? { connectionId: resource.connectionId } : {}),
|
|
230
473
|
};
|
|
231
474
|
}
|
|
232
475
|
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { claimExpiredFileUploadCleanup, completeExpiredFileUploadCleanup } from "@opengeni/db";
|
|
2
|
+
import type { ObjectStorage } from "@opengeni/storage";
|
|
3
|
+
import type { ActivityServices } from "./types";
|
|
4
|
+
|
|
5
|
+
export const FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1_000;
|
|
6
|
+
export const FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1_000;
|
|
7
|
+
export const FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
|
|
8
|
+
|
|
9
|
+
export type ReapExpiredFileUploadsResult = {
|
|
10
|
+
claimed: number;
|
|
11
|
+
deleted: number;
|
|
12
|
+
failed: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export type FileUploadReaperActivityOptions = {
|
|
16
|
+
graceMs?: number;
|
|
17
|
+
claimTimeoutMs?: number;
|
|
18
|
+
batchSize?: number;
|
|
19
|
+
/** Failure-injection seam; production uses the configured provider delete. */
|
|
20
|
+
deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Build the provider-neutral expired direct-upload reaper. Claims are durable
|
|
25
|
+
* and reclaimable; object deletion is idempotent; only a successful delete is
|
|
26
|
+
* settled terminally. One provider failure never aborts the rest of the batch.
|
|
27
|
+
*/
|
|
28
|
+
export function createFileUploadReaperActivities(
|
|
29
|
+
services: () => Promise<ActivityServices>,
|
|
30
|
+
options: FileUploadReaperActivityOptions = {},
|
|
31
|
+
) {
|
|
32
|
+
const graceMs = options.graceMs ?? FILE_UPLOAD_CLEANUP_GRACE_MS;
|
|
33
|
+
const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
|
|
34
|
+
const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
|
|
35
|
+
const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
|
|
36
|
+
|
|
37
|
+
async function reapExpiredFileUploads(): Promise<ReapExpiredFileUploadsResult> {
|
|
38
|
+
const { db, objectStorage, observability } = await services();
|
|
39
|
+
if (!objectStorage) {
|
|
40
|
+
return { claimed: 0, deleted: 0, failed: 0 };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const claims = await claimExpiredFileUploadCleanup(db, {
|
|
44
|
+
graceMs,
|
|
45
|
+
claimTimeoutMs,
|
|
46
|
+
limit: batchSize,
|
|
47
|
+
});
|
|
48
|
+
let deleted = 0;
|
|
49
|
+
let failed = 0;
|
|
50
|
+
for (const claim of claims) {
|
|
51
|
+
try {
|
|
52
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
53
|
+
const settled = await completeExpiredFileUploadCleanup(db, claim);
|
|
54
|
+
if (!settled) {
|
|
55
|
+
throw new Error("cleanup claim no longer owns a reclaimable upload");
|
|
56
|
+
}
|
|
57
|
+
deleted += 1;
|
|
58
|
+
} catch (error) {
|
|
59
|
+
failed += 1;
|
|
60
|
+
observability.warn("expired file upload cleanup failed; claim remains reclaimable", {
|
|
61
|
+
workspaceId: claim.workspaceId,
|
|
62
|
+
uploadId: claim.uploadId,
|
|
63
|
+
fileId: claim.fileId,
|
|
64
|
+
error: error instanceof Error ? error.message : String(error),
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (claims.length > 0) {
|
|
69
|
+
observability.info("expired file upload cleanup swept", {
|
|
70
|
+
claimed: claims.length,
|
|
71
|
+
deleted,
|
|
72
|
+
failed,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return { claimed: claims.length, deleted, failed };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return { reapExpiredFileUploads };
|
|
79
|
+
}
|