@defrex/autobuild-hosted-dispatcher 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -3
- package/package.json +5 -3
- package/src/dispatcher.ts +79 -38
package/README.md
CHANGED
|
@@ -40,9 +40,12 @@ Dispatcher variables (server-only, like every secret; none reach the browser):
|
|
|
40
40
|
for the kernel, the hosted ticket source, and guests. Absolute http(s)
|
|
41
41
|
origin; https required in production.
|
|
42
42
|
- `AB_DISPATCHER_REPOSITORIES`: comma-separated repositories the dispatcher
|
|
43
|
-
|
|
44
|
-
When unset,
|
|
45
|
-
|
|
43
|
+
selects in configured order, intersected with the current durable project
|
|
44
|
+
registry. When unset, every authorized invocation queries all active
|
|
45
|
+
registrations. An empty registry is a successful zero-project tick. Removing
|
|
46
|
+
a project stops subsequent ticks, without cancelling already-started work.
|
|
47
|
+
Registry failures fail closed. Configure `AB_POSTGRES_URL` or `DATABASE_URL`;
|
|
48
|
+
the registry pool is reused while membership is queried each invocation.
|
|
46
49
|
- `AB_DISPATCHER_BUDGET_SECONDS`: per-invocation work budget (default 240,
|
|
47
50
|
clamped 10–780); pair it with the route's `maxDuration` as described in the
|
|
48
51
|
operator procedure.
|
|
@@ -87,3 +90,12 @@ events keep growing by roughly one tick per minute per repository (intentional);
|
|
|
87
90
|
the run/config artifacts themselves are retention-bounded — the store keeps the
|
|
88
91
|
latest 200 revisions per dispatcher artifact kind and prunes older revisions at
|
|
89
92
|
deposit time, overridable with `AB_ARTIFACT_RETENTION_MAX_REVISIONS`.
|
|
93
|
+
|
|
94
|
+
Manage projects through the deployment's database-authorized
|
|
95
|
+
`ab-hosted-store projects list|add|remove` commands (mutations require
|
|
96
|
+
`--user IDENTITY`); see [connected projects](../../docs/setup.md#connected-projects).
|
|
97
|
+
Migrate PostgreSQL before deployment. Deprecated `AB_WEB_REPOSITORIES` seeds
|
|
98
|
+
registrations once per database, including standalone dispatcher deployments;
|
|
99
|
+
subsequent changes to it are not administration. Forge token override keys are
|
|
100
|
+
validated eagerly. In fallback mode a removed project's override is inert; an
|
|
101
|
+
explicit repository selection still rejects unknown keys.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@defrex/autobuild-hosted-dispatcher",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"engines": {
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
},
|
|
30
30
|
"peerDependencies": {
|
|
31
31
|
"@defrex/autobuild": ">=0.9.0",
|
|
32
|
-
"@defrex/autobuild-hosted-store-service": ">=0.9.0"
|
|
32
|
+
"@defrex/autobuild-hosted-store-service": ">=0.9.0",
|
|
33
|
+
"@defrex/autobuild-postgres-store": ">=0.11.0"
|
|
33
34
|
},
|
|
34
35
|
"publishConfig": {
|
|
35
36
|
"access": "public"
|
|
@@ -40,6 +41,7 @@
|
|
|
40
41
|
"!src/**/*.test.ts",
|
|
41
42
|
"!src/**/*.test.tsx",
|
|
42
43
|
"!src/**/*.spec.ts",
|
|
43
|
-
"!src/**/*.spec.tsx"
|
|
44
|
+
"!src/**/*.spec.tsx",
|
|
45
|
+
"!src/testing/**"
|
|
44
46
|
]
|
|
45
47
|
}
|
package/src/dispatcher.ts
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import { openPostgresCredentialStoreFromEnv } from '@defrex/autobuild-postgres-store/credentials'
|
|
2
|
+
import {
|
|
3
|
+
openPostgresProjectRegistryFromEnv,
|
|
4
|
+
normalizeProjectRepository,
|
|
5
|
+
type ProjectRegistryReader,
|
|
6
|
+
} from '@defrex/autobuild-postgres-store/projects'
|
|
1
7
|
/**
|
|
2
8
|
* The hosted dispatcher (AUT-303): a cron-authenticated HTTP surface that
|
|
3
9
|
* makes this deployment the kernel owner. One authorized invocation runs, per
|
|
@@ -12,12 +18,11 @@
|
|
|
12
18
|
* kernelRunId. That id is the repository-lease holder and it tags every tick
|
|
13
19
|
* report, run boundary, and deposited `dispatcher-effective-config` artifact,
|
|
14
20
|
* so the web dashboard's repository journal shows hosted activity exactly as
|
|
15
|
-
* it shows a local one.
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* secret and minted tokens never appear in logs, responses, or error messages.
|
|
21
|
+
* it shows a local one. Each project invocation issues an opaque stored
|
|
22
|
+
* credential with immutable dispatcher purpose and configured expiry. It
|
|
23
|
+
* grants that registered project's raw store and ticket access only. It stays
|
|
24
|
+
* valid after tick return for spawned builds. Credentials never enter logs,
|
|
25
|
+
* responses or errors.
|
|
21
26
|
*
|
|
22
27
|
* This surface is deliberately NOT routed through `createHostedStoreService`:
|
|
23
28
|
* the machine routes gate on Autobuild version headers a bare cron request
|
|
@@ -26,11 +31,9 @@
|
|
|
26
31
|
* operator servers.
|
|
27
32
|
*/
|
|
28
33
|
import { createHash, randomUUID, timingSafeEqual } from 'node:crypto'
|
|
29
|
-
import { normalizeGitRemoteUrl } from '@defrex/autobuild/origin'
|
|
30
34
|
import type { Clock } from '@defrex/autobuild/plugin-sdk'
|
|
31
35
|
import { spawnExec } from '@defrex/autobuild/git-worktree'
|
|
32
36
|
import { abDispatch, type DispatchOpts } from '@defrex/autobuild/cli'
|
|
33
|
-
import { mintToken } from '@defrex/autobuild/remote-store'
|
|
34
37
|
import { parseHostedStoreEnv, type HostedStoreEnv } from '@defrex/autobuild-hosted-store-service'
|
|
35
38
|
|
|
36
39
|
/** The service environment the dispatcher reads. Secrets are only ever taken
|
|
@@ -46,8 +49,8 @@ export interface HostedDispatcherConfig {
|
|
|
46
49
|
/** The deployment's public origin, used as `AB_STORE` for the kernel, the
|
|
47
50
|
* hosted ticket source, and guests. */
|
|
48
51
|
origin: string
|
|
49
|
-
/**
|
|
50
|
-
repositories
|
|
52
|
+
/** Explicit normalized repository selection, in configured order; absent queries all registered projects. */
|
|
53
|
+
repositories?: readonly string[]
|
|
51
54
|
/** Optional per-repository GitHub token overrides (`AB_DISPATCHER_GITHUB_TOKENS`),
|
|
52
55
|
* keyed by the same normalized https identities as `repositories`. A
|
|
53
56
|
* repository with an override authenticates its forge operations with it;
|
|
@@ -85,19 +88,18 @@ function integerSetting(env: HostedDispatcherEnv, name: string, fallback: number
|
|
|
85
88
|
|
|
86
89
|
/** Parse one repository-CSV variable into normalized https identities. */
|
|
87
90
|
function repositoryList(raw: string, name: string): string[] {
|
|
88
|
-
const values = raw
|
|
89
|
-
|
|
90
|
-
.map((value) => value.trim())
|
|
91
|
-
.filter((value) => value !== '')
|
|
92
|
-
if (values.length === 0) throw new Error(`${name} must name at least one repository`)
|
|
91
|
+
const values = raw.split(',').map((value) => value.trim())
|
|
92
|
+
|
|
93
93
|
const repositories: string[] = []
|
|
94
94
|
for (const value of values) {
|
|
95
95
|
// Entries are repository identities — normalized https origins — so an
|
|
96
96
|
// operator-spelled `git@github.com:acme/app.git` still matches the Store's
|
|
97
97
|
// identity for this repository (same rule as AB_WEB_REPOSITORIES).
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
let normalized: string
|
|
99
|
+
try {
|
|
100
|
+
normalized = normalizeProjectRepository(value)
|
|
101
|
+
} catch {
|
|
102
|
+
throw new Error(`${name} contains an unsafe repository name`)
|
|
101
103
|
}
|
|
102
104
|
if (!repositories.includes(normalized)) repositories.push(normalized)
|
|
103
105
|
}
|
|
@@ -114,7 +116,7 @@ function repositoryList(raw: string, name: string): string[] {
|
|
|
114
116
|
* Error messages name the variable and the offending identity — never a value. */
|
|
115
117
|
function githubTokenOverrides(
|
|
116
118
|
env: HostedDispatcherEnv,
|
|
117
|
-
repositories
|
|
119
|
+
repositories?: readonly string[],
|
|
118
120
|
): Map<string, string> {
|
|
119
121
|
const raw = env.AB_DISPATCHER_GITHUB_TOKENS?.trim()
|
|
120
122
|
if (raw === undefined || raw === '') return new Map()
|
|
@@ -138,8 +140,13 @@ function githubTokenOverrides(
|
|
|
138
140
|
// Entries are repository identities — normalized https origins, exactly
|
|
139
141
|
// like the repository set — so an operator-spelled
|
|
140
142
|
// `git@github.com:acme/app.git` matches the served `https://` identity.
|
|
141
|
-
|
|
142
|
-
|
|
143
|
+
let identity: string
|
|
144
|
+
try {
|
|
145
|
+
identity = normalizeProjectRepository(rawKey)
|
|
146
|
+
} catch {
|
|
147
|
+
throw new Error('AB_DISPATCHER_GITHUB_TOKENS contains an invalid repository identity')
|
|
148
|
+
}
|
|
149
|
+
if (repositories !== undefined && !repositories.includes(identity)) {
|
|
143
150
|
throw new Error(
|
|
144
151
|
`AB_DISPATCHER_GITHUB_TOKENS contains a key that is not a served repository: ${JSON.stringify(rawKey)}`,
|
|
145
152
|
)
|
|
@@ -191,12 +198,10 @@ export function parseHostedDispatcherEnv(env: HostedDispatcherEnv): HostedDispat
|
|
|
191
198
|
if (env.NODE_ENV === 'production' && origin.protocol !== 'https:') {
|
|
192
199
|
throw new Error('AB_DISPATCHER_ORIGIN must use https in production')
|
|
193
200
|
}
|
|
194
|
-
const rawRepositories = env.AB_DISPATCHER_REPOSITORIES
|
|
201
|
+
const rawRepositories = env.AB_DISPATCHER_REPOSITORIES
|
|
195
202
|
const repositories =
|
|
196
|
-
rawRepositories === undefined
|
|
197
|
-
?
|
|
198
|
-
// repository set once.
|
|
199
|
-
repositoryList(required(env, 'AB_WEB_REPOSITORIES'), 'AB_WEB_REPOSITORIES')
|
|
203
|
+
rawRepositories === undefined
|
|
204
|
+
? undefined
|
|
200
205
|
: repositoryList(rawRepositories, 'AB_DISPATCHER_REPOSITORIES')
|
|
201
206
|
const rawBudget = integerSetting(env, 'AB_DISPATCHER_BUDGET_SECONDS', DEFAULT_BUDGET_SECONDS)
|
|
202
207
|
const budgetSeconds = Math.min(MAX_BUDGET_SECONDS, Math.max(MIN_BUDGET_SECONDS, rawBudget))
|
|
@@ -216,7 +221,7 @@ export function parseHostedDispatcherEnv(env: HostedDispatcherEnv): HostedDispat
|
|
|
216
221
|
return {
|
|
217
222
|
secret,
|
|
218
223
|
origin: origin.origin,
|
|
219
|
-
repositories,
|
|
224
|
+
...(repositories !== undefined ? { repositories } : {}),
|
|
220
225
|
githubTokenOverrides: githubTokenOverridesMap,
|
|
221
226
|
budgetSeconds,
|
|
222
227
|
tokenTtlSeconds,
|
|
@@ -244,6 +249,8 @@ export interface HostedDispatcherTickSummary {
|
|
|
244
249
|
}
|
|
245
250
|
|
|
246
251
|
export interface HostedDispatcherOptions {
|
|
252
|
+
issueDispatcherCredential?: (repo: string, creator: string, expiresAt: Date) => Promise<string>
|
|
253
|
+
openProjectRegistry?: (env: HostedDispatcherEnv) => Promise<ProjectRegistryReader>
|
|
247
254
|
/** Service environment; defaults to `process.env`. */
|
|
248
255
|
env?: HostedDispatcherEnv
|
|
249
256
|
clock?: Clock
|
|
@@ -322,6 +329,18 @@ export function createHostedDispatcher(options: HostedDispatcherOptions = {}): {
|
|
|
322
329
|
): Promise<HostedDispatcherTickSummary>
|
|
323
330
|
} {
|
|
324
331
|
const env = options.env ?? process.env
|
|
332
|
+
let projectPromise: Promise<ProjectRegistryReader> | undefined
|
|
333
|
+
const openProjects = () => {
|
|
334
|
+
if (!projectPromise) {
|
|
335
|
+
const attempt = (options.openProjectRegistry ?? openPostgresProjectRegistryFromEnv)(env)
|
|
336
|
+
projectPromise = attempt
|
|
337
|
+
void attempt.catch(() => {
|
|
338
|
+
if (projectPromise === attempt) projectPromise = undefined
|
|
339
|
+
})
|
|
340
|
+
}
|
|
341
|
+
return projectPromise
|
|
342
|
+
}
|
|
343
|
+
|
|
325
344
|
const clock: Clock = options.clock ?? (() => new Date())
|
|
326
345
|
const dispatch = options.dispatch ?? ((opts: DispatchOpts) => abDispatch(opts))
|
|
327
346
|
const log = options.log ?? ((line: string) => console.log(line))
|
|
@@ -332,14 +351,17 @@ export function createHostedDispatcher(options: HostedDispatcherOptions = {}): {
|
|
|
332
351
|
credentials: HostedDispatcherInvocation = {},
|
|
333
352
|
): Promise<HostedDispatcherTickSummary> {
|
|
334
353
|
const config = parseHostedDispatcherEnv(env)
|
|
354
|
+
const registered = (await (await openProjects()).list()).map((entry) => entry.repo)
|
|
355
|
+
const selected =
|
|
356
|
+
config.repositories?.filter((repo) => registered.includes(repo)) ?? registered
|
|
335
357
|
const now = clock().getTime()
|
|
336
358
|
const deadlineAt = now + config.budgetSeconds * 1000
|
|
337
359
|
const sandboxAuth = sandboxAuthSource(env, credentials)
|
|
338
360
|
log(
|
|
339
|
-
`${LOG_PREFIX} ${invocation}: tick start repositories=${
|
|
361
|
+
`${LOG_PREFIX} ${invocation}: tick start repositories=${selected.length} budgetSeconds=${config.budgetSeconds} origin=${config.origin} sandboxAuth=${sandboxAuth}`,
|
|
340
362
|
)
|
|
341
363
|
const repositories: HostedDispatcherRepositoryOutcome[] = []
|
|
342
|
-
for (const repository of
|
|
364
|
+
for (const repository of selected) {
|
|
343
365
|
if (clock().getTime() + MIN_REMAINING_MS > deadlineAt) {
|
|
344
366
|
repositories.push({ repository, outcome: 'skipped' })
|
|
345
367
|
log(`${LOG_PREFIX} ${invocation}: ${repository} skipped (invocation budget exhausted)`)
|
|
@@ -351,15 +373,34 @@ export function createHostedDispatcher(options: HostedDispatcherOptions = {}): {
|
|
|
351
373
|
const stderr = options.stderr ?? ((line: string) => logError(`${kernelPrefix} ${line}`))
|
|
352
374
|
const startedAt = Date.now()
|
|
353
375
|
log(`${LOG_PREFIX} ${invocation}: ${repository} tick run=${runId}`)
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
376
|
+
// Stored project authority outlives the tick for spawned builds.
|
|
377
|
+
// This issuer cannot request owner/operator purpose.
|
|
378
|
+
const issue =
|
|
379
|
+
options.issueDispatcherCredential ??
|
|
380
|
+
(async (repo: string, creator: string, expiresAt: Date) => {
|
|
381
|
+
const credentials = await openPostgresCredentialStoreFromEnv(env)
|
|
382
|
+
try {
|
|
383
|
+
return (await credentials.issue(repo, creator, { purpose: 'dispatcher', expiresAt }))
|
|
384
|
+
.token
|
|
385
|
+
} finally {
|
|
386
|
+
await credentials.close()
|
|
387
|
+
}
|
|
388
|
+
})
|
|
389
|
+
let token: string
|
|
390
|
+
try {
|
|
391
|
+
token = await issue(
|
|
392
|
+
repository,
|
|
393
|
+
`system:${runId}`,
|
|
394
|
+
new Date(clock().getTime() + config.tokenTtlSeconds * 1000),
|
|
395
|
+
)
|
|
396
|
+
} catch (error) {
|
|
397
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
398
|
+
repositories.push({ repository, outcome: 'failed', runId, error: message })
|
|
399
|
+
logError(
|
|
400
|
+
`${LOG_PREFIX} ${invocation}: ${repository} credential issuance failed: ${errorDetail(error)}`,
|
|
401
|
+
)
|
|
402
|
+
continue
|
|
403
|
+
}
|
|
363
404
|
const childEnv: Record<string, string | undefined> = {
|
|
364
405
|
...env,
|
|
365
406
|
AB_STORE: config.origin,
|