@defrex/autobuild-hosted-dispatcher 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -40,9 +40,12 @@ Dispatcher variables (server-only, like every secret; none reach the browser):
40
40
  for the kernel, the hosted ticket source, and guests. Absolute http(s)
41
41
  origin; https required in production.
42
42
  - `AB_DISPATCHER_REPOSITORIES`: comma-separated repositories the dispatcher
43
- serves, normalized `https://` identities exactly like `AB_WEB_REPOSITORIES`.
44
- When unset, the deployment's `AB_WEB_REPOSITORIES` set is used, so a
45
- deployment configures its repository set once.
43
+ selects in configured order, intersected with the current durable project
44
+ registry. When unset, every authorized invocation queries all active
45
+ registrations. An empty registry is a successful zero-project tick. Removing
46
+ a project stops subsequent ticks, without cancelling already-started work.
47
+ Registry failures fail closed. Configure `AB_POSTGRES_URL` or `DATABASE_URL`;
48
+ the registry pool is reused while membership is queried each invocation.
46
49
  - `AB_DISPATCHER_BUDGET_SECONDS`: per-invocation work budget (default 240,
47
50
  clamped 10–780); pair it with the route's `maxDuration` as described in the
48
51
  operator procedure.
@@ -87,3 +90,12 @@ events keep growing by roughly one tick per minute per repository (intentional);
87
90
  the run/config artifacts themselves are retention-bounded — the store keeps the
88
91
  latest 200 revisions per dispatcher artifact kind and prunes older revisions at
89
92
  deposit time, overridable with `AB_ARTIFACT_RETENTION_MAX_REVISIONS`.
93
+
94
+ Manage projects through the deployment's database-authorized
95
+ `ab-hosted-store projects list|add|remove` commands (mutations require
96
+ `--user IDENTITY`); see [connected projects](../../docs/setup.md#connected-projects).
97
+ Migrate PostgreSQL before deployment. Deprecated `AB_WEB_REPOSITORIES` seeds
98
+ registrations once per database, including standalone dispatcher deployments;
99
+ subsequent changes to it are not administration. Forge token override keys are
100
+ validated eagerly. In fallback mode a removed project's override is inert; an
101
+ explicit repository selection still rejects unknown keys.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@defrex/autobuild-hosted-dispatcher",
3
- "version": "0.10.1",
3
+ "version": "0.12.0",
4
4
  "type": "module",
5
5
  "license": "Apache-2.0",
6
6
  "engines": {
@@ -29,7 +29,8 @@
29
29
  },
30
30
  "peerDependencies": {
31
31
  "@defrex/autobuild": ">=0.9.0",
32
- "@defrex/autobuild-hosted-store-service": ">=0.9.0"
32
+ "@defrex/autobuild-hosted-store-service": ">=0.9.0",
33
+ "@defrex/autobuild-postgres-store": ">=0.11.0"
33
34
  },
34
35
  "publishConfig": {
35
36
  "access": "public"
@@ -40,6 +41,7 @@
40
41
  "!src/**/*.test.ts",
41
42
  "!src/**/*.test.tsx",
42
43
  "!src/**/*.spec.ts",
43
- "!src/**/*.spec.tsx"
44
+ "!src/**/*.spec.tsx",
45
+ "!src/testing/**"
44
46
  ]
45
47
  }
package/src/dispatcher.ts CHANGED
@@ -1,3 +1,9 @@
1
+ import { openPostgresCredentialStoreFromEnv } from '@defrex/autobuild-postgres-store/credentials'
2
+ import {
3
+ openPostgresProjectRegistryFromEnv,
4
+ normalizeProjectRepository,
5
+ type ProjectRegistryReader,
6
+ } from '@defrex/autobuild-postgres-store/projects'
1
7
  /**
2
8
  * The hosted dispatcher (AUT-303): a cron-authenticated HTTP surface that
3
9
  * makes this deployment the kernel owner. One authorized invocation runs, per
@@ -12,12 +18,11 @@
12
18
  * kernelRunId. That id is the repository-lease holder and it tags every tick
13
19
  * report, run boundary, and deposited `dispatcher-effective-config` artifact,
14
20
  * so the web dashboard's repository journal shows hosted activity exactly as
15
- * it shows a local one. Guests authenticate with a short-TTL unattributed
16
- * deployment operator token (`operator: true`) signed with the deployment's
17
- * own store signing secret — the exact secret both the Store and ticket
18
- * servers verify tokens against — so one credential covers store, tickets, and
19
- * the guest session, and store identity still travels via the run id. The
20
- * secret and minted tokens never appear in logs, responses, or error messages.
21
+ * it shows a local one. Each project invocation issues an opaque stored
22
+ * credential with immutable dispatcher purpose and configured expiry. It
23
+ * grants that registered project's raw store and ticket access only. It stays
24
+ * valid after tick return for spawned builds. Credentials never enter logs,
25
+ * responses or errors.
21
26
  *
22
27
  * This surface is deliberately NOT routed through `createHostedStoreService`:
23
28
  * the machine routes gate on Autobuild version headers a bare cron request
@@ -26,11 +31,9 @@
26
31
  * operator servers.
27
32
  */
28
33
  import { createHash, randomUUID, timingSafeEqual } from 'node:crypto'
29
- import { normalizeGitRemoteUrl } from '@defrex/autobuild/origin'
30
34
  import type { Clock } from '@defrex/autobuild/plugin-sdk'
31
35
  import { spawnExec } from '@defrex/autobuild/git-worktree'
32
36
  import { abDispatch, type DispatchOpts } from '@defrex/autobuild/cli'
33
- import { mintToken } from '@defrex/autobuild/remote-store'
34
37
  import { parseHostedStoreEnv, type HostedStoreEnv } from '@defrex/autobuild-hosted-store-service'
35
38
 
36
39
  /** The service environment the dispatcher reads. Secrets are only ever taken
@@ -46,8 +49,8 @@ export interface HostedDispatcherConfig {
46
49
  /** The deployment's public origin, used as `AB_STORE` for the kernel, the
47
50
  * hosted ticket source, and guests. */
48
51
  origin: string
49
- /** Normalized https repository identities, configured order preserved. */
50
- repositories: readonly string[]
52
+ /** Explicit normalized repository selection, in configured order; absent queries all registered projects. */
53
+ repositories?: readonly string[]
51
54
  /** Optional per-repository GitHub token overrides (`AB_DISPATCHER_GITHUB_TOKENS`),
52
55
  * keyed by the same normalized https identities as `repositories`. A
53
56
  * repository with an override authenticates its forge operations with it;
@@ -85,19 +88,18 @@ function integerSetting(env: HostedDispatcherEnv, name: string, fallback: number
85
88
 
86
89
  /** Parse one repository-CSV variable into normalized https identities. */
87
90
  function repositoryList(raw: string, name: string): string[] {
88
- const values = raw
89
- .split(',')
90
- .map((value) => value.trim())
91
- .filter((value) => value !== '')
92
- if (values.length === 0) throw new Error(`${name} must name at least one repository`)
91
+ const values = raw.split(',').map((value) => value.trim())
92
+
93
93
  const repositories: string[] = []
94
94
  for (const value of values) {
95
95
  // Entries are repository identities — normalized https origins — so an
96
96
  // operator-spelled `git@github.com:acme/app.git` still matches the Store's
97
97
  // identity for this repository (same rule as AB_WEB_REPOSITORIES).
98
- const normalized = normalizeGitRemoteUrl(value)
99
- if (!/^https:\/\/[^\s]+$/.test(normalized)) {
100
- throw new Error(`${name} contains an unsafe repository name: ${JSON.stringify(value)}`)
98
+ let normalized: string
99
+ try {
100
+ normalized = normalizeProjectRepository(value)
101
+ } catch {
102
+ throw new Error(`${name} contains an unsafe repository name`)
101
103
  }
102
104
  if (!repositories.includes(normalized)) repositories.push(normalized)
103
105
  }
@@ -114,7 +116,7 @@ function repositoryList(raw: string, name: string): string[] {
114
116
  * Error messages name the variable and the offending identity — never a value. */
115
117
  function githubTokenOverrides(
116
118
  env: HostedDispatcherEnv,
117
- repositories: readonly string[],
119
+ repositories?: readonly string[],
118
120
  ): Map<string, string> {
119
121
  const raw = env.AB_DISPATCHER_GITHUB_TOKENS?.trim()
120
122
  if (raw === undefined || raw === '') return new Map()
@@ -138,8 +140,13 @@ function githubTokenOverrides(
138
140
  // Entries are repository identities — normalized https origins, exactly
139
141
  // like the repository set — so an operator-spelled
140
142
  // `git@github.com:acme/app.git` matches the served `https://` identity.
141
- const identity = normalizeGitRemoteUrl(rawKey)
142
- if (!/^https:\/\/[^\s]+$/.test(identity) || !repositories.includes(identity)) {
143
+ let identity: string
144
+ try {
145
+ identity = normalizeProjectRepository(rawKey)
146
+ } catch {
147
+ throw new Error('AB_DISPATCHER_GITHUB_TOKENS contains an invalid repository identity')
148
+ }
149
+ if (repositories !== undefined && !repositories.includes(identity)) {
143
150
  throw new Error(
144
151
  `AB_DISPATCHER_GITHUB_TOKENS contains a key that is not a served repository: ${JSON.stringify(rawKey)}`,
145
152
  )
@@ -191,12 +198,10 @@ export function parseHostedDispatcherEnv(env: HostedDispatcherEnv): HostedDispat
191
198
  if (env.NODE_ENV === 'production' && origin.protocol !== 'https:') {
192
199
  throw new Error('AB_DISPATCHER_ORIGIN must use https in production')
193
200
  }
194
- const rawRepositories = env.AB_DISPATCHER_REPOSITORIES?.trim()
201
+ const rawRepositories = env.AB_DISPATCHER_REPOSITORIES
195
202
  const repositories =
196
- rawRepositories === undefined || rawRepositories === ''
197
- ? // Fall back to the web configuration so a deployment configures its
198
- // repository set once.
199
- repositoryList(required(env, 'AB_WEB_REPOSITORIES'), 'AB_WEB_REPOSITORIES')
203
+ rawRepositories === undefined
204
+ ? undefined
200
205
  : repositoryList(rawRepositories, 'AB_DISPATCHER_REPOSITORIES')
201
206
  const rawBudget = integerSetting(env, 'AB_DISPATCHER_BUDGET_SECONDS', DEFAULT_BUDGET_SECONDS)
202
207
  const budgetSeconds = Math.min(MAX_BUDGET_SECONDS, Math.max(MIN_BUDGET_SECONDS, rawBudget))
@@ -216,7 +221,7 @@ export function parseHostedDispatcherEnv(env: HostedDispatcherEnv): HostedDispat
216
221
  return {
217
222
  secret,
218
223
  origin: origin.origin,
219
- repositories,
224
+ ...(repositories !== undefined ? { repositories } : {}),
220
225
  githubTokenOverrides: githubTokenOverridesMap,
221
226
  budgetSeconds,
222
227
  tokenTtlSeconds,
@@ -244,6 +249,8 @@ export interface HostedDispatcherTickSummary {
244
249
  }
245
250
 
246
251
  export interface HostedDispatcherOptions {
252
+ issueDispatcherCredential?: (repo: string, creator: string, expiresAt: Date) => Promise<string>
253
+ openProjectRegistry?: (env: HostedDispatcherEnv) => Promise<ProjectRegistryReader>
247
254
  /** Service environment; defaults to `process.env`. */
248
255
  env?: HostedDispatcherEnv
249
256
  clock?: Clock
@@ -322,6 +329,18 @@ export function createHostedDispatcher(options: HostedDispatcherOptions = {}): {
322
329
  ): Promise<HostedDispatcherTickSummary>
323
330
  } {
324
331
  const env = options.env ?? process.env
332
+ let projectPromise: Promise<ProjectRegistryReader> | undefined
333
+ const openProjects = () => {
334
+ if (!projectPromise) {
335
+ const attempt = (options.openProjectRegistry ?? openPostgresProjectRegistryFromEnv)(env)
336
+ projectPromise = attempt
337
+ void attempt.catch(() => {
338
+ if (projectPromise === attempt) projectPromise = undefined
339
+ })
340
+ }
341
+ return projectPromise
342
+ }
343
+
325
344
  const clock: Clock = options.clock ?? (() => new Date())
326
345
  const dispatch = options.dispatch ?? ((opts: DispatchOpts) => abDispatch(opts))
327
346
  const log = options.log ?? ((line: string) => console.log(line))
@@ -332,14 +351,17 @@ export function createHostedDispatcher(options: HostedDispatcherOptions = {}): {
332
351
  credentials: HostedDispatcherInvocation = {},
333
352
  ): Promise<HostedDispatcherTickSummary> {
334
353
  const config = parseHostedDispatcherEnv(env)
354
+ const registered = (await (await openProjects()).list()).map((entry) => entry.repo)
355
+ const selected =
356
+ config.repositories?.filter((repo) => registered.includes(repo)) ?? registered
335
357
  const now = clock().getTime()
336
358
  const deadlineAt = now + config.budgetSeconds * 1000
337
359
  const sandboxAuth = sandboxAuthSource(env, credentials)
338
360
  log(
339
- `${LOG_PREFIX} ${invocation}: tick start repositories=${config.repositories.length} budgetSeconds=${config.budgetSeconds} origin=${config.origin} sandboxAuth=${sandboxAuth}`,
361
+ `${LOG_PREFIX} ${invocation}: tick start repositories=${selected.length} budgetSeconds=${config.budgetSeconds} origin=${config.origin} sandboxAuth=${sandboxAuth}`,
340
362
  )
341
363
  const repositories: HostedDispatcherRepositoryOutcome[] = []
342
- for (const repository of config.repositories) {
364
+ for (const repository of selected) {
343
365
  if (clock().getTime() + MIN_REMAINING_MS > deadlineAt) {
344
366
  repositories.push({ repository, outcome: 'skipped' })
345
367
  log(`${LOG_PREFIX} ${invocation}: ${repository} skipped (invocation budget exhausted)`)
@@ -351,15 +373,34 @@ export function createHostedDispatcher(options: HostedDispatcherOptions = {}): {
351
373
  const stderr = options.stderr ?? ((line: string) => logError(`${kernelPrefix} ${line}`))
352
374
  const startedAt = Date.now()
353
375
  log(`${LOG_PREFIX} ${invocation}: ${repository} tick run=${runId}`)
354
- // Unattributed deployment operator credential: it covers store and
355
- // ticket operations (attributed operator tokens are operator-API-only
356
- // and cannot write store events). It names nothing because store
357
- // identity travels via the run id. Never logged, never returned.
358
- const token = mintToken(config.secret, {
359
- operator: true,
360
- session: '*',
361
- exp: clock().getTime() + config.tokenTtlSeconds * 1000,
362
- })
376
+ // Stored project authority outlives the tick for spawned builds.
377
+ // This issuer cannot request owner/operator purpose.
378
+ const issue =
379
+ options.issueDispatcherCredential ??
380
+ (async (repo: string, creator: string, expiresAt: Date) => {
381
+ const credentials = await openPostgresCredentialStoreFromEnv(env)
382
+ try {
383
+ return (await credentials.issue(repo, creator, { purpose: 'dispatcher', expiresAt }))
384
+ .token
385
+ } finally {
386
+ await credentials.close()
387
+ }
388
+ })
389
+ let token: string
390
+ try {
391
+ token = await issue(
392
+ repository,
393
+ `system:${runId}`,
394
+ new Date(clock().getTime() + config.tokenTtlSeconds * 1000),
395
+ )
396
+ } catch (error) {
397
+ const message = error instanceof Error ? error.message : String(error)
398
+ repositories.push({ repository, outcome: 'failed', runId, error: message })
399
+ logError(
400
+ `${LOG_PREFIX} ${invocation}: ${repository} credential issuance failed: ${errorDetail(error)}`,
401
+ )
402
+ continue
403
+ }
363
404
  const childEnv: Record<string, string | undefined> = {
364
405
  ...env,
365
406
  AB_STORE: config.origin,