@fabrika/app 0.9.0-cloudflare.2 → 0.9.0-cloudflare.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +20 -18
  3. package/skill/references/app-setup.md +29 -16
  4. package/skill/references/endpoint.md +24 -22
  5. package/skill/references/errors.md +3 -2
  6. package/skill/references/jobs.md +13 -3
  7. package/skill/references/tests.md +7 -0
  8. package/src/bun.ts +159 -6
  9. package/src/cloudflare/application.ts +543 -73
  10. package/src/cloudflare/export.ts +621 -0
  11. package/src/cloudflare/import.ts +1097 -0
  12. package/src/cloudflare/maintenance.ts +518 -0
  13. package/src/cloudflare/ops.ts +147 -18
  14. package/src/cloudflare/restore.ts +167 -0
  15. package/src/cloudflare/tracker.ts +192 -0
  16. package/src/cloudflare.ts +3 -2
  17. package/src/contract/artifact-schema.ts +7 -1
  18. package/src/contract/build.ts +6 -2
  19. package/src/contract/context.ts +26 -9
  20. package/src/contract/defaults.ts +22 -2
  21. package/src/contract/define.ts +1 -1
  22. package/src/contract/group.ts +1 -1
  23. package/src/files/kinds.ts +54 -24
  24. package/src/implement/anonymous.ts +1 -10
  25. package/src/implement/boot.ts +21 -9
  26. package/src/implement/dispatch.ts +171 -33
  27. package/src/implement/egress.ts +34 -8
  28. package/src/implement/invocation.ts +50 -10
  29. package/src/implement/workload.ts +18 -3
  30. package/src/jobs/runner.ts +639 -96
  31. package/src/testing/kit.ts +60 -0
  32. package/types/src/bun.d.ts +34 -1
  33. package/types/src/cloudflare/application.d.ts +36 -5
  34. package/types/src/cloudflare/export.d.ts +16 -0
  35. package/types/src/cloudflare/import.d.ts +58 -0
  36. package/types/src/cloudflare/maintenance.d.ts +117 -0
  37. package/types/src/cloudflare/ops.d.ts +39 -4
  38. package/types/src/cloudflare/restore.d.ts +9 -0
  39. package/types/src/cloudflare/tracker.d.ts +29 -0
  40. package/types/src/cloudflare.d.ts +2 -1
  41. package/types/src/contract/artifact-schema.d.ts +2 -0
  42. package/types/src/contract/context.d.ts +24 -9
  43. package/types/src/contract/defaults.d.ts +19 -1
  44. package/types/src/contract/define.d.ts +1 -1
  45. package/types/src/contract/group.d.ts +1 -1
  46. package/types/src/files/kinds.d.ts +17 -6
  47. package/types/src/implement/anonymous.d.ts +0 -4
  48. package/types/src/implement/boot.d.ts +11 -0
  49. package/types/src/implement/dispatch.d.ts +31 -0
  50. package/types/src/implement/invocation.d.ts +17 -7
  51. package/types/src/implement/workload.d.ts +5 -0
  52. package/types/src/jobs/runner.d.ts +60 -2
  53. package/types/src/testing/kit.d.ts +16 -0
@@ -36,15 +36,18 @@ import {
36
36
  sha256Hex,
37
37
  type WebhookScheme,
38
38
  } from '@fabrika/auth-core'
39
+ import { DatabaseError } from '@fabrika/db'
39
40
  import { type CapabilityAuthority, type ContainerGrant, MAX_WITHIN_VALUES } from '@fabrika/db/capability'
40
41
  import type { DataAccess, FileBinding, FrameworkTables } from '@fabrika/db/framework'
41
42
  import type { DataKind } from '@fabrika/db/tables'
42
43
  import type { TokenVerifier } from '@fabrika/jwks'
43
44
  import { applyAnswerCache, withResponseHeaders } from '@fabrika/proxy-core/headers'
45
+ import { AsyncLocalStorage } from 'node:async_hooks'
44
46
  import { createHash, timingSafeEqual } from 'node:crypto'
45
47
  import { z } from 'zod/v4'
46
48
  import { fileFieldsOf } from '../contract/build'
47
49
  import {
50
+ bodyDeadlineMs,
48
51
  BROWSER_CONFIG_FILE,
49
52
  CHALLENGE_FIELD,
50
53
  FILE_INPUT_PART,
@@ -68,7 +71,7 @@ import {
68
71
  import type { RequestExecutionContext } from '../fetch'
69
72
  import { FileWork, type StoredFileKinds, storedFilesHandle } from '../files/kinds'
70
73
  import { FileDownload, fileResponse, withFileTreatment } from '../files/treatment'
71
- import { abortedBody, bodyDeadlineMs, bootstrapPath, bootstrapResponse, readBodyInTime, withBootstrap } from './anonymous'
74
+ import { abortedBody, bootstrapPath, bootstrapResponse, readBodyInTime, withBootstrap } from './anonymous'
72
75
  import { attachmentResponse } from './attachment'
73
76
  import { boundedStream, type Deadline, deadline, handlerSignal, readBoundedBytes } from './bounded'
74
77
  import type { BrowserConfig } from './browser-config'
@@ -89,7 +92,17 @@ import type { FailureReports } from './failures'
89
92
  import { ServerFault } from './faults'
90
93
  import { type HeaderPolicy, withHeaderPolicy } from './headers'
91
94
  import type { ContractDatabase } from './implement'
92
- import { boundTo, boundWebhookApp, endpointData, Invocation, kickRecorder, siteData, type SiteDeclaration, undeclaredSql } from './invocation'
95
+ import {
96
+ boundTo,
97
+ boundWebhookApp,
98
+ endpointData,
99
+ Invocation,
100
+ type KickRecorder,
101
+ kickRecorder,
102
+ siteData,
103
+ type SiteDeclaration,
104
+ undeclaredSql,
105
+ } from './invocation'
93
106
  import {
94
107
  applicationDirectory,
95
108
  createScope,
@@ -121,6 +134,76 @@ import {
121
134
  } from './webhook'
122
135
  import type { WorkloadCredential } from './workload'
123
136
 
137
+ /** The classes of invocation an object's maintenance barrier admits per mode, counts and drains (ADR-0069 §3). */
138
+ export type InvocationKind = 'read' | 'write' | 'webhook' | 'kick' | 'job' | 'framework-job'
139
+
140
+ /** One admitted invocation, counted until it settles. */
141
+ export interface InvocationAdmission {
142
+ /** Aborts when a drain stops the invocation. */
143
+ readonly signal: AbortSignal
144
+ /** Counts `work` the invocation handed on until it settles, past the invocation's own end. */
145
+ track(work: Promise<unknown>): void
146
+ /** Ends the invocation's own count; called once it settled. */
147
+ release(): void
148
+ }
149
+
150
+ /** What a Cloudflare object's barrier puts around each of its events; `null` refuses an invocation. */
151
+ export interface InvocationGate {
152
+ admit(kind: InvocationKind): InvocationAdmission | null
153
+ }
154
+
155
+ const invocationGate = new AsyncLocalStorage<InvocationGate>()
156
+
157
+ const UNGATED: InvocationAdmission = Object.freeze({ signal: new AbortController().signal, track: () => {}, release: () => {} })
158
+
159
+ /** Runs `work` with `gate` admitting every invocation it starts, the background work it hands on included. */
160
+ export function runGated<T>(gate: InvocationGate, work: () => T): T {
161
+ return invocationGate.run(gate, work)
162
+ }
163
+
164
+ /** Admits an invocation of `kind` at the current event's gate; outside an object every invocation is admitted, uncounted. */
165
+ export function admitInvocation(kind: InvocationKind): InvocationAdmission | null {
166
+ const gate = invocationGate.getStore()
167
+ return gate === undefined ? UNGATED : gate.admit(kind)
168
+ }
169
+
170
+ /** How long a refused caller waits before it tries again, in seconds. */
171
+ const MAINTENANCE_RETRY_AFTER_S = 30
172
+
173
+ /** The answer while maintenance refuses an invocation: 503 with `Retry-After`, the fixed body of every 503. */
174
+ export function maintenanceRefusal(): Response {
175
+ const response = serverError(503)
176
+ response.headers.set('Retry-After', String(MAINTENANCE_RETRY_AFTER_S))
177
+ return response
178
+ }
179
+
180
+ /**
181
+ * A write the object's write fence refused: the invocation was admitted before a restrictive transition (ADR-0069 §3).
182
+ * A `DatabaseError`, so `@fabrika/db` passes it through its normalization of statement and commit failures unchanged;
183
+ * its fixed text names no statement or value. Dispatch answers it as `maintenanceRefusal()`.
184
+ */
185
+ export class MaintenanceFenceError extends DatabaseError {
186
+ override name = 'MaintenanceFenceError'
187
+
188
+ constructor() {
189
+ super(null)
190
+ }
191
+ }
192
+
193
+ /** The class an endpoint's invocation counts in: a writing endpoint, a webhook, or one that only reads. */
194
+ function invocationKind(served: ServedEndpoint): InvocationKind {
195
+ switch (served.kind) {
196
+ case 'webhook':
197
+ return 'webhook'
198
+ case 'procedure':
199
+ return served.site?.kind === 'mutation' && served.readOnly !== true ? 'write' : 'read'
200
+ case 'public':
201
+ return served.site.kind === 'mutation' ? 'write' : 'read'
202
+ default:
203
+ return 'read'
204
+ }
205
+ }
206
+
124
207
  /** SHA-256 of a proxy secret: 32 bytes whatever the secret's length, as `timingSafeEqual` needs. */
125
208
  export function proxySecretDigest(secret: string): Uint8Array {
126
209
  return createHash('sha256').update(secret).digest()
@@ -357,36 +440,21 @@ export class ContractRuntime {
357
440
  }
358
441
  where = served.row.id
359
442
  this.#checkEndpoint(served, request)
360
- switch (served.kind) {
361
- case 'health':
362
- return jsonResponse({ status: 'ok' }, 200)
363
- case 'spa': {
364
- const caller = await this.#authorize(served.row, request)
365
- return await this.#spaFile(served, request, caller)
366
- }
367
- case 'public-spa':
368
- return await this.#spaFile(served, request, null)
369
- case 'spa-prefix':
370
- return new Response(null, { status: 308, headers: { Location: `${served.prefix}/${new URL(request.url).search}` } })
371
- case 'link-spa':
372
- if (rawRequestTarget(request.url).pathname === bootstrapPath(served.prefix)) {
373
- return bootstrapResponse()
374
- }
375
- return await withBootstrap(await this.#asset(served, request), served.prefix)
376
- case 'procedure':
377
- return await this.#procedure(served, request, exec)
378
- case 'public':
379
- return await this.#public(served, request, exec)
380
- case 'challenge':
381
- return await this.#issueChallenge(served, request)
382
- case 'webhook':
383
- return await this.#webhook(served, request, plan.params, exec)
384
- case 'description':
385
- await this.#authorize(served.row, request)
386
- await readInput({ row: served.row, input: null }, request)
387
- return jsonResponse({ result: served.description }, 200)
443
+ // Counted from here, while it authorizes and uploads too, until it settled.
444
+ const admission = admitInvocation(invocationKind(served))
445
+ if (admission === null) {
446
+ return maintenanceRefusal()
447
+ }
448
+ try {
449
+ return await this.#served(served, request, plan, exec)
450
+ } finally {
451
+ admission.release()
388
452
  }
389
453
  } catch (error) {
454
+ if (error instanceof MaintenanceFenceError) {
455
+ // An expected refusal, not a failure: nothing it wrote was kept, and the caller retries.
456
+ return maintenanceRefusal()
457
+ }
390
458
  const response = errorResponse(error, where)
391
459
  if (response.status >= 500) {
392
460
  const failure = { endpoint: where, requestId: request.headers.get(REQUEST_ID_HEADER), status: response.status, error }
@@ -397,6 +465,43 @@ export class ContractRuntime {
397
465
  }
398
466
  }
399
467
 
468
+ async #served(
469
+ served: ServedEndpoint,
470
+ request: Request,
471
+ plan: Extract<EndpointPlan, { outcome: 'endpoint' }>,
472
+ exec: RequestExecutionContext | undefined,
473
+ ): Promise<Response | FileDownload> {
474
+ switch (served.kind) {
475
+ case 'health':
476
+ return jsonResponse({ status: 'ok' }, 200)
477
+ case 'spa': {
478
+ const caller = await this.#authorize(served.row, request)
479
+ return await this.#spaFile(served, request, caller)
480
+ }
481
+ case 'public-spa':
482
+ return await this.#spaFile(served, request, null)
483
+ case 'spa-prefix':
484
+ return new Response(null, { status: 308, headers: { Location: `${served.prefix}/${new URL(request.url).search}` } })
485
+ case 'link-spa':
486
+ if (rawRequestTarget(request.url).pathname === bootstrapPath(served.prefix)) {
487
+ return bootstrapResponse()
488
+ }
489
+ return await withBootstrap(await this.#asset(served, request), served.prefix)
490
+ case 'procedure':
491
+ return await this.#procedure(served, request, exec)
492
+ case 'public':
493
+ return await this.#public(served, request, exec)
494
+ case 'challenge':
495
+ return await this.#issueChallenge(served, request)
496
+ case 'webhook':
497
+ return await this.#webhook(served, request, plan.params, exec)
498
+ case 'description':
499
+ await this.#authorize(served.row, request)
500
+ await readInput({ row: served.row, input: null }, request)
501
+ return jsonResponse({ result: served.description }, 200)
502
+ }
503
+ }
504
+
400
505
  /**
401
506
  * An SPA mount's files, at `<prefix>/__fabrika-config.json` its browser config (`browser-config.ts`), and at
402
507
  * `<prefix>/__fabrika-me.json` the caller document of the person the mount admitted. A public mount admits no one:
@@ -596,6 +701,8 @@ export class ContractRuntime {
596
701
  } finally {
597
702
  frame.end()
598
703
  }
704
+ // Before the answer opens a file's stream: a failed write then has nothing to cancel.
705
+ await this.#recordKicks(invocation, kicks)
599
706
  const answer = await this.#answerOf(served, result, request, work, carried)
600
707
  outcome = 'committing'
601
708
  await commitAnswer(invocation, signal, answer)
@@ -603,6 +710,12 @@ export class ContractRuntime {
603
710
  })
604
711
  this.#kickAll(kicks.kicked(), exec)
605
712
  return answer
713
+ } catch (error) {
714
+ // The fence refuses inside the transaction, before its COMMIT: a known rollback, so what it staged goes now.
715
+ if (error instanceof MaintenanceFenceError) {
716
+ outcome = 'open'
717
+ }
718
+ throw error
606
719
  } finally {
607
720
  budget?.clear()
608
721
  await invocation?.rollback()
@@ -985,8 +1098,24 @@ export class ContractRuntime {
985
1098
  }
986
1099
 
987
1100
  /**
988
- * Sends each job once, after the commit: in the background, never awaited and never rejecting. Without an execution
989
- * context a kick runs detached.
1101
+ * Writes each kick's wake in the invocation's transaction, after the handler and before the commit: a rollback leaves
1102
+ * none, and a crash between the commit and the kick's start leaves the wake to the scheduler. A kick due now is due by
1103
+ * the database's clock, which takes the lease and marks the wakes a run owns.
1104
+ */
1105
+ async #recordKicks(invocation: Invocation | null, kicks: KickRecorder): Promise<void> {
1106
+ const framework = this.#parts.framework
1107
+ if (invocation === null || framework === null || !kicks.recorded()) {
1108
+ return
1109
+ }
1110
+ const transaction = await invocation.frameworkSite('ctx.kick').session()
1111
+ for (const wake of kicks.wakes(await framework.wakes.now(transaction))) {
1112
+ await framework.wakes.schedule(wake, transaction)
1113
+ }
1114
+ }
1115
+
1116
+ /**
1117
+ * Sends each job kicked without `at` once, after the commit: in the background, never awaited and never rejecting.
1118
+ * Without an execution context a kick runs detached.
990
1119
  */
991
1120
  #kickAll(jobs: readonly string[], exec: RequestExecutionContext | undefined): void {
992
1121
  for (const job of new Set(jobs)) {
@@ -1092,6 +1221,7 @@ export class ContractRuntime {
1092
1221
  } finally {
1093
1222
  frame.end()
1094
1223
  }
1224
+ await this.#recordKicks(invocation, kicks)
1095
1225
  const output = served.row.output
1096
1226
  const answer = output !== null && isFileShape(output)
1097
1227
  ? await this.#fileKinds(served.row.id).download(work, served.row.id, output.kind, result)
@@ -1102,6 +1232,12 @@ export class ContractRuntime {
1102
1232
  })
1103
1233
  this.#kickAll(kicks.kicked(), exec)
1104
1234
  return answer
1235
+ } catch (error) {
1236
+ // The fence refuses inside the transaction, before its COMMIT: a known rollback, so what it staged goes now.
1237
+ if (error instanceof MaintenanceFenceError) {
1238
+ outcome = 'open'
1239
+ }
1240
+ throw error
1105
1241
  } finally {
1106
1242
  budget?.clear()
1107
1243
  await invocation.rollback()
@@ -1244,6 +1380,7 @@ export class ContractRuntime {
1244
1380
  if (!(await tables.completeDelivery(claim, using))) {
1245
1381
  throw new ServerFault('webhook_claim_lost', `the claim of a delivery to '${served.webhook}' expired and another copy took it over`)
1246
1382
  }
1383
+ await this.#recordKicks(invocation, kicks)
1247
1384
  committing = true
1248
1385
  await invocation?.commit(signal)
1249
1386
  })
@@ -1251,7 +1388,8 @@ export class ContractRuntime {
1251
1388
  } catch (error) {
1252
1389
  await invocation?.rollback()
1253
1390
  // Only a known rollback deletes what the delivery staged; after a commit that may have happened, the sweep decides.
1254
- if (!committing) {
1391
+ // The fence refuses inside the transaction, before its COMMIT, so its refusal is a known rollback.
1392
+ if (!committing || error instanceof MaintenanceFenceError) {
1255
1393
  await this.#parts.fileKinds?.release(work)
1256
1394
  }
1257
1395
  // The provider redelivers a failed delivery; giving the claim back lets the redelivery run at once. The unsigned
@@ -1,5 +1,5 @@
1
1
  // `egress`: while a site's handler runs, the global `fetch` reaches only the HTTPS origins the site declares, or, for a
2
- // cron job that declares `egress: 'public'`, only public hosts (ADR-0070). It stops a mistake and makes a bypass
2
+ // cron job that declares `egress: 'public'`, only public hosts (ADR-0071). It stops a mistake and makes a bypass
3
3
  // conspicuous; it is not a sandbox (ADR-0012 §8): code that kept the `fetch` it found before boot, or opens its own
4
4
  // sockets, escapes it. The framework's own facilities (mail, connectors, IAM, the object store) run outside the site's
5
5
  // scope, under their own declarations.
@@ -299,13 +299,13 @@ async function follow(
299
299
  let method = request.method
300
300
  const headers = withoutForbiddenHeaders(new Headers(request.headers))
301
301
  // Read once, so that a 307 or a 308 sends it again.
302
- let body: ArrayBuffer | null = request.body === null ? null : await request.arrayBuffer()
302
+ let body: ArrayBuffer | null = request.body === null ? null : await untilAborted(signal, request.arrayBuffer())
303
303
  for (let hop = 0;; hop++) {
304
- await admit(site, rules, url)
304
+ await admit(site, rules, url, signal)
305
305
  const response = await send(url.href, { method, headers: new Headers(headers), body, redirect: 'manual', signal })
306
306
  const location = REDIRECTS.has(response.status) ? response.headers.get('location') : null
307
307
  if (location === null || mode === 'manual') {
308
- return await capped(site, response, url, hop > 0, done)
308
+ return await capped(site, response, url, hop > 0, signal, done)
309
309
  }
310
310
  await response.body?.cancel()
311
311
  if (mode === 'error') {
@@ -330,7 +330,7 @@ async function follow(
330
330
  }
331
331
 
332
332
  /** Refuses a hop public egress does not reach: by the URL, by the installation's hosts, by what its name resolves to. */
333
- async function admit(site: string, rules: PublicEgressRules, url: URL): Promise<void> {
333
+ async function admit(site: string, rules: PublicEgressRules, url: URL, signal: AbortSignal): Promise<void> {
334
334
  const refusal = publicEgressRefusal(url, rules.refusedHosts)
335
335
  if (refusal !== null) {
336
336
  throw new CapabilityRefusedError('refused_egress', `${site} calls ${url.origin}, which public egress refuses: ${refusal}`)
@@ -340,7 +340,7 @@ async function admit(site: string, rules: PublicEgressRules, url: URL): Promise<
340
340
  return
341
341
  }
342
342
  const host = normalizedHost(url.hostname)
343
- const addresses = await platformCall(() => resolve(host))
343
+ const addresses = await untilAborted(signal, platformCall(() => resolve(host)))
344
344
  const internal = addresses.find((address) => !isPublicAddress(address))
345
345
  if (addresses.length === 0 || internal !== undefined) {
346
346
  throw new CapabilityRefusedError(
@@ -356,7 +356,14 @@ async function admit(site: string, rules: PublicEgressRules, url: URL): Promise<
356
356
  * `response` with its body bounded by `maxResponseBytes`, and the URL and redirect flag of the request it answers,
357
357
  * which a response the guard builds would otherwise lose.
358
358
  */
359
- async function capped(site: string, response: Response, url: URL, redirected: boolean, done: () => void): Promise<Response> {
359
+ async function capped(
360
+ site: string,
361
+ response: Response,
362
+ url: URL,
363
+ redirected: boolean,
364
+ signal: AbortSignal,
365
+ done: () => void,
366
+ ): Promise<Response> {
360
367
  const max = PUBLIC_EGRESS_LIMITS.maxResponseBytes
361
368
  const tooLarge = () =>
362
369
  new CapabilityRefusedError(
@@ -376,8 +383,9 @@ async function capped(site: string, response: Response, url: URL, redirected: bo
376
383
  let seen = 0
377
384
  const bound = new ReadableStream<Uint8Array>({
378
385
  async pull(controller) {
379
- const read = await reader.read().catch((error: unknown) => {
386
+ const read = await untilAborted(signal, reader.read()).catch(async (error: unknown) => {
380
387
  done()
388
+ await reader.cancel(error).catch(() => undefined)
381
389
  throw error
382
390
  })
383
391
  if (read.done) {
@@ -407,6 +415,24 @@ async function capped(site: string, response: Response, url: URL, redirected: bo
407
415
  return withRequestUrl(answer, url, redirected)
408
416
  }
409
417
 
418
+ /**
419
+ * `work`, or the signal's reason once it aborts, whichever comes first: a stage the runtime does not abort by the
420
+ * signal (a resolver, a body read) still ends at the deadline. The work itself is left to settle unobserved.
421
+ */
422
+ async function untilAborted<T>(signal: AbortSignal, work: Promise<T>): Promise<T> {
423
+ signal.throwIfAborted()
424
+ let abort = (): void => {}
425
+ const aborted = new Promise<never>((_resolve, reject) => {
426
+ abort = () => reject(signal.reason)
427
+ })
428
+ signal.addEventListener('abort', abort, { once: true })
429
+ try {
430
+ return await Promise.race([work, aborted])
431
+ } finally {
432
+ signal.removeEventListener('abort', abort)
433
+ }
434
+ }
435
+
410
436
  function withRequestUrl(response: Response, url: URL, redirected: boolean): Response {
411
437
  try {
412
438
  Object.defineProperties(response, { url: { value: url.href }, redirected: { value: redirected } })
@@ -10,6 +10,7 @@ import {
10
10
  type DataSite,
11
11
  type FileBinding,
12
12
  type InvocationTransaction,
13
+ type NewWake,
13
14
  type QueryRefusal,
14
15
  QueryRefusedError,
15
16
  type Session,
@@ -467,22 +468,61 @@ export function jobSql(frame: RequestFrame, data: DataAccess, mode: SqlMode, ses
467
468
  return data.sql({ mode, check: boundTo(frame, ended, 'tx.sql'), session: async () => session, fail: () => session.fail() })
468
469
  }
469
470
 
471
+ /** The kicks of one invocation, as `ctx.kick` recorded them. */
472
+ export interface KickRecorder {
473
+ readonly kick: (job: string, options?: unknown) => void
474
+ /** The jobs kicked without `at`, each once: what the dispatcher starts after the commit. */
475
+ readonly kicked: () => readonly string[]
476
+ /** Whether the invocation kicked anything, now or for later. */
477
+ readonly recorded: () => boolean
478
+ /**
479
+ * Every kick's wake, one per job and due second (`kick:<job>:<due second>`); a kick without `at` is due at `now`. The
480
+ * site writes them in its transaction, so a rollback leaves none.
481
+ */
482
+ readonly wakes: (now: Date) => readonly NewWake[]
483
+ }
484
+
485
+ /** A kick's due time: `undefined` for one without options; anything but `{ at: <valid Date> }` is refused. */
486
+ function kickAt(site: string, options: unknown): Date | undefined {
487
+ if (options === undefined) {
488
+ return undefined
489
+ }
490
+ const keys = isPlainObject(options) ? Object.keys(options) : null
491
+ const at: unknown = isPlainObject(options) ? ownField(options, 'at') : undefined
492
+ if (keys === null || keys.length !== 1 || keys[0] !== 'at' || !(at instanceof Date) || Number.isNaN(at.getTime())) {
493
+ throw new QueryRefusedError('invalid_argument', `${site}: ctx.kick takes a job and optionally { at: <Date> }; a kick carries nothing else`)
494
+ }
495
+ return new Date(at.getTime())
496
+ }
497
+
498
+ const kickKey = (job: string, due: Date): string => `kick:${job}:${Math.floor(due.getTime() / 1_000)}`
499
+
470
500
  /**
471
- * `ctx.kick` of one invocation: it names only a declared job, and records it; the dispatcher sends each recorded job once,
472
- * after the commit, and nothing on a rollback.
501
+ * `ctx.kick` of one invocation: it names only a declared job, and records it; the site writes each kick's wake in its
502
+ * transaction, and the dispatcher starts each job kicked without `at` once, after the commit, and nothing on a rollback.
473
503
  */
474
- export function kickRecorder(
475
- site: string,
476
- declared: readonly string[],
477
- ): { readonly kick: (job: string) => void; readonly kicked: () => readonly string[] } {
478
- const kicked = new Set<string>()
504
+ export function kickRecorder(site: string, declared: readonly string[]): KickRecorder {
505
+ const kicks: { readonly job: string; readonly at: Date | undefined }[] = []
479
506
  return {
480
- kick: (job: string) => {
507
+ kick: (job: string, options?: unknown) => {
481
508
  if (!declared.includes(job)) {
482
509
  throw new CapabilityRefusedError('undeclared_kick', `${site} does not declare kicks: ['${String(job)}']`)
483
510
  }
484
- kicked.add(job)
511
+ kicks.push({ job, at: kickAt(site, options) })
512
+ },
513
+ kicked: () => [...new Set(kicks.filter((kick) => kick.at === undefined).map((kick) => kick.job))],
514
+ recorded: () => kicks.length > 0,
515
+ wakes: (now: Date) => {
516
+ const wakes = new Map<string, NewWake>()
517
+ for (const { job, at } of kicks) {
518
+ const dueAt = at ?? now
519
+ const dedupe = kickKey(job, dueAt)
520
+ const kept = wakes.get(dedupe)
521
+ if (kept === undefined || dueAt < kept.dueAt) {
522
+ wakes.set(dedupe, { job, reason: 'kick', dueAt, dedupe })
523
+ }
524
+ }
525
+ return [...wakes.values()]
485
526
  },
486
- kicked: () => [...kicked],
487
527
  }
488
528
  }
@@ -102,12 +102,27 @@ export class WorkloadCredential {
102
102
  if (cached !== null && cached.exp - Math.floor(Date.now() / 1000) > TOKEN_REFRESH_SKEW_SECONDS) {
103
103
  return Promise.resolve({ ok: true, perms: cached.perms, token: cached.token })
104
104
  }
105
- this.#pending ??= this.#exchange().finally(() => {
106
- this.#pending = null
107
- })
105
+ if (this.#pending === null) {
106
+ const pending: Promise<WorkloadAuthority> = this.#exchange().finally(() => {
107
+ if (this.#pending === pending) {
108
+ this.#pending = null
109
+ }
110
+ })
111
+ this.#pending = pending
112
+ }
108
113
  return this.#pending
109
114
  }
110
115
 
116
+ /**
117
+ * The caller stopped waiting for `pending`, an answer of `authority()` (its attempt timed out or was stopped): the
118
+ * next call exchanges afresh instead of waiting on an exchange that may never answer.
119
+ */
120
+ abandon(pending: Promise<WorkloadAuthority>): void {
121
+ if (this.#pending === pending) {
122
+ this.#pending = null
123
+ }
124
+ }
125
+
111
126
  async #exchange(): Promise<WorkloadAuthority> {
112
127
  const { kind, name, key, audience, source, verifier } = this.#options
113
128
  let minted: MintFromKeyResult