@fabrika/app 0.9.0-dev.4 → 0.9.0-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +41 -14
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +2 -1
  5. package/skill/references/endpoint.md +30 -24
  6. package/skill/references/errors.md +40 -35
  7. package/skill/references/files.md +4 -2
  8. package/skill/references/jobs.md +21 -4
  9. package/skill/references/table-and-migration.md +19 -2
  10. package/skill/references/tests.md +40 -1
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +641 -0
  18. package/src/cloudflare/import.ts +1021 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/compile.ts +1 -0
  30. package/src/contract/context.ts +35 -9
  31. package/src/contract/declaration.ts +2 -0
  32. package/src/contract/defaults.ts +22 -2
  33. package/src/contract/define.ts +29 -3
  34. package/src/contract/kit.ts +4 -1
  35. package/src/contract/platform.ts +3 -2
  36. package/src/contract/vocabulary.ts +26 -8
  37. package/src/dev/launcher.ts +50 -4
  38. package/src/disk-assets.ts +8 -3
  39. package/src/files/archive.ts +2 -2
  40. package/src/files/job-names.ts +12 -0
  41. package/src/files/kinds.ts +73 -31
  42. package/src/files/scan.ts +3 -4
  43. package/src/files/streams.ts +3 -1
  44. package/src/implement/anonymous.ts +12 -11
  45. package/src/implement/boot.ts +75 -6
  46. package/src/implement/bounded.ts +21 -2
  47. package/src/implement/capability.ts +15 -3
  48. package/src/implement/dispatch.ts +419 -177
  49. package/src/implement/failures.ts +17 -1
  50. package/src/implement/faults.ts +1 -0
  51. package/src/implement/index.ts +1 -0
  52. package/src/implement/invocation.ts +58 -10
  53. package/src/implement/principal.ts +25 -2
  54. package/src/implement/terminals.ts +93 -0
  55. package/src/implement/workload.ts +42 -16
  56. package/src/jobs/mail.ts +8 -0
  57. package/src/jobs/report.ts +5 -1
  58. package/src/jobs/runner.ts +672 -113
  59. package/src/jobs/schedule.ts +7 -1
  60. package/src/jobs/scrub.ts +14 -1
  61. package/src/jobs/types.ts +6 -0
  62. package/src/spa-fallback.ts +12 -0
  63. package/src/testing/endpoint-access.ts +307 -152
  64. package/src/testing/isolation.ts +8 -1
  65. package/src/testing/kit.ts +80 -2
  66. package/src/testing/proxy.ts +15 -10
  67. package/types/src/build-recipe.d.ts +70 -0
  68. package/types/src/build.d.ts +19 -1
  69. package/types/src/bun.d.ts +34 -1
  70. package/types/src/cloudflare/application.d.ts +73 -0
  71. package/types/src/cloudflare/bindings.d.ts +33 -0
  72. package/types/src/cloudflare/export.d.ts +16 -0
  73. package/types/src/cloudflare/import.d.ts +51 -0
  74. package/types/src/cloudflare/maintenance.d.ts +117 -0
  75. package/types/src/cloudflare/names.d.ts +11 -0
  76. package/types/src/cloudflare/ops.d.ts +81 -0
  77. package/types/src/cloudflare/r2-files.d.ts +9 -0
  78. package/types/src/cloudflare/restore.d.ts +9 -0
  79. package/types/src/cloudflare/tracker.d.ts +29 -0
  80. package/types/src/cloudflare.d.ts +12 -18
  81. package/types/src/contract/app-id.d.ts +2 -0
  82. package/types/src/contract/artifact-schema.d.ts +2 -0
  83. package/types/src/contract/context.d.ts +35 -11
  84. package/types/src/contract/declaration.d.ts +4 -0
  85. package/types/src/contract/defaults.d.ts +19 -1
  86. package/types/src/contract/define.d.ts +20 -3
  87. package/types/src/contract/kit.d.ts +2 -1
  88. package/types/src/contract/platform.d.ts +2 -2
  89. package/types/src/contract/vocabulary.d.ts +9 -7
  90. package/types/src/disk-assets.d.ts +4 -1
  91. package/types/src/files/job-names.d.ts +9 -0
  92. package/types/src/files/kinds.d.ts +21 -14
  93. package/types/src/files/scan.d.ts +1 -3
  94. package/types/src/implement/anonymous.d.ts +2 -4
  95. package/types/src/implement/boot.d.ts +17 -1
  96. package/types/src/implement/bounded.d.ts +11 -1
  97. package/types/src/implement/capability.d.ts +4 -2
  98. package/types/src/implement/dispatch.d.ts +34 -1
  99. package/types/src/implement/failures.d.ts +5 -0
  100. package/types/src/implement/faults.d.ts +1 -0
  101. package/types/src/implement/index.d.ts +1 -0
  102. package/types/src/implement/invocation.d.ts +22 -7
  103. package/types/src/implement/principal.d.ts +6 -1
  104. package/types/src/implement/terminals.d.ts +33 -0
  105. package/types/src/implement/workload.d.ts +5 -0
  106. package/types/src/jobs/runner.d.ts +66 -2
  107. package/types/src/jobs/schedule.d.ts +2 -0
  108. package/types/src/jobs/types.d.ts +6 -0
  109. package/types/src/spa-fallback.d.ts +2 -0
  110. package/types/src/testing/endpoint-access.d.ts +37 -5
  111. package/types/src/testing/isolation.d.ts +1 -1
  112. package/types/src/testing/kit.d.ts +22 -1
  113. package/types/src/testing/proxy.d.ts +4 -4
@@ -16,6 +16,7 @@ import {
16
16
  type EndpointRow,
17
17
  type ExportDescription,
18
18
  FORM_URLENCODED_TYPE,
19
+ type IamTerminalRpc,
19
20
  isAttachmentShape,
20
21
  isFileShape,
21
22
  isRedirectShape,
@@ -35,15 +36,18 @@ import {
35
36
  sha256Hex,
36
37
  type WebhookScheme,
37
38
  } from '@fabrika/auth-core'
39
+ import { DatabaseError } from '@fabrika/db'
38
40
  import { type CapabilityAuthority, type ContainerGrant, MAX_WITHIN_VALUES } from '@fabrika/db/capability'
39
41
  import type { DataAccess, FileBinding, FrameworkTables } from '@fabrika/db/framework'
40
42
  import type { DataKind } from '@fabrika/db/tables'
41
43
  import type { TokenVerifier } from '@fabrika/jwks'
42
44
  import { applyAnswerCache, withResponseHeaders } from '@fabrika/proxy-core/headers'
45
+ import { AsyncLocalStorage } from 'node:async_hooks'
43
46
  import { createHash, timingSafeEqual } from 'node:crypto'
44
47
  import { z } from 'zod/v4'
45
48
  import { fileFieldsOf } from '../contract/build'
46
49
  import {
50
+ bodyDeadlineMs,
47
51
  BROWSER_CONFIG_FILE,
48
52
  CHALLENGE_FIELD,
49
53
  FILE_INPUT_PART,
@@ -67,9 +71,9 @@ import {
67
71
  import type { RequestExecutionContext } from '../fetch'
68
72
  import { FileWork, type StoredFileKinds, storedFilesHandle } from '../files/kinds'
69
73
  import { FileDownload, fileResponse, withFileTreatment } from '../files/treatment'
70
- import { abortedBody, bodyDeadlineMs, bootstrapPath, bootstrapResponse, readBoundedBody, withBootstrap } from './anonymous'
74
+ import { abortedBody, bootstrapPath, bootstrapResponse, readBodyInTime, withBootstrap } from './anonymous'
71
75
  import { attachmentResponse } from './attachment'
72
- import { boundedStream, handlerSignal, readBoundedBytes } from './bounded'
76
+ import { boundedStream, type Deadline, deadline, handlerSignal, readBoundedBytes } from './bounded'
73
77
  import type { BrowserConfig } from './browser-config'
74
78
  import {
75
79
  type AppCapability,
@@ -88,7 +92,17 @@ import type { FailureReports } from './failures'
88
92
  import { ServerFault } from './faults'
89
93
  import { type HeaderPolicy, withHeaderPolicy } from './headers'
90
94
  import type { ContractDatabase } from './implement'
91
- import { boundTo, boundWebhookApp, endpointData, Invocation, kickRecorder, siteData, type SiteDeclaration, undeclaredSql } from './invocation'
95
+ import {
96
+ boundTo,
97
+ boundWebhookApp,
98
+ endpointData,
99
+ Invocation,
100
+ type KickRecorder,
101
+ kickRecorder,
102
+ siteData,
103
+ type SiteDeclaration,
104
+ undeclaredSql,
105
+ } from './invocation'
92
106
  import {
93
107
  applicationDirectory,
94
108
  createScope,
@@ -104,6 +118,7 @@ import { paramsObject, urlParams } from './params'
104
118
  import { pageCallerOf, presentedBy, type Principal } from './principal'
105
119
  import { project, ProjectionRefusedError } from './project'
106
120
  import { bucketsOf, DEFAULT_ADMISSION, type InFlightShare, type ProcedureAdmission, requestClient } from './public'
121
+ import { type TerminalIdentification, terminalIdentification } from './terminals'
107
122
  import type { SignedTokens } from './tokens'
108
123
  import type { AppUrl } from './urls'
109
124
  import { githubAccounts, verifyDelivery } from './verifiers'
@@ -119,6 +134,76 @@ import {
119
134
  } from './webhook'
120
135
  import type { WorkloadCredential } from './workload'
121
136
 
137
+ /** The classes of invocation an object's maintenance barrier admits per mode, counts and drains (ADR-0069 §3). */
138
+ export type InvocationKind = 'read' | 'write' | 'webhook' | 'kick' | 'job' | 'framework-job'
139
+
140
+ /** One admitted invocation, counted until it settles. */
141
+ export interface InvocationAdmission {
142
+ /** Aborts when a drain stops the invocation. */
143
+ readonly signal: AbortSignal
144
+ /** Counts `work` the invocation handed on until it settles, past the invocation's own end. */
145
+ track(work: Promise<unknown>): void
146
+ /** Ends the invocation's own count; called once it settled. */
147
+ release(): void
148
+ }
149
+
150
+ /** What a Cloudflare object's barrier puts around each of its events; `null` refuses an invocation. */
151
+ export interface InvocationGate {
152
+ admit(kind: InvocationKind): InvocationAdmission | null
153
+ }
154
+
155
+ const invocationGate = new AsyncLocalStorage<InvocationGate>()
156
+
157
+ const UNGATED: InvocationAdmission = Object.freeze({ signal: new AbortController().signal, track: () => {}, release: () => {} })
158
+
159
+ /** Runs `work` with `gate` admitting every invocation it starts, the background work it hands on included. */
160
+ export function runGated<T>(gate: InvocationGate, work: () => T): T {
161
+ return invocationGate.run(gate, work)
162
+ }
163
+
164
+ /** Admits an invocation of `kind` at the current event's gate; outside an object every invocation is admitted, uncounted. */
165
+ export function admitInvocation(kind: InvocationKind): InvocationAdmission | null {
166
+ const gate = invocationGate.getStore()
167
+ return gate === undefined ? UNGATED : gate.admit(kind)
168
+ }
169
+
170
+ /** How long a refused caller waits before it tries again, in seconds. */
171
+ const MAINTENANCE_RETRY_AFTER_S = 30
172
+
173
+ /** The answer while maintenance refuses an invocation: 503 with `Retry-After`, the fixed body of every 503. */
174
+ export function maintenanceRefusal(): Response {
175
+ const response = serverError(503)
176
+ response.headers.set('Retry-After', String(MAINTENANCE_RETRY_AFTER_S))
177
+ return response
178
+ }
179
+
180
+ /**
181
+ * A write the object's write fence refused: the invocation was admitted before a restrictive transition (ADR-0069 §3).
182
+ * A `DatabaseError`, so `@fabrika/db` passes it through its normalization of statement and commit failures unchanged;
183
+ * its fixed text names no statement or value. Dispatch answers it as `maintenanceRefusal()`.
184
+ */
185
+ export class MaintenanceFenceError extends DatabaseError {
186
+ override name = 'MaintenanceFenceError'
187
+
188
+ constructor() {
189
+ super(null)
190
+ }
191
+ }
192
+
193
+ /** The class an endpoint's invocation counts in: a writing endpoint, a webhook, or one that only reads. */
194
+ function invocationKind(served: ServedEndpoint): InvocationKind {
195
+ switch (served.kind) {
196
+ case 'webhook':
197
+ return 'webhook'
198
+ case 'procedure':
199
+ return served.site?.kind === 'mutation' && served.readOnly !== true ? 'write' : 'read'
200
+ case 'public':
201
+ return served.site.kind === 'mutation' ? 'write' : 'read'
202
+ default:
203
+ return 'read'
204
+ }
205
+ }
206
+
122
207
  /** SHA-256 of a proxy secret: 32 bytes whatever the secret's length, as `timingSafeEqual` needs. */
123
208
  export function proxySecretDigest(secret: string): Uint8Array {
124
209
  return createHash('sha256').update(secret).digest()
@@ -241,6 +326,8 @@ export interface RuntimeParts {
241
326
  readonly assets: SpaAssets | null
242
327
  /** IAM's membership operations; `boot` refuses a contract that needs them when the adapter supplies none. */
243
328
  readonly membership: IamMembership | null
329
+ /** IAM's identification operations, where the contract declares a terminal; `boot` refuses one without them. */
330
+ readonly terminals: IamTerminalRpc | null
244
331
  /** The framework's own writes (admission buckets, webhook tables); `boot` refuses a webhook or a bucket without them. */
245
332
  readonly framework: FrameworkTables | null
246
333
  /** This runtime's capability mint, which `boot` alone holds, and the authority its stores are bound to. */
@@ -353,36 +440,21 @@ export class ContractRuntime {
353
440
  }
354
441
  where = served.row.id
355
442
  this.#checkEndpoint(served, request)
356
- switch (served.kind) {
357
- case 'health':
358
- return jsonResponse({ status: 'ok' }, 200)
359
- case 'spa': {
360
- const caller = await this.#authorize(served.row, request)
361
- return await this.#spaFile(served, request, caller)
362
- }
363
- case 'public-spa':
364
- return await this.#spaFile(served, request, null)
365
- case 'spa-prefix':
366
- return new Response(null, { status: 308, headers: { Location: `${served.prefix}/${new URL(request.url).search}` } })
367
- case 'link-spa':
368
- if (rawRequestTarget(request.url).pathname === bootstrapPath(served.prefix)) {
369
- return bootstrapResponse()
370
- }
371
- return await withBootstrap(await this.#asset(served, request), served.prefix)
372
- case 'procedure':
373
- return await this.#procedure(served, request, exec)
374
- case 'public':
375
- return await this.#public(served, request, exec)
376
- case 'challenge':
377
- return await this.#issueChallenge(served, request)
378
- case 'webhook':
379
- return await this.#webhook(served, request, plan.params, exec)
380
- case 'description':
381
- await this.#authorize(served.row, request)
382
- await readInput({ row: served.row, input: null }, request)
383
- return jsonResponse({ result: served.description }, 200)
443
+ // Counted from here, while it authorizes and uploads too, until it settled.
444
+ const admission = admitInvocation(invocationKind(served))
445
+ if (admission === null) {
446
+ return maintenanceRefusal()
447
+ }
448
+ try {
449
+ return await this.#served(served, request, plan, exec)
450
+ } finally {
451
+ admission.release()
384
452
  }
385
453
  } catch (error) {
454
+ if (error instanceof MaintenanceFenceError) {
455
+ // An expected refusal, not a failure: nothing it wrote was kept, and the caller retries.
456
+ return maintenanceRefusal()
457
+ }
386
458
  const response = errorResponse(error, where)
387
459
  if (response.status >= 500) {
388
460
  const failure = { endpoint: where, requestId: request.headers.get(REQUEST_ID_HEADER), status: response.status, error }
@@ -393,6 +465,43 @@ export class ContractRuntime {
393
465
  }
394
466
  }
395
467
 
468
+ async #served(
469
+ served: ServedEndpoint,
470
+ request: Request,
471
+ plan: Extract<EndpointPlan, { outcome: 'endpoint' }>,
472
+ exec: RequestExecutionContext | undefined,
473
+ ): Promise<Response | FileDownload> {
474
+ switch (served.kind) {
475
+ case 'health':
476
+ return jsonResponse({ status: 'ok' }, 200)
477
+ case 'spa': {
478
+ const caller = await this.#authorize(served.row, request)
479
+ return await this.#spaFile(served, request, caller)
480
+ }
481
+ case 'public-spa':
482
+ return await this.#spaFile(served, request, null)
483
+ case 'spa-prefix':
484
+ return new Response(null, { status: 308, headers: { Location: `${served.prefix}/${new URL(request.url).search}` } })
485
+ case 'link-spa':
486
+ if (rawRequestTarget(request.url).pathname === bootstrapPath(served.prefix)) {
487
+ return bootstrapResponse()
488
+ }
489
+ return await withBootstrap(await this.#asset(served, request), served.prefix)
490
+ case 'procedure':
491
+ return await this.#procedure(served, request, exec)
492
+ case 'public':
493
+ return await this.#public(served, request, exec)
494
+ case 'challenge':
495
+ return await this.#issueChallenge(served, request)
496
+ case 'webhook':
497
+ return await this.#webhook(served, request, plan.params, exec)
498
+ case 'description':
499
+ await this.#authorize(served.row, request)
500
+ await readInput({ row: served.row, input: null }, request)
501
+ return jsonResponse({ result: served.description }, 200)
502
+ }
503
+ }
504
+
396
505
  /**
397
506
  * An SPA mount's files, at `<prefix>/__fabrika-config.json` its browser config (`browser-config.ts`), and at
398
507
  * `<prefix>/__fabrika-me.json` the caller document of the person the mount admitted. A public mount admits no one:
@@ -518,6 +627,22 @@ export class ContractRuntime {
518
627
  const admission = served.admission ?? DEFAULT_ADMISSION
519
628
  await this.#admit(served.row, admission, request, caller.subject)
520
629
  const body = await procedureBody(served, request, this.#parts.fileKinds)
630
+ try {
631
+ return await this.#procedureInvocation(served, request, exec, caller, admission, body)
632
+ } finally {
633
+ // Whatever refuses the request once the body is open, a multipart body's stream and deadline end here.
634
+ await body.multipart?.close()
635
+ }
636
+ }
637
+
638
+ async #procedureInvocation(
639
+ served: Extract<ServedEndpoint, { kind: 'procedure' }>,
640
+ request: Request,
641
+ exec: RequestExecutionContext | undefined,
642
+ caller: Caller,
643
+ admission: ProcedureAdmission,
644
+ body: Awaited<ReturnType<typeof procedureBody>>,
645
+ ): Promise<Response | FileDownload> {
521
646
  // Signed tokens are verified as the input is parsed; the texts they carry are what the request may redirect to.
522
647
  const carried = new Set<string>()
523
648
  const fields = this.#parts.tokens.verifyFields(served.row.input, body.multipart === null ? body.input : body.multipart.json, carried)
@@ -528,58 +653,72 @@ export class ContractRuntime {
528
653
  const invocation = site === undefined ? null : new Invocation(frame, this.#data(), site.kind === 'query' || served.readOnly === true)
529
654
  const kicks = kickRecorder(`'${served.row.id}'`, site?.declaration.kicks ?? [])
530
655
  let outcome: 'open' | 'committing' = 'open'
656
+ let budget: Deadline | null = null
531
657
  try {
532
658
  const input = body.multipart === null
533
659
  ? fields
534
660
  : this.#parts.tokens.verifyFields(served.row.input, await this.#fileInput(served, body.multipart, work), carried)
535
661
  // The handler's time starts after the last body byte, so a slow upload does not eat it.
536
- const signal = handlerSignal(request, admission.timeoutMs)
537
- let result: unknown
538
- try {
539
- result = await runInRequest(frame, async () => {
540
- const membership = this.#membershipRequest(served.row, caller, request, frame)
541
- const access = await this.#access(served.row, caller, input, frame, membership)
542
- const binding = this.#parts.fileKinds?.binding(work)
543
- const ctx = Object.freeze({
544
- principal: caller.principal,
545
- requestId: frame.id,
546
- signal,
547
- client: requestClient(request, this.#parts.clientVouched),
548
- url: this.#parts.url,
549
- ...access.context,
550
- ...this.#facilities(served.facilities, served.row, membership, access.context['target'], {
551
- credential: caller.token,
552
- requestId: request.headers.get(REQUEST_ID_HEADER) ?? frame.id,
553
- }),
554
- ...(served.connect === undefined ? {} : { connect: connectContext(served.connect, frame) }),
555
- ...(site === undefined || invocation === null ? {} : siteContext(served.row, site, invocation, caller.subject, access.context, binding)),
556
- // An export's consumer reaches its rows only, never the files of their owners.
557
- ...(site === undefined || invocation === null || served.readOnly === true
558
- ? {}
559
- : { files: this.#files(served.row.id, invocation, site.kind, work) }),
560
- ...(served.binds === undefined || invocation === null
561
- ? {}
562
- : { webhooks: this.#webhookBindings(served.row.id, served.binds, invocation, caller.subject) }),
563
- ...this.#signing(`'${served.row.id}'`, site?.declaration.signs ?? []),
564
- kick: kicks.kick,
662
+ budget = handlerSignal(request, admission.timeoutMs)
663
+ const { signal } = budget
664
+ const answer = await enclosedAnswer(invocation, this.#hold(served.row.id, exec), async () => {
665
+ let result: unknown
666
+ try {
667
+ result = await runInRequest(frame, async () => {
668
+ const membership = this.#membershipRequest(served.row, caller, request, frame)
669
+ const access = await this.#access(served.row, caller, input, frame, membership)
670
+ const binding = this.#parts.fileKinds?.binding(work)
671
+ const ctx = Object.freeze({
672
+ principal: caller.principal,
673
+ requestId: frame.id,
674
+ signal,
675
+ client: requestClient(request, this.#parts.clientVouched),
676
+ url: this.#parts.url,
677
+ ...access.context,
678
+ ...this.#facilities(served.facilities, served.row, membership, access.context['target'], {
679
+ credential: caller.token,
680
+ requestId: request.headers.get(REQUEST_ID_HEADER) ?? frame.id,
681
+ }),
682
+ ...this.#terminals(caller.token, request.headers.get(REQUEST_ID_HEADER) ?? frame.id),
683
+ ...(served.connect === undefined ? {} : { connect: connectContext(served.connect, frame) }),
684
+ ...(site === undefined || invocation === null ? {} : siteContext(served.row, site, invocation, caller.subject, access.context, binding)),
685
+ // An export's consumer reaches its rows only, never the files of their owners.
686
+ ...(site === undefined || invocation === null || served.readOnly === true
687
+ ? {}
688
+ : { files: this.#files(served.row.id, invocation, site.kind, work) }),
689
+ ...(served.binds === undefined || invocation === null
690
+ ? {}
691
+ : { webhooks: this.#webhookBindings(served.row.id, served.binds, invocation, caller.subject) }),
692
+ ...this.#signing(`'${served.row.id}'`, site?.declaration.signs ?? []),
693
+ kick: kicks.kick,
694
+ })
695
+ const args = { input: access.input, ctx, ...(query === undefined ? {} : { query }) }
696
+ const egress = served.egress
697
+ return egress === undefined
698
+ ? await served.handler(args)
699
+ : await runAsSite(`'${served.row.id}'`, egress, async () => await served.handler(args))
565
700
  })
566
- const args = { input: access.input, ctx, ...(query === undefined ? {} : { query }) }
567
- const egress = served.egress
568
- return egress === undefined
569
- ? await served.handler(args)
570
- : await runAsSite(`'${served.row.id}'`, egress, async () => await served.handler(args))
571
- })
572
- } finally {
573
- frame.end()
574
- }
575
- const answer = await this.#answerOf(served, result, request, work, carried)
576
- outcome = 'committing'
577
- await commitAnswer(invocation, signal, answer)
701
+ } finally {
702
+ frame.end()
703
+ }
704
+ // Before the answer opens a file's stream: a failed write then has nothing to cancel.
705
+ await this.#recordKicks(invocation, kicks)
706
+ const answer = await this.#answerOf(served, result, request, work, carried)
707
+ outcome = 'committing'
708
+ await commitAnswer(invocation, signal, answer)
709
+ return answer
710
+ })
578
711
  this.#kickAll(kicks.kicked(), exec)
579
712
  return answer
713
+ } catch (error) {
714
+ // The fence refuses inside the transaction, before its COMMIT: a known rollback, so what it staged goes now.
715
+ if (error instanceof MaintenanceFenceError) {
716
+ outcome = 'open'
717
+ }
718
+ throw error
580
719
  } finally {
720
+ budget?.clear()
581
721
  await invocation?.rollback()
582
- await body.multipart?.close()
583
722
  // Only a known rollback deletes what the request staged; after a commit that may have happened, the sweep decides.
584
723
  if (outcome === 'open') {
585
724
  await this.#parts.fileKinds?.release(work)
@@ -671,6 +810,22 @@ export class ContractRuntime {
671
810
  return data
672
811
  }
673
812
 
813
+ /** Where an invocation of `site` reports a long hold of its transaction: `FailureReports`, its delivery kept alive like a failure's. */
814
+ #hold(site: string, exec: RequestExecutionContext | undefined): Hold {
815
+ return {
816
+ site,
817
+ held: (heldMs) => {
818
+ // The transaction already settled: a report that cannot be made, kept or sent never changes the answer.
819
+ try {
820
+ const delivery = this.#parts.failures.held(site, heldMs).catch(() => undefined)
821
+ exec?.waitUntil(delivery)
822
+ } catch {
823
+ // Dropped, like a delivery that fails.
824
+ }
825
+ },
826
+ }
827
+ }
828
+
674
829
  /** The caller's authority at IAM for this request, when the endpoint's access form names a dimension and an action. */
675
830
  #membershipRequest(row: CredentialedEndpoint, caller: Caller, request: Request, frame: RequestFrame): MembershipRequest | null {
676
831
  const iam = this.#parts.membership
@@ -687,6 +842,12 @@ export class ContractRuntime {
687
842
  }
688
843
  }
689
844
 
845
+ /** `ctx.terminals`, with the credential IAM takes the request's application from: the caller's, or the workload's. */
846
+ #terminals(credential: string, requestId: string): { readonly terminals?: TerminalIdentification } {
847
+ const rpc = this.#parts.terminals
848
+ return rpc === null ? {} : { terminals: terminalIdentification(rpc, { app: this.#parts.audience, credential, requestId }) }
849
+ }
850
+
690
851
  /** `ctx.members` and `ctx.links`, for exactly the endpoints the artifact gives them. */
691
852
  #facilities(
692
853
  facilities: Facilities,
@@ -755,17 +916,21 @@ export class ContractRuntime {
755
916
  }
756
917
  }
757
918
 
758
- /** The values of `dimension` in which the caller holds `action`, directly or through a container; all for a global grant. */
919
+ /**
920
+ * The values of `dimension` in which the caller holds `action`, directly or through a container, and its grants on the
921
+ * dimensions inside it, so a list returns every row a single-object read admits (S4); all for a global grant.
922
+ */
759
923
  #within(caller: Caller, action: string, dimension: string, frame: RequestFrame): Within<string, string> {
760
924
  const direct = scopedValues(caller.perms, action, dimension)
761
925
  if (direct === null) {
762
926
  return this.#parts.mint.within(frame, dimension, action, caller.subject, 'all')
763
927
  }
764
928
  const held = this.#heldContainers(caller.perms, action, dimension)
765
- if (direct.length === 0 && held.length === 0) {
929
+ const inside = this.#heldContained(caller.perms, action, dimension)
930
+ if (direct.length === 0 && held.length === 0 && inside.length === 0) {
766
931
  throw new ForbiddenError(`The caller holds '${action}' in no ${dimension}`)
767
932
  }
768
- return this.#parts.mint.within(frame, dimension, action, caller.subject, direct, held)
933
+ return this.#parts.mint.within(frame, dimension, action, caller.subject, direct, held, inside)
769
934
  }
770
935
 
771
936
  /** A scope value the caller acts in: without a grant on it or on a container of it, 403 whether it exists or not. */
@@ -908,7 +1073,7 @@ export class ContractRuntime {
908
1073
  admission.repeatableToken,
909
1074
  )
910
1075
  }
911
- return await this.#publicInvocation(served, request, body, fields, query, carried, authority.perms, exec)
1076
+ return await this.#publicInvocation(served, request, body, fields, query, carried, authority, exec)
912
1077
  } finally {
913
1078
  await multipart?.close()
914
1079
  release()
@@ -933,8 +1098,24 @@ export class ContractRuntime {
933
1098
  }
934
1099
 
935
1100
  /**
936
- * Sends each job once, after the commit: in the background, never awaited and never rejecting. Without an execution
937
- * context a kick runs detached.
1101
+ * Writes each kick's wake in the invocation's transaction, after the handler and before the commit: a rollback leaves
1102
+ * none, and a crash between the commit and the kick's start leaves the wake to the scheduler. A kick due now is due by
1103
+ * the database's clock, which takes the lease and marks the wakes a run owns.
1104
+ */
1105
+ async #recordKicks(invocation: Invocation | null, kicks: KickRecorder): Promise<void> {
1106
+ const framework = this.#parts.framework
1107
+ if (invocation === null || framework === null || !kicks.recorded()) {
1108
+ return
1109
+ }
1110
+ const transaction = await invocation.frameworkSite('ctx.kick').session()
1111
+ for (const wake of kicks.wakes(await framework.wakes.now(transaction))) {
1112
+ await framework.wakes.schedule(wake, transaction)
1113
+ }
1114
+ }
1115
+
1116
+ /**
1117
+ * Sends each job kicked without `at` once, after the commit: in the background, never awaited and never rejecting.
1118
+ * Without an execution context a kick runs detached.
938
1119
  */
939
1120
  #kickAll(jobs: readonly string[], exec: RequestExecutionContext | undefined): void {
940
1121
  for (const job of new Set(jobs)) {
@@ -974,9 +1155,10 @@ export class ContractRuntime {
974
1155
  fields: Readonly<Record<string, unknown>>,
975
1156
  query: Readonly<Record<string, unknown>> | undefined,
976
1157
  carried: Set<string>,
977
- perms: readonly PermissionEntry[],
1158
+ authority: { readonly perms: readonly PermissionEntry[]; readonly token: string },
978
1159
  exec: RequestExecutionContext | undefined,
979
1160
  ): Promise<Response | FileDownload> {
1161
+ const { perms } = authority
980
1162
  const kicks = kickRecorder(`'${served.row.id}'`, served.site.declaration.kicks)
981
1163
  const frame = new RequestFrame(crypto.randomUUID(), { readOnly: served.site.kind === 'query', endpoint: served.row.id })
982
1164
  const work = new FileWork(frame, frame.id)
@@ -995,55 +1177,69 @@ export class ContractRuntime {
995
1177
  return this.#parts.mint.app(frame, action, subject)
996
1178
  }
997
1179
  let outcome: 'open' | 'committing' = 'open'
1180
+ let budget: Deadline | null = null
998
1181
  try {
999
1182
  // The file parts come after the challenge, so a refused visitor stores nothing.
1000
1183
  const input = body.multipart === null
1001
1184
  ? fields
1002
1185
  : this.#parts.tokens.verifyFields(served.row.input, await this.#fileInput(served, body.multipart, work), carried)
1003
1186
  // The handler's time starts after the last body byte, so a slow upload does not eat it.
1004
- const signal = handlerSignal(request, served.admission.timeoutMs)
1005
- let result: unknown
1006
- try {
1007
- result = await runInRequest(frame, async () => {
1008
- const handles = invocation.handles(
1009
- served.site.kind,
1010
- served.site.declaration,
1011
- subject,
1012
- { carried },
1013
- this.#parts.fileKinds?.binding(work),
1014
- )
1015
- const where = `'${served.row.id}'`
1016
- const ctx = Object.freeze({
1017
- principal: Object.freeze({ kind: 'public', endpoint: served.row.id }),
1018
- requestId: frame.id,
1019
- signal,
1020
- client: requestClient(request, this.#parts.clientVouched),
1021
- url: this.#parts.url,
1022
- app,
1023
- db: Object.freeze({ app: (action: string) => handles.app(app(action)) }),
1024
- ...this.#signing(`'${served.row.id}'`, served.site.declaration.signs),
1025
- files: storedFilesHandle(this.#parts.fileKinds, work, {
1026
- where,
1027
- check: boundTo(frame, () => false, 'ctx.files'),
1028
- writes: served.site.kind !== 'query',
1029
- }),
1030
- kick: kicks.kick,
1187
+ budget = handlerSignal(request, served.admission.timeoutMs)
1188
+ const { signal } = budget
1189
+ const answer = await enclosedAnswer(invocation, this.#hold(served.row.id, exec), async () => {
1190
+ let result: unknown
1191
+ try {
1192
+ result = await runInRequest(frame, async () => {
1193
+ const handles = invocation.handles(
1194
+ served.site.kind,
1195
+ served.site.declaration,
1196
+ subject,
1197
+ { carried },
1198
+ this.#parts.fileKinds?.binding(work),
1199
+ )
1200
+ const where = `'${served.row.id}'`
1201
+ const ctx = Object.freeze({
1202
+ principal: Object.freeze({ kind: 'public', endpoint: served.row.id }),
1203
+ requestId: frame.id,
1204
+ signal,
1205
+ client: requestClient(request, this.#parts.clientVouched),
1206
+ url: this.#parts.url,
1207
+ app,
1208
+ db: Object.freeze({ app: (action: string) => handles.app(app(action)) }),
1209
+ ...this.#signing(`'${served.row.id}'`, served.site.declaration.signs),
1210
+ files: storedFilesHandle(this.#parts.fileKinds, work, {
1211
+ where,
1212
+ check: boundTo(frame, () => false, 'ctx.files'),
1213
+ writes: served.site.kind !== 'query',
1214
+ }),
1215
+ ...this.#terminals(authority.token, request.headers.get(REQUEST_ID_HEADER) ?? frame.id),
1216
+ kick: kicks.kick,
1217
+ })
1218
+ const args = { input, ctx, ...(query === undefined ? {} : { query }) }
1219
+ return await runAsSite(`'${served.row.id}'`, served.site.declaration.egress, async () => await served.handler(args))
1031
1220
  })
1032
- const args = { input, ctx, ...(query === undefined ? {} : { query }) }
1033
- return await runAsSite(`'${served.row.id}'`, served.site.declaration.egress, async () => await served.handler(args))
1034
- })
1035
- } finally {
1036
- frame.end()
1037
- }
1038
- const output = served.row.output
1039
- const answer = output !== null && isFileShape(output)
1040
- ? await this.#fileKinds(served.row.id).download(work, served.row.id, output.kind, result)
1041
- : outputResponse(served.row, served.output, result, request, carried)
1042
- outcome = 'committing'
1043
- await commitAnswer(invocation, signal, answer)
1221
+ } finally {
1222
+ frame.end()
1223
+ }
1224
+ await this.#recordKicks(invocation, kicks)
1225
+ const output = served.row.output
1226
+ const answer = output !== null && isFileShape(output)
1227
+ ? await this.#fileKinds(served.row.id).download(work, served.row.id, output.kind, result)
1228
+ : outputResponse(served.row, served.output, result, request, carried)
1229
+ outcome = 'committing'
1230
+ await commitAnswer(invocation, signal, answer)
1231
+ return answer
1232
+ })
1044
1233
  this.#kickAll(kicks.kicked(), exec)
1045
1234
  return answer
1235
+ } catch (error) {
1236
+ // The fence refuses inside the transaction, before its COMMIT: a known rollback, so what it staged goes now.
1237
+ if (error instanceof MaintenanceFenceError) {
1238
+ outcome = 'open'
1239
+ }
1240
+ throw error
1046
1241
  } finally {
1242
+ budget?.clear()
1047
1243
  await invocation.rollback()
1048
1244
  if (outcome === 'open') {
1049
1245
  await this.#parts.fileKinds?.release(work)
@@ -1066,7 +1262,7 @@ export class ContractRuntime {
1066
1262
  ): Promise<Response> {
1067
1263
  admitContentType(served.row, request)
1068
1264
  const limit = maxBytesOf(served.row)
1069
- const body = await readBoundedBody(request, limit, AbortSignal.timeout(bodyDeadlineMs(limit)))
1265
+ const body = await readBodyInTime(request, limit)
1070
1266
  const tables = this.#framework().webhooks
1071
1267
  const bindingId = params[WEBHOOK_BINDING_PARAMETER] ?? ''
1072
1268
  const binding = BINDING_ID.test(bindingId) ? await tables.binding(served.webhook, bindingId) : null
@@ -1130,65 +1326,70 @@ export class ContractRuntime {
1130
1326
  // The claim stays outside the delivery's transaction, and its completion commits inside it, with the work.
1131
1327
  const invocation = site === undefined ? null : new Invocation(frame, this.#data(), false)
1132
1328
  const kicks = kickRecorder(`webhook '${served.webhook}'`, site?.kicks ?? [])
1133
- const signal = handlerSignal(request, budgetMs)
1329
+ const budget = handlerSignal(request, budgetMs)
1330
+ const { signal } = budget
1134
1331
  let committing = false
1135
1332
  try {
1136
- try {
1137
- await runInRequest(frame, async () => {
1138
- const subject = webhookSubject(binding.id)
1139
- const reach = this.#webhookReach(served, frame, subject, binding.scope.value, chain)
1140
- const principal: WebhookPrincipal = Object.freeze({ kind: 'webhook', bindingId: binding.id })
1141
- const delivery = Object.freeze({ key })
1142
- const handles = site === undefined || invocation === null
1143
- ? null
1144
- : invocation.handles('mutation', site, subject, 'declared', this.#parts.fileKinds?.binding(work))
1145
- const data = site === undefined || invocation === null || handles === null
1146
- ? {}
1147
- : {
1148
- db: siteData(handles, (action) => handles.app(reach.app(action))),
1149
- sql: site.sql === null ? undeclaredSql(served.row.id, 'mutation') : invocation.sql(site.sql),
1150
- }
1151
- const client = requestClient(request, this.#parts.clientVouched)
1152
- const args = {
1153
- input,
1154
- ctx: Object.freeze({
1155
- principal,
1156
- requestId: frame.id,
1157
- signal,
1158
- client,
1159
- url: this.#parts.url,
1160
- ...reach.context,
1161
- delivery,
1162
- files: storedFilesHandle(this.#parts.fileKinds, work, {
1163
- where: `webhook '${served.webhook}'`,
1164
- check: boundTo(frame, () => false, 'ctx.files'),
1165
- writes: true,
1333
+ await enclosed(invocation, this.#hold(served.row.id, exec), async () => {
1334
+ try {
1335
+ await runInRequest(frame, async () => {
1336
+ const subject = webhookSubject(binding.id)
1337
+ const reach = this.#webhookReach(served, frame, subject, binding.scope.value, chain)
1338
+ const principal: WebhookPrincipal = Object.freeze({ kind: 'webhook', bindingId: binding.id })
1339
+ const delivery = Object.freeze({ key })
1340
+ const handles = site === undefined || invocation === null
1341
+ ? null
1342
+ : invocation.handles('mutation', site, subject, 'declared', this.#parts.fileKinds?.binding(work))
1343
+ const data = site === undefined || invocation === null || handles === null
1344
+ ? {}
1345
+ : {
1346
+ db: siteData(handles, (action) => handles.app(reach.app(action))),
1347
+ sql: site.sql === null ? undeclaredSql(served.row.id, 'mutation') : invocation.sql(site.sql),
1348
+ }
1349
+ const client = requestClient(request, this.#parts.clientVouched)
1350
+ const args = {
1351
+ input,
1352
+ ctx: Object.freeze({
1353
+ principal,
1354
+ requestId: frame.id,
1355
+ signal,
1356
+ client,
1357
+ url: this.#parts.url,
1358
+ ...reach.context,
1359
+ delivery,
1360
+ files: storedFilesHandle(this.#parts.fileKinds, work, {
1361
+ where: `webhook '${served.webhook}'`,
1362
+ check: boundTo(frame, () => false, 'ctx.files'),
1363
+ writes: true,
1364
+ }),
1365
+ ...this.#signing(`webhook '${served.webhook}'`, site?.signs ?? []),
1366
+ kick: kicks.kick,
1367
+ ...data,
1166
1368
  }),
1167
- ...this.#signing(`webhook '${served.webhook}'`, site?.signs ?? []),
1168
- kick: kicks.kick,
1169
- ...data,
1170
- }),
1171
- }
1172
- if (site === undefined) {
1173
- await served.handler(args)
1174
- } else {
1175
- await runAsSite(`webhook '${served.webhook}'`, site.egress, async () => await served.handler(args))
1176
- }
1177
- })
1178
- } finally {
1179
- frame.end()
1180
- }
1181
- const using = invocation === null ? undefined : await invocation.frameworkSite('the delivery claim').session()
1182
- if (!(await tables.completeDelivery(claim, using))) {
1183
- throw new ServerFault('webhook_claim_lost', `the claim of a delivery to '${served.webhook}' expired and another copy took it over`)
1184
- }
1185
- committing = true
1186
- await invocation?.commit(signal)
1369
+ }
1370
+ if (site === undefined) {
1371
+ await served.handler(args)
1372
+ } else {
1373
+ await runAsSite(`webhook '${served.webhook}'`, site.egress, async () => await served.handler(args))
1374
+ }
1375
+ })
1376
+ } finally {
1377
+ frame.end()
1378
+ }
1379
+ const using = invocation === null ? undefined : await invocation.frameworkSite('the delivery claim').session()
1380
+ if (!(await tables.completeDelivery(claim, using))) {
1381
+ throw new ServerFault('webhook_claim_lost', `the claim of a delivery to '${served.webhook}' expired and another copy took it over`)
1382
+ }
1383
+ await this.#recordKicks(invocation, kicks)
1384
+ committing = true
1385
+ await invocation?.commit(signal)
1386
+ })
1187
1387
  this.#kickAll(kicks.kicked(), exec)
1188
1388
  } catch (error) {
1189
1389
  await invocation?.rollback()
1190
1390
  // Only a known rollback deletes what the delivery staged; after a commit that may have happened, the sweep decides.
1191
- if (!committing) {
1391
+ // The fence refuses inside the transaction, before its COMMIT, so its refusal is a known rollback.
1392
+ if (!committing || error instanceof MaintenanceFenceError) {
1192
1393
  await this.#parts.fileKinds?.release(work)
1193
1394
  }
1194
1395
  // The provider redelivers a failed delivery; giving the claim back lets the redelivery run at once. The unsigned
@@ -1196,6 +1397,8 @@ export class ContractRuntime {
1196
1397
  console.error(`fabrika: ${served.row.id}: delivery_failed; the provider's redelivery runs it again`)
1197
1398
  await tables.releaseDelivery(claim)
1198
1399
  throw error
1400
+ } finally {
1401
+ budget.clear()
1199
1402
  }
1200
1403
  return jsonResponse({ result: { received: true } }, 200)
1201
1404
  }
@@ -1257,6 +1460,14 @@ export class ContractRuntime {
1257
1460
  return values.length === 0 ? [] : [{ dimension: container, values }]
1258
1461
  })
1259
1462
  }
1463
+
1464
+ /** The caller's grants of `action` on every dimension that lies inside `dimension`. */
1465
+ #heldContained(perms: PermissionEntry[], action: string, dimension: string): ContainerGrant[] {
1466
+ return [...this.#parts.containers].flatMap(([inner, containers]) => {
1467
+ const values = containers.includes(dimension) ? scopedValues(perms, action, inner) ?? [] : []
1468
+ return values.length === 0 ? [] : [{ dimension: inner, values }]
1469
+ })
1470
+ }
1260
1471
  }
1261
1472
 
1262
1473
  /** `ctx.db` (by the access form) and `ctx.sql` of a staged-kit endpoint: the declared tag, or one that refuses. */
@@ -1475,7 +1686,7 @@ async function readInput(
1475
1686
  const limit = served.row.admission?.maxBytes ?? null
1476
1687
  const text = limit === null
1477
1688
  ? await request.text()
1478
- : new TextDecoder().decode(await readBoundedBody(request, limit, AbortSignal.timeout(bodyDeadlineMs(limit))))
1689
+ : new TextDecoder().decode(await readBodyInTime(request, limit))
1479
1690
  if (mediaType(request.headers.get('Content-Type')) === FORM_URLENCODED_TYPE) {
1480
1691
  body = paramsObject(new URLSearchParams(text), served.row.input)
1481
1692
  } else if (text.trim() !== '') {
@@ -1547,16 +1758,17 @@ async function openMultipart(
1547
1758
  if (Number.isFinite(declared) && declared > bodyBytes) {
1548
1759
  throw tooLarge()
1549
1760
  }
1550
- const deadline = AbortSignal.any([request.signal, AbortSignal.timeout(bodyDeadlineMs(maxBytes))])
1761
+ const timer = deadline(bodyDeadlineMs(maxBytes), request.signal)
1551
1762
  const body = boundedStream(request.body ?? new Blob([]).stream(), bodyBytes)
1552
1763
  const parts = multipartParts(body.stream, boundary, { maxHeaderBytes: PROCEDURE_LIMITS.maxPartHeaderBytes })
1553
- const failure = (error: unknown) => (body.exceeded() ? tooLarge() : multipartFailure(error, parts, deadline, request))
1764
+ const failure = (error: unknown) => (body.exceeded() ? tooLarge() : multipartFailure(error, parts, timer.signal, request))
1554
1765
  const stop = () => {
1555
1766
  parts.cancel().catch(() => {})
1556
1767
  }
1557
- deadline.addEventListener('abort', stop, { once: true })
1768
+ timer.signal.addEventListener('abort', stop, { once: true })
1558
1769
  const close = async () => {
1559
- deadline.removeEventListener('abort', stop)
1770
+ timer.signal.removeEventListener('abort', stop)
1771
+ timer.clear()
1560
1772
  await parts.cancel()
1561
1773
  }
1562
1774
  let json: Readonly<Record<string, unknown>>
@@ -1617,6 +1829,36 @@ async function openMultipart(
1617
1829
  }
1618
1830
  }
1619
1831
 
1832
+ /** The site an invocation's long hold is logged under, and where it is reported. */
1833
+ interface Hold {
1834
+ readonly site: string
1835
+ readonly held: (heldMs: number) => void
1836
+ }
1837
+
1838
+ /** Runs `work` in the invocation's transaction (`Invocation.enclose`), or as is without one. */
1839
+ function enclosed<T>(invocation: Invocation | null, hold: Hold, work: () => Promise<T>): Promise<T> {
1840
+ return invocation === null ? work() : invocation.enclose(work, hold.site, hold.held)
1841
+ }
1842
+
1843
+ /**
1844
+ * An answer's work, from the handler to its commit, in the invocation's transaction. Where the transaction encloses
1845
+ * the work (SQLite) its COMMIT runs after `commitAnswer`, so a failed one cancels a file's object stream here.
1846
+ */
1847
+ async function enclosedAnswer<A extends Response | FileDownload>(invocation: Invocation | null, hold: Hold, work: () => Promise<A>): Promise<A> {
1848
+ let answer: A | undefined
1849
+ try {
1850
+ return await enclosed(invocation, hold, async () => {
1851
+ answer = await work()
1852
+ return answer
1853
+ })
1854
+ } catch (error) {
1855
+ if (answer instanceof FileDownload) {
1856
+ await answer.body.cancel()
1857
+ }
1858
+ throw error
1859
+ }
1860
+ }
1861
+
1620
1862
  /**
1621
1863
  * Commits the invocation of an answer that is ready: a file's object stream was opened before the commit, so a failed
1622
1864
  * commit cancels it and nothing of it is sent.