@fabrika/app 0.9.0-dev.3 → 0.9.0-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/package.json +14 -9
  2. package/skill/SKILL.md +41 -14
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +2 -1
  5. package/skill/references/endpoint.md +30 -24
  6. package/skill/references/errors.md +40 -35
  7. package/skill/references/files.md +4 -2
  8. package/skill/references/jobs.md +21 -4
  9. package/skill/references/table-and-migration.md +19 -2
  10. package/skill/references/tests.md +40 -1
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +641 -0
  18. package/src/cloudflare/import.ts +1021 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/compile.ts +1 -0
  30. package/src/contract/context.ts +35 -9
  31. package/src/contract/declaration.ts +2 -0
  32. package/src/contract/defaults.ts +22 -2
  33. package/src/contract/define.ts +29 -3
  34. package/src/contract/kit.ts +4 -1
  35. package/src/contract/platform.ts +3 -2
  36. package/src/contract/vocabulary.ts +26 -8
  37. package/src/dev/launcher.ts +84 -13
  38. package/src/dev/output.ts +13 -3
  39. package/src/disk-assets.ts +8 -3
  40. package/src/files/archive.ts +2 -2
  41. package/src/files/job-names.ts +12 -0
  42. package/src/files/kinds.ts +73 -31
  43. package/src/files/scan.ts +3 -4
  44. package/src/files/streams.ts +3 -1
  45. package/src/implement/anonymous.ts +12 -11
  46. package/src/implement/boot.ts +75 -6
  47. package/src/implement/bounded.ts +21 -2
  48. package/src/implement/capability.ts +15 -3
  49. package/src/implement/dispatch.ts +419 -177
  50. package/src/implement/failures.ts +17 -1
  51. package/src/implement/faults.ts +1 -0
  52. package/src/implement/index.ts +1 -0
  53. package/src/implement/invocation.ts +58 -10
  54. package/src/implement/principal.ts +25 -2
  55. package/src/implement/terminals.ts +93 -0
  56. package/src/implement/workload.ts +42 -16
  57. package/src/jobs/mail.ts +8 -0
  58. package/src/jobs/report.ts +5 -1
  59. package/src/jobs/runner.ts +672 -113
  60. package/src/jobs/schedule.ts +7 -1
  61. package/src/jobs/scrub.ts +14 -1
  62. package/src/jobs/types.ts +6 -0
  63. package/src/spa-fallback.ts +12 -0
  64. package/src/testing/endpoint-access.ts +307 -152
  65. package/src/testing/isolation.ts +8 -1
  66. package/src/testing/kit.ts +80 -2
  67. package/src/testing/proxy.ts +15 -10
  68. package/types/src/build-recipe.d.ts +70 -0
  69. package/types/src/build.d.ts +19 -1
  70. package/types/src/bun.d.ts +34 -1
  71. package/types/src/cloudflare/application.d.ts +73 -0
  72. package/types/src/cloudflare/bindings.d.ts +33 -0
  73. package/types/src/cloudflare/export.d.ts +16 -0
  74. package/types/src/cloudflare/import.d.ts +51 -0
  75. package/types/src/cloudflare/maintenance.d.ts +117 -0
  76. package/types/src/cloudflare/names.d.ts +11 -0
  77. package/types/src/cloudflare/ops.d.ts +81 -0
  78. package/types/src/cloudflare/r2-files.d.ts +9 -0
  79. package/types/src/cloudflare/restore.d.ts +9 -0
  80. package/types/src/cloudflare/tracker.d.ts +29 -0
  81. package/types/src/cloudflare.d.ts +12 -18
  82. package/types/src/contract/app-id.d.ts +2 -0
  83. package/types/src/contract/artifact-schema.d.ts +2 -0
  84. package/types/src/contract/context.d.ts +35 -11
  85. package/types/src/contract/declaration.d.ts +4 -0
  86. package/types/src/contract/defaults.d.ts +19 -1
  87. package/types/src/contract/define.d.ts +20 -3
  88. package/types/src/contract/kit.d.ts +2 -1
  89. package/types/src/contract/platform.d.ts +2 -2
  90. package/types/src/contract/vocabulary.d.ts +9 -7
  91. package/types/src/dev/output.d.ts +8 -2
  92. package/types/src/disk-assets.d.ts +4 -1
  93. package/types/src/files/job-names.d.ts +9 -0
  94. package/types/src/files/kinds.d.ts +21 -14
  95. package/types/src/files/scan.d.ts +1 -3
  96. package/types/src/implement/anonymous.d.ts +2 -4
  97. package/types/src/implement/boot.d.ts +17 -1
  98. package/types/src/implement/bounded.d.ts +11 -1
  99. package/types/src/implement/capability.d.ts +4 -2
  100. package/types/src/implement/dispatch.d.ts +34 -1
  101. package/types/src/implement/failures.d.ts +5 -0
  102. package/types/src/implement/faults.d.ts +1 -0
  103. package/types/src/implement/index.d.ts +1 -0
  104. package/types/src/implement/invocation.d.ts +22 -7
  105. package/types/src/implement/principal.d.ts +6 -1
  106. package/types/src/implement/terminals.d.ts +33 -0
  107. package/types/src/implement/workload.d.ts +5 -0
  108. package/types/src/jobs/runner.d.ts +66 -2
  109. package/types/src/jobs/schedule.d.ts +2 -0
  110. package/types/src/jobs/types.d.ts +6 -0
  111. package/types/src/spa-fallback.d.ts +2 -0
  112. package/types/src/testing/endpoint-access.d.ts +37 -5
  113. package/types/src/testing/isolation.d.ts +1 -1
  114. package/types/src/testing/kit.d.ts +22 -1
  115. package/types/src/testing/proxy.d.ts +4 -4
@@ -3,8 +3,11 @@
3
3
  // runtime credential only. It holds a long poll on Control's source receiver (ADR-0064); a new head brings the changed
4
4
  // files, which it writes, then it migrates when `db/migrations/` changed, runs the SPA commands when a web source
5
5
  // changed, and respawns the application. A change the build owns is refused with one line: it needs a bootstrap run.
6
+ // A new container migrates and verifies its schema here, not in an init command: only after Control said whether a reset
7
+ // is pending, since the database still holds what the previous container left.
6
8
 
7
9
  import {
10
+ DEV_LAUNCHER_BOOTED_FILE,
8
11
  DEV_LAUNCHER_CONFIG_FILE,
9
12
  DEV_LAUNCHER_STATE_FILE,
10
13
  type DevLauncherConfig,
@@ -18,7 +21,7 @@ import { fileURLToPath } from 'node:url'
18
21
  import { z } from 'zod/v4'
19
22
  import { makeProcessUndumpable } from '../undumpable.js'
20
23
  import { bootstrapReason, migrationsChanged, webSourceChanged } from './changes.js'
21
- import { OutputRing, type OutputSink, processOutput, readLines } from './output.js'
24
+ import { OutputRing, type OutputSink, type OutputStream, processOutput, readLines } from './output.js'
22
25
  import {
23
26
  DEV_SOURCE_REPORT_PATH,
24
27
  DEV_SOURCE_WAIT_MS,
@@ -91,6 +94,13 @@ async function readState(root: string, config: DevLauncherConfig): Promise<Launc
91
94
  return parsed?.success === true ? parsed.data : { head: config.head, reset: null, checks: null }
92
95
  }
93
96
 
97
+ /**
98
+ * `fabrika db codegen --verify` of the CLI that runs the configured `fabrika db migrate`. The configuration names no
99
+ * command for it, and a key of its own would fail the strict reader of every earlier 0.9 launcher.
100
+ */
101
+ const verifyCommand = (migrate: readonly string[]): readonly string[] | null =>
102
+ migrate.at(-2) === 'db' && migrate.at(-1) === 'migrate' ? [...migrate.slice(0, -1), 'codegen', '--verify'] : null
103
+
94
104
  export interface DevSourceTransport {
95
105
  readonly url: string
96
106
  readonly key: string
@@ -123,9 +133,14 @@ interface RunningApplication {
123
133
 
124
134
  interface CommandResult {
125
135
  readonly exitCode: number | null
136
+ /** The signal that ended the command, when one did: it never exited on its own. */
137
+ readonly signal: string | null
126
138
  readonly lines: readonly string[]
127
139
  }
128
140
 
141
+ /** Bun echoes the script it runs (`$ bun build-web.ts`) on stderr; it names the command, not a failure. */
142
+ const BUN_SCRIPT_ECHO = /^\$ \S/
143
+
129
144
  type SyncOutcome =
130
145
  | { readonly kind: 'applied'; readonly migrations: boolean; readonly spaFailure: string | null }
131
146
  | { readonly kind: 'refused'; readonly reason: string }
@@ -183,6 +198,8 @@ class Launcher {
183
198
  private reportFailing = false
184
199
  /** A newer bootstrap run replaced this container: it keeps serving until the platform stops it, and asks nothing. */
185
200
  private superseded = false
201
+ /** A new container checks the schema types at boot, as a bootstrap does; a restarted one holds synced files instead. */
202
+ private verifyAtBoot = false
186
203
 
187
204
  constructor(private readonly options: DevLauncherOptions) {
188
205
  this.root = options.root
@@ -202,9 +219,25 @@ class Launcher {
202
219
  }
203
220
 
204
221
  async start(): Promise<void> {
222
+ const booted = join(this.root, DEV_LAUNCHER_BOOTED_FILE)
223
+ await rm(booted, { force: true })
205
224
  this.state = await readState(this.root, this.config)
206
- const answer = await this.firstAnswer()
207
- await this.enqueue(() => this.apply(answer ?? { sync: null, reset: this.state.reset ?? 0, checks: this.state.checks ?? 0 }, true))
225
+ const fresh = this.state.reset === null
226
+ this.verifyAtBoot = fresh
227
+ let answer = await this.firstAnswer()
228
+ // A new container's first answer carries the reset generation the previous container applied; the second, any
229
+ // reset still pending, which the boot then runs before it migrates over the previous container's schema.
230
+ if (fresh && answer !== null && !this.superseded) {
231
+ this.state = { ...this.state, reset: answer.reset, checks: answer.checks }
232
+ answer = (await this.firstAnswer()) ?? answer
233
+ }
234
+ const boot = answer ?? { sync: null, reset: this.state.reset ?? 0, checks: this.state.checks ?? 0 }
235
+ try {
236
+ await this.enqueue(() => this.apply(boot, true))
237
+ } finally {
238
+ await mkdir(dirname(booted), { recursive: true })
239
+ await writeFile(booted, '')
240
+ }
208
241
  void this.follow()
209
242
  }
210
243
 
@@ -329,6 +362,8 @@ class Launcher {
329
362
  let changed = boot
330
363
  let migrate = boot
331
364
  let spaFailure: string | null = null
365
+ const reset = answer.reset > (this.state.reset ?? answer.reset)
366
+ if (answer.sync !== null || reset) this.ring.mark()
332
367
  if (answer.sync !== null) {
333
368
  const outcome = await this.writeSync(answer.sync)
334
369
  if (outcome.kind === 'refused') {
@@ -343,13 +378,16 @@ class Launcher {
343
378
  spaFailure = outcome.spaFailure
344
379
  }
345
380
  }
346
- const reset = answer.reset > (this.state.reset ?? answer.reset)
347
- if (changed || reset) await this.converge({ migrate: migrate || reset, reset, spaFailure, resetTo: answer.reset })
381
+ if (changed || reset) {
382
+ await this.converge({ migrate: migrate || reset, verify: (boot && this.verifyAtBoot) || reset, reset, spaFailure, resetTo: answer.reset })
383
+ }
348
384
  if (answer.checks > (this.state.checks ?? answer.checks)) await this.check(answer.checks)
349
385
  }
350
386
 
351
387
  /** Bring the database, the SPA and the running application to the files on disk, then report. */
352
- private async converge(steps: { migrate: boolean; reset: boolean; spaFailure: string | null; resetTo: number }): Promise<void> {
388
+ private async converge(
389
+ steps: { migrate: boolean; verify: boolean; reset: boolean; spaFailure: string | null; resetTo: number },
390
+ ): Promise<void> {
353
391
  this.status = steps.reset ? 'resetting' : 'syncing'
354
392
  void this.report()
355
393
  let failure: string | null = null
@@ -360,6 +398,7 @@ class Launcher {
360
398
  this.state = { ...this.state, reset: steps.resetTo }
361
399
  }
362
400
  if (failure === null && steps.migrate) failure = await this.timed('the migration', () => this.migrate())
401
+ if (failure === null && steps.verify) failure = await this.timed('the schema check', () => this.verifySchema())
363
402
  if (failure === null) failure = steps.spaFailure
364
403
  const served = await this.timed('the restart until healthy', () => this.restartApplication())
365
404
  failure ??= served
@@ -468,6 +507,19 @@ class Launcher {
468
507
  return `fabrika db migrate failed: ${lastLine(result)}; if an applied migration changed, call reset_dev`
469
508
  }
470
509
 
510
+ /** The committed schema types against the migrated schema, as a bootstrap verifies them: at a new container's boot and after a reset. */
511
+ private async verifySchema(): Promise<string | null> {
512
+ const command = verifyCommand(this.config.migrate)
513
+ if (this.provisioningUrl === null || command === null) return null
514
+ const result = await this.run(command, {
515
+ ...this.commandEnvironment,
516
+ FABRIKA_APP_ID: this.config.appId,
517
+ [PROVISIONING_DATABASE_URL]: this.provisioningUrl,
518
+ })
519
+ if (result.exitCode === 0) return null
520
+ return `fabrika db codegen --verify failed: ${lastLine(result)}; if the database holds another commit's schema, call reset_dev`
521
+ }
522
+
471
523
  private async resetDatabase(): Promise<string | null> {
472
524
  if (this.provisioningUrl === null) return null
473
525
  this.say(`dropping the schema and roles of ${this.config.appId}`)
@@ -483,7 +535,12 @@ class Launcher {
483
535
 
484
536
  private async buildSpa(): Promise<string | null> {
485
537
  for (const command of this.config.spaBuild) {
486
- const result = await this.run(['sh', '-c', command], this.commandEnvironment)
538
+ const result = await this.run(
539
+ ['sh', '-c', command],
540
+ this.commandEnvironment,
541
+ CHECK_TIMEOUT_MS,
542
+ (stream, line) => stream === 'stderr' && BUN_SCRIPT_ECHO.test(line) ? 'stdout' : stream,
543
+ )
487
544
  if (result.exitCode !== 0) return `the SPA command \`${command}\` failed: ${lastLine(result)}`
488
545
  }
489
546
  return null
@@ -498,12 +555,15 @@ class Launcher {
498
555
  this.say(`check ${check.name}: ${check.command}`)
499
556
  const started = performance.now()
500
557
  const result = await this.run(['sh', '-c', check.command], this.commandEnvironment, CHECK_TIMEOUT_MS)
558
+ const durationMs = Math.round(performance.now() - started)
559
+ const killed = result.signal === null ? [] : [killedLine(check.name, result.signal, durationMs >= CHECK_TIMEOUT_MS)]
560
+ for (const line of killed) this.say(line, 'stderr')
501
561
  results.push({
502
562
  name: check.name,
503
563
  ok: result.exitCode === 0,
504
564
  exitCode: result.exitCode,
505
- durationMs: Math.round(performance.now() - started),
506
- output: result.lines.slice(-CHECK_OUTPUT_LINES).map((line) => line.slice(0, 2_000)),
565
+ durationMs,
566
+ output: [...result.lines.slice(-(CHECK_OUTPUT_LINES - killed.length)), ...killed].map((line) => line.slice(0, 2_000)),
507
567
  })
508
568
  }
509
569
  this.checkResults = { request, head: this.state.head, results }
@@ -515,17 +575,22 @@ class Launcher {
515
575
  }
516
576
 
517
577
  /** Run a command in the root; its output goes to the log, and its lines are returned. */
518
- private async run(argv: readonly string[], env: Record<string, string>, timeoutMs = CHECK_TIMEOUT_MS): Promise<CommandResult> {
578
+ private async run(
579
+ argv: readonly string[],
580
+ env: Record<string, string>,
581
+ timeoutMs = CHECK_TIMEOUT_MS,
582
+ classify: (stream: OutputStream, line: string) => OutputStream = (stream) => stream,
583
+ ): Promise<CommandResult> {
519
584
  const child = Bun.spawn([...argv], { cwd: this.root, env, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' })
520
585
  const lines: string[] = []
521
- const collect = (stream: 'stdout' | 'stderr') => (line: string) => {
586
+ const collect = (stream: OutputStream) => (line: string) => {
522
587
  lines.push(line)
523
- this.ring.push(stream, line)
588
+ this.ring.push(classify(stream, line), line)
524
589
  }
525
590
  const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs)
526
591
  await Promise.all([readLines(child.stdout, collect('stdout')), readLines(child.stderr, collect('stderr')), child.exited])
527
592
  clearTimeout(timer)
528
- return { exitCode: child.exitCode, lines }
593
+ return { exitCode: child.exitCode, signal: child.signalCode === null ? null : String(child.signalCode), lines }
529
594
  }
530
595
 
531
596
  private async restartApplication(): Promise<string | null> {
@@ -609,6 +674,12 @@ class Launcher {
609
674
  }
610
675
  }
611
676
 
677
+ /** Why a check has no exit code: the launcher's timeout, or a signal from outside, most often the out-of-memory killer. */
678
+ const killedLine = (name: string, signal: string, timedOut: boolean): string =>
679
+ timedOut
680
+ ? `check ${name} was killed (${signal}) after the ${CHECK_TIMEOUT_MS / 60_000}-minute timeout; it failed`
681
+ : `check ${name} was killed (${signal}) before it finished, most likely because the container ran out of memory; it failed`
682
+
612
683
  const lastLine = (result: CommandResult): string => [...result.lines].reverse().find((line) => line.trim() !== '')?.trim() ?? 'no output'
613
684
 
614
685
  /** The process environment with its string values only. */
package/src/dev/output.ts CHANGED
@@ -16,10 +16,14 @@ export const processOutput: OutputSink = {
16
16
  },
17
17
  }
18
18
 
19
- /** The newest `capacity` lines, numbered from 0 since the launcher started, with every known secret value replaced. */
19
+ /**
20
+ * The newest `capacity` lines, numbered from 0 since the launcher started, with every known secret value replaced. A
21
+ * mark starts the lines a report carries: those of the newest sync or reset, and the output after it.
22
+ */
20
23
  export class OutputRing {
21
24
  private readonly lines: DevLogLine[] = []
22
25
  private next = 0
26
+ private marked = 0
23
27
 
24
28
  constructor(private readonly capacity: number, private readonly secrets: readonly string[], private readonly sink: OutputSink) {}
25
29
 
@@ -31,9 +35,15 @@ export class OutputRing {
31
35
  if (this.lines.length > this.capacity) this.lines.shift()
32
36
  }
33
37
 
34
- /** The newest `count` lines, oldest first. */
38
+ /** Report from the next line on: the lines before it belong to an older head. */
39
+ mark(): void {
40
+ this.marked = this.next
41
+ }
42
+
43
+ /** The newest `count` lines since the mark, oldest first. */
35
44
  tail(count: number): DevLogLine[] {
36
- return this.lines.slice(Math.max(0, this.lines.length - count))
45
+ const since = this.lines.filter((line) => line.n >= this.marked)
46
+ return since.slice(Math.max(0, since.length - count))
37
47
  }
38
48
 
39
49
  private redact(text: string): string {
@@ -4,8 +4,12 @@
4
4
  import { rawRequestTarget } from '@fabrika/auth-core'
5
5
  import { join, resolve, sep } from 'node:path'
6
6
  import type { SpaAssets } from './implement/dispatch.js'
7
+ import { isPageRoute } from './spa-fallback.js'
7
8
 
8
- /** Serves each SPA mount from its build directory under `root`; a path that names no file gets the SPA's `index.html`. */
9
+ /**
10
+ * Serves each SPA mount from its build directory under `root`; a page route that names no file gets the SPA's
11
+ * `index.html` (`isPageRoute`), and a missing asset is 404.
12
+ */
9
13
  export function diskAssets(root: string): SpaAssets {
10
14
  return {
11
15
  async serve(mount, request) {
@@ -15,8 +19,9 @@ export function diskAssets(root: string): SpaAssets {
15
19
  // The path is canonical, so every segment decodes and none is `.` or `..`.
16
20
  const file = join(dir, ...relative.split('/').filter((segment) => segment !== '').map((segment) => decodeURIComponent(segment)))
17
21
  const candidate = file.startsWith(`${dir}${sep}`) ? Bun.file(file) : null
18
- const served = candidate !== null && (await candidate.exists()) ? candidate : Bun.file(join(dir, 'index.html'))
19
- if (!(await served.exists())) {
22
+ const found = candidate !== null && (await candidate.exists()) ? candidate : null
23
+ const served = found ?? (isPageRoute(request) ? Bun.file(join(dir, 'index.html')) : null)
24
+ if (served === null || !(await served.exists())) {
20
25
  return new Response('Not found', { status: 404 })
21
26
  }
22
27
  return new Response(served)
@@ -4,7 +4,7 @@
4
4
  // stored or deflated entries, no encryption, a layout in which every byte belongs to an entry or the directory, sizes
5
5
  // and CRCs that match, bounded counts and sizes, and nested ZIPs to a small depth. Any other archive is refused.
6
6
 
7
- import { inflateRawSync } from 'node:zlib'
7
+ import { crc32, inflateRawSync } from 'node:zlib'
8
8
 
9
9
  export const ARCHIVE_LIMITS = {
10
10
  /** Entries of one archive, nested ones counted with their parent's. */
@@ -236,7 +236,7 @@ function unpacked(entry: Entry, data: Uint8Array, budget: Budget): Uint8Array {
236
236
  throw new Refused('malformed')
237
237
  }
238
238
  }
239
- if (content.length !== entry.uncompressed || Bun.hash.crc32(content) !== entry.crc) throw new Refused('malformed')
239
+ if (content.length !== entry.uncompressed || crc32(content) !== entry.crc) throw new Refused('malformed')
240
240
  return content
241
241
  }
242
242
 
@@ -0,0 +1,12 @@
1
+ // The framework's own file jobs, in a module of their own: the build recipe schedules them without the file stack.
2
+
3
+ /** The reserved name of the scan job: a `c.cron` job name cannot contain `-`, so no application job takes it. */
4
+ export const FILE_SCAN_JOB = 'fabrika-file-scan'
5
+ export const FILE_SCAN_SCHEDULE = '* * * * *'
6
+
7
+ /**
8
+ * The framework's sweep job: hourly, it deletes the files no declared column references and the pending ones past the
9
+ * grace, so none waits for the next request. A reserved name with `-`, which no `c.cron` name can take.
10
+ */
11
+ export const FILE_SWEEP_JOB = 'fabrika-file-sweep'
12
+ export const FILE_SWEEP_SCHEDULE = '41 * * * *'
@@ -2,7 +2,8 @@
2
2
  // the table's policy is its authority. This module stages a file (an incoming part or a `ctx.files.put`): its pending
3
3
  // row first, on a connection of its own, then its object, sniffed and bounded by the kind. It serves and opens a file
4
4
  // only through a reference the work holds and may read, and only a live one, by its scanned version. The sweep
5
- // deletes the files no declared column references, and the scan job decides about the quarantined ones (`scan.ts`).
5
+ // deletes the files no declared column references, and their objects once past the runtime's retention window, and the
6
+ // scan job decides about the quarantined ones (`scan.ts`).
6
7
 
7
8
  import type { ContractArtifact, FileKindArtifact } from '@fabrika/auth-core'
8
9
  import {
@@ -17,6 +18,7 @@ import {
17
18
  type StoredFiles,
18
19
  } from '@fabrika/db/framework'
19
20
  import { type FileObjectStore, type StoredObject, uuidv7 } from '@fabrika/platform'
21
+ import { RESERVED_FILE_KIND_PREFIX } from '../contract/artifact-schema'
20
22
  import type { FileRef } from '../contract/file'
21
23
  import { HttpError, NotFoundError } from '../errors'
22
24
  import { CapabilityRefusedError } from '../implement/capability'
@@ -29,16 +31,15 @@ import { bytesAreOfType, SNIFF_BYTES, SNIFFABLE_TYPES } from './sniff'
29
31
  import { cleanName, digesting, isNamed, readExactly, readHead } from './streams'
30
32
  import { FileDownload } from './treatment'
31
33
 
32
- /**
33
- * The framework's sweep job: hourly, it deletes the files no declared column references and the pending ones past the
34
- * grace, so none waits for the next request. A reserved name with `-`, which no `c.cron` name can take;
35
- * `@fabrika/provider-zerops` writes it into the crontab on this schedule (`ZEROPS_FILE_SWEEP_JOB`).
36
- */
37
- export const FILE_SWEEP_JOB = 'fabrika-file-sweep'
38
- export const FILE_SWEEP_SCHEDULE = '41 * * * *'
34
+ export { FILE_SWEEP_JOB, FILE_SWEEP_SCHEDULE } from './job-names'
39
35
  export const FILE_SWEEP_TIMEOUT_MS = 10 * 60_000
40
36
  /** How long a staged file waits for its first reference before the sweep may delete it. */
41
37
  export const FILE_PENDING_GRACE_MS = 60 * 60 * 1000
38
+ /**
39
+ * How long a removed file's object outlives its row on Cloudflare, so the rows of a restored backup still find their
40
+ * files: the 30 days backups are kept, which never exceeds this window.
41
+ */
42
+ export const FILE_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000
42
43
  /** How many files one sweep batch handles; the next batch, or the next run, takes the rest. */
43
44
  const SWEEP_BATCH = 20
44
45
 
@@ -67,6 +68,7 @@ export class FileWork {
67
68
  readonly holder: object
68
69
  readonly request: string
69
70
  readonly #staged: string[] = []
71
+ readonly #written = new Map<string, string>()
70
72
 
71
73
  constructor(holder: object, request: string) {
72
74
  this.holder = holder
@@ -77,9 +79,18 @@ export class FileWork {
77
79
  this.#staged.push(id)
78
80
  }
79
81
 
82
+ /** The work wrote the object of a file it staged, under a key no other file has. */
83
+ wrote(id: string, objectKey: string): void {
84
+ this.#written.set(id, objectKey)
85
+ }
86
+
80
87
  get stagedIds(): readonly string[] {
81
88
  return [...this.#staged]
82
89
  }
90
+
91
+ writtenKey(id: string): string | null {
92
+ return this.#written.get(id) ?? null
93
+ }
83
94
  }
84
95
 
85
96
  interface Kind {
@@ -92,12 +103,16 @@ export class StoredFileKinds {
92
103
  readonly #store: FileObjectStore
93
104
  readonly #files: StoredFiles
94
105
  readonly #columns: readonly FileColumnRef[]
106
+ readonly #retentionMs: number
107
+ /** Serializes each object deletion of the sweep with a cancel, so no deletion is in flight once a cancel resolves. */
108
+ #deletions: Promise<void> = Promise.resolve()
95
109
 
96
- constructor(kinds: readonly Kind[], store: FileObjectStore, files: StoredFiles, columns: readonly FileColumnRef[]) {
110
+ constructor(kinds: readonly Kind[], store: FileObjectStore, files: StoredFiles, columns: readonly FileColumnRef[], retentionMs: number) {
97
111
  this.#kinds = new Map(kinds.map((kind) => [kind.name, kind]))
98
112
  this.#store = store
99
113
  this.#files = files
100
114
  this.#columns = columns
115
+ this.#retentionMs = retentionMs
101
116
  }
102
117
 
103
118
  /** Whether the contract declares kind `name`. */
@@ -188,6 +203,7 @@ export class StoredFileKinds {
188
203
  const body = digesting(stream, null)
189
204
  try {
190
205
  const { size } = await this.#store.write(objectKey, body.stream, { contentType: type, maxBytes: artifact.maxBytes })
206
+ work.wrote(id, objectKey)
191
207
  const sha256 = body.digest()
192
208
  await this.#files.written(id, { size, sha256 })
193
209
  return fileReference({ id, kind: kind.name, name, type, size, status: 'pending' }, work.holder, true)
@@ -212,6 +228,13 @@ export class StoredFileKinds {
212
228
  try {
213
229
  const objectKey = await this.#files.abandon(id, work.request)
214
230
  if (objectKey === null) {
231
+ // On SQLite the pending row is written in the work's own transaction and rolled back with it, so only the
232
+ // work knows the object; a row still there is a file something claimed, whose object stays.
233
+ const written = work.writtenKey(id)
234
+ if (written !== null && (await this.#files.find(id)) === null) {
235
+ step = 'object_delete_failed'
236
+ await this.#store.delete(written)
237
+ }
215
238
  return
216
239
  }
217
240
  step = 'object_delete_failed'
@@ -304,41 +327,54 @@ export class StoredFileKinds {
304
327
  return new FileDownload({ body: object.body, size: file.size, contentType: file.type, name: file.name })
305
328
  }
306
329
 
330
+ #serialized<T>(work: () => Promise<T>): Promise<T> {
331
+ const result = this.#deletions.then(work)
332
+ this.#deletions = result.then(() => undefined, () => undefined)
333
+ return result
334
+ }
335
+
336
+ /**
337
+ * Cancels the tombstones of `objectKeys`, so their objects stay: a restore cancels every key its restored rows
338
+ * reference. Once it resolves, no deletion of this sweep is in flight. Answers how many it cancelled.
339
+ */
340
+ cancelDeletions(objectKeys: readonly string[]): Promise<number> {
341
+ return this.#serialized(() => this.#files.cancelDeletions(objectKeys))
342
+ }
343
+
307
344
  /**
308
- * The sweep: it finishes deletions a crash interrupted, adopts the migrated files a declared column references, then
309
- * deletes, batch by batch, every file past the grace that no declared column references: its row and the record of
310
- * its object in one transaction (the foreign key keeps a referenced one), then the object, then the record.
345
+ * The sweep: it adopts the migrated files a declared column references, then removes, batch by batch, every file past
346
+ * the grace that no declared column references: its row and the record of its object in one transaction (the foreign
347
+ * key keeps a referenced one). Last it claims each recorded object past the retention window, then deletes it.
311
348
  */
312
349
  async sweep(signal: AbortSignal): Promise<void> {
313
350
  let step = 'rows_read_failed'
314
351
  try {
315
- for (let keys = await this.#files.unfinishedDeletions(SWEEP_BATCH); keys.length > 0 && !signal.aborted;) {
316
- for (const key of keys) {
317
- step = 'object_delete_failed'
318
- await this.#store.delete(key)
319
- step = 'deletion_record_failed'
320
- await this.#files.forgetDeletion(key)
321
- }
322
- step = 'rows_read_failed'
323
- keys = keys.length === SWEEP_BATCH ? await this.#files.unfinishedDeletions(SWEEP_BATCH) : []
324
- }
325
352
  await this.#files.adoptReferenced(this.#columns)
326
353
  for (let more = true; more && !signal.aborted;) {
327
354
  step = 'rows_read_failed'
328
355
  const batch = await this.#files.unreferenced(this.#columns, SWEEP_BATCH, FILE_PENDING_GRACE_MS)
329
356
  for (const id of batch) {
330
357
  step = 'row_delete_failed'
331
- const key = await this.#files.remove(id, FILE_PENDING_GRACE_MS)
332
- if (key === null) {
333
- continue
334
- }
335
- step = 'object_delete_failed'
336
- await this.#store.delete(key)
337
- step = 'deletion_record_failed'
338
- await this.#files.forgetDeletion(key)
358
+ await this.#files.remove(id, FILE_PENDING_GRACE_MS)
339
359
  }
340
360
  more = batch.length === SWEEP_BATCH
341
361
  }
362
+ const cutoff = new Date(Date.now() - this.#retentionMs)
363
+ for (let more = true; more && !signal.aborted;) {
364
+ step = 'rows_read_failed'
365
+ const keys = await this.#files.dueDeletions(SWEEP_BATCH, cutoff)
366
+ for (const key of keys) {
367
+ await this.#serialized(async () => {
368
+ step = 'deletion_claim_failed'
369
+ // A crash or a failed delete after the claim leaks this object's bytes; it loses no data.
370
+ if (await this.#files.claimDeletion(key, cutoff)) {
371
+ step = 'object_delete_failed'
372
+ await this.#store.delete(key)
373
+ }
374
+ })
375
+ }
376
+ more = keys.length === SWEEP_BATCH
377
+ }
342
378
  } catch {
343
379
  console.error(`fabrika: stored files sweep: ${step}`)
344
380
  throw new ServerFault('file_sweep_failed', 'the sweep of stored files failed; the next run retries it')
@@ -418,13 +454,15 @@ function fileColumns(artifact: ContractArtifact): FileColumnRef[] {
418
454
 
419
455
  /**
420
456
  * The file kinds of `artifact`, or `null` when it declares none. Every reason a kind cannot be served goes into
421
- * `problems`, so `boot` refuses to start rather than take a file it cannot check.
457
+ * `problems`, so `boot` refuses to start rather than take a file it cannot check. A removed file's object outlives its
458
+ * row by `retentionMs`.
422
459
  */
423
460
  export function storedFileKinds(
424
461
  artifact: ContractArtifact,
425
462
  database: ImplementationDatabase | null,
426
463
  store: FileObjectStore | undefined,
427
464
  problems: string[],
465
+ retentionMs: number,
428
466
  ): StoredFileKinds | null {
429
467
  const entries = Object.entries(artifact.fileKinds ?? {})
430
468
  if (entries.length === 0) {
@@ -438,6 +476,9 @@ export function storedFileKinds(
438
476
  return null
439
477
  }
440
478
  for (const [name, kind] of entries) {
479
+ if (name.startsWith(RESERVED_FILE_KIND_PREFIX)) {
480
+ problems.push(`files '${name}' is a file kind name the framework reserves (${RESERVED_FILE_KIND_PREFIX}*)`)
481
+ }
441
482
  for (const type of kind.types.filter((candidate) => !SNIFFABLE_TYPES.includes(candidate))) {
442
483
  problems.push(`files '${name}' admits '${type}', whose bytes the framework cannot verify (allowed: ${SNIFFABLE_TYPES.join(', ')})`)
443
484
  }
@@ -450,5 +491,6 @@ export function storedFileKinds(
450
491
  store,
451
492
  frameworkTablesOf(database).files,
452
493
  fileColumns(artifact),
494
+ retentionMs,
453
495
  )
454
496
  }
package/src/files/scan.ts CHANGED
@@ -7,13 +7,12 @@
7
7
 
8
8
  import type { QuarantinedStoredFile } from '@fabrika/db/framework'
9
9
  import type { FileObjectStore } from '@fabrika/platform'
10
+ import { createHash } from 'node:crypto'
10
11
  import type { OperationsReporter } from '../jobs/report'
11
12
  import { type ArchiveRefusal, checkArchive } from './archive'
12
13
  import { ZIP_TYPES } from './sniff'
13
14
 
14
- /** The reserved name of the scan job: a `c.cron` job name cannot contain `-`, so no application job takes it. */
15
- export const FILE_SCAN_JOB = 'fabrika-file-scan'
16
- export const FILE_SCAN_SCHEDULE = '* * * * *'
15
+ export { FILE_SCAN_JOB, FILE_SCAN_SCHEDULE } from './job-names'
17
16
 
18
17
  export const SCAN_LIMITS = {
19
18
  /** One pass scans at most this many files, and at most `batchBytes` of them; the next pass takes the rest. */
@@ -181,7 +180,7 @@ class ScanPass {
181
180
  }
182
181
  const bytes = object.size === file.size ? await readExactly(object.body, file.size, signal) : null
183
182
  if (object.size !== file.size) await object.body.cancel()
184
- if (bytes === null || new Bun.CryptoHasher('sha256').update(bytes).digest('hex') !== file.sha256) {
183
+ if (bytes === null || createHash('sha256').update(bytes).digest('hex') !== file.sha256) {
185
184
  if (!signal.aborted) this.#states.set(file.id, { kind: 'mismatch' })
186
185
  throw new Error('the object differs from its upload')
187
186
  }
@@ -1,6 +1,8 @@
1
1
  // The byte streams of a stored file on its way in and out: the head the sniff reads, the digest and the deadline of a
2
2
  // write, the exact read of a file a site opens, and the client's file name as metadata.
3
3
 
4
+ import { createHash } from 'node:crypto'
5
+
4
6
  const MAX_NAME_CODE_POINTS = 255
5
7
 
6
8
  /** The client's file name, metadata only: at most 255 code points, never half a character, no control characters. */
@@ -62,7 +64,7 @@ export function digesting(body: ReadableStream<Uint8Array>, timeoutMs: number |
62
64
  timedOut(): boolean
63
65
  stop(): void
64
66
  } {
65
- const hasher = new Bun.CryptoHasher('sha256')
67
+ const hasher = createHash('sha256')
66
68
  let timer: ReturnType<typeof setTimeout> | undefined
67
69
  let expired = false
68
70
  const stream = body.pipeThrough(
@@ -2,18 +2,9 @@
2
2
  // body, so the framework enforces it) and the share SPA's fragment bootstrap. A form's admission is
3
3
  // `form-admission.ts`, a webhook's signature `verifiers.ts`.
4
4
 
5
- import { LINK_BOOTSTRAP_FILE, SHARE_KEY_GLOBAL } from '../contract/defaults'
5
+ import { bodyDeadlineMs, LINK_BOOTSTRAP_FILE, SHARE_KEY_GLOBAL } from '../contract/defaults'
6
6
  import { HttpError } from '../errors'
7
- import { readBoundedBytes } from './bounded'
8
-
9
- /** The slowest body a procedure waits for, in bytes per second: its deadline is `maxBytes` at this rate, 30 s at least. */
10
- export const BODY_RATE_FLOOR = 16 * 1024
11
- const MIN_BODY_DEADLINE_MS = 30_000
12
-
13
- /** How long a body of at most `maxBytes` may take to arrive. */
14
- export function bodyDeadlineMs(maxBytes: number): number {
15
- return Math.max(MIN_BODY_DEADLINE_MS, Math.ceil((maxBytes / BODY_RATE_FLOOR) * 1000))
16
- }
7
+ import { deadline, readBoundedBytes } from './bounded'
17
8
 
18
9
  /**
19
10
  * The body as bytes, read until `maxBytes`; a longer one stops the read and is a 413. A body whose stream fails before
@@ -36,6 +27,16 @@ export async function readBoundedBody(request: Request, maxBytes: number, deadli
36
27
  return body
37
28
  }
38
29
 
30
+ /** `readBoundedBody` within the deadline `maxBytes` derives; the deadline ends with the read. */
31
+ export async function readBodyInTime(request: Request, maxBytes: number): Promise<Uint8Array<ArrayBuffer>> {
32
+ const timer = deadline(bodyDeadlineMs(maxBytes))
33
+ try {
34
+ return await readBoundedBody(request, maxBytes, timer.signal)
35
+ } finally {
36
+ timer.clear()
37
+ }
38
+ }
39
+
39
40
  /** What a request answers whose body did not arrive whole: nobody reads it, and no failure is reported. */
40
41
  export function abortedBody(): HttpError {
41
42
  return new HttpError(400, 'aborted', 'The body could not be read to its end')