@fabrika/app 0.5.5 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +29 -21
  3. package/skill/references/app-setup.md +54 -36
  4. package/skill/references/dev.md +5 -5
  5. package/skill/references/endpoint.md +152 -25
  6. package/skill/references/errors.md +26 -23
  7. package/skill/references/files.md +187 -137
  8. package/skill/references/jobs.md +82 -35
  9. package/skill/references/public-intake.md +387 -236
  10. package/skill/references/spa.md +5 -5
  11. package/skill/references/start.md +2 -1
  12. package/skill/references/table-and-migration.md +24 -2
  13. package/skill/references/tests.md +8 -5
  14. package/skill/references/webhooks-links-connectors.md +65 -12
  15. package/src/bun.ts +3 -2
  16. package/src/client/contract.ts +184 -74
  17. package/src/client/index.ts +2 -1
  18. package/src/client/transport.ts +7 -2
  19. package/src/cloudflare.ts +2 -0
  20. package/src/contract/artifact-schema.ts +110 -15
  21. package/src/contract/build.ts +1184 -613
  22. package/src/contract/compile.ts +44 -10
  23. package/src/contract/connect.ts +71 -5
  24. package/src/contract/context.ts +132 -23
  25. package/src/contract/declaration.ts +78 -63
  26. package/src/contract/defaults.ts +64 -58
  27. package/src/contract/define.ts +76 -124
  28. package/src/contract/file.ts +78 -0
  29. package/src/contract/group.ts +244 -37
  30. package/src/contract/index.ts +18 -0
  31. package/src/contract/kit.ts +206 -126
  32. package/src/contract/platform.ts +8 -9
  33. package/src/contract/redirect.ts +49 -0
  34. package/src/contract/schema-from-shape.ts +7 -0
  35. package/src/contract/schema-shape.ts +10 -0
  36. package/src/contract/snapshot-source.ts +5 -0
  37. package/src/contract/token.ts +89 -0
  38. package/src/contract/webhook-binding.ts +26 -0
  39. package/src/contract/webhook-scheme.ts +32 -0
  40. package/src/files/handle.ts +22 -0
  41. package/src/files/kinds.ts +454 -0
  42. package/src/files/scan.ts +15 -9
  43. package/src/files/streams.ts +111 -0
  44. package/src/honeypot.ts +7 -0
  45. package/src/implement/anonymous.ts +18 -42
  46. package/src/implement/boot.ts +177 -159
  47. package/src/implement/bounded.ts +23 -0
  48. package/src/implement/browser-config.ts +4 -4
  49. package/src/implement/capability.ts +78 -2
  50. package/src/implement/{form-admission.ts → client-address.ts} +3 -38
  51. package/src/implement/dispatch.ts +810 -409
  52. package/src/implement/egress.ts +84 -0
  53. package/src/implement/faults.ts +11 -1
  54. package/src/implement/index.ts +3 -2
  55. package/src/implement/invocation.ts +135 -35
  56. package/src/implement/params.ts +88 -0
  57. package/src/implement/principal.ts +7 -0
  58. package/src/implement/public.ts +142 -0
  59. package/src/implement/tokens.ts +219 -0
  60. package/src/implement/types.ts +24 -3
  61. package/src/implement/urls.ts +78 -0
  62. package/src/implement/verifiers.ts +299 -0
  63. package/src/implement/webhook.ts +136 -55
  64. package/src/implement/workload.ts +16 -14
  65. package/src/index.ts +3 -0
  66. package/src/jobs/mail.ts +98 -10
  67. package/src/jobs/runner.ts +103 -71
  68. package/src/jobs/scrub.ts +4 -1
  69. package/src/jobs/types.ts +19 -2
  70. package/src/react/hooks.ts +2 -2
  71. package/src/react/provider.tsx +2 -2
  72. package/src/testing/dev.ts +2 -2
  73. package/src/testing/endpoint-access.ts +167 -128
  74. package/src/testing/kit.ts +79 -24
  75. package/src/testing/policy-isolation.ts +55 -29
  76. package/src/testing/proxy.ts +6 -2
  77. package/src/testing/sample-file.ts +30 -0
  78. package/src/client/files.ts +0 -135
  79. package/src/contract/file-schemas.ts +0 -25
  80. package/src/files/collections.ts +0 -848
@@ -1,848 +0,0 @@
1
- // The runtime of `c.files`: the generated upload, download, list and remove endpoints, the locator of every
2
- // `file:<name>` dimension, the scan pass of `scan: true` collections (`scan.ts`), and the sweep that deletes the objects
3
- // of removed, orphaned and abandoned files. Access is the core's:
4
- // the dispatcher checks the access form and mints the target, and every database operation here takes that target,
5
- // so `@fabrika/db` scopes it by the whole chain the access check saw.
6
-
7
- import type { ContractArtifact, CredentialedEndpoint, EndpointRow, FileCollectionArtifact } from '@fabrika/auth-core'
8
- import type { CapabilityAuthority, RefGrant } from '@fabrika/db/capability'
9
- import {
10
- type FileCollectionTable,
11
- fileTableName,
12
- fileTablesOf,
13
- type FormFiles,
14
- frameworkTablesOf,
15
- isScopedDatabase,
16
- QueryRefusedError,
17
- type Session,
18
- type StoredFile,
19
- type WrittenFormFile,
20
- } from '@fabrika/db/framework'
21
- import { type FileObjectStore, type StoredObject, uuidv7 } from '@fabrika/platform'
22
- import { z } from 'zod/v4'
23
- import { FILE_LIMITS } from '../contract/defaults'
24
- import { FileLinkListInput, FileListInput, FileListOutput } from '../contract/file-schemas'
25
- import { HttpError, jsonResponse, NotFoundError } from '../errors'
26
- import { type CapabilityMint, CapabilityRefusedError, type RequestFrame } from '../implement/capability'
27
- import { NO_FACILITIES, type ServedEndpoint } from '../implement/dispatch'
28
- import { ServerFault } from '../implement/faults'
29
- import type { ContractDatabase, ImplementationDatabase } from '../implement/implement'
30
- import type { OperationsReporter } from '../jobs/report'
31
- import type { FileCollectionJobHandle, FileCollections as FilesOfSite, FileListing, FileListPage, OpenedFile, OpenOptions } from './handle'
32
- import { type FileScanner, runScanPass } from './scan'
33
- import { bytesAreOfType, SNIFF_BYTES, SNIFFABLE_TYPES } from './sniff'
34
- import { FileDownload } from './treatment'
35
-
36
- /** Which step of a file cleanup or sweep failed: the whole of what its log line says about the failure. */
37
- type FileCleanupStep = 'rows_read_failed' | 'object_delete_failed' | 'row_delete_failed'
38
-
39
- /** How many removable files one sweep handles; the next remove picks up the rest. */
40
- const SWEEP_BATCH = 20
41
- const MAX_NAME_CODE_POINTS = 255
42
- /** An upload whose body has not arrived by then fails, so its pending row is abandoned well before the grace ends. */
43
- export const UPLOAD_TIMEOUT_MS = 10 * 60 * 1000
44
- /** A pending row older than this is an upload that will never complete; the sweep deletes its object and the row. */
45
- export const PENDING_GRACE_MS = 60 * 60 * 1000
46
- /**
47
- * The framework's sweep job: hourly, it deletes the objects of every collection's removed and orphaned files and of
48
- * abandoned uploads, so none waits for the next request. A reserved name with `-`, which no `c.cron` name can take;
49
- * `@fabrika/provider-zerops` writes it into the crontab on this schedule (`ZEROPS_FILE_SWEEP_JOB`).
50
- */
51
- export const FILE_SWEEP_JOB = 'fabrika-file-sweep'
52
- export const FILE_SWEEP_SCHEDULE = '41 * * * *'
53
- export const FILE_SWEEP_TIMEOUT_MS = 10 * 60_000
54
- const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
55
-
56
- const ById = z.strictObject({ id: z.uuid() })
57
-
58
- /** Where `ctx.files` runs: the site's frame, the session its statements run in, and whether it may remove. */
59
- export interface FileSite {
60
- /** `job 'erase'`, an endpoint id: named in a refusal. */
61
- readonly where: string
62
- readonly frame: RequestFrame
63
- /** Throws once the site's work ended, or when called from other work. */
64
- readonly check: () => void
65
- /** The invocation's transaction, a job's fenced transaction, or a job's statement-at-a-time session. */
66
- readonly session: () => Promise<Session>
67
- /** `null` where `removeAll` runs (a mutation, a job's transaction); otherwise why it does not. */
68
- readonly readOnly: string | null
69
- /** Whether `attach` runs here: only in a job's transaction, the job that drains a form's inbox. */
70
- readonly attaches: boolean
71
- }
72
-
73
- interface Collection {
74
- readonly name: string
75
- readonly artifact: FileCollectionArtifact
76
- readonly dimension: string
77
- /** The owner table's dimension, which `file:<name>` lies in. */
78
- readonly ownerDimension: string
79
- readonly table: FileCollectionTable
80
- /** The link mounts bound to the owner's dimension, for a `link: true` collection: each serves its link holders. */
81
- readonly linkMounts: readonly string[]
82
- }
83
-
84
- /** The media type of a `Content-Type` value, without parameters, lower-cased. */
85
- function mediaType(header: string | null): string | null {
86
- const type = header?.split(';')[0]?.trim().toLowerCase()
87
- return type === undefined || type === '' ? null : type
88
- }
89
-
90
- /** The client's file name, metadata only: at most 255 code points, never half a character, no control characters. */
91
- export function originalName(request: Request): string | null {
92
- return cleanName(new URL(request.url).searchParams.get('name') ?? '')
93
- }
94
-
95
- function cleanName(raw: string): string | null {
96
- const name = Array.from(raw.replace(/\p{Cc}/gu, '').trim()).slice(0, MAX_NAME_CODE_POINTS).join('')
97
- return name === '' ? null : name
98
- }
99
-
100
- /** One file part of a form's submission, as the multipart reader hands it over. */
101
- export interface FormFilePart {
102
- readonly filename: string | null
103
- /** The media type the part declares; the bytes must be of it. */
104
- readonly contentType: string | null
105
- readonly body: ReadableStream<Uint8Array>
106
- }
107
-
108
- /** A form's file whose object is written under its pending row: what the inbox append holds, or a failure cleans up. */
109
- export interface ReceivedFormFile extends WrittenFormFile {
110
- readonly objectKey: string
111
- }
112
-
113
- /** The first `size` bytes of `body`, and a stream that yields the whole body again. */
114
- async function readHead(
115
- body: ReadableStream<Uint8Array>,
116
- size: number,
117
- ): Promise<{ readonly head: Uint8Array; readonly stream: ReadableStream<Uint8Array> }> {
118
- const reader = body.getReader()
119
- const chunks: Uint8Array[] = []
120
- let length = 0
121
- let done = false
122
- while (length < size) {
123
- const next = await reader.read()
124
- if (next.done) {
125
- done = true
126
- break
127
- }
128
- chunks.push(next.value)
129
- length += next.value.byteLength
130
- }
131
- const buffered = new Uint8Array(length)
132
- let offset = 0
133
- for (const chunk of chunks) {
134
- buffered.set(chunk, offset)
135
- offset += chunk.byteLength
136
- }
137
- const stream = new ReadableStream<Uint8Array>({
138
- start(controller) {
139
- if (buffered.byteLength > 0) {
140
- controller.enqueue(buffered)
141
- }
142
- if (done) {
143
- controller.close()
144
- }
145
- },
146
- async pull(controller) {
147
- const next = await reader.read()
148
- if (next.done) {
149
- controller.close()
150
- } else {
151
- controller.enqueue(next.value)
152
- }
153
- },
154
- cancel: (reason) => reader.cancel(reason),
155
- })
156
- return { head: buffered.subarray(0, size), stream }
157
- }
158
-
159
- /** The body as it is stored: hashed on the way, and failed once `timeoutMs` (`null`: no timer) passes before it ends. */
160
- function digesting(body: ReadableStream<Uint8Array>, timeoutMs: number | null): {
161
- readonly stream: ReadableStream<Uint8Array>
162
- digest(): string
163
- timedOut(): boolean
164
- stop(): void
165
- } {
166
- const hasher = new Bun.CryptoHasher('sha256')
167
- let timer: ReturnType<typeof setTimeout> | undefined
168
- let expired = false
169
- const stream = body.pipeThrough(
170
- new TransformStream<Uint8Array, Uint8Array>({
171
- start(controller) {
172
- if (timeoutMs !== null) {
173
- timer = setTimeout(() => {
174
- expired = true
175
- controller.error(new Error('the upload did not finish in time'))
176
- }, timeoutMs)
177
- }
178
- },
179
- transform(chunk, controller) {
180
- hasher.update(chunk)
181
- controller.enqueue(chunk)
182
- },
183
- flush() {
184
- clearTimeout(timer)
185
- },
186
- }),
187
- )
188
- return { stream, digest: () => hasher.digest('hex'), timedOut: () => expired, stop: () => clearTimeout(timer) }
189
- }
190
-
191
- /** The whole body when it holds exactly `size` bytes; `null` as soon as it proves longer, or once it ends shorter. */
192
- async function readExactly(body: ReadableStream<Uint8Array>, size: number): Promise<Uint8Array | null> {
193
- const bytes = new Uint8Array(size)
194
- let length = 0
195
- const reader = body.getReader()
196
- try {
197
- for (let next = await reader.read(); !next.done; next = await reader.read()) {
198
- if (length + next.value.byteLength > size) {
199
- return null
200
- }
201
- bytes.set(next.value, length)
202
- length += next.value.byteLength
203
- }
204
- } finally {
205
- reader.releaseLock()
206
- }
207
- return length === size ? bytes : null
208
- }
209
-
210
- function isNamed(error: unknown, name: string): boolean {
211
- return error instanceof Error && error.name === name
212
- }
213
-
214
- export class FileCollections {
215
- readonly #collections: readonly Collection[]
216
- readonly #byDimension: ReadonlyMap<string, Collection>
217
- readonly #store: FileObjectStore
218
- readonly #capabilities: CapabilityAuthority
219
- readonly #mint: CapabilityMint
220
- /** The files forms received, before they have an owner (`fabrika_form_files`). */
221
- readonly #formFiles: FormFiles
222
-
223
- constructor(
224
- collections: readonly Collection[],
225
- store: FileObjectStore,
226
- capabilities: CapabilityAuthority,
227
- mint: CapabilityMint,
228
- formFiles: FormFiles,
229
- ) {
230
- this.#collections = collections
231
- this.#byDimension = new Map(collections.map((collection) => [collection.dimension, collection]))
232
- this.#store = store
233
- this.#capabilities = capabilities
234
- this.#mint = mint
235
- this.#formFiles = formFiles
236
- }
237
-
238
- /** Whether the contract declares the collection `name`, so a form's file can land in it. */
239
- has(name: string): boolean {
240
- return this.#collections.some((collection) => collection.name === name)
241
- }
242
-
243
- /** The largest file collection `name` admits; 0 for a collection it does not serve. */
244
- maxBytesOf(name: string): number {
245
- return this.#collections.find((collection) => collection.name === name)?.artifact.maxBytes ?? 0
246
- }
247
-
248
- /** `ctx.files` of one site: each collection's handle, by its name. */
249
- site(site: FileSite): FilesOfSite<FileCollectionJobHandle> {
250
- const handles = new Map(this.#collections.map((collection) => [collection.name, this.#handle(collection, site)]))
251
- return (name) => {
252
- site.check()
253
- const handle = typeof name === 'string' ? handles.get(name) : undefined
254
- if (handle === undefined) {
255
- throw new ServerFault(
256
- 'file_collection_unknown',
257
- `${site.where}: ctx.files('${String(name)}') names no collection of the contract (collections: ${[...handles.keys()].join(', ')})`,
258
- )
259
- }
260
- return handle
261
- }
262
- }
263
-
264
- #handle(collection: Collection, site: FileSite): FileCollectionJobHandle {
265
- return Object.freeze({
266
- list: async (owner: unknown, page: FileListPage = {}) => {
267
- site.check()
268
- this.#ownerGrant(collection, site, owner, ['read'])
269
- return await this.#listOwned(collection, site, owner, page)
270
- },
271
- open: async (owner: unknown, id: unknown, options: OpenOptions) => {
272
- site.check()
273
- const grant = this.#ownerGrant(collection, site, owner, ['read'])
274
- return await this.#open(collection, site, grant, id, options)
275
- },
276
- removeAll: async (owner: unknown) => {
277
- site.check()
278
- if (site.readOnly !== null) {
279
- throw new QueryRefusedError('read_only', `${site.where}: files '${collection.name}' removeAll: ${site.readOnly}`)
280
- }
281
- this.#ownerGrant(collection, site, owner, ['delete'])
282
- return await collection.table.removeOwned(owner, new Date(), await site.session())
283
- },
284
- attach: async (owner: unknown, id: unknown) => {
285
- site.check()
286
- if (!site.attaches) {
287
- throw new QueryRefusedError(
288
- 'undeclared_reach',
289
- `${site.where}: files '${collection.name}' attach runs only in a job's ctx.transaction((tx) => …), as tx.files`,
290
- )
291
- }
292
- const grant = this.#ownerGrant(collection, site, owner, ['write'])
293
- if (typeof id !== 'string') {
294
- throw new QueryRefusedError('invalid_argument', `${site.where}: files '${collection.name}' attach takes the id of a form's file`)
295
- }
296
- return await this.#attach(collection, grant, owner, id, await site.session())
297
- },
298
- })
299
- }
300
-
301
- /**
302
- * Moves the file a form received under `id` into the collection, owned by `owner`, in `session`: the transaction
303
- * that wrote the owner, so a rollback leaves the file held. A scanning collection holds it `quarantined` until the
304
- * scan job's verdict. `false` when no held file of this collection has the id (never received, of another collection,
305
- * or its hold ended); `true` again for a file this owner attached before, so a retried attempt answers alike.
306
- */
307
- async #attach(collection: Collection, grant: RefGrant, owner: unknown, id: string, session: Session): Promise<boolean> {
308
- const claimed = await this.#formFiles.claim(collection.name, id, session)
309
- if (claimed === null) {
310
- const located = UUID.test(id) ? await collection.table.locate(id, session) : null
311
- return located !== null && located.get(grant.dimension) === grant.id
312
- && [...grant.chain].every(([dimension, value]) => located.get(dimension) === value)
313
- }
314
- // Thrown, not answered: the transaction rolls back, and the claimed file stays held for a later attempt.
315
- const limit = collection.artifact.maxPerOwner
316
- if ((await collection.table.adopt(owner, claimed, session, limit)) === 'limit') {
317
- throw tooMany(collection)
318
- }
319
- // Counted again: an upload of a person may have passed its own count beside this one.
320
- if (limit !== null && (await collection.table.countHeld(owner, session)) > limit) {
321
- throw tooMany(collection)
322
- }
323
- return true
324
- }
325
-
326
- /**
327
- * Receives one file of a form's submission into collection `name`: the part's declared type must be one the
328
- * collection admits and its first bytes that type, its row in `fabrika_form_files` is written pending before its
329
- * object `<name>/<id>`, and the object stops at the collection's `maxBytes`. Answers `null` for an empty part without
330
- * a file name: a file input the visitor left empty. Whatever fails after the row cleans the object and the row up.
331
- */
332
- async receiveFormFile(form: string, name: string, part: FormFilePart): Promise<ReceivedFormFile | null> {
333
- const collection = this.#collections.find((candidate) => candidate.name === name)
334
- if (collection === undefined) {
335
- await part.body.cancel()
336
- throw new ServerFault('contract_inconsistent', `form '${form}' takes a file into files '${name}', which is not served`)
337
- }
338
- const { artifact } = collection
339
- const { head, stream } = await readHead(part.body, SNIFF_BYTES)
340
- if (head.byteLength === 0 && (part.filename === null || part.filename === '')) {
341
- // The part ended with no byte: it was read to its end, and the next part may follow.
342
- return null
343
- }
344
- const type = part.contentType
345
- if (type === null || !artifact.types.includes(type)) {
346
- await stream.cancel()
347
- throw new HttpError(415, 'unsupported_media_type', `files '${name}' admits ${artifact.types.join(', ')}`)
348
- }
349
- if (!bytesAreOfType(type, head)) {
350
- await stream.cancel()
351
- throw new HttpError(415, 'unsupported_media_type', `The bytes of the file are not ${type}`)
352
- }
353
- if (head.byteLength === 0 && artifact.scan === true) {
354
- await stream.cancel()
355
- throw new HttpError(400, 'empty', `files '${name}' scans its files and admits no empty file`)
356
- }
357
- const id = uuidv7()
358
- const objectKey = `${name}/${id}`
359
- await this.#formFiles.stage({ id, form, collection: name, objectKey, contentType: type, originalName: cleanName(part.filename ?? '') })
360
- // The form's own deadline bounds the whole body, so the digest needs no timer of its own.
361
- const body = digesting(stream, null)
362
- try {
363
- const { size } = await this.#store.write(objectKey, body.stream, { contentType: type, maxBytes: artifact.maxBytes })
364
- return Object.freeze({ id, objectKey, size, sha256: body.digest() })
365
- } catch (error) {
366
- if (isNamed(error, 'ObjectExistsError')) {
367
- // The key holds an object this submission did not write: never delete it, only the pending row.
368
- await this.#formFiles.abandon(id)
369
- throw new ServerFault('file_key_taken', `files '${name}' found an object already under a new file's key`)
370
- }
371
- await this.releaseFormFiles([{ id, objectKey }])
372
- if (isNamed(error, 'ObjectTooLargeError')) {
373
- throw tooLarge(collection)
374
- }
375
- throw error
376
- } finally {
377
- body.stop()
378
- }
379
- }
380
-
381
- /**
382
- * Deletes the objects of files a submission received and then their pending rows: it did not commit. A failed step is
383
- * logged by its name and left for the sweep, which deletes a pending row after `PENDING_GRACE_MS`.
384
- */
385
- async releaseFormFiles(files: readonly { readonly id: string; readonly objectKey: string }[]): Promise<void> {
386
- for (const file of files) {
387
- let step: FileCleanupStep = 'object_delete_failed'
388
- try {
389
- await this.#store.delete(file.objectKey)
390
- step = 'row_delete_failed'
391
- await this.#formFiles.abandon(file.id)
392
- } catch {
393
- console.error(`fabrika: form files cleanup: ${step}`)
394
- }
395
- }
396
- }
397
-
398
- /** One batch of the form files whose hold ended or whose submission never committed: the object, then the row. */
399
- async #sweepFormFiles(): Promise<'more' | 'done' | 'failed'> {
400
- let step: FileCleanupStep = 'rows_read_failed'
401
- try {
402
- const batch = await this.#formFiles.removable(SWEEP_BATCH, PENDING_GRACE_MS)
403
- for (const file of batch) {
404
- step = 'object_delete_failed'
405
- await this.#store.delete(file.objectKey)
406
- step = 'row_delete_failed'
407
- await this.#formFiles.purge(file.id, PENDING_GRACE_MS)
408
- }
409
- return batch.length === SWEEP_BATCH ? 'more' : 'done'
410
- } catch {
411
- console.error(`fabrika: form files sweep: ${step}`)
412
- return 'failed'
413
- }
414
- }
415
-
416
- /**
417
- * The grant of `owner`: a `Ref` of the collection's owner, of the work running now, for one of the collection's
418
- * `actions`: `read` lists and opens, `delete` removes. A write capability alone reads no file's name or bytes.
419
- */
420
- #ownerGrant(collection: Collection, site: FileSite, owner: unknown, actions: readonly ('read' | 'write' | 'delete')[]): RefGrant {
421
- const grant = this.#capabilities.resolve(owner)
422
- const needed = actions.map((action) => collection.artifact[action])
423
- if (grant.kind !== 'ref' || grant.dimension !== collection.ownerDimension || !needed.includes(grant.action)) {
424
- throw new CapabilityRefusedError(
425
- 'wrong_capability',
426
- `${site.where}: files '${collection.name}' takes a Ref of ${collection.ownerDimension} for ${
427
- needed.map((action) => `'${action}'`).join(' or ')
428
- }, got one for '${grant.action}'`,
429
- )
430
- }
431
- return grant
432
- }
433
-
434
- async #listOwned(collection: Collection, site: FileSite, owner: unknown, page: FileListPage): Promise<FileListing> {
435
- const limit = page.limit ?? FILE_LIMITS.listPageDefault
436
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > FILE_LIMITS.listPageMax) {
437
- throw new QueryRefusedError('list_page', `${site.where}: files '${collection.name}' lists 1 to ${FILE_LIMITS.listPageMax} files a page`)
438
- }
439
- if (page.after !== undefined && !UUID.test(page.after)) {
440
- throw new QueryRefusedError('list_page', `${site.where}: files '${collection.name}' takes the id of a file as after`)
441
- }
442
- const listed = await collection.table.list(owner, { after: page.after ?? null, limit }, await site.session())
443
- return Object.freeze({ items: Object.freeze([...listed.items]), next: listed.next })
444
- }
445
-
446
- /**
447
- * A file of the owner `grant` names, read whole: located now, it must lie beneath that owner where the `Ref` says,
448
- * and is then read under a file `Ref` of the same principal and action, as the download endpoint reads it.
449
- */
450
- async #open(collection: Collection, site: FileSite, grant: RefGrant, id: unknown, options: OpenOptions): Promise<OpenedFile> {
451
- const maxBytes = options.maxBytes
452
- if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
453
- throw new QueryRefusedError('invalid_argument', `${site.where}: files '${collection.name}' open takes maxBytes, a positive whole number`)
454
- }
455
- if (typeof id !== 'string' || !UUID.test(id)) {
456
- throw new NotFoundError()
457
- }
458
- const located = await collection.table.locate(id, await site.session())
459
- const beneath = located !== null && located.get(grant.dimension) === grant.id
460
- && [...grant.chain].every(([dimension, value]) => located.get(dimension) === value)
461
- if (located === null || !beneath) {
462
- throw new NotFoundError()
463
- }
464
- const target = this.#mint.ref(site.frame, collection.dimension, grant.action, grant.principal, id, located)
465
- const file = await collection.table.find(target, await site.session())
466
- if (file.status === 'quarantined') {
467
- throw new HttpError(409, 'quarantined', 'The file is being checked for malware; try again in a few minutes')
468
- }
469
- if (file.status === 'rejected') {
470
- throw new NotFoundError()
471
- }
472
- if (file.size > maxBytes) {
473
- throw new HttpError(413, 'too_large', `The file has ${file.size} bytes, more than the ${maxBytes} this site reads`)
474
- }
475
- const object = collection.artifact.scan === true ? await this.#scannedObject(collection, target, file) : await this.#store.read(file.objectKey)
476
- if (object === null) {
477
- throw new ServerFault('file_object_missing', `the object of a file in '${collection.name}' is missing from storage`)
478
- }
479
- const bytes = object.size === file.size ? await readExactly(object.body, file.size) : null
480
- if (bytes === null) {
481
- await object.body.cancel()
482
- throw new ServerFault('file_object_mismatch', `the object of a file in '${collection.name}' has another size than its row records`)
483
- }
484
- return Object.freeze({ id: file.id, name: file.originalName, type: file.contentType, size: file.size, bytes })
485
- }
486
-
487
- /** The `file:<name>` dimensions; the dispatcher locates them like a table's. */
488
- get dimensions(): readonly string[] {
489
- return [...this.#byDimension.keys()]
490
- }
491
-
492
- /** The application's database, with the locator extended to every file dimension: a live file's row, read now. */
493
- locating(database: ContractDatabase): ContractDatabase {
494
- return {
495
- tables: database.tables,
496
- locate: (dimension, id, using) => {
497
- const collection = this.#byDimension.get(dimension)
498
- if (collection === undefined) {
499
- return database.locate(dimension, id, using)
500
- }
501
- return UUID.test(id) ? collection.table.locate(id, using) : Promise.resolve(null)
502
- },
503
- }
504
- }
505
-
506
- /** The served endpoints of every collection, by endpoint id. */
507
- endpoints(rows: ReadonlyMap<string, EndpointRow>, problems: string[]): ReadonlyMap<string, ServedEndpoint> {
508
- const served = new Map<string, ServedEndpoint>()
509
- for (const collection of this.#collections) {
510
- const row = (suffix: string, id = `files.${collection.name}.${suffix}`): CredentialedEndpoint | null => {
511
- const found = rows.get(id)
512
- if (found === undefined || found.requirement === null) {
513
- problems.push(`files '${collection.name}' has no '${id}' endpoint row`)
514
- return null
515
- }
516
- return found
517
- }
518
- const upload = row('upload')
519
- const download = row('download')
520
- const list = row('list')
521
- const remove = row('remove')
522
- if (upload !== null) {
523
- served.set(upload.id, { kind: 'file', row: upload, serve: (request, target) => this.#upload(collection, request, target) })
524
- }
525
- if (download !== null) {
526
- served.set(download.id, { kind: 'file', row: download, serve: (_request, target) => this.#download(collection, target) })
527
- }
528
- if (list !== null) {
529
- served.set(list.id, {
530
- kind: 'procedure',
531
- row: list,
532
- input: FileListInput,
533
- output: FileListOutput,
534
- handler: async ({ input, ctx }) => this.#list(collection, 'target' in ctx ? ctx.target : undefined, input),
535
- facilities: NO_FACILITIES,
536
- })
537
- }
538
- for (const mount of collection.linkMounts) {
539
- const linkRow = (suffix: string): CredentialedEndpoint | null => row(suffix, `${mount}.files.${collection.name}.${suffix}`)
540
- const linkUpload = linkRow('upload')
541
- const linkDownload = linkRow('download')
542
- const linkList = linkRow('list')
543
- if (linkList !== null) {
544
- served.set(linkList.id, {
545
- kind: 'procedure',
546
- row: linkList,
547
- input: FileLinkListInput,
548
- output: FileListOutput,
549
- handler: async ({ input, ctx }) => this.#list(collection, 'target' in ctx ? ctx.target : undefined, input),
550
- facilities: NO_FACILITIES,
551
- })
552
- }
553
- if (linkUpload !== null) {
554
- served.set(linkUpload.id, { kind: 'file', row: linkUpload, serve: (request, target) => this.#upload(collection, request, target) })
555
- }
556
- if (linkDownload !== null) {
557
- served.set(linkDownload.id, {
558
- kind: 'file',
559
- row: linkDownload,
560
- serve: (_request, target) => this.#download(collection, target),
561
- beneath: collection.dimension,
562
- })
563
- }
564
- }
565
- if (remove !== null) {
566
- served.set(remove.id, {
567
- kind: 'procedure',
568
- row: remove,
569
- input: ById,
570
- output: ById,
571
- handler: async ({ ctx }) => this.#remove(collection, 'target' in ctx ? ctx.target : undefined),
572
- facilities: NO_FACILITIES,
573
- })
574
- }
575
- }
576
- return served
577
- }
578
-
579
- /**
580
- * The pending row comes first and is the intent to clean the object up: whatever fails after it — the write, a
581
- * lost answer of the store, the owner moving, the final write of the row — leaves a row the sweep finds.
582
- */
583
- async #upload(collection: Collection, request: Request, owner: unknown): Promise<Response> {
584
- const { artifact, table } = collection
585
- const type = mediaType(request.headers.get('Content-Type'))
586
- if (type === null || !artifact.types.includes(type)) {
587
- throw new HttpError(415, 'unsupported_media_type', `files '${collection.name}' admits ${artifact.types.join(', ')}`)
588
- }
589
- if (Number(request.headers.get('Content-Length') ?? 0) > artifact.maxBytes) {
590
- throw tooLarge(collection)
591
- }
592
- // A quarantined or rejected file holds its place too: uploads waiting for the scan are not a way past the limit.
593
- if (artifact.maxPerOwner !== null && (await table.countHeld(owner)) >= artifact.maxPerOwner) {
594
- throw tooMany(collection)
595
- }
596
- const { head, stream } = await readHead(request.body ?? new Blob([]).stream(), SNIFF_BYTES)
597
- if (!bytesAreOfType(type, head)) {
598
- await stream.cancel()
599
- throw new HttpError(415, 'unsupported_media_type', `The bytes of the file are not ${type}`)
600
- }
601
- const id = uuidv7()
602
- const objectKey = `${collection.name}/${id}`
603
- await table.begin(owner, { id, objectKey, contentType: type, originalName: originalName(request), createdAt: new Date() })
604
- const body = digesting(stream, UPLOAD_TIMEOUT_MS)
605
- try {
606
- const { size } = await this.#store.write(objectKey, body.stream, { contentType: type, maxBytes: artifact.maxBytes })
607
- if (size === 0 && artifact.scan === true) {
608
- // A scanner gives no verdict on nothing, and a file goes live only on a clean verdict.
609
- throw new HttpError(400, 'empty', `files '${collection.name}' scans its files and admits no empty file`)
610
- }
611
- await table.complete(owner, id, { size, sha256: body.digest() }, artifact.scan === true ? 'quarantined' : 'live')
612
- } catch (error) {
613
- if (isNamed(error, 'ObjectExistsError')) {
614
- // The key holds an object this upload did not write: never delete it, only the pending row.
615
- await table.abandon(id)
616
- throw new ServerFault('file_key_taken', `files '${collection.name}' found an object already under a new file's key`)
617
- }
618
- await this.#abandon(collection, id, objectKey)
619
- if (isNamed(error, 'ObjectTooLargeError')) {
620
- throw tooLarge(collection)
621
- }
622
- if (body.timedOut()) {
623
- throw new HttpError(408, 'timeout', `The upload did not finish within ${UPLOAD_TIMEOUT_MS / 1000} seconds`)
624
- }
625
- throw error
626
- } finally {
627
- body.stop()
628
- }
629
- // Counted again once held: two concurrent uploads may both have passed the first count.
630
- if (artifact.maxPerOwner !== null && (await table.countHeld(owner)) > artifact.maxPerOwner) {
631
- await table.retract(owner, id, new Date())
632
- await this.#sweep(collection)
633
- throw tooMany(collection)
634
- }
635
- return jsonResponse({ result: { id } }, 200)
636
- }
637
-
638
- /**
639
- * Deletes the object, then the pending row; a failure of either leaves the row for the sweep. The log names the step
640
- * that failed, never the error: a storage or driver error's name and message are not the framework's.
641
- */
642
- async #abandon(collection: Collection, id: string, objectKey: string): Promise<void> {
643
- let step: FileCleanupStep = 'object_delete_failed'
644
- try {
645
- await this.#store.delete(objectKey)
646
- step = 'row_delete_failed'
647
- await collection.table.abandon(id)
648
- } catch {
649
- console.error(`fabrika: files '${collection.name}' cleanup: ${step}`)
650
- }
651
- }
652
-
653
- /** Only a live file is served: a quarantined one answers 409 until the scan pass decides, a rejected one 404. */
654
- async #download(collection: Collection, target: unknown): Promise<FileDownload> {
655
- const file = await collection.table.find(target)
656
- if (file.status === 'quarantined') {
657
- throw new HttpError(409, 'quarantined', 'The file is being checked for malware; try again in a few minutes')
658
- }
659
- if (file.status === 'rejected') {
660
- throw new NotFoundError()
661
- }
662
- const object = collection.artifact.scan === true ? await this.#scannedObject(collection, target, file) : await this.#store.read(file.objectKey)
663
- if (object === null) {
664
- throw new ServerFault('file_object_missing', `the object of a file in '${collection.name}' is missing from storage`)
665
- }
666
- if (object.size !== file.size) {
667
- await object.body.cancel()
668
- throw new ServerFault('file_object_mismatch', `the object of a file in '${collection.name}' has another size than its row records`)
669
- }
670
- return new FileDownload({ body: object.body, size: file.size, contentType: file.contentType, name: file.originalName })
671
- }
672
-
673
- /**
674
- * The object of a live file of a scanning collection, only as the version the scan passed, in one conditional read:
675
- * no byte of a replaced object is served. A file without that version — replaced, or live before the collection
676
- * scanned — is quarantined again, so the scan job decides about the bytes stored now.
677
- */
678
- async #scannedObject(collection: Collection, target: unknown, file: StoredFile): Promise<StoredObject> {
679
- const object = file.objectVersion === null ? null : await this.#store.readVersion(file.objectKey, file.objectVersion)
680
- if (object === null) {
681
- await collection.table.requarantine(target, file.objectVersion)
682
- throw new HttpError(409, 'quarantined', 'The file is being checked for malware; try again in a few minutes')
683
- }
684
- return object
685
- }
686
-
687
- async #list(collection: Collection, owner: unknown, input: Readonly<Record<string, unknown>>): Promise<z.input<typeof FileListOutput>> {
688
- // The dispatcher parsed the input with `FileListInput` (`FileLinkListInput` for a link holder); the target is the owner.
689
- const after = input['after']
690
- const limit = input['limit']
691
- const page = await collection.table.list(owner, {
692
- after: typeof after === 'string' ? after : null,
693
- limit: typeof limit === 'number' ? limit : FILE_LIMITS.listPageDefault,
694
- })
695
- return {
696
- items: page.items.map((file) => ({ ...file, createdAt: file.createdAt.toISOString() })),
697
- next: page.next,
698
- }
699
- }
700
-
701
- async #remove(collection: Collection, target: unknown): Promise<{ readonly id: string }> {
702
- const grant = this.#capabilities.resolve(target)
703
- if (grant.kind !== 'ref') {
704
- throw new ServerFault('contract_inconsistent', `files '${collection.name}' remove was handed a Within`)
705
- }
706
- await collection.table.markRemoved(target, new Date())
707
- await this.#sweep(collection)
708
- return { id: grant.id }
709
- }
710
-
711
- /**
712
- * Deletes the objects of one batch of removed, orphaned and abandoned files, then their rows. Retry-safe: a row stays
713
- * until its object is gone, so a failed delete is retried by the next sweep. It never fails the request that
714
- * triggered it; it answers whether a full batch went (`more`), a smaller one (`done`), or a step failed.
715
- */
716
- async #sweep(collection: Collection): Promise<'more' | 'done' | 'failed'> {
717
- const pendingBefore = new Date(Date.now() - PENDING_GRACE_MS)
718
- let step: FileCleanupStep = 'rows_read_failed'
719
- try {
720
- const batch = await collection.table.removable(SWEEP_BATCH, pendingBefore)
721
- for (const file of batch) {
722
- step = 'object_delete_failed'
723
- await this.#store.delete(file.objectKey)
724
- step = 'row_delete_failed'
725
- await collection.table.purge(file.id, pendingBefore)
726
- }
727
- return batch.length === SWEEP_BATCH ? 'more' : 'done'
728
- } catch {
729
- console.error(`fabrika: files '${collection.name}' sweep: ${step}`)
730
- return 'failed'
731
- }
732
- }
733
-
734
- /**
735
- * Sweeps every collection, batch by batch, until none holds a removable file or `signal` aborts: the body of the sweep
736
- * job. A collection whose sweep failed waits for the next run, and the run fails once every other one was swept.
737
- */
738
- async sweep(signal: AbortSignal): Promise<void> {
739
- const failed: string[] = []
740
- for (const collection of this.#collections) {
741
- let state: 'more' | 'done' | 'failed' = 'more'
742
- while (state === 'more' && !signal.aborted) {
743
- state = await this.#sweep(collection)
744
- }
745
- if (state === 'failed') {
746
- failed.push(collection.name)
747
- }
748
- }
749
- // A form's file nobody attached before its hold ended, or whose submission never committed.
750
- let held: 'more' | 'done' | 'failed' = 'more'
751
- while (held === 'more' && !signal.aborted) {
752
- held = await this.#sweepFormFiles()
753
- }
754
- if (held === 'failed') {
755
- failed.push('received by forms')
756
- }
757
- if (failed.length > 0) {
758
- throw new ServerFault('file_sweep_failed', `the sweep of files ${failed.map((name) => `'${name}'`).join(', ')} failed; the next run retries it`)
759
- }
760
- }
761
-
762
- /** Whether a collection scans its uploads, so the scan job runs. */
763
- get scans(): boolean {
764
- return this.#collections.some((collection) => collection.artifact.scan === true)
765
- }
766
-
767
- /** One scan pass over every `scan: true` collection, then their sweep: the body of the scan job. */
768
- async scanPass(scanner: FileScanner | null, reporter: OperationsReporter, signal: AbortSignal): Promise<void> {
769
- const scanning = this.#collections.filter((collection) => collection.artifact.scan === true)
770
- await runScanPass({ collections: scanning, store: this.#store, scanner, reporter, signal })
771
- for (const collection of scanning) {
772
- await this.#sweep(collection)
773
- }
774
- }
775
- }
776
-
777
- function tooLarge(collection: Collection): HttpError {
778
- return new HttpError(413, 'too_large', `files '${collection.name}' admits at most ${collection.artifact.maxBytes} bytes per file`)
779
- }
780
-
781
- function tooMany(collection: Collection): HttpError {
782
- return new HttpError(
783
- 409,
784
- 'limit',
785
- `files '${collection.name}' admits at most ${collection.artifact.maxPerOwner} files per ${collection.artifact.owner}`,
786
- )
787
- }
788
-
789
- /**
790
- * The file collections of `artifact`, or `null` when it declares none. Every reason a collection cannot be served
791
- * goes into `problems`, so `boot` refuses to start rather than answer an upload it cannot check.
792
- */
793
- /** Whether a collection's `link` reaches its files through the link mount `mount`. */
794
- const servesLink = (link: boolean | readonly string[], mount: string): boolean => (typeof link === 'boolean' ? link : link.includes(mount))
795
-
796
- export function fileCollections(
797
- artifact: ContractArtifact,
798
- database: ImplementationDatabase | null,
799
- store: FileObjectStore | undefined,
800
- capabilities: CapabilityAuthority,
801
- mint: CapabilityMint,
802
- problems: string[],
803
- ): FileCollections | null {
804
- const entries = Object.entries(artifact.files)
805
- if (entries.length === 0) {
806
- return null
807
- }
808
- if (store === undefined) {
809
- problems.push('files are declared, and the runtime adapter supplies no object storage (FABRIKA_APP_STORAGE_* on Zerops)')
810
- }
811
- if (database === null || !isScopedDatabase(database)) {
812
- problems.push('files need the scoped(tables) data layer of @fabrika/db: a file row lives beside its owner row')
813
- return null
814
- }
815
- const tables = fileTablesOf(database)
816
- const collections: Collection[] = []
817
- for (const [name, collection] of entries) {
818
- if (fileTableName(name) === null) {
819
- problems.push(`files '${name}' needs a lowercase snake_case name of at most 39 characters: it names the table fabrika_files_<name>`)
820
- continue
821
- }
822
- for (const type of collection.types.filter((candidate) => !SNIFFABLE_TYPES.includes(candidate))) {
823
- problems.push(`files '${name}' admits '${type}', whose bytes the framework cannot verify (allowed: ${SNIFFABLE_TYPES.join(', ')})`)
824
- }
825
- const dimension = `file:${name}`
826
- try {
827
- const table = tables.collection({
828
- name,
829
- owner: collection.owner,
830
- dimension,
831
- actions: { read: collection.read, write: collection.write, delete: collection.delete },
832
- scan: collection.scan === true,
833
- })
834
- const ownerDimension = artifact.dimensions[dimension]?.in ?? null
835
- if (ownerDimension === null) {
836
- problems.push(`files '${name}' has no dimension ${dimension} inside its owner's in the compiled contract`)
837
- continue
838
- }
839
- const linkMounts = Object.entries(artifact.mounts).flatMap(([mount, declared]) =>
840
- declared.kind === 'link' && declared.dimension === ownerDimension && servesLink(collection.link, mount) ? [mount] : []
841
- )
842
- collections.push({ name, artifact: collection, dimension, ownerDimension, table, linkMounts })
843
- } catch (error) {
844
- problems.push(error instanceof Error ? error.message : `files '${name}' cannot be served`)
845
- }
846
- }
847
- return store === undefined ? null : new FileCollections(collections, store, capabilities, mint, frameworkTablesOf(database).formFiles)
848
- }