@fabrika/app 0.5.4 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -9
- package/skill/SKILL.md +11 -10
- package/skill/references/app-setup.md +8 -8
- package/skill/references/files.md +23 -0
- package/skill/references/public-intake.md +98 -16
- package/skill/references/spa.md +5 -3
- package/src/contract/artifact-schema.ts +2 -1
- package/src/contract/build.ts +109 -16
- package/src/contract/context.ts +14 -5
- package/src/contract/declaration.ts +8 -3
- package/src/contract/defaults.ts +27 -0
- package/src/contract/define.ts +15 -1
- package/src/contract/kit.ts +8 -2
- package/src/files/collections.ts +195 -16
- package/src/files/handle.ts +21 -2
- package/src/implement/anonymous.ts +15 -2
- package/src/implement/boot.ts +31 -6
- package/src/implement/dispatch.ts +194 -5
- package/src/implement/implement.ts +6 -2
- package/src/implement/invocation.ts +1 -0
- package/src/implement/multipart.ts +332 -0
- package/src/jobs/runner.ts +38 -15
- package/src/react/testing.ts +3 -0
- package/src/testing/endpoint-access.ts +12 -6
- package/src/testing/kit.ts +16 -3
package/src/contract/defaults.ts
CHANGED
|
@@ -27,6 +27,33 @@ export const FORM_LIMITS = {
|
|
|
27
27
|
/** The input field that carries a form's challenge token; a form with a challenge declares it as a string. */
|
|
28
28
|
export const FORM_CHALLENGE_FIELD = 'challengeToken'
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* A form that takes files is `multipart/form-data`: its input travels as JSON in the first part, named `input`, and each
|
|
32
|
+
* file in a part named after its field. The proxy admits only that type for such a form.
|
|
33
|
+
*/
|
|
34
|
+
export { FORM_MULTIPART_TYPE } from '@fabrika/auth-core'
|
|
35
|
+
export const FORM_INPUT_PART = 'input'
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The bounds of a form's files: at most `maxFields` file fields, a file each; a received file waits `holdMs` for the
|
|
39
|
+
* job that drains the inbox to attach it, then the sweep deletes it; a submission's body must arrive within
|
|
40
|
+
* `timeoutMs`; a part's headers hold at most `maxPartHeaderBytes`.
|
|
41
|
+
*/
|
|
42
|
+
export const FORM_FILE_LIMITS = {
|
|
43
|
+
maxFields: 4,
|
|
44
|
+
holdMs: 7 * 24 * 60 * 60 * 1000,
|
|
45
|
+
timeoutMs: 5 * 60 * 1000,
|
|
46
|
+
maxPartHeaderBytes: 8 * 1024,
|
|
47
|
+
/**
|
|
48
|
+
* A form with files accepts this many submissions per minute unless it declares `rate.perForm`, and its files write
|
|
49
|
+
* at most `bytesPerHour` per hour unless it declares `rate.bytesPerHour` (at most `maxBytesPerHour`). Each file
|
|
50
|
+
* reserves its collection's `maxBytes` before its first byte is written, and gives back what it did not use.
|
|
51
|
+
*/
|
|
52
|
+
perForm: 20,
|
|
53
|
+
bytesPerHour: 512 * 1024 * 1024,
|
|
54
|
+
maxBytesPerHour: 64 * 1024 * 1024 * 1024,
|
|
55
|
+
} as const
|
|
56
|
+
|
|
30
57
|
/** Each challenge provider's fixed browser origins: its widget's script and its frame. */
|
|
31
58
|
export const CHALLENGE_ORIGINS: Readonly<Record<ChallengeProvider, { readonly scriptSrc: readonly string[]; readonly frameSrc: readonly string[] }>> =
|
|
32
59
|
{
|
package/src/contract/define.ts
CHANGED
|
@@ -313,19 +313,32 @@ export interface FormChallengeOption {
|
|
|
313
313
|
readonly siteKey?: string
|
|
314
314
|
}
|
|
315
315
|
|
|
316
|
+
/** A file field of a form: the collection its file lands in (a `scan: true` one), and whether a submission must carry it. */
|
|
317
|
+
export interface FormFileOption {
|
|
318
|
+
readonly collection: string
|
|
319
|
+
/** `true`: a submission without the file is refused (400). Absent or `false`: the column gets `null`. */
|
|
320
|
+
readonly required?: boolean
|
|
321
|
+
}
|
|
322
|
+
|
|
316
323
|
/** What `c.form(path, options)` takes. */
|
|
317
324
|
export type FormOptions<I, T> =
|
|
318
325
|
& {
|
|
319
326
|
readonly input: I & NotLoose<I>
|
|
320
327
|
readonly appendTo: string
|
|
321
328
|
readonly maxBytes?: number
|
|
322
|
-
|
|
329
|
+
/** Per minute; `bytesPerHour` only on a form with files, the bytes all its submissions write per hour. */
|
|
330
|
+
readonly rate?: { readonly perClient?: number; readonly perForm?: number; readonly bytesPerHour?: number }
|
|
323
331
|
readonly honeypot?: string
|
|
324
332
|
readonly challenge?: FormChallengeOption
|
|
325
333
|
/** The cron job each accepted submission kicks, in the background of the constant answer. */
|
|
326
334
|
readonly then?: T
|
|
327
335
|
/** A page on another origin may submit it: `true` from the environment's listed origins, `'*'` from any. */
|
|
328
336
|
readonly crossOrigin?: true | '*'
|
|
337
|
+
/**
|
|
338
|
+
* Files a submission carries beside its input, by field. The form becomes `multipart/form-data` and needs a
|
|
339
|
+
* challenge; each file waits, owned by nobody, until the job that drains the inbox attaches it.
|
|
340
|
+
*/
|
|
341
|
+
readonly files?: { readonly [field: string]: FormFileOption }
|
|
329
342
|
}
|
|
330
343
|
& Owned<'kind' | 'path'>
|
|
331
344
|
|
|
@@ -494,6 +507,7 @@ export function surfaceKit<A extends string, D extends string, CD extends string
|
|
|
494
507
|
challenge: options.challenge,
|
|
495
508
|
then: options.then,
|
|
496
509
|
...(options.crossOrigin === undefined ? {} : { crossOrigin: options.crossOrigin }),
|
|
510
|
+
...(options.files === undefined ? {} : { files: options.files }),
|
|
497
511
|
unknownOptions: unknownOptions(options, 'form'),
|
|
498
512
|
}),
|
|
499
513
|
webhook: (path, options) => ({
|
package/src/contract/kit.ts
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
type CronJob,
|
|
35
35
|
type Export,
|
|
36
36
|
type FormChallengeOption,
|
|
37
|
+
type FormFileOption,
|
|
37
38
|
type FormJobs,
|
|
38
39
|
type FormMount,
|
|
39
40
|
type FormOptions,
|
|
@@ -192,11 +193,14 @@ type InboxTarget<Ix extends AnyDbIndex, T, Fields, H, Ch> = unknown extends OneN
|
|
|
192
193
|
: OneName<T, LiteralInbox>
|
|
193
194
|
|
|
194
195
|
type ExtraFields<T extends string, Extra extends string, Columns extends string> = [Extra] extends [never] ? unknown
|
|
195
|
-
: Unavailable<`the inbox ${T} has no column ${ListOf<Extra>}: each input field is named like its column (columns: ${ListOf<Columns>})`>
|
|
196
|
+
: Unavailable<`the inbox ${T} has no column ${ListOf<Extra>}: each input and file field is named like its column (columns: ${ListOf<Columns>})`>
|
|
196
197
|
|
|
197
198
|
/** The challenge token's field, exempt from the inbox's columns where the form declares a challenge. */
|
|
198
199
|
type ChallengeField<Ch> = [Ch] extends [never] ? never : typeof FORM_CHALLENGE_FIELD
|
|
199
200
|
|
|
201
|
+
/** A form's file fields: each is a column of its inbox too, which receives the held file's id. */
|
|
202
|
+
type FileFields<F> = [F] extends [never] ? never : keyof F
|
|
203
|
+
|
|
200
204
|
/** The builders of the contract's callback: the mounts that take no handler, and `rpc`, which mounts groups. */
|
|
201
205
|
export interface ContractKit<K extends KitTypes> extends Pick<Kit<K['action'], K['dimension'], K['creatable']>, 'spa' | 'health' | 'files'> {
|
|
202
206
|
/** A form that appends each submission to its inbox `appendTo`, one column per input field, and answers a constant. */
|
|
@@ -206,12 +210,14 @@ export interface ContractKit<K extends KitTypes> extends Pick<Kit<K['action'], K
|
|
|
206
210
|
const H extends string = never,
|
|
207
211
|
const Ch extends FormChallengeOption = never,
|
|
208
212
|
const Then extends string = never,
|
|
213
|
+
const F extends { readonly [field: string]: FormFileOption } = never,
|
|
209
214
|
>(
|
|
210
215
|
path: Path,
|
|
211
216
|
options: FormOptions<I, Then> & {
|
|
212
|
-
readonly appendTo: T & NoInfer<InboxTarget<K['index'], T, keyof z.output<I>, H, ChallengeField<Ch>>>
|
|
217
|
+
readonly appendTo: T & NoInfer<InboxTarget<K['index'], T, keyof z.output<I> | FileFields<F>, H, ChallengeField<Ch>>>
|
|
213
218
|
readonly honeypot?: H & NoInfer<OneName<H, LiteralHoneypot>>
|
|
214
219
|
readonly challenge?: Ch
|
|
220
|
+
readonly files?: F
|
|
215
221
|
},
|
|
216
222
|
): FormMount<I, NoInfer<Then>>
|
|
217
223
|
/** The groups' endpoints at `POST <prefix>/<group>.<endpoint>`, to the callers the groups name. */
|
package/src/files/collections.ts
CHANGED
|
@@ -10,10 +10,13 @@ import {
|
|
|
10
10
|
type FileCollectionTable,
|
|
11
11
|
fileTableName,
|
|
12
12
|
fileTablesOf,
|
|
13
|
+
type FormFiles,
|
|
14
|
+
frameworkTablesOf,
|
|
13
15
|
isScopedDatabase,
|
|
14
16
|
QueryRefusedError,
|
|
15
17
|
type Session,
|
|
16
18
|
type StoredFile,
|
|
19
|
+
type WrittenFormFile,
|
|
17
20
|
} from '@fabrika/db/framework'
|
|
18
21
|
import { type FileObjectStore, type StoredObject, uuidv7 } from '@fabrika/platform'
|
|
19
22
|
import { z } from 'zod/v4'
|
|
@@ -25,7 +28,7 @@ import { NO_FACILITIES, type ServedEndpoint } from '../implement/dispatch'
|
|
|
25
28
|
import { ServerFault } from '../implement/faults'
|
|
26
29
|
import type { ContractDatabase, ImplementationDatabase } from '../implement/implement'
|
|
27
30
|
import type { OperationsReporter } from '../jobs/report'
|
|
28
|
-
import type {
|
|
31
|
+
import type { FileCollectionJobHandle, FileCollections as FilesOfSite, FileListing, FileListPage, OpenedFile, OpenOptions } from './handle'
|
|
29
32
|
import { type FileScanner, runScanPass } from './scan'
|
|
30
33
|
import { bytesAreOfType, SNIFF_BYTES, SNIFFABLE_TYPES } from './sniff'
|
|
31
34
|
import { FileDownload } from './treatment'
|
|
@@ -63,6 +66,8 @@ export interface FileSite {
|
|
|
63
66
|
readonly session: () => Promise<Session>
|
|
64
67
|
/** `null` where `removeAll` runs (a mutation, a job's transaction); otherwise why it does not. */
|
|
65
68
|
readonly readOnly: string | null
|
|
69
|
+
/** Whether `attach` runs here: only in a job's transaction, the job that drains a form's inbox. */
|
|
70
|
+
readonly attaches: boolean
|
|
66
71
|
}
|
|
67
72
|
|
|
68
73
|
interface Collection {
|
|
@@ -84,11 +89,27 @@ function mediaType(header: string | null): string | null {
|
|
|
84
89
|
|
|
85
90
|
/** The client's file name, metadata only: at most 255 code points, never half a character, no control characters. */
|
|
86
91
|
export function originalName(request: Request): string | null {
|
|
87
|
-
|
|
92
|
+
return cleanName(new URL(request.url).searchParams.get('name') ?? '')
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function cleanName(raw: string): string | null {
|
|
88
96
|
const name = Array.from(raw.replace(/\p{Cc}/gu, '').trim()).slice(0, MAX_NAME_CODE_POINTS).join('')
|
|
89
97
|
return name === '' ? null : name
|
|
90
98
|
}
|
|
91
99
|
|
|
100
|
+
/** One file part of a form's submission, as the multipart reader hands it over. */
|
|
101
|
+
export interface FormFilePart {
|
|
102
|
+
readonly filename: string | null
|
|
103
|
+
/** The media type the part declares; the bytes must be of it. */
|
|
104
|
+
readonly contentType: string | null
|
|
105
|
+
readonly body: ReadableStream<Uint8Array>
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** A form's file whose object is written under its pending row: what the inbox append holds, or a failure cleans up. */
|
|
109
|
+
export interface ReceivedFormFile extends WrittenFormFile {
|
|
110
|
+
readonly objectKey: string
|
|
111
|
+
}
|
|
112
|
+
|
|
92
113
|
/** The first `size` bytes of `body`, and a stream that yields the whole body again. */
|
|
93
114
|
async function readHead(
|
|
94
115
|
body: ReadableStream<Uint8Array>,
|
|
@@ -135,8 +156,8 @@ async function readHead(
|
|
|
135
156
|
return { head: buffered.subarray(0, size), stream }
|
|
136
157
|
}
|
|
137
158
|
|
|
138
|
-
/** The body as it is stored: hashed on the way, and failed once `timeoutMs` passes before it ends. */
|
|
139
|
-
function digesting(body: ReadableStream<Uint8Array>, timeoutMs: number): {
|
|
159
|
+
/** The body as it is stored: hashed on the way, and failed once `timeoutMs` (`null`: no timer) passes before it ends. */
|
|
160
|
+
function digesting(body: ReadableStream<Uint8Array>, timeoutMs: number | null): {
|
|
140
161
|
readonly stream: ReadableStream<Uint8Array>
|
|
141
162
|
digest(): string
|
|
142
163
|
timedOut(): boolean
|
|
@@ -148,10 +169,12 @@ function digesting(body: ReadableStream<Uint8Array>, timeoutMs: number): {
|
|
|
148
169
|
const stream = body.pipeThrough(
|
|
149
170
|
new TransformStream<Uint8Array, Uint8Array>({
|
|
150
171
|
start(controller) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
172
|
+
if (timeoutMs !== null) {
|
|
173
|
+
timer = setTimeout(() => {
|
|
174
|
+
expired = true
|
|
175
|
+
controller.error(new Error('the upload did not finish in time'))
|
|
176
|
+
}, timeoutMs)
|
|
177
|
+
}
|
|
155
178
|
},
|
|
156
179
|
transform(chunk, controller) {
|
|
157
180
|
hasher.update(chunk)
|
|
@@ -194,17 +217,36 @@ export class FileCollections {
|
|
|
194
217
|
readonly #store: FileObjectStore
|
|
195
218
|
readonly #capabilities: CapabilityAuthority
|
|
196
219
|
readonly #mint: CapabilityMint
|
|
197
|
-
|
|
198
|
-
|
|
220
|
+
/** The files forms received, before they have an owner (`fabrika_form_files`). */
|
|
221
|
+
readonly #formFiles: FormFiles
|
|
222
|
+
|
|
223
|
+
constructor(
|
|
224
|
+
collections: readonly Collection[],
|
|
225
|
+
store: FileObjectStore,
|
|
226
|
+
capabilities: CapabilityAuthority,
|
|
227
|
+
mint: CapabilityMint,
|
|
228
|
+
formFiles: FormFiles,
|
|
229
|
+
) {
|
|
199
230
|
this.#collections = collections
|
|
200
231
|
this.#byDimension = new Map(collections.map((collection) => [collection.dimension, collection]))
|
|
201
232
|
this.#store = store
|
|
202
233
|
this.#capabilities = capabilities
|
|
203
234
|
this.#mint = mint
|
|
235
|
+
this.#formFiles = formFiles
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Whether the contract declares the collection `name`, so a form's file can land in it. */
|
|
239
|
+
has(name: string): boolean {
|
|
240
|
+
return this.#collections.some((collection) => collection.name === name)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** The largest file collection `name` admits; 0 for a collection it does not serve. */
|
|
244
|
+
maxBytesOf(name: string): number {
|
|
245
|
+
return this.#collections.find((collection) => collection.name === name)?.artifact.maxBytes ?? 0
|
|
204
246
|
}
|
|
205
247
|
|
|
206
248
|
/** `ctx.files` of one site: each collection's handle, by its name. */
|
|
207
|
-
site(site: FileSite): FilesOfSite<
|
|
249
|
+
site(site: FileSite): FilesOfSite<FileCollectionJobHandle> {
|
|
208
250
|
const handles = new Map(this.#collections.map((collection) => [collection.name, this.#handle(collection, site)]))
|
|
209
251
|
return (name) => {
|
|
210
252
|
site.check()
|
|
@@ -219,7 +261,7 @@ export class FileCollections {
|
|
|
219
261
|
}
|
|
220
262
|
}
|
|
221
263
|
|
|
222
|
-
#handle(collection: Collection, site: FileSite):
|
|
264
|
+
#handle(collection: Collection, site: FileSite): FileCollectionJobHandle {
|
|
223
265
|
return Object.freeze({
|
|
224
266
|
list: async (owner: unknown, page: FileListPage = {}) => {
|
|
225
267
|
site.check()
|
|
@@ -239,9 +281,138 @@ export class FileCollections {
|
|
|
239
281
|
this.#ownerGrant(collection, site, owner, ['delete'])
|
|
240
282
|
return await collection.table.removeOwned(owner, new Date(), await site.session())
|
|
241
283
|
},
|
|
284
|
+
attach: async (owner: unknown, id: unknown) => {
|
|
285
|
+
site.check()
|
|
286
|
+
if (!site.attaches) {
|
|
287
|
+
throw new QueryRefusedError(
|
|
288
|
+
'undeclared_reach',
|
|
289
|
+
`${site.where}: files '${collection.name}' attach runs only in a job's ctx.transaction((tx) => …), as tx.files`,
|
|
290
|
+
)
|
|
291
|
+
}
|
|
292
|
+
const grant = this.#ownerGrant(collection, site, owner, ['write'])
|
|
293
|
+
if (typeof id !== 'string') {
|
|
294
|
+
throw new QueryRefusedError('invalid_argument', `${site.where}: files '${collection.name}' attach takes the id of a form's file`)
|
|
295
|
+
}
|
|
296
|
+
return await this.#attach(collection, grant, owner, id, await site.session())
|
|
297
|
+
},
|
|
242
298
|
})
|
|
243
299
|
}
|
|
244
300
|
|
|
301
|
+
/**
|
|
302
|
+
* Moves the file a form received under `id` into the collection, owned by `owner`, in `session`: the transaction
|
|
303
|
+
* that wrote the owner, so a rollback leaves the file held. A scanning collection holds it `quarantined` until the
|
|
304
|
+
* scan job's verdict. `false` when no held file of this collection has the id (never received, of another collection,
|
|
305
|
+
* or its hold ended); `true` again for a file this owner attached before, so a retried attempt answers alike.
|
|
306
|
+
*/
|
|
307
|
+
async #attach(collection: Collection, grant: RefGrant, owner: unknown, id: string, session: Session): Promise<boolean> {
|
|
308
|
+
const claimed = await this.#formFiles.claim(collection.name, id, session)
|
|
309
|
+
if (claimed === null) {
|
|
310
|
+
const located = UUID.test(id) ? await collection.table.locate(id, session) : null
|
|
311
|
+
return located !== null && located.get(grant.dimension) === grant.id
|
|
312
|
+
&& [...grant.chain].every(([dimension, value]) => located.get(dimension) === value)
|
|
313
|
+
}
|
|
314
|
+
// Thrown, not answered: the transaction rolls back, and the claimed file stays held for a later attempt.
|
|
315
|
+
const limit = collection.artifact.maxPerOwner
|
|
316
|
+
if ((await collection.table.adopt(owner, claimed, session, limit)) === 'limit') {
|
|
317
|
+
throw tooMany(collection)
|
|
318
|
+
}
|
|
319
|
+
// Counted again: an upload of a person may have passed its own count beside this one.
|
|
320
|
+
if (limit !== null && (await collection.table.countHeld(owner, session)) > limit) {
|
|
321
|
+
throw tooMany(collection)
|
|
322
|
+
}
|
|
323
|
+
return true
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Receives one file of a form's submission into collection `name`: the part's declared type must be one the
|
|
328
|
+
* collection admits and its first bytes that type, its row in `fabrika_form_files` is written pending before its
|
|
329
|
+
* object `<name>/<id>`, and the object stops at the collection's `maxBytes`. Answers `null` for an empty part without
|
|
330
|
+
* a file name: a file input the visitor left empty. Whatever fails after the row cleans the object and the row up.
|
|
331
|
+
*/
|
|
332
|
+
async receiveFormFile(form: string, name: string, part: FormFilePart): Promise<ReceivedFormFile | null> {
|
|
333
|
+
const collection = this.#collections.find((candidate) => candidate.name === name)
|
|
334
|
+
if (collection === undefined) {
|
|
335
|
+
await part.body.cancel()
|
|
336
|
+
throw new ServerFault('contract_inconsistent', `form '${form}' takes a file into files '${name}', which is not served`)
|
|
337
|
+
}
|
|
338
|
+
const { artifact } = collection
|
|
339
|
+
const { head, stream } = await readHead(part.body, SNIFF_BYTES)
|
|
340
|
+
if (head.byteLength === 0 && (part.filename === null || part.filename === '')) {
|
|
341
|
+
// The part ended with no byte: it was read to its end, and the next part may follow.
|
|
342
|
+
return null
|
|
343
|
+
}
|
|
344
|
+
const type = part.contentType
|
|
345
|
+
if (type === null || !artifact.types.includes(type)) {
|
|
346
|
+
await stream.cancel()
|
|
347
|
+
throw new HttpError(415, 'unsupported_media_type', `files '${name}' admits ${artifact.types.join(', ')}`)
|
|
348
|
+
}
|
|
349
|
+
if (!bytesAreOfType(type, head)) {
|
|
350
|
+
await stream.cancel()
|
|
351
|
+
throw new HttpError(415, 'unsupported_media_type', `The bytes of the file are not ${type}`)
|
|
352
|
+
}
|
|
353
|
+
if (head.byteLength === 0 && artifact.scan === true) {
|
|
354
|
+
await stream.cancel()
|
|
355
|
+
throw new HttpError(400, 'empty', `files '${name}' scans its files and admits no empty file`)
|
|
356
|
+
}
|
|
357
|
+
const id = uuidv7()
|
|
358
|
+
const objectKey = `${name}/${id}`
|
|
359
|
+
await this.#formFiles.stage({ id, form, collection: name, objectKey, contentType: type, originalName: cleanName(part.filename ?? '') })
|
|
360
|
+
// The form's own deadline bounds the whole body, so the digest needs no timer of its own.
|
|
361
|
+
const body = digesting(stream, null)
|
|
362
|
+
try {
|
|
363
|
+
const { size } = await this.#store.write(objectKey, body.stream, { contentType: type, maxBytes: artifact.maxBytes })
|
|
364
|
+
return Object.freeze({ id, objectKey, size, sha256: body.digest() })
|
|
365
|
+
} catch (error) {
|
|
366
|
+
if (isNamed(error, 'ObjectExistsError')) {
|
|
367
|
+
// The key holds an object this submission did not write: never delete it, only the pending row.
|
|
368
|
+
await this.#formFiles.abandon(id)
|
|
369
|
+
throw new ServerFault('file_key_taken', `files '${name}' found an object already under a new file's key`)
|
|
370
|
+
}
|
|
371
|
+
await this.releaseFormFiles([{ id, objectKey }])
|
|
372
|
+
if (isNamed(error, 'ObjectTooLargeError')) {
|
|
373
|
+
throw tooLarge(collection)
|
|
374
|
+
}
|
|
375
|
+
throw error
|
|
376
|
+
} finally {
|
|
377
|
+
body.stop()
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Deletes the objects of files a submission received and then their pending rows: it did not commit. A failed step is
|
|
383
|
+
* logged by its name and left for the sweep, which deletes a pending row after `PENDING_GRACE_MS`.
|
|
384
|
+
*/
|
|
385
|
+
async releaseFormFiles(files: readonly { readonly id: string; readonly objectKey: string }[]): Promise<void> {
|
|
386
|
+
for (const file of files) {
|
|
387
|
+
let step: FileCleanupStep = 'object_delete_failed'
|
|
388
|
+
try {
|
|
389
|
+
await this.#store.delete(file.objectKey)
|
|
390
|
+
step = 'row_delete_failed'
|
|
391
|
+
await this.#formFiles.abandon(file.id)
|
|
392
|
+
} catch {
|
|
393
|
+
console.error(`fabrika: form files cleanup: ${step}`)
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** One batch of the form files whose hold ended or whose submission never committed: the object, then the row. */
|
|
399
|
+
async #sweepFormFiles(): Promise<'more' | 'done' | 'failed'> {
|
|
400
|
+
let step: FileCleanupStep = 'rows_read_failed'
|
|
401
|
+
try {
|
|
402
|
+
const batch = await this.#formFiles.removable(SWEEP_BATCH, PENDING_GRACE_MS)
|
|
403
|
+
for (const file of batch) {
|
|
404
|
+
step = 'object_delete_failed'
|
|
405
|
+
await this.#store.delete(file.objectKey)
|
|
406
|
+
step = 'row_delete_failed'
|
|
407
|
+
await this.#formFiles.purge(file.id, PENDING_GRACE_MS)
|
|
408
|
+
}
|
|
409
|
+
return batch.length === SWEEP_BATCH ? 'more' : 'done'
|
|
410
|
+
} catch {
|
|
411
|
+
console.error(`fabrika: form files sweep: ${step}`)
|
|
412
|
+
return 'failed'
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
245
416
|
/**
|
|
246
417
|
* The grant of `owner`: a `Ref` of the collection's owner, of the work running now, for one of the collection's
|
|
247
418
|
* `actions`: `read` lists and opens, `delete` removes. A write capability alone reads no file's name or bytes.
|
|
@@ -322,12 +493,12 @@ export class FileCollections {
|
|
|
322
493
|
locating(database: ContractDatabase): ContractDatabase {
|
|
323
494
|
return {
|
|
324
495
|
tables: database.tables,
|
|
325
|
-
locate: (dimension, id) => {
|
|
496
|
+
locate: (dimension, id, using) => {
|
|
326
497
|
const collection = this.#byDimension.get(dimension)
|
|
327
498
|
if (collection === undefined) {
|
|
328
|
-
return database.locate(dimension, id)
|
|
499
|
+
return database.locate(dimension, id, using)
|
|
329
500
|
}
|
|
330
|
-
return UUID.test(id) ? collection.table.locate(id) : Promise.resolve(null)
|
|
501
|
+
return UUID.test(id) ? collection.table.locate(id, using) : Promise.resolve(null)
|
|
331
502
|
},
|
|
332
503
|
}
|
|
333
504
|
}
|
|
@@ -575,6 +746,14 @@ export class FileCollections {
|
|
|
575
746
|
failed.push(collection.name)
|
|
576
747
|
}
|
|
577
748
|
}
|
|
749
|
+
// A form's file nobody attached before its hold ended, or whose submission never committed.
|
|
750
|
+
let held: 'more' | 'done' | 'failed' = 'more'
|
|
751
|
+
while (held === 'more' && !signal.aborted) {
|
|
752
|
+
held = await this.#sweepFormFiles()
|
|
753
|
+
}
|
|
754
|
+
if (held === 'failed') {
|
|
755
|
+
failed.push('received by forms')
|
|
756
|
+
}
|
|
578
757
|
if (failed.length > 0) {
|
|
579
758
|
throw new ServerFault('file_sweep_failed', `the sweep of files ${failed.map((name) => `'${name}'`).join(', ')} failed; the next run retries it`)
|
|
580
759
|
}
|
|
@@ -665,5 +844,5 @@ export function fileCollections(
|
|
|
665
844
|
problems.push(error instanceof Error ? error.message : `files '${name}' cannot be served`)
|
|
666
845
|
}
|
|
667
846
|
}
|
|
668
|
-
return store === undefined ? null : new FileCollections(collections, store, capabilities, mint)
|
|
847
|
+
return store === undefined ? null : new FileCollections(collections, store, capabilities, mint, frameworkTablesOf(database).formFiles)
|
|
669
848
|
}
|
package/src/files/handle.ts
CHANGED
|
@@ -62,7 +62,7 @@ export interface FileCollectionReader {
|
|
|
62
62
|
}
|
|
63
63
|
|
|
64
64
|
/** A site that writes also removes: a mutation, and a job inside `ctx.transaction`. */
|
|
65
|
-
export interface
|
|
65
|
+
export interface FileCollectionWriter extends FileCollectionReader {
|
|
66
66
|
/**
|
|
67
67
|
* Marks every file of the owner removed, in this site's transaction, and answers how many; `owner` is a `Ref` for the
|
|
68
68
|
* collection's `delete` action. The framework's sweep deletes their objects after the commit.
|
|
@@ -70,8 +70,27 @@ export interface FileCollectionHandle extends FileCollectionReader {
|
|
|
70
70
|
removeAll(owner: Ref<string, string>): Promise<number>
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/** A mutation's handle: it writes, and attaches nothing, since only the job that drains a form's inbox does. */
|
|
74
|
+
export type FileCollectionHandle = FileCollectionWriter & {
|
|
75
|
+
readonly attach: Unavailable<"only a job attaches a form's file, inside ctx.transaction((tx) => …), as tx.files">
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** A job's transaction also attaches the files a form received: the job that drains the form's inbox. */
|
|
79
|
+
export interface FileCollectionJobHandle extends FileCollectionWriter {
|
|
80
|
+
/**
|
|
81
|
+
* Attaches the file a form received under `id` (the id its inbox column holds) to `owner`, a `Ref` for the
|
|
82
|
+
* collection's `write` action, in this site's transaction: the one that wrote the owner, located with `tx.ref`. A
|
|
83
|
+
* scanning collection holds it `quarantined` until the scan job's verdict. `false` when no file of this collection
|
|
84
|
+
* waits under the id (none was sent, or its hold of 7 days ended); `true` again for one this owner attached before.
|
|
85
|
+
*/
|
|
86
|
+
attach(owner: Ref<string, string>, id: string): Promise<boolean>
|
|
87
|
+
}
|
|
88
|
+
|
|
73
89
|
/** A site that only reads: a query, and a job outside `ctx.transaction`. */
|
|
74
|
-
export type FileCollectionReadHandle<Why extends string> = FileCollectionReader & {
|
|
90
|
+
export type FileCollectionReadHandle<Why extends string> = FileCollectionReader & {
|
|
91
|
+
readonly removeAll: Unavailable<Why>
|
|
92
|
+
readonly attach: Unavailable<Why>
|
|
93
|
+
}
|
|
75
94
|
|
|
76
95
|
/** `ctx.files(<collection>)`: the handle of one collection of the contract, by its name in `files`. */
|
|
77
96
|
export type FileCollections<H> = (collection: string) => H
|
|
@@ -6,15 +6,28 @@ import { LINK_BOOTSTRAP_FILE, SHARE_KEY_GLOBAL } from '../contract/defaults'
|
|
|
6
6
|
import { HttpError } from '../errors'
|
|
7
7
|
import { readBoundedBytes } from './bounded'
|
|
8
8
|
|
|
9
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* The body as bytes, read until `maxBytes`; a longer one stops the read and is a 413. A body whose stream fails before
|
|
11
|
+
* its end (the client closed the connection) is the sender's 400 `aborted`, never a failure of the application.
|
|
12
|
+
*/
|
|
10
13
|
export async function readBoundedBody(request: Request, maxBytes: number): Promise<Uint8Array<ArrayBuffer>> {
|
|
11
|
-
|
|
14
|
+
let body: Uint8Array<ArrayBuffer> | null
|
|
15
|
+
try {
|
|
16
|
+
body = await readBoundedBytes(request.body, { maxBytes, declaredLength: request.headers.get('Content-Length') })
|
|
17
|
+
} catch {
|
|
18
|
+
throw abortedBody()
|
|
19
|
+
}
|
|
12
20
|
if (body === null) {
|
|
13
21
|
throw new HttpError(413, 'too_large', `The body is larger than ${maxBytes} bytes`)
|
|
14
22
|
}
|
|
15
23
|
return body
|
|
16
24
|
}
|
|
17
25
|
|
|
26
|
+
/** What a request answers whose body did not arrive whole: nobody reads it, and no failure is reported. */
|
|
27
|
+
export function abortedBody(): HttpError {
|
|
28
|
+
return new HttpError(400, 'aborted', 'The body could not be read to its end')
|
|
29
|
+
}
|
|
30
|
+
|
|
18
31
|
/**
|
|
19
32
|
* Runs before any other script of the share SPA: it takes the key out of the fragment, removes the fragment from the
|
|
20
33
|
* address with `history.replaceState`, and keeps the key in a closure. Telemetry that reads `location` later finds
|
package/src/implement/boot.ts
CHANGED
|
@@ -41,11 +41,11 @@ import type { FileObjectStore } from '@fabrika/platform'
|
|
|
41
41
|
import { compileContract } from '../contract/compile'
|
|
42
42
|
import { exportDescription } from '../contract/connect'
|
|
43
43
|
import { isProcedure, type ProcedureDeclaration, type ProcedureTreeDeclaration, SURFACE_SECTIONS } from '../contract/declaration'
|
|
44
|
-
import { exportBase, FORM_CHALLENGE_FIELD, FORM_LIMITS, spaPrefixRowId } from '../contract/defaults'
|
|
44
|
+
import { exportBase, FORM_CHALLENGE_FIELD, FORM_FILE_LIMITS, FORM_LIMITS, spaPrefixRowId } from '../contract/defaults'
|
|
45
45
|
import type { Schema } from '../contract/define'
|
|
46
46
|
import { kitOriginOf } from '../contract/handled'
|
|
47
47
|
import type { RequestExecutionContext } from '../fetch'
|
|
48
|
-
import { FILE_SWEEP_JOB, FILE_SWEEP_SCHEDULE, FILE_SWEEP_TIMEOUT_MS, fileCollections } from '../files/collections'
|
|
48
|
+
import { FILE_SWEEP_JOB, FILE_SWEEP_SCHEDULE, FILE_SWEEP_TIMEOUT_MS, type FileCollections, fileCollections } from '../files/collections'
|
|
49
49
|
import { FILE_SCAN_JOB, FILE_SCAN_SCHEDULE, type FileScanner, SCAN_LIMITS } from '../files/scan'
|
|
50
50
|
import { type MailStandIn, resolveMailTransports } from '../jobs/mail'
|
|
51
51
|
import { logReporter, type OperationsReporter, operationsReporter, operationsSettings, type ReportSource } from '../jobs/report'
|
|
@@ -126,6 +126,8 @@ export interface BootOptions {
|
|
|
126
126
|
* Turnstile's always-passing test widget hands out for every submission. Every other token stays single use.
|
|
127
127
|
*/
|
|
128
128
|
readonly repeatableChallengeToken?: string
|
|
129
|
+
/** How long a form's multipart body may take to arrive; `FORM_FILE_LIMITS.timeoutMs` unless a framework test shortens it. */
|
|
130
|
+
readonly formFileTimeoutMs?: number
|
|
129
131
|
/**
|
|
130
132
|
* The developer's own machine only (`fabrika app dev`): what a job's handler threw, which every log of a deployed
|
|
131
133
|
* application withholds because it may carry a credential.
|
|
@@ -518,7 +520,8 @@ function frameworkProblems(artifact: ContractArtifact, framework: FrameworkTable
|
|
|
518
520
|
const fields = (input?.type === 'object' ? Object.keys(input.fields) : []).filter((field) =>
|
|
519
521
|
field !== submitted.honeypot && !(submitted.challenge && field === FORM_CHALLENGE_FIELD)
|
|
520
522
|
)
|
|
521
|
-
|
|
523
|
+
// A file field's column receives the id of the file the submission carried.
|
|
524
|
+
problems.push(...framework.inboxProblems(name, [...fields, ...Object.keys(mount.files ?? {})]))
|
|
522
525
|
}
|
|
523
526
|
}
|
|
524
527
|
}
|
|
@@ -535,11 +538,26 @@ function submissionFieldsOf(mount: FormMountArtifact): SubmissionFields {
|
|
|
535
538
|
* A form's admission from its approved mount: the declared rate or the defaults, and a challenge verified with the
|
|
536
539
|
* secret the application variable it names holds. Without that variable the application does not boot.
|
|
537
540
|
*/
|
|
538
|
-
function formAdmission(
|
|
541
|
+
function formAdmission(
|
|
542
|
+
name: string,
|
|
543
|
+
artifact: ContractArtifact,
|
|
544
|
+
options: BootOptions,
|
|
545
|
+
files: FileCollections | null,
|
|
546
|
+
problems: string[],
|
|
547
|
+
): FormAdmission | null {
|
|
539
548
|
const mount = artifact.mounts[name]
|
|
540
549
|
if (mount?.kind !== 'form') {
|
|
541
550
|
return null
|
|
542
551
|
}
|
|
552
|
+
const fileFields = new Map(Object.entries(mount.files ?? {}))
|
|
553
|
+
for (const [field, file] of fileFields) {
|
|
554
|
+
if (files === null || !files.has(file.collection)) {
|
|
555
|
+
problems.push(`form '${name}' takes its file '${field}' into files '${file.collection}', which this runtime does not serve`)
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
if (fileFields.size > 0 && mount.challenge === undefined) {
|
|
559
|
+
problems.push(`form '${name}' takes files and declares no challenge; an anonymous upload needs a solved one`)
|
|
560
|
+
}
|
|
543
561
|
let challenge: ChallengeVerifier | null = null
|
|
544
562
|
if (mount.challenge !== undefined) {
|
|
545
563
|
const secret: unknown = options.environment.application[mount.challenge.secret]
|
|
@@ -553,8 +571,15 @@ function formAdmission(name: string, artifact: ContractArtifact, options: BootOp
|
|
|
553
571
|
}
|
|
554
572
|
return Object.freeze({
|
|
555
573
|
perClient: mount.rate?.perClient ?? FORM_LIMITS.perClient,
|
|
556
|
-
|
|
574
|
+
// A form with files accepts fewer submissions by default: each may write its collections' maxBytes.
|
|
575
|
+
perForm: mount.rate?.perForm ?? (fileFields.size === 0 ? FORM_LIMITS.perForm : FORM_FILE_LIMITS.perForm),
|
|
557
576
|
fields: Object.freeze(submissionFieldsOf(mount)),
|
|
577
|
+
files: fileFields.size === 0 || files === null ? null : {
|
|
578
|
+
fields: fileFields,
|
|
579
|
+
collections: files,
|
|
580
|
+
bytesPerHour: mount.rate?.bytesPerHour ?? FORM_FILE_LIMITS.bytesPerHour,
|
|
581
|
+
timeoutMs: options.formFileTimeoutMs ?? FORM_FILE_LIMITS.timeoutMs,
|
|
582
|
+
},
|
|
558
583
|
challenge,
|
|
559
584
|
repeatableToken: challenge === null ? null : (options.repeatableChallengeToken ?? null),
|
|
560
585
|
})
|
|
@@ -719,7 +744,7 @@ export function boot(app: ContractImplementation, options: BootOptions): Contrac
|
|
|
719
744
|
}
|
|
720
745
|
} else if (mount.kind === 'form') {
|
|
721
746
|
const row = rows.get(name)
|
|
722
|
-
const admission = formAdmission(name, artifact, options, problems)
|
|
747
|
+
const admission = formAdmission(name, artifact, options, files, problems)
|
|
723
748
|
const approved = artifact.mounts[name]
|
|
724
749
|
const then = approved?.kind === 'form' ? (approved.then ?? null) : null
|
|
725
750
|
if (row !== undefined && admission !== null) {
|