@fabrika/db 0.6.0 → 0.7.0

package/src/framework-tables.ts

@@ -1,14 +1,15 @@
 // The framework's side of its own tables (`@fabrika/db/migrations`): webhook bindings and the delivery keys that
 // make a replay a no-op. A delivery key is derived from authenticated content (`@fabrika/app` hashes the binding
-// and the signed body), never from a header; the `delivery_id` column holds it. Only `@fabrika/app`'s webhook runtime reaches them, through `frameworkTablesOf`. Every
+// and the signed body), never from a header; the `delivery_id` column holds it, claimed under a lease and completed
+// in the handler's transaction. Only `@fabrika/app`'s webhook runtime reaches them, through `frameworkTablesOf`. Every
 // statement is a guarded tree like a store's, keyed by equalities on the framework table itself.
 
 import { BinaryOperationNode, type OperationNode, OperatorNode, ValueNode } from 'kysely'
-import { sessionOf } from './database'
-import { DatabaseError, QueryRefusedError } from './errors'
+import { runFixedStatement, type Session, sessionOf } from './database'
+import { QueryRefusedError } from './errors'
 import type { GuardContext } from './guard'
 import { WEBHOOK_BINDINGS_TABLE, WEBHOOK_DELIVERIES_TABLE } from './migrations'
-import { acknowledgement, allOf, column, deleteNode, equals, insertNode, locateNode, scopedWhere } from './nodes'
+import { acknowledgement, allOf, column, deleteNode, equals, insertNode, locateNode, scopedWhere, updateNode } from './nodes'
 import type { DataLayerRuntime } from './store'
 
 /** One webhook binding: the scope value it reaches, the provider account it accepts, and its signing secret. */
@@ -17,7 +18,7 @@ export interface WebhookBinding {
 	/** The webhook mount's name. */
 	readonly webhook: string
 	readonly scope: { readonly type: string; readonly value: string }
-	/** `installation:<id>` or `repository:<id>`: the GitHub account a delivery must name. */
+	/** The provider account: for GitHub, `installation:<id>` or `repository:<id>`, which a delivery must name. */
 	readonly account: string
 	readonly secret: string
 }
@@ -28,22 +29,61 @@ export interface NewWebhookBinding extends WebhookBinding {
 }
 
 export interface WebhookTables {
-	createBinding(binding: NewWebhookBinding): Promise<void>
+	/** Records a binding, in `using` when given: the transaction of the procedure that binds. */
+	createBinding(binding: NewWebhookBinding, using?: Session): Promise<void>
 	/** The binding `id` of `webhook`, or `null`. */
 	binding(webhook: string, id: string): Promise<WebhookBinding | null>
-	/** Removes the binding when it belongs to `webhook` and `scope`; `false` when there was none. */
-	removeBinding(webhook: string, id: string, scope: { readonly type: string; readonly value: string }): Promise<boolean>
 	/**
-	 * Records a delivery key; `false` when the binding already received it within the retention window. Keys of the
-	 * binding older than `retainedSince` are dropped first, so the table stays bounded.
+	 * Replaces the secret of the binding when it belongs to `webhook` and `scope`, in `using` when given: a provider that
+	 * issues its own signing secret shows it only after the binding's URL is registered. `false` when there was none.
 	 */
-	recordDelivery(bindingId: string, deliveryKey: string, receivedAt: number, retainedSince: number): Promise<boolean>
-	/** Forgets a delivery whose handler failed, so the provider's redelivery runs it again. */
-	forgetDelivery(bindingId: string, deliveryKey: string): Promise<void>
+	replaceSecret(
+		webhook: string,
+		id: string,
+		scope: { readonly type: string; readonly value: string },
+		secret: string,
+		using?: Session,
+	): Promise<boolean>
+	/** Removes the binding when it belongs to `webhook` and `scope`, in `using` when given; `false` when there was none. */
+	removeBinding(webhook: string, id: string, scope: { readonly type: string; readonly value: string }, using?: Session): Promise<boolean>
+	/**
+	 * Claims a delivery key for `leaseMs`: `null` when the binding completed it within the retention window, or another
+	 * copy holds an unexpired claim. A claim whose lease expired is taken over. Keys of the binding older than
+	 * `retainedSince` are dropped first, so the table stays bounded.
+	 */
+	claimDelivery(bindingId: string, deliveryKey: string, now: number, leaseMs: number, retainedSince: number): Promise<DeliveryClaim | null>
+	/**
+	 * Completes a claimed delivery in `using`, the transaction its handler wrote in, so the work and the completion
+	 * commit together; `false` when another copy took the claim over meanwhile, and that transaction must not commit.
+	 * Without `using` (a handler with no transaction) it completes on its own.
+	 */
+	completeDelivery(claim: DeliveryClaim, using?: Session): Promise<boolean>
+	/** Gives a claim back after its handler failed, so the provider's redelivery runs it again at once. */
+	releaseDelivery(claim: DeliveryClaim): Promise<void>
+}
+
+/** One copy's claim on a delivery: its lease end fences the completion against a copy that took the claim over. */
+export interface DeliveryClaim {
+	readonly bindingId: string
+	readonly key: string
+	readonly leaseUntil: number
 }
 
+const D = WEBHOOK_DELIVERIES_TABLE
+
+/** A new key, or one whose claim expired without completion; a completed one (no lease) is never taken. */
+const SQL_CLAIM = `insert into ${D} (binding_id, delivery_id, received_at, lease_until) values ($1, $2, $3::bigint, $4::bigint)
+on conflict (binding_id, delivery_id) do update set received_at = excluded.received_at, lease_until = excluded.lease_until
+where ${D}.lease_until is not null and ${D}.lease_until < excluded.received_at
+returning delivery_id`
+
+const SQL_COMPLETE = `update ${D} set lease_until = null
+where binding_id = $1 and delivery_id = $2 and lease_until = $3::bigint
+returning delivery_id`
+
+const SQL_RELEASE = `delete from ${D} where binding_id = $1 and delivery_id = $2 and lease_until = $3::bigint returning delivery_id`
+
 const BINDING_COLUMNS = ['id', 'webhook', 'scope_type', 'scope_value', 'account', 'secret']
-const UNIQUE_VIOLATION = '23505'
 
 function guard(root: GuardContext['root'], table: string, scope: OperationNode | undefined): GuardContext {
 	return { root, table, joins: {}, scope }
@@ -57,14 +97,20 @@ function text(row: Readonly<Record<string, unknown>>, column: string): string {
 	return value
 }
 
-function isUniqueViolation(error: unknown): boolean {
-	return error instanceof DatabaseError && error.sqlState === UNIQUE_VIOLATION
+/** The binding `id` of `webhook`, only when it reaches `scope`. */
+function bindingOf(webhook: string, id: string, scope: { readonly type: string; readonly value: string }): OperationNode {
+	return allOf([
+		equals(WEBHOOK_BINDINGS_TABLE, 'id', id),
+		equals(WEBHOOK_BINDINGS_TABLE, 'webhook', webhook),
+		equals(WEBHOOK_BINDINGS_TABLE, 'scope_type', scope.type),
+		equals(WEBHOOK_BINDINGS_TABLE, 'scope_value', scope.value),
+	])
 }
 
 export function webhookTables(runtime: () => DataLayerRuntime): WebhookTables {
 	const session = () => sessionOf(runtime())
 	return Object.freeze({
-		async createBinding(binding: NewWebhookBinding): Promise<void> {
+		async createBinding(binding: NewWebhookBinding, using?: Session): Promise<void> {
 			const values = new Map<string, string | number>([
 				['id', binding.id],
 				['webhook', binding.webhook],
@@ -76,7 +122,7 @@ export function webhookTables(runtime: () => DataLayerRuntime): WebhookTables {
 				['created_at', binding.createdAt],
 			])
 			const node = insertNode(WEBHOOK_BINDINGS_TABLE, values, acknowledgement(WEBHOOK_BINDINGS_TABLE, 'id', undefined))
-			await session().run(node, guard('InsertQueryNode', WEBHOOK_BINDINGS_TABLE, undefined))
+			await (using ?? session()).run(node, guard('InsertQueryNode', WEBHOOK_BINDINGS_TABLE, undefined))
 		},
 
 		async binding(webhook: string, id: string): Promise<WebhookBinding | null> {
@@ -97,21 +143,37 @@ export function webhookTables(runtime: () => DataLayerRuntime): WebhookTables {
 			})
 		},
 
-		async removeBinding(webhook: string, id: string, scope: { readonly type: string; readonly value: string }): Promise<boolean> {
-			const predicate = allOf([
-				equals(WEBHOOK_BINDINGS_TABLE, 'id', id),
-				equals(WEBHOOK_BINDINGS_TABLE, 'webhook', webhook),
-				equals(WEBHOOK_BINDINGS_TABLE, 'scope_type', scope.type),
-				equals(WEBHOOK_BINDINGS_TABLE, 'scope_value', scope.value),
-			])
-			const removed = await session().run(
+		async replaceSecret(
+			webhook: string,
+			id: string,
+			scope: { readonly type: string; readonly value: string },
+			secret: string,
+			using?: Session,
+		): Promise<boolean> {
+			const predicate = bindingOf(webhook, id, scope)
+			const replaced = await (using ?? session()).run(
+				updateNode(
+					WEBHOOK_BINDINGS_TABLE,
+					new Map([['secret', secret]]),
+					undefined,
+					scopedWhere(undefined, predicate),
+					acknowledgement(WEBHOOK_BINDINGS_TABLE, 'id', undefined),
+				),
+				guard('UpdateQueryNode', WEBHOOK_BINDINGS_TABLE, predicate),
+			)
+			return replaced.length > 0
+		},
+
+		async removeBinding(webhook: string, id: string, scope: { readonly type: string; readonly value: string }, using?: Session): Promise<boolean> {
+			const predicate = bindingOf(webhook, id, scope)
+			const removed = await (using ?? session()).run(
 				deleteNode(WEBHOOK_BINDINGS_TABLE, scopedWhere(undefined, predicate), 'id'),
 				guard('DeleteQueryNode', WEBHOOK_BINDINGS_TABLE, predicate),
 			)
 			return removed.length > 0
 		},
 
-		async recordDelivery(bindingId: string, deliveryKey: string, receivedAt: number, retainedSince: number): Promise<boolean> {
+		async claimDelivery(bindingId: string, deliveryKey: string, now: number, leaseMs: number, retainedSince: number): Promise<DeliveryClaim | null> {
 			const binding = equals(WEBHOOK_DELIVERIES_TABLE, 'binding_id', bindingId)
 			const expired = BinaryOperationNode.create(
 				column(WEBHOOK_DELIVERIES_TABLE, 'received_at'),
@@ -122,26 +184,19 @@ export function webhookTables(runtime: () => DataLayerRuntime): WebhookTables {
 				deleteNode(WEBHOOK_DELIVERIES_TABLE, scopedWhere(expired, binding), 'delivery_id'),
 				guard('DeleteQueryNode', WEBHOOK_DELIVERIES_TABLE, binding),
 			)
-			const values = new Map<string, string | number>([['binding_id', bindingId], ['delivery_id', deliveryKey], ['received_at', receivedAt]])
-			const node = insertNode(WEBHOOK_DELIVERIES_TABLE, values, acknowledgement(WEBHOOK_DELIVERIES_TABLE, 'delivery_id', undefined))
-			try {
-				await session().run(node, guard('InsertQueryNode', WEBHOOK_DELIVERIES_TABLE, undefined))
-				return true
-			} catch (error) {
-				// The primary key decides between two concurrent copies of one delivery: exactly one insert wins.
-				if (isUniqueViolation(error)) {
-					return false
-				}
-				throw error
-			}
+			const leaseUntil = now + leaseMs
+			// The primary key decides between concurrent copies of one delivery: exactly one inserts or takes over.
+			const claimed = await runFixedStatement(session(), SQL_CLAIM, [bindingId, deliveryKey, now, leaseUntil])
+			return claimed.length === 0 ? null : Object.freeze({ bindingId, key: deliveryKey, leaseUntil })
 		},
 
-		async forgetDelivery(bindingId: string, deliveryKey: string): Promise<void> {
-			const predicate = allOf([equals(WEBHOOK_DELIVERIES_TABLE, 'binding_id', bindingId), equals(WEBHOOK_DELIVERIES_TABLE, 'delivery_id', deliveryKey)])
-			await session().run(
-				deleteNode(WEBHOOK_DELIVERIES_TABLE, scopedWhere(undefined, predicate), 'delivery_id'),
-				guard('DeleteQueryNode', WEBHOOK_DELIVERIES_TABLE, predicate),
-			)
+		async completeDelivery(claim: DeliveryClaim, using?: Session): Promise<boolean> {
+			const completed = await runFixedStatement(using ?? session(), SQL_COMPLETE, [claim.bindingId, claim.key, claim.leaseUntil])
+			return completed.length > 0
+		},
+
+		async releaseDelivery(claim: DeliveryClaim): Promise<void> {
+			await runFixedStatement(session(), SQL_RELEASE, [claim.bindingId, claim.key, claim.leaseUntil])
 		},
 	})
 }

package/src/framework.ts

@@ -2,11 +2,11 @@
 // the data layer an application passed to `implement` to the framework's connection and capability authority, and
 // hands the framework the locator its access check uses. Browser- and Worker-safe: the Worker adapter imports `boot`.
 
-import type { FileTables } from './file-tables'
 import type { DataAccess } from './orm'
 import { frameworkSideOf, type FrameworkTables, type Locator } from './registry'
 import type { DataLayerRuntime } from './store'
 
+export type { Admissions } from './admissions'
 export type { LeaseFence } from './capability'
 export type { InvocationTransaction, Session } from './database'
 export {
@@ -18,27 +18,14 @@ export {
 	type QueryRefusal,
 	QueryRefusedError,
 } from './errors'
-export { fileTableName } from './file-table-sql'
-export type {
-	AdoptedFile,
-	FileCollectionSpec,
-	FileCollectionTable,
-	FilePage,
-	FileTables,
-	HeldStatus,
-	ListedFile,
-	PendingFile,
-	QuarantinedFile,
-	RemovableFile,
-	StoredFile,
-} from './file-tables'
-export type { FormAdmissions, InboxQuota } from './form-admissions'
-export type { ClaimedFormFile, FormFiles, HeldFormFiles, RemovableFormFile, StagedFormFile, WrittenFormFile } from './form-files'
-export type { NewWebhookBinding, WebhookBinding, WebhookTables } from './framework-tables'
+export type { FileBinding } from './file-columns'
+export { type FileHolding, fileHoldingOf, type FileReference, fileReference, holdsFile } from './file-references'
+export type { DeliveryClaim, NewWebhookBinding, WebhookBinding, WebhookTables } from './framework-tables'
 export type { JobLeases, JobLeaseState, JobOutcome, OverdueClaim } from './job-leases'
 export type { DataAccess, DataHandles, DataSite } from './orm'
 export type { FrameworkTables, Locator } from './registry'
 export type { SqlMode, SqlSite, SqlTag, SqlValue } from './sql'
+export type { FileColumnRef, QuarantinedStoredFile, ServedFile, StagedFile, StoredFileMeta, StoredFiles, StoredFileStatus } from './stored-files'
 
 /** Whether `value` is an object `scoped(tables)` created, rather than a look-alike with the same fields. */
 export function isScopedDatabase(value: unknown): boolean {
@@ -57,18 +44,9 @@ export function bindScopedDatabase(value: unknown, runtime: DataLayerRuntime): L
 	return side.bind(runtime)
 }
 
-/** The file collections' tables of the layer `scoped(tables)` created; each operation needs the layer bound first. */
-export function fileTablesOf(value: unknown): FileTables {
-	const side = frameworkSideOf(value)
-	if (side === undefined) {
-		throw new TypeError('not a data layer scoped(tables) created')
-	}
-	return side.files
-}
-
 /**
- * The framework's own writes over the layer `scoped(tables)` created: inbox appends for `c.form` and the webhook
- * tables. Each operation needs the layer bound first. Throws for any other value.
+ * The framework's own writes over the layer `scoped(tables)` created: the webhook tables, the job leases and the
+ * admission counters. Each operation needs the layer bound first. Throws for any other value.
  */
 export function frameworkTablesOf(value: unknown): FrameworkTables {
 	const side = frameworkSideOf(value)

package/src/guard.ts

@@ -21,9 +21,11 @@ export interface GuardContext {
 	readonly scope: OperationNode | undefined
 	/**
 	 * A select that locks its rows: `share` is the layer's check of a mutation's destination, `update` a mutation's
-	 * `getForUpdate` of its own target. No other lock is admitted.
+	 * `getForUpdate` of its own target, `claim` a queue claim's `for update skip locked`. No other lock is admitted.
 	 */
-	readonly lock?: 'share' | 'update'
+	readonly lock?: 'share' | 'update' | 'claim'
+	/** An insert that does nothing on a conflict of its listed columns: `on conflict (columns) do nothing`, and no other. */
+	readonly onConflict?: 'nothing'
 }
 
 function lookup(...names: string[]): NameSet {
@@ -72,8 +74,10 @@ const NODE_KINDS = lookup(
 	'ReturningNode',
 	// Only as an orderBy direction; checked below.
 	'RawNode',
-	// Only as the one lock of a destination check or a getForUpdate; checked below.
+	// Only as the one lock of a destination check or a getForUpdate, or a claim's two; checked below.
 	'SelectModifierNode',
+	// Only as an insert's `on conflict (columns) do nothing`; checked below.
+	'OnConflictNode',
 	// Only as a selected own column's text, the exact position a keyset page continues from; checked below.
 	'CastNode',
 	'DataTypeNode',
@@ -99,7 +103,8 @@ export function assertAllowedTree(root: OperationNode, context: GuardContext): v
 	for (let index = 0; index < keys.length; index++) {
 		const key = keys[index] ?? ''
 		const lockModifier = key === 'endModifiers' && context.lock !== undefined && context.root === 'SelectQueryNode'
-		if (read(root, key) !== undefined && !inSet(allowed, key) && !lockModifier) {
+		const conflict = key === 'onConflict' && context.onConflict === 'nothing' && context.root === 'InsertQueryNode'
+		if (read(root, key) !== undefined && !inSet(allowed, key) && !lockModifier && !conflict) {
 			throw new QueryRefusedError('statement_shape', `${context.root}.${key} is not allowed`)
 		}
 	}
@@ -236,6 +241,8 @@ function checkNode(node: OperationNode, parent: Visit | undefined, context: Guar
 			return checkRaw(node, parent)
 		case 'SelectModifierNode':
 			return checkLock(node, parent, context)
+		case 'OnConflictNode':
+			return checkConflict(node, parent, context)
 		case 'CastNode':
 			return checkCast(node, parent, context)
 		case 'DataTypeNode':
@@ -265,7 +272,8 @@ function checkNode(node: OperationNode, parent: Visit | undefined, context: Guar
 		case 'ColumnNode':
 			if (
 				parent === undefined
-				|| !(parent.node.kind === 'ReferenceNode' || parent.node.kind === 'InsertQueryNode' || parent.node.kind === 'ColumnUpdateNode')
+				|| !(parent.node.kind === 'ReferenceNode' || parent.node.kind === 'InsertQueryNode' || parent.node.kind === 'ColumnUpdateNode'
+					|| parent.node.kind === 'OnConflictNode')
 			) {
 				throw new QueryRefusedError('statement_shape', 'unqualified column')
 			}
@@ -320,17 +328,43 @@ function checkValue(node: OperationNode): void {
 	}
 }
 
+/** The lock modifiers each lock kind admits, in the order the compiler prints them. */
+const LOCKS: Readonly<Record<'share' | 'update' | 'claim', readonly string[]>> = freeze({
+	share: freeze(['ForShare']),
+	update: freeze(['ForUpdate']),
+	claim: freeze(['ForUpdate', 'SkipLocked']),
+})
+
 function checkLock(node: OperationNode, parent: Visit | undefined, context: GuardContext): void {
 	const modifiers = parent === undefined ? undefined : read(parent.node, 'endModifiers')
-	const onRoot = parent !== undefined && parent.up === undefined && parent.key === 'endModifiers' && isArray(modifiers) && modifiers.length === 1
-	const expected = context.lock === 'share' ? 'ForShare' : context.lock === 'update' ? 'ForUpdate' : undefined
-	const plain = expected !== undefined && read(node, 'modifier') === expected && read(node, 'of') === undefined
+	const expected = context.lock === undefined ? [] : LOCKS[context.lock]
+	const onRoot = parent !== undefined && parent.up === undefined && parent.key === 'endModifiers' && isArray(modifiers)
+		&& modifiers.length === expected.length
+	let position = -1
+	for (let index = 0; onRoot && index < modifiers.length; index++) {
+		if (modifiers[index] === node) {
+			position = index
+		}
+	}
+	const plain = position !== -1 && read(node, 'modifier') === expected[position] && read(node, 'of') === undefined
 		&& read(node, 'rawModifier') === undefined
 	if (!onRoot || !plain) {
 		throw new QueryRefusedError('statement_shape', 'row locks are not allowed')
 	}
 }
 
+/** `on conflict (<columns>) do nothing` on the root insert: no constraint name, index expression, WHERE or update. */
+function checkConflict(node: OperationNode, parent: Visit | undefined, context: GuardContext): void {
+	const columns = read(node, 'columns')
+	const onRoot = parent !== undefined && parent.up === undefined && parent.key === 'onConflict' && context.onConflict === 'nothing'
+	const plain = read(node, 'doNothing') === true && isArray(columns) && columns.length > 0 && read(node, 'constraint') === undefined
+		&& read(node, 'indexExpression') === undefined && read(node, 'indexWhere') === undefined && read(node, 'updates') === undefined
+		&& read(node, 'updateWhere') === undefined
+	if (!onRoot || !plain) {
+		throw new QueryRefusedError('statement_shape', 'an insert admits only on conflict (columns) do nothing')
+	}
+}
+
 /** `cast(<own column> as text) as <alias>`, selected at the top of a select: nothing else casts. */
 function checkCast(node: OperationNode, parent: Visit | undefined, context: GuardContext): void {
 	const selected = parent?.node.kind === 'AliasNode' && parent.key === 'node' && parent.up?.node.kind === 'SelectionNode'

package/src/index.ts

@@ -7,6 +7,8 @@ export type {
 	LeaseFence,
 	Ref,
 	RefGrant,
+	RowGrant,
+	RowRef,
 	Within,
 	WithinGrant,
 } from './capability'
@@ -35,17 +37,6 @@ export {
 	type QueryRefusal,
 	QueryRefusedError,
 } from './errors'
-export {
-	FILE_STATUSES,
-	FILE_TABLE_PREFIX,
-	type FileColumn,
-	type FileColumnType,
-	type FileOwner,
-	fileScanSql,
-	type FileStatus,
-	fileTableName,
-	fileTableSql,
-} from './file-table-sql'
 export type {
 	AnyDbIndex,
 	ChildRows,
@@ -85,8 +76,6 @@ export {
 	defineTables,
 	type DimensionPolicy,
 	type DimensionScope,
-	type InboxPolicy,
-	type InboxScope,
 	type InsertValues,
 	isTables,
 	readTables,

package/src/job-leases.ts

@@ -254,6 +254,17 @@ function refuseEnded(fence: LeaseFence): void {
 	}
 }
 
+const SQL_CLOCK = 'select clock_timestamp() as now'
+
+/**
+ * The database's clock inside `session`'s transaction: a queue claim compares and moves `due` by it, so no process
+ * clock decides when a row is due.
+ */
+export async function databaseNow(session: Session): Promise<Date> {
+	const [row] = await rows(session, SQL_CLOCK, [])
+	return requiredDate(row ?? {}, 'now')
+}
+
 const lost = (fence: LeaseFence): LeaseLostError =>
 	new LeaseLostError(`job '${fence.job}' lost its lease or its attempt's deadline (generation ${fence.generation}); nothing was written`)
 

package/src/links.ts

@@ -1,5 +1,5 @@
 // A Ref's predicate over a table's containment links: the first link takes the Ref's id, every other link a value
-// of its chain, and a chain entry no link covers is refused. Shared by the stores and the file collections' tables.
+// of its chain, and a chain entry no link covers is refused. Shared by the stores and the handles.
 
 import type { OperationNode } from 'kysely'
 import { freeze, inSet, nameSet, ownEntry, ownKeys, recordOf } from './builtins'

package/src/migrations.ts

@@ -13,6 +13,15 @@ export const FORM_ADMISSIONS_TABLE = 'fabrika_form_admissions'
 export const FORM_CHALLENGES_TABLE = 'fabrika_form_challenges'
 export const FORM_FILES_TABLE = 'fabrika_form_files'
 export const FORM_BYTES_TABLE = 'fabrika_form_bytes'
+export const FILES_TABLE = 'fabrika_files'
+export const FILE_DELETIONS_TABLE = 'fabrika_file_deletions'
+/** The function an application's last adoption migration calls with the name of a file collection it adopted. */
+export const FILES_RELEASE_FUNCTION = 'fabrika_files_release'
+/**
+ * The function an adoption migration calls right before it reads a collection's table: it copies the rows an older
+ * instance wrote since 0008 ran, so each one it adopts has its `fabrika_files` row. Idempotent.
+ */
+export const FILES_COPY_FUNCTION = 'fabrika_files_copy_legacy'
 
 export interface FrameworkMigration {
 	/** Ordered by name; an applied one is immutable, like an application's. */
@@ -128,6 +137,123 @@ CREATE TABLE ${FORM_BYTES_TABLE} (
 	bytes BIGINT NOT NULL,
 	PRIMARY KEY (form, window_start)
 );
+`,
+	}),
+	Object.freeze({
+		name: '0007_webhook_delivery_claims.sql',
+		sql: `-- A delivery is claimed with a lease and completed in its handler's transaction, which clears the lease. A claim
+-- whose lease expired without completion is taken over by the provider's redelivery. Every row recorded before has
+-- no lease, so it stays a completed delivery, as does one an older framework release records during a rollout.
+ALTER TABLE ${WEBHOOK_DELIVERIES_TABLE} ADD COLUMN lease_until BIGINT;
+`,
+	}),
+	Object.freeze({
+		name: '0008_files.sql',
+		sql: `-- Every stored file: a value a table's file column references by id. A row is 'pending' from before its object
+-- (<kind>/<id>) is written until the first reference claims it, then 'quarantined' (a scanning kind) or 'live'; the scan
+-- makes a quarantined one 'live' or 'rejected'. scope is the top-level value of its first dimensional reference; legacy
+-- names the collection a migrated file came from, which the sweep never deletes.
+CREATE TABLE ${FILES_TABLE} (
+	id UUID PRIMARY KEY,
+	kind TEXT NOT NULL,
+	status TEXT NOT NULL CONSTRAINT ${FILES_TABLE}_status CHECK (status IN ('pending', 'quarantined', 'live', 'rejected')),
+	object_key TEXT NOT NULL UNIQUE,
+	size BIGINT,
+	sha256 TEXT,
+	content_type TEXT NOT NULL,
+	name TEXT,
+	scope TEXT,
+	request TEXT,
+	created_at TIMESTAMPTZ NOT NULL,
+	object_version TEXT,
+	scan_attempts INTEGER NOT NULL DEFAULT 0,
+	scan_after TIMESTAMPTZ,
+	legacy TEXT
+);
+CREATE INDEX ${FILES_TABLE}_created_at ON ${FILES_TABLE} (created_at) WHERE legacy IS NULL;
+CREATE INDEX ${FILES_TABLE}_scan ON ${FILES_TABLE} (scan_after) WHERE status = 'quarantined';
+CREATE INDEX ${FILES_TABLE}_legacy ON ${FILES_TABLE} (legacy) WHERE legacy IS NOT NULL;
+
+-- An object whose file row is deleted, until the object is: the sweep finishes a deletion a crash interrupted.
+CREATE TABLE ${FILE_DELETIONS_TABLE} (
+	object_key TEXT PRIMARY KEY,
+	recorded_at TIMESTAMPTZ NOT NULL
+);
+
+-- Copies the held rows of a collection's table (fabrika_files_<collection>) into ${FILES_TABLE}, marked legacy with
+-- the collection: same id, object, status and scan state. A row copied before is skipped, and so is an orphan, whose
+-- owner was deleted: no application can adopt it, so the release records its object for deletion.
+CREATE FUNCTION ${FILES_COPY_FUNCTION}(collection TEXT) RETURNS void LANGUAGE plpgsql AS $$
+DECLARE
+	source TEXT := 'fabrika_files_' || collection;
+	scanned BOOLEAN;
+BEGIN
+	IF collection !~ '^[a-z_][a-z0-9_]*$' OR to_regclass(source) IS NULL THEN
+		RAISE EXCEPTION 'no file collection table %', source;
+	END IF;
+	SELECT count(*) = 3 INTO scanned FROM information_schema.columns
+	WHERE table_schema = current_schema() AND table_name = source AND column_name IN ('object_version', 'scan_attempts', 'scan_after');
+	EXECUTE format(
+		'INSERT INTO ${FILES_TABLE} (id, kind, status, object_key, size, sha256, content_type, name, created_at, object_version, scan_attempts, scan_after, legacy) '
+		|| 'SELECT id, $1, status, object_key, size, sha256, content_type, original_name, created_at, %s, $1 FROM %I '
+		|| 'WHERE deleted_at IS NULL AND status <> ''pending'' AND owner_id IS NOT NULL ON CONFLICT (id) DO NOTHING',
+		CASE WHEN scanned THEN 'object_version, scan_attempts, scan_after' ELSE 'NULL, 0, NULL' END,
+		source
+	) USING collection;
+END
+$$;
+
+-- An application's last adoption migration of a collection: it copies what an older version wrote meanwhile, refuses
+-- while a copied file is referenced by no file column, clears the marker of the rest (the sweep may then delete one),
+-- records the objects only the old table knew for deletion, and drops the old table.
+CREATE FUNCTION ${FILES_RELEASE_FUNCTION}(collection TEXT) RETURNS void LANGUAGE plpgsql AS $$
+DECLARE
+	source TEXT := 'fabrika_files_' || collection;
+	reference RECORD;
+	referenced TEXT := 'false';
+	unreferenced BIGINT;
+BEGIN
+	PERFORM ${FILES_COPY_FUNCTION}(collection);
+	FOR reference IN
+		SELECT c.conrelid::regclass::text AS relation, a.attname AS column_name
+		FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
+		WHERE c.contype = 'f' AND c.confrelid = '${FILES_TABLE}'::regclass AND cardinality(c.conkey) = 1
+	LOOP
+		referenced := referenced || format(' OR EXISTS (SELECT 1 FROM %s r WHERE r.%I = f.id)', reference.relation, reference.column_name);
+	END LOOP;
+	EXECUTE format('SELECT count(*) FROM ${FILES_TABLE} f WHERE f.legacy = $1 AND NOT (%s)', referenced) INTO unreferenced USING collection;
+	IF unreferenced > 0 THEN
+		RAISE EXCEPTION '${FILES_RELEASE_FUNCTION}(%): % files of the collection are referenced by no file column; adopt them in a migration first', collection, unreferenced;
+	END IF;
+	UPDATE ${FILES_TABLE} SET legacy = NULL WHERE legacy = collection;
+	EXECUTE format(
+		'INSERT INTO ${FILE_DELETIONS_TABLE} (object_key, recorded_at) SELECT s.object_key, now() FROM %I s '
+		|| 'WHERE NOT EXISTS (SELECT 1 FROM ${FILES_TABLE} f WHERE f.object_key = s.object_key) ON CONFLICT (object_key) DO NOTHING',
+		source
+	);
+	EXECUTE format('DROP TABLE %I', source);
+END
+$$;
+
+-- Every collection table this schema holds now. A held form file moves too: its kind is its collection, and the grace
+-- counts from the end of its hold, so the sweep keeps it as long as the hold did.
+DO $$
+DECLARE
+	found RECORD;
+BEGIN
+	FOR found IN
+		SELECT substr(table_name, length('fabrika_files_') + 1) AS name FROM information_schema.tables
+		WHERE table_schema = current_schema() AND table_type = 'BASE TABLE' AND table_name LIKE 'fabrika\\_files\\_%'
+	LOOP
+		PERFORM ${FILES_COPY_FUNCTION}(found.name);
+	END LOOP;
+	INSERT INTO ${FILES_TABLE} (id, kind, status, object_key, size, sha256, content_type, name, created_at)
+	SELECT f.id, f.collection, 'quarantined', f.object_key, f.size, f.sha256, f.content_type, f.original_name, f.held_until
+	FROM ${FORM_FILES_TABLE} f WHERE f.status = 'held'
+	ON CONFLICT (id) DO NOTHING;
+	DELETE FROM ${FORM_FILES_TABLE} f WHERE status = 'held' AND EXISTS (SELECT 1 FROM ${FILES_TABLE} c WHERE c.id = f.id);
+END
+$$;
 `,
 	}),
 ])

package/src/nodes.ts

@@ -16,6 +16,7 @@ import {
 	JoinNode,
 	LimitNode,
 	OffsetNode,
+	type OnConflictNode,
 	type OperationNode,
 	OperatorNode,
 	OrderByItemNode,
@@ -228,15 +229,24 @@ export function aggregateNode(
 	return Object.freeze(node)
 }
 
-/** `answer` is `undefined` only for an inbox append, which answers nothing. */
-export function insertNode(table: string, values: ReadonlyMap<string, Primitive | null>, answer: ReturningNode | undefined): InsertQueryNode {
+/** `conflict` names the columns of a unique index the insert does nothing on: `on conflict (<columns>) do nothing`. */
+export function insertNode(
+	table: string,
+	values: ReadonlyMap<string, Primitive | null>,
+	answer: ReturningNode,
+	conflict?: readonly string[],
+): InsertQueryNode {
 	const names = [...values.keys()]
+	const onConflict: OnConflictNode | undefined = conflict === undefined
+		? undefined
+		: Object.freeze({ kind: 'OnConflictNode', columns: conflict.map((name) => ColumnNode.create(name)), doNothing: true })
 	const node: InsertQueryNode = {
 		kind: 'InsertQueryNode',
 		into: TableNode.create(table),
 		columns: names.map((name) => ColumnNode.create(name)),
 		values: ValuesNode.create([ValueListNode.create(names.map((name) => ValueNode.create(values.get(name) ?? null)))]),
-		...(answer === undefined ? {} : { returning: answer }),
+		...(onConflict === undefined ? {} : { onConflict }),
+		returning: answer,
 	}
 	return Object.freeze(node)
 }
@@ -331,6 +341,28 @@ export function withCursorColumns(node: SelectQueryNode, table: string, columns:
 	return Object.freeze({ ...node, selections: [...(node.selections ?? []), ...cursor] })
 }
 
+/**
+ * A queue claim's lease: `select <columns> from table where (<conditions> and (<due> is not null and <due> <= $now and
+ * <attempts> < $max)) and scope order by <due>, <key> limit $n for update skip locked`. A row another claim holds is
+ * skipped, never waited for.
+ */
+export function claimNode(
+	state: SelectState,
+	columns: readonly string[],
+	due: { readonly due: string; readonly attempts: string; readonly now: Date; readonly maxAttempts: number },
+	scope: OperationNode,
+	days: NameSet,
+): SelectQueryNode {
+	const table = state.table
+	const waiting = allOf([
+		BinaryOperationNode.create(column(table, due.due), OperatorNode.create('is not'), ValueNode.createImmediate(null)),
+		BinaryOperationNode.create(column(table, due.due), OperatorNode.create('<='), ValueNode.create(due.now)),
+		BinaryOperationNode.create(column(table, due.attempts), OperatorNode.create('<'), ValueNode.create(due.maxAttempts)),
+	])
+	const node = selectNode(state, columns, scope, waiting, days)
+	return Object.freeze({ ...node, endModifiers: [SelectModifierNode.create('ForUpdate'), SelectModifierNode.create('SkipLocked')] })
+}
+
 /** `select <columns> from table where scope for update`: a mutation's precondition read of its own target. */
 export function lockedSelectNode(table: string, columns: readonly string[], scope: OperationNode, days: NameSet = NO_DAYS): SelectQueryNode {
 	const node: SelectQueryNode = {