@fabrika/app 0.4.8 → 0.5.0

package/src/jobs/runner.ts

@@ -6,9 +6,10 @@
 // to Operations, each under a fixed failure code.
 
 import type { IamMembership } from '@fabrika/auth'
-import { type ContractArtifact, JOB_SITE_PREFIX, type JobArtifact, type PermissionEntry, permits } from '@fabrika/auth-core'
+import { type ContractArtifact, JOB_SITE_PREFIX, type JobArtifact, type JobMailer, type PermissionEntry, permits } from '@fabrika/auth-core'
 import type { CapabilityAuthority } from '@fabrika/db/capability'
 import { type DataAccess, type FrameworkTables, type JobLeases, type LeaseFence, LeaseLostError } from '@fabrika/db/framework'
+import { EmailDeliveryError, type EmailSenderCapabilities, mayRetryDelivery } from '@fabrika/email'
 import { NotFoundError } from '../errors'
 import type { FileCollections } from '../files/collections'
 import { type CapabilityMint, CapabilityRefusedError, type Ref, RequestFrame, runInRequest, type Within } from '../implement/capability'
@@ -30,9 +31,10 @@ import {
 } from '../implement/invocation'
 import { jobLinkIssuing, type JobLinkMount } from '../implement/membership'
 import type { WorkloadCredential } from '../implement/workload'
+import { jobMail, type MailTransports } from './mail'
 import type { OperationsReporter } from './report'
 import { type CronSchedule, parseCron, periodAt } from './schedule'
-import type { JobContext, JobPrincipal } from './types'
+import type { AppMail, JobContext, JobPrincipal } from './types'
 
 /** An attempt of a job without a `timeout` is cancelled after this long. */
 export const DEFAULT_JOB_TIMEOUT_MS = 15 * 60_000
@@ -66,6 +68,10 @@ export type JobFailureCode =
 	| 'unauthorized'
 	/** IAM could not be asked for the job's token. */
 	| 'iam_unavailable'
+	/** The process stopped a kicked run while shutting down; the schedule runs the job again. */
+	| 'stopped'
+	/** The transport refused a message the handler did not catch; `reason` is the `EmailDeliveryError` code. */
+	| 'mail_failed'
 
 export type JobRunResult =
 	| { readonly status: 'succeeded'; readonly attempts: number }
@@ -124,11 +130,16 @@ export const systemClock: JobClock = Object.freeze({
 				resolve()
 				return
 			}
-			const timer = setTimeout(resolve, ms)
+			// The listener goes with the wait: a long-lived signal (a process's stop) would otherwise collect one per wait.
+			const done = new AbortController()
+			const timer = setTimeout(() => {
+				done.abort()
+				resolve()
+			}, ms)
 			signal.addEventListener('abort', () => {
 				clearTimeout(timer)
 				resolve()
-			}, { once: true })
+			}, { once: true, signal: done.signal })
 		}),
 	random: () => Math.random(),
 })
@@ -144,6 +155,8 @@ interface BoundJob {
 	readonly site: SiteDeclaration | null
 	/** The link mount the approved artifact says the job `issues`: its `ctx.links`. */
 	readonly links: JobLinkMount | null
+	/** The mail transport the approved artifact says the job reaches: its `ctx.mail`. */
+	readonly mailer: JobMailer | null
 }
 
 interface RunnerContext {
@@ -159,6 +172,7 @@ interface RunnerContext {
 	readonly capabilities: CapabilityAuthority
 	readonly containers: ReadonlyMap<string, readonly string[]>
 	readonly revealHandlerErrors: HandlerErrorSink | null
+	readonly mail: MailTransports
 }
 
 /** Where `fabrika app dev` shows what a handler threw, on the developer's own machine; nothing else sets one. */
@@ -192,6 +206,8 @@ export interface JobBindingInput {
 	readonly containers: ReadonlyMap<string, readonly string[]>
 	/** What a job's handler threw, for `fabrika app dev`; without it the error is only its fixed code. */
 	readonly revealHandlerErrors?: HandlerErrorSink
+	/** The transports of the jobs that declare `mailer`, which `boot` resolved. */
+	readonly mail: MailTransports
 }
 
 interface BoundFrameworkJob {
@@ -210,6 +226,11 @@ export interface BoundJobs {
 	 * waits for no lock, settles when that work has ended and never rejects; the schedule stays the recovery path.
 	 */
 	readonly kick: (name: string) => Promise<void>
+	/**
+	 * Aborts every kicked run in flight; a later kick starts none and only sets the pending flag, for a process that is
+	 * not stopping; the schedule recovers their work.
+	 */
+	readonly stopKicks: () => void
 }
 
 function ownChild(node: unknown, key: string): unknown {
@@ -253,12 +274,23 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
 		const site = isSiteHandler(node) ? approvedDeclaration(where, `${JOB_SITE_PREFIX}${name}`, node.declared, 'job', input.artifact, problems) : null
 		const credential = input.credentials.get(`job:${name}`)
 		const links = jobLinks(where, `${JOB_SITE_PREFIX}${name}`, site, input, problems)
+		const mailer = input.artifact.declared?.[`${JOB_SITE_PREFIX}${name}`]?.mailer ?? null
 		if (typeof handler !== 'function') {
 			problems.push(`${where} has no handler; add it to handlers.jobs`)
 		} else if (site !== null && input.data === null) {
 			problems.push(`${where} reaches data through ctx.db, which runs on @fabrika/db: pass serve(contract, { database: scoped(tables) })`)
 		} else if (schedule !== null && timeoutMs !== null && credential !== undefined) {
-			bound.set(name, { name, artifact, schedule, timeoutMs, handler: (args) => Reflect.apply(handler, undefined, [args]), credential, site, links })
+			bound.set(name, {
+				name,
+				artifact,
+				schedule,
+				timeoutMs,
+				handler: (args) => Reflect.apply(handler, undefined, [args]),
+				credential,
+				site,
+				links,
+				mailer,
+			})
 		}
 	}
 	const frameworkJobs = new Map<string, BoundFrameworkJob>()
@@ -285,6 +317,7 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
 		capabilities: input.capabilities,
 		containers: input.containers,
 		revealHandlerErrors: input.revealHandlerErrors ?? null,
+		mail: input.mail,
 	})
 	const jobs: ContractJobs = Object.freeze({
 		names,
@@ -320,22 +353,28 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
 			}
 		},
 	})
+	const stopping = new AbortController()
 	const kick = async (name: string): Promise<void> => {
 		const job = bound.get(name)
 		if (job !== undefined && framework !== null) {
-			await kickOnce(job, contextFor(framework.jobs, undefined))
+			await kickOnce(job, contextFor(framework.jobs, undefined), stopping.signal)
 		}
 	}
-	return { jobs, declared, kick }
+	return { jobs, declared, kick, stopKicks: () => stopping.abort() }
 }
 
 /**
  * One kick: a run while the lease is free, else the shared pending flag. Neither waits for a lock, so a kick never
  * holds a connection requests share while a run writes. After a failed run the flag stays for the schedule.
  */
-async function kickOnce(job: BoundJob, context: RunnerContext): Promise<void> {
-	if ((await runJob(job, context, systemClock, 'try')).status === 'skipped') {
-		await handOver(job, context, systemClock)
+async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal): Promise<void> {
+	if (stop.aborted) {
+		// The flag waits for no lock, and another process that is not stopping runs the job.
+		await markPending(job, context)
+		return
+	}
+	if ((await runJob(job, context, systemClock, 'try', stop)).status === 'skipped') {
+		await handOver(job, context, systemClock, stop)
 	}
 }
 
@@ -343,11 +382,11 @@ async function kickOnce(job: BoundJob, context: RunnerContext): Promise<void> {
  * Leaves the flag to the run holding the lease, which takes it after it succeeded. That run may have finished before
  * the flag was set, so the lease is tried once more; `null` when another run still holds it.
  */
-async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock): Promise<JobRunResult | null> {
+async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, stop?: AbortSignal): Promise<JobRunResult | null> {
 	if (!(await markPending(job, context))) {
 		return null
 	}
-	const result = await runJob(job, context, clock, 'try')
+	const result = await runJob(job, context, clock, 'try', stop)
 	return result.status === 'skipped' ? null : result
 }
 
@@ -428,17 +467,20 @@ interface Progress {
  */
 type Acquisition = 'wait' | 'try'
 
-/** A run of the job, then one more while a kick arrived after the last one started and it succeeded. */
-async function runJob(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition): Promise<JobRunResult> {
-	let result = await runOnce(job, context, clock, acquisition)
-	while (result.status === 'succeeded' && (await pendingTaken(job, context))) {
-		const next = await runOnce(job, context, clock, 'try')
+/**
+ * A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once `stop`
+ * aborted, no follow-up run starts: the pending flag stays for the schedule.
+ */
+async function runJob(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition, stop?: AbortSignal): Promise<JobRunResult> {
+	let result = await runOnce(job, context, clock, acquisition, stop)
+	while (result.status === 'succeeded' && stop?.aborted !== true && (await pendingTaken(job, context))) {
+		const next = await runOnce(job, context, clock, 'try', stop)
 		if (next.status !== 'skipped') {
 			result = next
 			continue
 		}
 		// A newer run holds the lease and may have read before the kick's row: the flag goes back to it.
-		const handed = await handOver(job, context, clock)
+		const handed = await handOver(job, context, clock, stop)
 		return handed ?? result
 	}
 	return result
@@ -454,10 +496,10 @@ async function pendingTaken(job: BoundJob, context: RunnerContext): Promise<bool
 	}
 }
 
-async function runOnce(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition): Promise<JobRunResult> {
+async function runOnce(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition, stop?: AbortSignal): Promise<JobRunResult> {
 	const progress: Progress = { attempts: 0 }
 	try {
-		return await runLeased(job, context, clock, progress, acquisition)
+		return await runLeased(job, context, clock, progress, acquisition, stop)
 	} catch {
 		// Only the lease store throws here: the handler's errors stay inside its attempt. Its message may quote the URL.
 		return failed(job, progress.attempts, failure('driver_failed'), context.reporter)
@@ -470,10 +512,14 @@ async function runLeased(
 	clock: JobClock,
 	progress: Progress,
 	acquisition: Acquisition,
+	stop?: AbortSignal,
 ): Promise<JobRunResult> {
 	const { leases } = context
 	await leases.register(job.name)
 	const leaseMs = job.timeoutMs + LEASE_GRACE_MS
+	if (stop?.aborted === true) {
+		return { status: 'skipped' }
+	}
 	let fence: LeaseFence | null
 	if (acquisition === 'wait') {
 		await surveil(job, context)
@@ -488,8 +534,8 @@ async function runLeased(
 	await pendingTaken(job, context)
 	const attempts = job.artifact.retries + 1
 	progress.attempts = 1
-	let outcome = await attemptOnce(job, fence, progress.attempts, clock, context)
-	while (outcome !== null && progress.attempts < attempts) {
+	let outcome = await attemptOnce(job, fence, progress.attempts, clock, context, stop)
+	while (outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
 		console.error(`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}`)
 		const delay = backoffMs(progress.attempts, clock.random())
 		// A new generation at once: nothing the failed attempt still has in flight commits from here on.
@@ -498,9 +544,9 @@ async function runLeased(
 			return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
 		}
 		fence = next
-		await clock.wait(delay, new AbortController().signal)
+		await clock.wait(delay, stop ?? new AbortController().signal)
 		progress.attempts++
-		outcome = await attemptOnce(job, fence, progress.attempts, clock, context)
+		outcome = await attemptOnce(job, fence, progress.attempts, clock, context, stop)
 	}
 	if (outcome === null) {
 		if (await leases.finish(fence, 'succeeded')) {
@@ -509,6 +555,11 @@ async function runLeased(
 		return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
 	}
 	await leases.finish(fence, 'failed')
+	if (outcome.code === 'stopped') {
+		// Not a failure of the job: the schedule runs it again, so Operations hears nothing.
+		console.error(`fabrika: job '${job.name}' stopped: the process is shutting down; its schedule runs it again`)
+		return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
+	}
 	return failed(job, progress.attempts, outcome, context.reporter)
 }
 
@@ -518,9 +569,11 @@ interface JobFailure {
 	readonly why: FailureReason | null
 	/** What the attempt threw, which only the report carries, scrubbed (ADR-0059); never a log line or the result. */
 	readonly thrown?: unknown
+	/** Another attempt must not run: it could send a message the transport may already have taken. */
+	readonly final: boolean
 }
 
-const failure = (code: JobFailureCode): JobFailure => ({ code, why: null })
+const failure = (code: JobFailureCode): JobFailure => ({ code, why: null, final: false })
 
 /** The code, then a refusal's reason and its fixed text: the same words a request's log line uses. */
 const describeFailure = ({ code, why }: JobFailure): string => (why === null ? code : `${code}: ${why.reason}: ${why.text}`)
@@ -565,7 +618,7 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
 	const timer = new AbortController()
 	let outcome: JobFailure | null
 	try {
-		const ran = runInRequest(frame, () => bound.job.run(context.reporter, cancel.signal)).then(() => null, failureOf)
+		const ran = runInRequest(frame, () => bound.job.run(context.reporter, cancel.signal)).then(() => null, (error: unknown) => failureOf(error, null))
 		const timeout = clock.wait(bound.job.timeoutMs, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
 		const result = await Promise.race([ran, timeout])
 		outcome = result === TIMED_OUT ? failure('timeout') : result
@@ -590,6 +643,9 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
 	return failed(bound, 1, outcome, context.reporter)
 }
 
+/** Read afresh at each call: the signal may abort while the caller awaits. */
+const isStopped = (stop: AbortSignal | undefined): boolean => stop?.aborted === true
+
 /**
  * `null` when the handler finished, else the attempt's failure code. Either way the attempt's frame ends and the
  * statements it still has running are cancelled before the runner touches the lease again.
@@ -600,26 +656,40 @@ async function attemptOnce(
 	attempt: number,
 	clock: JobClock,
 	context: RunnerContext,
+	stop?: AbortSignal,
 ): Promise<JobFailure | null> {
+	if (isStopped(stop)) {
+		return failure('stopped')
+	}
 	// The job's authority is what IAM signs for its key at the start of the attempt; a revoked key stops the next one.
 	const authority = await job.credential.authority()
+	if (isStopped(stop)) {
+		return failure('stopped')
+	}
 	if (!authority.ok) {
 		return failure(authority.reason === 'unavailable' ? 'iam_unavailable' : 'unauthorized')
 	}
 	const frame = new RequestFrame(fence.holder, { fence })
 	const cancel = new AbortController()
 	const timer = new AbortController()
-	const ctx = jobContext(job, frame, attempt, cancel.signal, context, authority.perms)
+	const signal = stop === undefined ? cancel.signal : AbortSignal.any([cancel.signal, stop])
+	const ctx = jobContext(job, frame, attempt, signal, context, authority.perms)
 	try {
 		const ran = runInRequest(frame, async () => {
 			await job.handler({ ctx })
 		}).then(() => null, (error: unknown) => {
-			const failed = failureOf(error)
+			const failed = failureOf(error, job.mailer === null ? null : (context.mail[job.mailer]?.sender.capabilities ?? null))
 			if (failed.code === 'handler_failed') context.revealHandlerErrors?.(`job '${job.name}' attempt ${attempt}`, error)
 			return failed
 		})
 		const timeout = clock.wait(job.timeoutMs, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
-		const result = await Promise.race([ran, timeout])
+		const stopped = new Promise<JobFailure>((resolve) => {
+			if (stop?.aborted === true) {
+				resolve(failure('stopped'))
+			}
+			stop?.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: timer.signal })
+		})
+		const result = await Promise.race([ran, timeout, stopped])
 		return result === TIMED_OUT ? failure('timeout') : result
 	} finally {
 		frame.end()
@@ -629,18 +699,26 @@ async function attemptOnce(
 	}
 }
 
+/** What a job without a declared transport sends through, if it sends at all: nothing it promises, so nothing is resent. */
+const UNKNOWN_TRANSPORT: EmailSenderCapabilities = Object.freeze({ idempotent: false, confirmation: 'queued', maxBatch: 1 })
+
 /**
  * By class, never by name: an application error can call itself anything. A refusal is classified as a request's is
  * (`failureCode`), so both name it alike; every other thrown value is `handler_failed`.
  */
-function failureOf(error: unknown): JobFailure {
+function failureOf(error: unknown, transport: EmailSenderCapabilities | null): JobFailure {
 	if (error instanceof LeaseLostError) {
 		return failure('lease_lost')
 	}
+	if (error instanceof EmailDeliveryError) {
+		const text = error.outcome === 'unknown' ? 'the transport may have accepted the message' : 'the transport did not send the message'
+		// ADR-0019's one retry rule: a repeat over a non-idempotent transport may deliver the message twice.
+		return { code: 'mail_failed', why: { reason: error.code, text }, final: !mayRetryDelivery(error, transport ?? UNKNOWN_TRANSPORT) }
+	}
 	const code = failureCode(error)
 	return code === 'query_refused' || code === 'capability_refused'
-		? { code, why: failureReason(error), thrown: error }
-		: { code: 'handler_failed', why: null, thrown: error }
+		? { code, why: failureReason(error), thrown: error, final: false }
+		: { code: 'handler_failed', why: null, thrown: error, final: false }
 }
 
 function jobContext(
@@ -676,6 +754,7 @@ function jobContext(
 		signal,
 		...(job.site === null || context.data === null ? {} : kitJobContext(frame, context.data, context.files, job.name, job.site, principal.id)),
 		...(job.links === null || context.membership === null ? {} : { links: jobLinksOf(job, job.links, context.membership, frame, context) }),
+		...(job.mailer === null ? {} : { mail: jobMailOf(job, job.mailer, frame, signal, context) }),
 		within<const Act extends string, const Dim extends string>(action: Act, dimension: Dim): Within<Dim, Act> {
 			check(action, dimension)
 			return context.mint.within(frame, dimension, action, principal.id, 'all')
@@ -725,6 +804,24 @@ function jobLinksOf(job: BoundJob, mount: JobLinkMount, iam: IamMembership, fram
 	})
 }
 
+/** `ctx.mail` of a job that declares `mailer`: the transport `boot` resolved for it, until the attempt ends. */
+function jobMailOf(job: BoundJob, mailer: JobMailer, frame: RequestFrame, signal: AbortSignal, context: RunnerContext): AppMail {
+	const transport = context.mail[mailer]
+	if (transport === null) {
+		throw new ServerFault('facility_unbound', `job '${job.name}' declares mailer: '${mailer}', and boot resolved no transport for it`)
+	}
+	return jobMail({
+		transport,
+		keyPrefix: context.mail.keyPrefix,
+		signal,
+		assertLive: () => {
+			if (frame.ended) {
+				throw new CapabilityRefusedError('ended', `attempt ${frame.id} of job '${job.name}' has ended; it sends no mail`)
+			}
+		},
+	})
+}
+
 /**
  * What a staged-kit job adds: `ctx.db`, whose reads run each on its own, and `ctx.transaction(work)`, the one place it
  * writes and runs `tx.sql` — fenced by the attempt's lease, so an attempt that timed out or lost it commits nothing.

package/src/jobs/types.ts

@@ -1,6 +1,7 @@
 // The handler types of a contract's jobs: one handler per `c.cron` job under `handlers.jobs`, each with a context
 // that reaches only the actions the job declares. A job has no request and no caller, so there is no `input`.
 
+import type { EmailDeliveryResult, EmailSenderCapabilities } from '@fabrika/email'
 import type { CronJob, DimensionOf } from '../contract/define'
 import type { Ref, Within } from '../implement/capability'
 
@@ -32,6 +33,37 @@ export interface JobContext<A extends string, D extends string> {
 	ref<const Act extends A, const Dim extends D>(action: Act, dimension: Dim, id: string): Promise<Ref<Dim, Act>>
 }
 
+/** One message a job sends. The framework prefixes `idempotencyKey` before the transport sees it. */
+export interface AppMailMessage {
+	readonly to: string
+	readonly subject: string
+	readonly text: string
+	readonly html?: string
+	/** 1..200 visible ASCII characters, unique per message within this application environment. */
+	readonly idempotencyKey: string
+	readonly replyTo?: string
+	readonly fromName?: string
+}
+
+/** `ctx.mail` of a job that declares `mailer`. */
+export interface AppMail {
+	/**
+	 * false where the installation has no mail transport (`FABRIKA_EMAIL_PROVIDER=none`, or a local one without any):
+	 * every `send` fails `mail_not_configured`, outcome `not_sent`. An outbox then leaves its rows queued and spends no
+	 * attempt; they leave on the first run after the operator adds a transport and the application is deployed again.
+	 */
+	readonly available: boolean
+	/** What the transport promises: whether a repeat under one key is deduplicated, sync or queued acceptance. */
+	readonly capabilities: EmailSenderCapabilities
+	/**
+	 * Hands one message to the transport. Rejects with `EmailDeliveryError`; its `outcome` says whether the message may
+	 * have left. `signal` adds to the attempt's own `ctx.signal`, which always aborts the send.
+	 */
+	send(message: AppMailMessage, signal?: AbortSignal): Promise<EmailDeliveryResult>
+	/** `mayRetryDelivery` over this transport: true only for an `EmailDeliveryError` that is safe to send again. */
+	mayRetry(error: unknown): boolean
+}
+
 export type JobHandler<A extends string, D extends string> = (args: { readonly ctx: JobContext<A, D> }) => void | Promise<void>
 
 type JobsOf<C> = C extends { readonly surface: { readonly jobs: infer J } } ? J : never

package/src/testing/dev.ts

@@ -1,7 +1,10 @@
 // The kit `fabrika app dev` previews an application through, and the types of the seed module it runs first.
 
+import { EMAIL_DELIVERY_ERRORS, type EmailDeliveryErrorCode, type EmailDeliveryOutcome } from '@fabrika/email'
+import type { MemoryHandoff } from '@fabrika/email/memory'
 import type { AnyContract, DimensionOf } from '../contract/define'
 import type { ContractImplementation } from '../implement/implement'
+import { mailKeyPrefix } from '../jobs/mail'
 import type { PersonTestCaller, TestCaller } from './caller'
 import { passingChallenge, TURNSTILE_TEST_TOKEN } from './challenge'
 import { bootTestKit, type TestKit, type TestKitOptions } from './kit'
@@ -51,15 +54,40 @@ export interface DevOutboxMessage {
 	readonly body: string
 }
 
+/** A message a job handed to the kit's mail stand-in, as `fabrika app dev` shows it. */
+export interface DevMail {
+	/** Its position among the handoffs, from 1. */
+	readonly id: number
+	/** When it was handed off, ISO 8601. */
+	readonly at: string
+	readonly to: string
+	readonly subject: string
+	readonly text: string
+	readonly html: string | null
+	readonly replyTo: string | null
+	readonly fromName: string | null
+	/** The idempotency key as the job passed it. */
+	readonly key: string
+	/** The key the transport got: `key` behind the application environment's prefix. */
+	readonly transportKey: string
+	readonly answer: 'accepted' | EmailDeliveryErrorCode
+	readonly outcome: 'accepted' | EmailDeliveryOutcome
+	/** Whether this handoff put the message in the inbox; a repeat under its key does not. */
+	readonly delivered: boolean
+}
+
 /** What `fabrika app dev` stands in for outside the application, for the seed to read. */
 export interface DevOutside {
 	/** Every request sent to `DEV_OUTBOX_ORIGIN` so far, oldest first. */
 	outbox(): readonly DevOutboxMessage[]
+	/** Every message a job handed to the mail stand-in so far, oldest first. */
+	mails(): readonly DevMail[]
 }
 
 /**
  * The default export of `dev/seed.ts`: it receives the application's test kit, the API its tests use, writes through the
- * endpoints, and answers who and what the picker offers. `outside` reads what the application sent to the dev outbox.
+ * endpoints, and answers who and what the picker offers. `outside` reads what the application sent to the dev outbox
+ * and every message a job handed to the mail stand-in, oldest first.
  */
 export type DevSeed<C extends AnyContract> = (dev: TestKit<C>, outside: DevOutside) => Promise<DevSeedResult<C>> | DevSeedResult<C>
 
@@ -71,6 +99,8 @@ export interface DevKit<C extends AnyContract> {
 	membershipsOf(caller: TestCaller): readonly TestMembership[]
 	/** The invitation e-mails the stand-in IAM sent, in the order it first sent them. */
 	invitations(): readonly SentInvitation[]
+	/** Every message the application's jobs handed to the mail stand-in, oldest first; one refused before the handoff is absent. */
+	mails(): readonly DevMail[]
 }
 
 const isPerson = (caller: TestCaller): caller is PersonTestCaller => caller.kind === 'human' && caller.email !== null
@@ -95,5 +125,29 @@ export function devKit<C extends AnyContract>(app: ContractImplementation<C>, op
 		},
 		membershipsOf: (caller: TestCaller) => standIn.membershipsOf(caller),
 		invitations: () => standIn.invitations(),
+		mails: () => {
+			const prefix = mailKeyPrefix(kit.mailKeyScope)
+			return kit.mail.handoffs.map((handoff, index) => devMail(handoff, index + 1, prefix))
+		},
 	})
 }
+
+function devMail(handoff: MemoryHandoff, id: number, prefix: string): DevMail {
+	const { message, answer } = handoff
+	const transportKey = message.idempotencyKey
+	return {
+		id,
+		at: new Date(handoff.at).toISOString(),
+		to: message.to,
+		subject: message.subject,
+		text: message.text,
+		html: message.html ?? null,
+		replyTo: message.replyTo ?? null,
+		fromName: message.fromName ?? null,
+		key: transportKey.startsWith(prefix) ? transportKey.slice(prefix.length) : transportKey,
+		transportKey,
+		answer,
+		outcome: answer === 'accepted' ? 'accepted' : EMAIL_DELIVERY_ERRORS[answer].outcome,
+		delivered: handoff.taken,
+	}
+}

package/src/testing/endpoint-access.ts

@@ -805,5 +805,20 @@ export function endpointAccessKit<DB, P>(app: ContractImplementation<AnyContract
 			return status >= 500 || status === 401 || status === 403 ? { answered: false, status } : { answered: true }
 		},
 	}
-	return Object.freeze(accessKit)
+	// Each case waits for the runs its requests kicked, so the suite's teardown never closes the database under one.
+	const drained = <A extends readonly unknown[], R>(method: (...args: A) => Promise<R>) => async (...args: A): Promise<R> => {
+		try {
+			return await method(...args)
+		} finally {
+			await kit.drain()
+		}
+	}
+	return Object.freeze({
+		denied: drained(accessKit.denied),
+		leaked: drained(accessKit.leaked),
+		own: drained(accessKit.own),
+		file: drained(accessKit.file),
+		scoped: drained(accessKit.scoped),
+		global: drained(accessKit.global),
+	})
 }

package/src/testing/index.ts

@@ -4,8 +4,10 @@ import { refuseInProduction } from '@fabrika/db/production'
 
 refuseInProduction('@fabrika/app/testing')
 export type { IamMembershipRpc } from '@fabrika/auth-core'
+export { type MemoryEmail, MemoryEmailSender, type MemoryHandoff } from '@fabrika/email/memory'
 export type { FileScanner } from '../files/scan'
 export type { ChallengeFetch } from '../implement/challenge'
+export { mailKeyPrefix, type MailKeyScope, transportMailKey } from '../jobs/mail'
 export type { HandlerErrorSink } from '../jobs/runner'
 export type {
 	LabelledTestCaller,
@@ -24,6 +26,7 @@ export {
 	type DevKit,
 	devKit,
 	type DevLink,
+	type DevMail,
 	type DevOutboxMessage,
 	type DevOutside,
 	type DevPersona,