@protobase/server 0.8.1 → 0.9.0

package/dist/jobs/failed-jobs.js

@@ -0,0 +1,67 @@
+import { jobsSchema } from "./schema.js";
+import { f, l, resource, view } from "@protobase/schema";
+//#region src/jobs/failed-jobs.ts
+const admins = ({ user }) => user?.roles?.includes("admin") ?? false;
+/**
+* The failed jobs, over the `jobs.failed` view: the admin app lists them with their last error, and retries or deletes
+* one through the view's triggers. Admins only, whatever the project's roles say.
+*/
+const failedJobs = resource("failedJobs").table(`${jobsSchema}.failed`).fields({
+	id: f.uuid().readOnly().filterable(),
+	name: f.text().readOnly().filterable().sortable(),
+	error: f.text().optional().readOnly(),
+	attempts: f.integer().readOnly(),
+	failedAt: f.timestamp().readOnly().filterable().sortable(),
+	createdAt: f.timestamp().readOnly().sortable(),
+	key: f.text().optional().readOnly().filterable(),
+	payload: f.json().optional().readOnly(),
+	output: f.json().optional().readOnly(),
+	retry: f.boolean()
+}).primaryKey((r) => r.id).access({
+	list: admins,
+	read: admins,
+	update: admins,
+	delete: admins,
+	create: () => false
+});
+const failedJobsView = view("failedJobs").names({
+	singular: "Failed job",
+	plural: "Failed jobs"
+}).icon("triangle-alert").title((r) => r.name).help("Jobs that used up their attempts, or failed for good. Retry one once its cause is fixed: it gets one more attempt.").fields((r) => ({
+	name: r.name.label("Job"),
+	error: r.error.label("Last error"),
+	failedAt: r.failedAt.label("Failed"),
+	createdAt: r.createdAt.label("Enqueued"),
+	output: r.output.label("Error details")
+})).list((r) => ({
+	columns: [
+		r.name,
+		r.error,
+		r.attempts,
+		r.failedAt
+	],
+	sort: [[r.failedAt, "desc"]]
+})).layout((r) => [
+	l.section("Failure", [r.error, r.output]),
+	l.section("Job", [
+		r.name,
+		r.key,
+		r.payload
+	]),
+	l.sidebar([
+		r.attempts,
+		r.failedAt,
+		r.createdAt
+	])
+]).actions((a) => [a.update("retry", {
+	label: "Retry",
+	icon: "rotate-ccw",
+	set: { retry: true },
+	confirm: "Run this job once more, from now?"
+}), a.remove("delete", {
+	label: "Delete",
+	icon: "trash-2",
+	confirm: "Delete this job? It will not run again."
+})]);
+//#endregion
+export { failedJobs, failedJobsView };

package/dist/jobs/functions-sql.js

@@ -0,0 +1,126 @@
+/** The comment each Protobase function in the jobs schema carries, which the startup check reads. */
+const functionsComment = `Protobase jobs 2`;
+/** Protobase's own functions beside pg-boss's tables, by name; each carries `functionsComment`. */
+const functionNames = [
+	"enqueue",
+	"next_due",
+	"failed_retry",
+	"failed_delete"
+];
+/**
+* `jobs.enqueue(name, payload, run_at, key, group_key, priority)`: a job from SQL, such as a trigger, in the caller's
+* transaction. It inserts what pg-boss's `send()` does, with the queue's retry settings; it creates a queue that no
+* worker has created yet. A `key` keeps one waiting job, as `mode: 'dedupe'` does, and an ordered job takes its group
+* as `group_key`. Tied to the pg-boss schema version, which the startup check pins.
+*/
+const enqueue = (schema) => `create or replace function ${schema}.enqueue(
+  job_name text,
+  payload jsonb default null,
+  run_at timestamptz default null,
+  key text default null,
+  group_key text default null,
+  priority integer default 0
+) returns uuid
+language plpgsql as $$
+declare
+  queue ${schema}.queue;
+  start timestamptz := coalesce(run_at, ${schema}.job_now());
+  job_id uuid;
+begin
+  if key is not null and group_key is not null then
+    raise exception 'Job "%": an ordered job takes a group_key and no key', job_name;
+  end if;
+  perform ${schema}.create_queue(job_name, jsonb_build_object('policy', case when group_key is null then 'standard' else 'key_strict_fifo' end));
+  select * into queue from ${schema}.queue where name = job_name;
+  if (queue.policy = 'key_strict_fifo') <> (group_key is not null) then
+    raise exception 'Job "%" %', job_name, case when group_key is null then 'is ordered: pass its group_key' else 'is not ordered: it takes no group_key' end;
+  end if;
+  if key is not null then
+    select id into job_id from ${schema}.job
+    where name = job_name and singleton_key = key and state < 'active'
+    order by created_on desc limit 1;
+    if found then
+      return job_id;
+    end if;
+  end if;
+  execute format(
+    'insert into ${schema}.%I (name, data, priority, start_after, singleton_key, retry_limit, retry_delay, retry_backoff,
+       retry_delay_max, expire_seconds, deletion_seconds, keep_until, policy, dead_letter, heartbeat_seconds, upsert_by_key)
+     values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $4 + $12 * interval ''1s'', $13, $14, $15, $16)
+     on conflict do nothing
+     returning id',
+    queue.table_name)
+  using job_name, payload, priority, start, coalesce(key, group_key), queue.retry_limit, queue.retry_delay,
+    queue.retry_backoff, queue.retry_delay_max, queue.expire_seconds, queue.deletion_seconds, queue.retention_seconds,
+    queue.policy, queue.dead_letter, queue.heartbeat_seconds, key is not null
+  into job_id;
+  -- A concurrent enqueue of the same key inserted first: its job is the one waiting.
+  if job_id is null and key is not null then
+    select id into job_id from ${schema}.job
+    where name = job_name and singleton_key = key and state < 'active'
+    order by created_on desc limit 1;
+  end if;
+  return job_id;
+end
+$$;
+comment on function ${schema}.enqueue(text, jsonb, timestamptz, text, text, integer) is '${functionsComment}';`;
+/**
+* `jobs.next_due()`: when the next job is due, or `null` when none waits: what a host that sleeps reads to know when to
+* wake the app, through any pooler. A waiting job counts from its start time, unless a failed job holds back its ordered
+* group; a running one from when its try times out, since its worker may have died with the app. Recurring jobs wait
+* as their next run, so they count too.
+*/
+const nextDue = (schema) => `create or replace function ${schema}.next_due() returns timestamptz
+language sql stable as $$
+  select min(due) from (
+    select j.start_after as due from ${schema}.job j
+    where j.state < 'active' and not j.blocked
+      and not (j.policy = 'key_strict_fifo' and exists (
+        select 1 from ${schema}.job h where h.name = j.name and h.singleton_key = j.singleton_key and h.state = 'failed'))
+    union all
+    select started_on + expire_seconds * interval '1s' from ${schema}.job where state = 'active'
+  ) jobs
+$$;
+comment on function ${schema}.next_due() is '${functionsComment}';`;
+/**
+* `jobs.failed`: the failed jobs, for the admin app's page, with their last error. Updating `retry` to true retries a
+* job, as pg-boss's `retry()` does: one more attempt, from now; deleting a row deletes the job.
+*/
+const failed = (schema) => `create index if not exists job_failed_i on ${schema}.job (completed_on) where state = 'failed';
+drop view if exists ${schema}.failed;
+create view ${schema}.failed as
+select id, name, data as payload, retry_count + 1 as attempts,
+  coalesce(output ->> 'message', output -> 'value' ->> 'message', output ->> 'value') as error,
+  output, completed_on as failed_at, created_on as created_at, singleton_key as key, false as retry
+from ${schema}.job where state = 'failed';
+create or replace function ${schema}.failed_retry() returns trigger
+language plpgsql as $$
+begin
+  if new.retry then
+    update ${schema}.job
+    set state = 'retry', retry_limit = retry_limit + 1, completed_on = null,
+      start_after = greatest(start_after, ${schema}.job_now()), keep_until = greatest(keep_until, ${schema}.job_now() + interval '14 days')
+    where name = old.name and id = old.id and state = 'failed';
+  end if;
+  return new;
+end
+$$;
+comment on function ${schema}.failed_retry() is '${functionsComment}';
+create or replace function ${schema}.failed_delete() returns trigger
+language plpgsql as $$
+begin
+  delete from ${schema}.job where name = old.name and id = old.id and state = 'failed';
+  return old;
+end
+$$;
+comment on function ${schema}.failed_delete() is '${functionsComment}';
+create trigger failed_retry instead of update on ${schema}.failed for each row execute function ${schema}.failed_retry();
+create trigger failed_delete instead of delete on ${schema}.failed for each row execute function ${schema}.failed_delete();`;
+/** The statements that create or replace every Protobase object in `schema`. */
+const functionsSql = (schema) => [
+	enqueue(schema),
+	nextDue(schema),
+	failed(schema)
+].join("\n\n");
+//#endregion
+export { functionNames, functionsComment, functionsSql };

package/dist/jobs/job-types.d.ts

@@ -0,0 +1,73 @@
+import { StandardSchema } from "../functions/read-json.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/job-types.d.ts
+/** What `run` gets beside the payload. */
+export type JobContext = {
+  /** The database pool. The enqueuer's transaction has committed by the time a job runs. */
+  db: Db;
+  job: {
+    id: string;
+    name: string;
+    attempt: number;
+    maxAttempts: number;
+  };
+  /** Aborted when the job's timeout passes, or when the worker shuts down and its grace period ends. */
+  signal: AbortSignal;
+};
+export type JobOptions<Payload> = {
+  /** Letters, digits, `_`, `-`, `.` and `/`; the job's queue in the `jobs` schema. */
+  name: string;
+  /** Checks the payload when it is enqueued and again before it runs, such as a zod schema. */
+  payload?: StandardSchema<Payload>;
+  run: (payload: Payload, context: JobContext) => Promise<void> | void;
+  /** Tries before the job is kept as failed, listed in the admin app. Default 10. */
+  maxAttempts?: number;
+  /** The wait before the first retry, doubling with each one: milliseconds or text such as `30 seconds`. Default `2s`. */
+  retryDelay?: number | string;
+  /** The longest wait between two tries. Default `1 hour`. */
+  maxRetryDelay?: number | string;
+  /** How long one try may run before it counts as failed. Default `15 minutes`. */
+  timeout?: number | string;
+  /** Jobs of this name running at once in each worker process. Default 1. */
+  concurrency?: number;
+  /**
+   * Jobs run one at a time per group, in the order they were enqueued, and every enqueue names its group. A job that is
+   * retrying or failed holds back the rest of its group until it succeeds, or is retried or deleted in the admin app.
+   */
+  ordered?: boolean;
+  /**
+   * Runs the job on this cron schedule, such as `0 7 * * *` (with an optional seconds field first), with no payload: one
+   * run per time, however many workers run. A run missed while no worker ran happens once, as soon as one does.
+   */
+  schedule?: string;
+  /** The time zone `schedule` is read in, such as `Europe/Amsterdam`. Default `UTC`. */
+  timezone?: string;
+};
+/** Where and how one enqueue lands; a job runs as soon as possible by default. */
+export type EnqueueOptions = {
+  /** When the job may run first. */
+  runAt?: Date;
+  /** How long from now before the job may run first: milliseconds or text such as `5 minutes`. */
+  delay?: number | string;
+  /**
+   * Keeps one waiting job per key; a running job does not count. With `mode: 'dedupe'` (the default) an enqueue while
+   * one waits leaves it as it is, with `'replace'` it takes this payload and these options. Not for ordered jobs.
+   */
+  key?: string;
+  mode?: 'dedupe' | 'replace';
+  /** The group of an ordered job; required there, refused elsewhere. */
+  group?: string;
+  /** Higher runs first. Default 0. */
+  priority?: number;
+};
+export type JobDefinition<Payload = any> = {
+  readonly kind: 'job';
+  readonly name: string;
+  readonly options: JobOptions<Payload>;
+  /**
+   * Adds a job and returns its id: in `db`'s own transaction when it is one, so the job commits with the caller's
+   * writes or not at all. With a `key` it returns the waiting job's id when that one is kept.
+   */
+  enqueue(db: Db, ...args: Payload extends undefined ? [payload?: Payload, options?: EnqueueOptions] : [payload: Payload, options?: EnqueueOptions]): Promise<string>;
+};
+//#endregion

package/dist/jobs/options.js

@@ -0,0 +1,44 @@
+import { parseDuration } from "../files/duration.js";
+//#region src/jobs/options.ts
+/** pg-boss counts in whole seconds; a duration rounds up, so a wait is never shorter than asked. */
+const seconds = (value, label) => Math.ceil(parseDuration(value, label) / 1e3);
+const positive = (value, label) => {
+	if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a whole number of at least 1, not ${value}`);
+	return value;
+};
+/** The pg-boss queue a job definition runs in: its policy, retries with backoff and timeout. */
+const queueOptions = (options) => {
+	const label = `Job "${options.name}"`;
+	return {
+		policy: options.ordered ? "key_strict_fifo" : "standard",
+		retryLimit: positive(options.maxAttempts ?? 10, `${label} maxAttempts`) - 1,
+		retryDelay: positive(seconds(options.retryDelay ?? "2s", `${label} retryDelay`), `${label} retryDelay in seconds`),
+		retryBackoff: true,
+		retryDelayMax: positive(seconds(options.maxRetryDelay ?? "1 hour", `${label} maxRetryDelay`), `${label} maxRetryDelay in seconds`),
+		expireInSeconds: positive(seconds(options.timeout ?? "15 minutes", `${label} timeout`), `${label} timeout in seconds`)
+	};
+};
+/** The queue settings `queueOptions` sets, as `getQueue` reads them back, so a changed definition is noticed. */
+const queueSettingsDiffer = (wanted, stored) => Object.keys(wanted).some((key) => key !== "policy" && stored[key] !== wanted[key]);
+/**
+* The pg-boss send options of one enqueue: when, how urgent, and the key a waiting job is found by. An ordered job's
+* group is its key in pg-boss, which keeps one group's jobs in order.
+*/
+const sendOptions = (options, enqueue = {}) => {
+	const label = `Job "${options.name}"`;
+	if (enqueue.runAt && enqueue.delay !== void 0) throw new Error(`${label}: pass runAt or delay, not both`);
+	if (options.ordered && enqueue.group === void 0) throw new Error(`${label} is ordered: name the group each job runs in with { group }`);
+	if (options.ordered && enqueue.key !== void 0) throw new Error(`${label} is ordered: its jobs are found by their group, so it takes no key`);
+	if (!options.ordered && enqueue.group !== void 0) throw new Error(`${label} is not ordered: a group needs ordered: true in defineJob`);
+	if (enqueue.mode && enqueue.key === void 0) throw new Error(`${label}: mode needs a key`);
+	if (enqueue.priority !== void 0 && !Number.isSafeInteger(enqueue.priority)) throw new Error(`${label}: priority must be a whole number`);
+	const startAfter = enqueue.runAt ?? (enqueue.delay === void 0 ? void 0 : seconds(enqueue.delay, `${label} delay`));
+	const singletonKey = options.ordered ? enqueue.group : enqueue.key;
+	return {
+		...startAfter !== void 0 && { startAfter },
+		...enqueue.priority !== void 0 && { priority: enqueue.priority },
+		...singletonKey !== void 0 && { singletonKey }
+	};
+};
+//#endregion
+export { queueOptions, queueSettingsDiffer, sendOptions };

package/dist/jobs/run-job.d.ts

@@ -0,0 +1,9 @@
+import "./job-types.js";
+import { Db } from "@protobase/query";
+import "pg-boss";
+//#region src/jobs/run-job.d.ts
+/** Thrown from a job's `run`, it fails the job now: the attempts it has left are not used. */
+export declare class FatalJobError extends Error {
+  name: string;
+}
+//#endregion

package/dist/jobs/run-job.js

@@ -0,0 +1,78 @@
+//#region src/jobs/run-job.ts
+/** Thrown from a job's `run`, it fails the job now: the attempts it has left are not used. */
+var FatalJobError = class extends Error {
+	name = "FatalJobError";
+};
+/** The payload as the job's `payload` schema returns it, or why it does not pass. */
+const validatePayload = async (definition, payload) => {
+	const schema = definition.options.payload;
+	if (!schema) return { value: payload };
+	const result = await schema["~standard"].validate(payload);
+	if (!result.issues) return { value: result.value };
+	const issues = result.issues.map((issue) => {
+		const field = (issue.path ?? []).map((part) => String(typeof part === "object" ? part.key : part)).join(".");
+		return field ? `${field}: ${issue.message}` : issue.message;
+	});
+	return { invalid: `Job "${definition.name}" has an invalid payload: ${issues.join("; ")}` };
+};
+const checkPayload = async (definition, payload) => {
+	const checked = await validatePayload(definition, payload);
+	if ("invalid" in checked) throw new FatalJobError(checked.invalid);
+	return checked.value;
+};
+const readOnly = (error) => error?.code === "25006";
+const pause = (ms, signal) => new Promise((resolve) => {
+	if (signal.aborted) return resolve();
+	const timer = setTimeout(resolve, ms);
+	signal.addEventListener("abort", () => {
+		clearTimeout(timer);
+		resolve();
+	}, { once: true });
+});
+/** Runs `run` again every `waitMs` while the database is read-only, until the signal ends the try. */
+const throughWriteFreeze = async (run, signal, waitMs = 5e3) => {
+	for (;;) try {
+		return await run();
+	} catch (error) {
+		if (!readOnly(error) || signal.aborted) throw error;
+		await pause(waitMs, signal);
+	}
+};
+/** What a failed try keeps as the job's output, which the admin app shows as its last error. */
+const errorOutput = (error) => error instanceof Error ? {
+	name: error.name,
+	message: error.message,
+	...error.stack && { stack: error.stack }
+} : { message: String(error) };
+/**
+* One try of one job: completed when `run` returns, failed (to be retried while attempts are left) when it throws, and
+* failed for good on a `FatalJobError` or an invalid payload.
+*/
+const runJob = async (definition, job, db) => {
+	try {
+		const payload = await checkPayload(definition, job.data ?? void 0);
+		const context = {
+			db,
+			job: {
+				id: job.id,
+				name: job.name,
+				attempt: job.retryCount + 1,
+				maxAttempts: job.retryLimit + 1
+			},
+			signal: job.signal
+		};
+		await throughWriteFreeze(() => definition.options.run(payload, context), job.signal);
+		return {
+			id: job.id,
+			status: "completed"
+		};
+	} catch (error) {
+		return {
+			id: job.id,
+			status: error instanceof FatalJobError ? "deadletter" : "failed",
+			output: errorOutput(error)
+		};
+	}
+};
+//#endregion
+export { FatalJobError, runJob, throughWriteFreeze, validatePayload };

package/dist/jobs/runtime.d.ts

@@ -0,0 +1,46 @@
+import { JobDefinition } from "./job-types.js";
+import { JobsEnqueuer } from "./current.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/runtime.d.ts
+export type StartJobsOptions = {
+  db: Db;
+  jobs: JobDefinition[];
+  /** Runs the jobs in this process; `false` only enqueues them, for a process whose jobs another one runs. Default true. */
+  work?: boolean;
+  /** Seconds between a worker's checks for due jobs; an enqueue outside a transaction wakes this process's worker at once. Default 2. */
+  pollSeconds?: number;
+  /** Seconds running jobs get to finish when the jobs stop, before they are failed and retried later. Default 30. */
+  shutdownSeconds?: number;
+  /** Gets every error the workers and the queue maintenance meet outside a job's own `run`. */
+  onError: (error: unknown) => void;
+  /** The time recurring jobs are scheduled from; tests pass their own. */
+  now?: () => Date;
+};
+/** What a drain left: whether no job is due or running any more, how many are, and when the next one is due. */
+export type DrainResult = {
+  drained: boolean;
+  remaining: number;
+  nextDue: Date | null;
+};
+export type JobsRuntime = JobsEnqueuer & {
+  /** Whether this process runs the jobs, or only enqueues them. */
+  readonly works: boolean;
+  /** Takes the project's jobs again, as `protobase dev` does when their files change. */
+  update(jobs: JobDefinition[]): Promise<void>;
+  /**
+   * Runs the jobs that are due until none is due or running, or `seconds` have passed: for a host that wakes a sleeping
+   * app to run its jobs. Needs this process to run jobs.
+   */
+  drain(seconds: number): Promise<DrainResult>;
+  /** When the next job is due, or `null` when none waits (`jobs.next_due()`). */
+  nextDue(): Promise<Date | null>;
+  /** Stops taking jobs, lets running ones finish within `shutdownSeconds`, then fails and releases the rest. */
+  stop(): Promise<void>;
+};
+/**
+ * Starts the app's jobs in this process: their queues, a worker per job unless `work` is false, and the maintenance
+ * that retries jobs whose worker died. It reads the jobs schema, never changes it: check it first with
+ * `checkJobsSchema`. Enqueueing through `defineJob(...).enqueue` reaches the runtime started here.
+ */
+export declare const startJobs: ({ db, jobs, work, pollSeconds, shutdownSeconds, onError, now }: StartJobsOptions) => Promise<JobsRuntime>;
+//#endregion

package/dist/jobs/runtime.js

@@ -0,0 +1,158 @@
+import { clearCurrentJobs, setCurrentJobs } from "./current.js";
+import { jobsSchema } from "./schema.js";
+import "./schedule.js";
+import { runJob, validatePayload } from "./run-job.js";
+import { jobsDatabase } from "./database.js";
+import { queueOptions, queueSettingsDiffer, sendOptions } from "./options.js";
+import { createSchedules } from "./schedules.js";
+import { sql } from "kysely";
+import { PgBoss } from "pg-boss";
+//#region src/jobs/runtime.ts
+const createBoss = (db, work) => new PgBoss({
+	db: jobsDatabase(db),
+	schema: jobsSchema,
+	migrate: false,
+	createSchema: false,
+	schedule: false,
+	supervise: work,
+	registerInstance: false,
+	reindex: false,
+	monitorVacuum: false,
+	useListenNotify: false
+});
+/**
+* Starts the app's jobs in this process: their queues, a worker per job unless `work` is false, and the maintenance
+* that retries jobs whose worker died. It reads the jobs schema, never changes it: check it first with
+* `checkJobsSchema`. Enqueueing through `defineJob(...).enqueue` reaches the runtime started here.
+*/
+const startJobs = async ({ db, jobs, work = true, pollSeconds = 2, shutdownSeconds = 30, onError, now = () => /* @__PURE__ */ new Date() }) => {
+	const boss = createBoss(db, work);
+	boss.on("error", onError);
+	await boss.start();
+	const schedules = createSchedules(boss, db, now);
+	const definitions = /* @__PURE__ */ new Map();
+	const workers = /* @__PURE__ */ new Map();
+	const ensureQueue = async (definition) => {
+		const wanted = queueOptions(definition.options);
+		const queue = await boss.getQueue(definition.name);
+		if (!queue) return boss.createQueue(definition.name, wanted);
+		if (queue.policy !== wanted.policy) throw new Error(`Job "${definition.name}" ${definition.options.ordered ? "became" : "is no longer"} ordered, but its queue keeps the old order. Let its jobs finish or delete them, drop the queue with \`select ${jobsSchema}.delete_queue('${definition.name}')\`, then start again.`);
+		if (queueSettingsDiffer(wanted, queue)) {
+			const { policy, ...settings } = wanted;
+			await boss.updateQueue(definition.name, settings);
+		}
+	};
+	const startWorker = async (definition) => {
+		const concurrency = definition.options.concurrency ?? 1;
+		const running = workers.get(definition.name);
+		if (running?.concurrency === concurrency) return;
+		if (running) await boss.offWork(definition.name);
+		const id = await boss.work(definition.name, {
+			batchSize: 1,
+			localConcurrency: concurrency,
+			pollingIntervalSeconds: pollSeconds,
+			includeMetadata: true,
+			perJobResults: true
+		}, async ([job]) => {
+			const current = definitions.get(definition.name) ?? definition;
+			if (current.options.schedule !== void 0 && job.singletonKey === "protobase.schedule") await schedules.following(current, job.startAfter);
+			return [await runJob(current, job, db)];
+		});
+		workers.set(definition.name, {
+			id,
+			concurrency
+		});
+	};
+	const update = async (next) => {
+		for (const definition of next) await ensureQueue(definition);
+		for (const name of definitions.keys()) {
+			if (next.some((definition) => definition.name === name)) continue;
+			definitions.delete(name);
+			if (workers.delete(name)) await boss.offWork(name);
+		}
+		for (const definition of next) definitions.set(definition.name, definition);
+		if (!work) return;
+		for (const definition of next) await startWorker(definition);
+		await schedules.sync(next);
+	};
+	const enqueue = async (definition, target, payload, options = {}) => {
+		const { name } = definition;
+		if (!definitions.has(name)) throw new Error(`Job "${name}" is not among the app's jobs; export it from the module protobase.config.ts passes as \`jobs\``);
+		const checked = await validatePayload(definition, payload);
+		if ("invalid" in checked) throw new Error(checked.invalid);
+		const data = checked.value ?? null;
+		const send = {
+			...sendOptions(definition.options, options),
+			db: jobsDatabase(target)
+		};
+		const keyed = async (key) => {
+			if (options.mode !== "replace") {
+				const [waiting] = await boss.findJobs(name, {
+					key,
+					queued: true,
+					db: send.db
+				});
+				if (waiting) return waiting.id;
+			}
+			return (await boss.upsert(name, data, send)).jobs[0];
+		};
+		const id = options.key === void 0 ? await boss.send(name, data, send) : await keyed(options.key);
+		if (!id) throw new Error(`Job "${name}" was not enqueued`);
+		if (!target.isTransaction) wake(name);
+		return id;
+	};
+	const nextDue = async () => (await sql`select ${sql.id(jobsSchema, "next_due")}() as due`.execute(db)).rows[0].due;
+	const pending = async () => {
+		const job = sql.id(jobsSchema, "job");
+		const { rows } = await sql`
+      select count(*)::int as count from ${job} j where j.name = any(${[...definitions.keys()]}::text[]) and (
+        (j.state < 'active' and not j.blocked and j.start_after <= now()
+          and not (j.policy = 'key_strict_fifo' and exists (
+            select 1 from ${job} h where h.name = j.name and h.singleton_key = j.singleton_key and h.state = 'failed')))
+        or (j.state = 'active' and j.started_on + j.expire_seconds * interval '1s' > now()))`.execute(db);
+		return rows[0].count;
+	};
+	const drain = async (seconds) => {
+		if (!work) throw new Error("This process runs no jobs, so it cannot drain them");
+		const deadline = Date.now() + seconds * 1e3;
+		for (;;) {
+			for (const worker of workers.values()) boss.notifyWorker(worker.id);
+			const remaining = await pending();
+			if (remaining === 0 || Date.now() >= deadline) return {
+				drained: remaining === 0,
+				remaining,
+				nextDue: await nextDue()
+			};
+			await new Promise((resolve) => setTimeout(resolve, 100));
+		}
+	};
+	const wake = (name) => {
+		const worker = workers.get(name);
+		if (worker) boss.notifyWorker(worker.id);
+	};
+	const runtime = {
+		works: work,
+		enqueue,
+		wake,
+		update,
+		drain,
+		nextDue,
+		stop: async () => {
+			await boss.stop({
+				graceful: true,
+				timeout: shutdownSeconds * 1e3
+			});
+			clearCurrentJobs(runtime);
+		}
+	};
+	try {
+		await update(jobs);
+		setCurrentJobs(runtime);
+	} catch (error) {
+		await boss.stop({ graceful: false });
+		throw error;
+	}
+	return runtime;
+};
+//#endregion
+export { startJobs };

package/dist/jobs/schedule.js

@@ -0,0 +1,25 @@
+import { createHash } from "node:crypto";
+import { CronExpressionParser } from "cron-parser";
+//#region src/jobs/schedule.ts
+/**
+* The key every run of a recurring job waits under. Each run is a job of its own, enqueued for its time: it is what
+* `jobs.next_due()` sees while the app sleeps, so a host that wakes the app for due jobs wakes it for these too.
+*/
+const scheduleKey = "protobase.schedule";
+/** The first run of `schedule` strictly after `after`, in `timezone`; throws on an expression cron-parser refuses. */
+const nextRun = (schedule, timezone, after) => CronExpressionParser.parse(schedule, {
+	currentDate: after,
+	tz: timezone
+}).next().toDate();
+/** Whether `at` is a run of `schedule`, so a run left from an earlier schedule can be told apart. */
+const isRun = (schedule, timezone, at) => nextRun(schedule, timezone, /* @__PURE__ */ new Date(at.getTime() - 1)).getTime() === at.getTime();
+/**
+* The id of the run of job `name` at `at`: the same in every process, so however many workers enqueue a run, the
+* database keeps one (its primary key), and the run happens once.
+*/
+const runId = (name, at) => {
+	const hex = createHash("sha256").update(`${name}@${at.toISOString()}`).digest("hex");
+	return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-8${hex.slice(13, 16)}-${(parseInt(hex[16], 16) & 3 | 8).toString(16)}${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
+};
+//#endregion
+export { isRun, nextRun, runId, scheduleKey };

package/dist/jobs/schedules.js

@@ -0,0 +1,41 @@
+import { jobsSchema } from "./schema.js";
+import { isRun, nextRun, runId, scheduleKey } from "./schedule.js";
+import { jobsDatabase } from "./database.js";
+import { sql } from "kysely";
+//#region src/jobs/schedules.ts
+const job = sql.id(jobsSchema, "job");
+/** The recurring jobs' runs: enqueued ahead, one per time, and kept in step with their schedules. */
+const createSchedules = (boss, db, now) => {
+	const timezone = (definition) => definition.options.timezone ?? "UTC";
+	const enqueueRun = (definition, at) => boss.send(definition.name, null, {
+		id: runId(definition.name, at),
+		startAfter: at,
+		singletonKey: scheduleKey,
+		db: jobsDatabase(db)
+	});
+	/**
+	* Makes the waiting runs match `definitions`: drops those of jobs without a schedule now, and those at a time the
+	* schedule no longer has, then enqueues each schedule's next run. A run that is due stays, to run as soon as it can.
+	*/
+	const sync = async (definitions) => {
+		const scheduled = definitions.filter((definition) => definition.options.schedule !== void 0);
+		await sql`delete from ${job} where singleton_key = ${scheduleKey} and state < 'active' and not (name = any(${scheduled.map(({ name }) => name)}::text[]))`.execute(db);
+		for (const definition of scheduled) {
+			const stale = (await sql`
+        select id, start_after from ${job} where name = ${definition.name} and singleton_key = ${scheduleKey} and state < 'active'`.execute(db)).rows.filter((run) => !isRun(definition.options.schedule, timezone(definition), new Date(run.start_after))).map((run) => run.id);
+			if (stale.length > 0) await sql`delete from ${job} where name = ${definition.name} and id = any(${stale}::uuid[])`.execute(db);
+			await enqueueRun(definition, nextRun(definition.options.schedule, timezone(definition), now()));
+		}
+	};
+	/** As a run starts, the one after it: from its own time, or from now when it ran late, so missed times are skipped. */
+	const following = async (definition, ranAt) => {
+		const from = new Date(Math.max(ranAt.getTime(), now().getTime()));
+		await enqueueRun(definition, nextRun(definition.options.schedule, timezone(definition), from));
+	};
+	return {
+		sync,
+		following
+	};
+};
+//#endregion
+export { createSchedules };