@protobase/server 0.8.1 → 0.8.2

package/dist/create-admin.d.ts

@@ -5,6 +5,7 @@ import { AdminEnv, Authenticator, PageSource, ViewSource } from "./types.js";
 import { ResourceSource } from "./resource-source.js";
 import { FunctionSource } from "./functions/define-function.js";
 import { AdminOptions } from "./admin-options.js";
+import { JobDefinition } from "./jobs/job-types.js";
 import { Hono } from "hono";
 import { UserMenuSource } from "@protobase/schema";
 import { Db } from "@protobase/query";
@@ -26,8 +27,10 @@ export type CreateAdminInput = {
   files?: FilesOptions;
   /** Where the changes that resources' events are raised from are captured (see `EventsOptions`). */
   events?: EventsOptions;
+  /** The app's jobs (see `defineJob`): with any, admins get the failed jobs page (`failedJobs`). */
+  jobs?: JobDefinition[];
   options?: AdminOptions;
 };
 /** The admin REST API as a Hono app. It uses only Web APIs, so it runs on Node, Workers and anywhere else Hono does. */
-export declare const createAdmin: ({ resources, views, pages, userMenu, db, authenticate, auth, functions, files: filesOptions, events, options }: CreateAdminInput) => Hono<AdminEnv, import("hono/types").BlankSchema, "/">;
+export declare const createAdmin: ({ resources: appResources, views: appViews, pages, userMenu, db, authenticate, auth, functions, files: filesOptions, events, jobs, options }: CreateAdminInput) => Hono<AdminEnv, import("hono/types").BlankSchema, "/">;
 //#endregion

package/dist/create-admin.js

@@ -23,12 +23,15 @@ import { docsRoutes } from "./routes/docs.js";
 import { metaRoutes } from "./routes/meta.js";
 import { recordRoutes } from "./routes/record.js";
 import { createScanGuard } from "./scan-guard.js";
+import { failedJobs, failedJobsView } from "./jobs/failed-jobs.js";
 import { Hono } from "hono";
 import { checkPages } from "@protobase/layout";
 //#region src/create-admin.ts
 /** The admin REST API as a Hono app. It uses only Web APIs, so it runs on Node, Workers and anywhere else Hono does. */
-const createAdmin = ({ resources, views = [], pages = [], userMenu, db, authenticate, auth, functions = {}, files: filesOptions, events, options = {} }) => {
+const createAdmin = ({ resources: appResources, views: appViews = [], pages = [], userMenu, db, authenticate, auth, functions = {}, files: filesOptions, events, jobs = [], options = {} }) => {
 	const basePath = options.basePath ?? "/api/v1";
+	const resources = jobs.length > 0 ? [...appResources, failedJobs] : appResources;
+	const views = jobs.length > 0 ? [...appViews, failedJobsView] : appViews;
 	const registry = buildRegistry(resources);
 	const apiFunctions = Object.fromEntries(Object.entries(functions).map(([name, source]) => [name, toApiFunction(name, source)]));
 	const viewModels = views.map((view) => view.toModel());

package/dist/functions/require-env.d.ts

@@ -3,5 +3,5 @@
  * An environment variable the function cannot work without, such as an API key: `protobase dev` reads it from `.env`,
  * `protobase serve` from its environment. A missing one is an error, so the call is a 500 the app reports.
  */
-export declare const requireEnv: (name: string) => any;
+export declare const requireEnv: (name: string) => string;
 //#endregion

package/dist/index.d.ts

@@ -24,6 +24,8 @@ import { AssistantTool, ToolContext, defineTool } from "./assistant/tools.js";
 import { AssistantOptions, AssistantSettings, assistantRoles, assistantSettings, assistantVariables, seesAssistant } from "./assistant/assistant-settings.js";
 import { RuntimeOptions, runtimeUrl, runtimeVariable, seesRuntime } from "./runtime/runtime-settings.js";
 import { AdminOptions } from "./admin-options.js";
+import { StandardSchema, readJson } from "./functions/read-json.js";
+import { EnqueueOptions, JobContext, JobDefinition, JobOptions } from "./jobs/job-types.js";
 import { CreateAdminInput, createAdmin } from "./create-admin.js";
 import { ProjectConfig, defineConfig, mergeConfig } from "./project-config.js";
 import { consoleAuditQueue } from "./audit/console-queue.js";
@@ -47,11 +49,15 @@ import { WidgetOptions, maxWidgetPropsBytes, widget } from "./assistant/widget.j
 import { QueryOptions, QueryResult, runReadOnlyQuery, runReadWriteQuery } from "./assistant/run-query.js";
 import { readOnlyQueryTool, readWriteQueryTool } from "./assistant/query-tools.js";
 import { FunctionEnv, PublicFunctionEnv, functionApp } from "./functions/function-app.js";
-import { StandardSchema, readJson } from "./functions/read-json.js";
 import { SignatureOptions, StandardWebhookOptions, verifySignature, verifyStandardWebhook } from "./functions/verify-signature.js";
 import { requireEnv } from "./functions/require-env.js";
 import { LocalFilesOptions, localFileStore, localFiles } from "./files/local-files.js";
 import { FileCleanupInput, createFileCleanup } from "./files/file-cleanup.js";
 import { FileObject } from "./files/present.js";
 import { aspectRatio, fileSize, fileType, imageSize, processor } from "./files/processors.js";
-export { type Admin, type AdminAuth, type AdminOptions, type ApiFunction, type ApprovalAnswer, type ApprovalRequest, type AssistantBackend, type AssistantOptions, type AssistantSettings, type AssistantTool, type AuditEvent, type AuditQueue, type Authenticator, type BetterAuthAuthenticatorOptions, type CapturedRow, type ChatMessage, type ChatTextPart, type CleanupEntry, type CleanupReport, type CleanupSchedule, type CoalescedChange, type Completion, type CompletionOptions, type Conversation, type ConversationRecord, type ConversationStore, type ConversationsOptions, type CorsOptions, type CreateAdminInput, type CreateAuthOptions, type EventsOptions, type FileCleanupInput, type FileObject, type FileProvider, type FileStore, type FilesOptions, type FunctionApp, type FunctionCaller, type FunctionContext, type FunctionEnv, type FunctionHandler, type FunctionOptions, type FunctionRecords, type FunctionSource, HttpProblem, type IssueTokenOptions, type JwtOptions, type LinkedAccount, type ListParams, type LocalFilesOptions, type MemberInput, ModelError, type ModelSettings, type NewOrganization, type NewUser, type OperatorProvider, type OrganizationRole, type OrganizationsOptions, type PipelineHook, type PlatformSignIn, type ProjectConfig, type ProtectedFunctionOptions, type PublicFunctionContext, type PublicFunctionEnv, type PublicFunctionHandler, type PublicFunctionOptions, type QueryOptions, type QueryResult, type ReasoningDetail, type ReasoningEffort, type RecordKeyInput, type RecordPage, type ResourceChange, type ResourceSource, type RoleDefinition, type RolesInput, type RuntimeOptions, type ScanGuardMode, type ScheduledDelete, type Session, type SessionOrganization, type SignInPolicy, type SignInRule, type SignatureOptions, type StaffAccessRule, type StaffSignIn, type StandardSchema, type StandardWebhookOptions, type StoredMember, type StoredObject, type StoredOrganization, type StoredUser, type TenantValue, type ToolCall, type ToolContext, type ToolRecord, type ToolRecords, type ToolSpec, type TurnOptions, type User, type UserCreatedHook, type ViewSource, type WidgetOptions, type WriteEvent, type WriteOperation, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, changesMigration, checkAuthSchema, checkChanges, coalesce, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineTool, deleteUser, eventMatcher, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, issueToken, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, migrationCommand, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, recordOf, removeMember, requestApproval, requireEnv, resourceChange, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };
+import { defineJob, isJobDefinition, jobDefinitions } from "./jobs/define-job.js";
+import { FatalJobError } from "./jobs/run-job.js";
+import { DrainResult, JobsRuntime, StartJobsOptions, startJobs } from "./jobs/runtime.js";
+import { jobsDrainApp } from "./jobs/drain-route.js";
+import { checkJobsSchema, jobsSchema, jobsSchemaMigration } from "./jobs/schema.js";
+export { type Admin, type AdminAuth, type AdminOptions, type ApiFunction, type ApprovalAnswer, type ApprovalRequest, type AssistantBackend, type AssistantOptions, type AssistantSettings, type AssistantTool, type AuditEvent, type AuditQueue, type Authenticator, type BetterAuthAuthenticatorOptions, type CapturedRow, type ChatMessage, type ChatTextPart, type CleanupEntry, type CleanupReport, type CleanupSchedule, type CoalescedChange, type Completion, type CompletionOptions, type Conversation, type ConversationRecord, type ConversationStore, type ConversationsOptions, type CorsOptions, type CreateAdminInput, type CreateAuthOptions, type DrainResult, type EnqueueOptions, type EventsOptions, FatalJobError, type FileCleanupInput, type FileObject, type FileProvider, type FileStore, type FilesOptions, type FunctionApp, type FunctionCaller, type FunctionContext, type FunctionEnv, type FunctionHandler, type FunctionOptions, type FunctionRecords, type FunctionSource, HttpProblem, type IssueTokenOptions, type JobContext, type JobDefinition, type JobOptions, type JobsRuntime, type JwtOptions, type LinkedAccount, type ListParams, type LocalFilesOptions, type MemberInput, ModelError, type ModelSettings, type NewOrganization, type NewUser, type OperatorProvider, type OrganizationRole, type OrganizationsOptions, type PipelineHook, type PlatformSignIn, type ProjectConfig, type ProtectedFunctionOptions, type PublicFunctionContext, type PublicFunctionEnv, type PublicFunctionHandler, type PublicFunctionOptions, type QueryOptions, type QueryResult, type ReasoningDetail, type ReasoningEffort, type RecordKeyInput, type RecordPage, type ResourceChange, type ResourceSource, type RoleDefinition, type RolesInput, type RuntimeOptions, type ScanGuardMode, type ScheduledDelete, type Session, type SessionOrganization, type SignInPolicy, type SignInRule, type SignatureOptions, type StaffAccessRule, type StaffSignIn, type StandardSchema, type StandardWebhookOptions, type StartJobsOptions, type StoredMember, type StoredObject, type StoredOrganization, type StoredUser, type TenantValue, type ToolCall, type ToolContext, type ToolRecord, type ToolRecords, type ToolSpec, type TurnOptions, type User, type UserCreatedHook, type ViewSource, type WidgetOptions, type WriteEvent, type WriteOperation, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, changesMigration, checkAuthSchema, checkChanges, checkJobsSchema, coalesce, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineJob, defineTool, deleteUser, eventMatcher, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, isJobDefinition, issueToken, jobDefinitions, jobsDrainApp, jobsSchema, jobsSchemaMigration, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, migrationCommand, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, recordOf, removeMember, requestApproval, requireEnv, resourceChange, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, startJobs, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };

package/dist/index.js

@@ -22,8 +22,10 @@ import { storageCleanupSchedule } from "./files/cleanup-schedule.js";
 import { localFileStore, localFiles } from "./files/local-files.js";
 import { defineFunction } from "./functions/define-function.js";
 import { runtimeUrl, runtimeVariable, seesRuntime } from "./runtime/runtime-settings.js";
+import { checkJobsSchema, jobsSchema, jobsSchemaMigration } from "./jobs/schema.js";
 import { createAdmin } from "./create-admin.js";
 import { configExports } from "./resource-source.js";
+import { defineJob, isJobDefinition, jobDefinitions } from "./jobs/define-job.js";
 import { defineConfig, mergeConfig } from "./project-config.js";
 import { jwtAuthenticator } from "./auth/jwt.js";
 import { betterAuthAuthenticator } from "./better-auth/authenticator.js";
@@ -40,4 +42,7 @@ import { verifySignature, verifyStandardWebhook } from "./functions/verify-signa
 import { requireEnv } from "./functions/require-env.js";
 import { createFileCleanup } from "./files/file-cleanup.js";
 import { aspectRatio, fileSize, fileType, imageSize, processor } from "./files/processors.js";
-export { HttpProblem, ModelError, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, changesMigration, checkAuthSchema, checkChanges, coalesce, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineTool, deleteUser, eventMatcher, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, issueToken, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, migrationCommand, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, recordOf, removeMember, requestApproval, requireEnv, resourceChange, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };
+import { FatalJobError } from "./jobs/run-job.js";
+import { startJobs } from "./jobs/runtime.js";
+import { jobsDrainApp } from "./jobs/drain-route.js";
+export { FatalJobError, HttpProblem, ModelError, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, changesMigration, checkAuthSchema, checkChanges, checkJobsSchema, coalesce, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineJob, defineTool, deleteUser, eventMatcher, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, isJobDefinition, issueToken, jobDefinitions, jobsDrainApp, jobsSchema, jobsSchemaMigration, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, migrationCommand, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, recordOf, removeMember, requestApproval, requireEnv, resourceChange, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, startJobs, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };

package/dist/jobs/current.d.ts

@@ -0,0 +1,8 @@
+import { EnqueueOptions, JobDefinition } from "./job-types.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/current.d.ts
+/** What `defineJob(...).enqueue` reaches: the jobs runtime this process started. */
+export type JobsEnqueuer = {
+  enqueue(definition: JobDefinition, db: Db, payload: unknown, options?: EnqueueOptions): Promise<string>;
+};
+//#endregion

package/dist/jobs/current.js

@@ -0,0 +1,18 @@
+//#region src/jobs/current.ts
+const key = Symbol.for("protobase.jobs");
+const holder = globalThis;
+/** The running jobs runtime; enqueueing needs one, started by `protobase dev`, `serve` and `worker`, or by `startJobs`. */
+const currentJobs = () => {
+	const runtime = holder[key];
+	if (!runtime) throw new Error("Jobs are not running in this process: start them with startJobs(), as protobase dev, serve and worker do");
+	return runtime;
+};
+const setCurrentJobs = (runtime) => {
+	if (holder[key]) throw new Error("Jobs are already running in this process; stop them before starting them again");
+	holder[key] = runtime;
+};
+const clearCurrentJobs = (runtime) => {
+	if (holder[key] === runtime) delete holder[key];
+};
+//#endregion
+export { clearCurrentJobs, currentJobs, setCurrentJobs };

package/dist/jobs/database.js

@@ -0,0 +1,36 @@
+import { CompiledQuery } from "kysely";
+//#region src/jobs/database.ts
+const block = /^\s*BEGIN;\s*(SET LOCAL lock_timeout = \d+);\s*(SET LOCAL idle_in_transaction_session_timeout = \d+);\s*(SELECT pg_advisory_xact_lock\([^;]*\);\s*)?([\s\S]+?);\s*COMMIT;\s*$/;
+/**
+* The statements of one of pg-boss's transaction texts, in order, or `undefined` for any other text. A text that
+* starts a transaction and is not in that shape is an error, so a pg-boss upgrade that changes it fails loudly.
+*/
+const transactionStatements = (text) => {
+	if (!/^\s*BEGIN;/.test(text)) return void 0;
+	const match = block.exec(text);
+	if (!match) throw new Error(`Unexpected pg-boss transaction:\n${text}`);
+	const [, lockTimeout, idleTimeout, lock, statement] = match;
+	return [
+		lockTimeout,
+		idleTimeout,
+		...lock ? [lock.trim().replace(/;$/, "")] : [],
+		statement
+	];
+};
+/**
+* pg-boss's database over the app's own Kysely pool or transaction: one connection pool for the app and its jobs, the
+* same driver on Node, Bun and PGlite. Every statement runs on its own, or inside one transaction for pg-boss's
+* transaction texts, which works through a pooler in transaction mode: pg-boss takes only transaction-scoped locks.
+*/
+const jobsDatabase = (db) => ({ executeSql: async (text, values) => {
+	if (values && values.length > 0) return { rows: [...(await db.executeQuery(CompiledQuery.raw(text, values))).rows] };
+	const statements = transactionStatements(text);
+	if (!statements) return { rows: [...(await db.executeQuery(CompiledQuery.raw(text))).rows] };
+	return db.transaction().execute(async (trx) => {
+		let rows = [];
+		for (const statement of statements) rows = [...(await trx.executeQuery(CompiledQuery.raw(statement))).rows];
+		return { rows };
+	});
+} });
+//#endregion
+export { jobsDatabase, transactionStatements };

package/dist/jobs/define-job.d.ts

@@ -0,0 +1,8 @@
+import { JobDefinition, JobOptions } from "./job-types.js";
+//#region src/jobs/define-job.d.ts
+/** A job the app's workers run: export it from the module that `protobase.config.ts` passes as `jobs`. */
+export declare const defineJob: <Payload = undefined>(options: JobOptions<Payload>) => JobDefinition<Payload>;
+export declare const isJobDefinition: (value: unknown) => value is JobDefinition;
+/** The job definitions among a module's exports, such as `import * as jobs from './config/jobs'`, each name once. */
+export declare const jobDefinitions: (module: Record<string, unknown>) => JobDefinition[];
+//#endregion

package/dist/jobs/define-job.js

@@ -0,0 +1,38 @@
+import { currentJobs } from "./current.js";
+import { nextRun } from "./schedule.js";
+//#region src/jobs/define-job.ts
+const namePattern = /^[\w.\-/]+$/;
+const checkSchedule = (name, schedule, timezone) => {
+	try {
+		nextRun(schedule, timezone, /* @__PURE__ */ new Date());
+	} catch (error) {
+		if (!(error instanceof Error)) throw error;
+		throw new Error(`Job "${name}" has the schedule "${schedule}" in ${timezone}, which is not a cron schedule there: ${error.message}`, { cause: error });
+	}
+};
+/** A job the app's workers run: export it from the module that `protobase.config.ts` passes as `jobs`. */
+const defineJob = (options) => {
+	if (!namePattern.test(options.name)) throw new Error(`Job name "${options.name}" may only hold letters, digits, "_", "-", "." and "/"`);
+	if (options.schedule !== void 0) checkSchedule(options.name, options.schedule, options.timezone ?? "UTC");
+	const definition = {
+		kind: "job",
+		name: options.name,
+		options,
+		enqueue: async (db, payload, enqueueOptions) => currentJobs().enqueue(definition, db, payload, enqueueOptions)
+	};
+	return definition;
+};
+const isJobDefinition = (value) => typeof value === "object" && value !== null && value.kind === "job" && typeof value.enqueue === "function";
+/** The job definitions among a module's exports, such as `import * as jobs from './config/jobs'`, each name once. */
+const jobDefinitions = (module) => {
+	const byName = /* @__PURE__ */ new Map();
+	for (const value of Object.values(module)) {
+		if (!isJobDefinition(value)) continue;
+		const known = byName.get(value.name);
+		if (known && known !== value) throw new Error(`Two jobs are named "${value.name}"; rename one of them`);
+		byName.set(value.name, value);
+	}
+	return [...byName.values()];
+};
+//#endregion
+export { defineJob, isJobDefinition, jobDefinitions };

package/dist/jobs/drain-route.d.ts

@@ -0,0 +1,21 @@
+import { JobsRuntime } from "./runtime.js";
+//#region src/jobs/drain-route.d.ts
+/**
+ * `POST /api/jobs/drain?seconds=25`: runs the app's due jobs until none is left or `seconds` have passed, and answers
+ * `{ drained, remaining, nextDue }`. For the operator of a host that lets the app sleep and wakes it for its jobs: it
+ * takes the operator's `token` as a bearer token, never a user's.
+ */
+export declare const jobsDrainApp: ({ runtime, token }: {
+  runtime: JobsRuntime;
+  token: string;
+}) => import("hono/hono-base").HonoBase<import("hono/types").BlankEnv, {
+  "/api/jobs/drain": {
+    $post: {
+      input: {};
+      output: {};
+      outputFormat: string;
+      status: import("hono/utils/http-status").StatusCode;
+    };
+  };
+}, "/", "/api/jobs/drain">;
+//#endregion

package/dist/jobs/drain-route.js

@@ -0,0 +1,28 @@
+import { HttpProblem, badRequest, problemResponse, unauthorized } from "../problem.js";
+import { Hono } from "hono";
+import { createHash, timingSafeEqual } from "node:crypto";
+//#region src/jobs/drain-route.ts
+const drainPath = "/api/jobs/drain";
+const digest = (text) => createHash("sha256").update(text).digest();
+const sameToken = (given, token) => given !== void 0 && timingSafeEqual(digest(given), digest(token));
+const maxSeconds = 300;
+/**
+* `POST /api/jobs/drain?seconds=25`: runs the app's due jobs until none is left or `seconds` have passed, and answers
+* `{ drained, remaining, nextDue }`. For the operator of a host that lets the app sleep and wakes it for its jobs: it
+* takes the operator's `token` as a bearer token, never a user's.
+*/
+const jobsDrainApp = ({ runtime, token }) => new Hono().post(drainPath, async (c) => {
+	const problem = (error) => problemResponse(error, c.req.path);
+	if (!sameToken(c.req.header("authorization")?.replace(/^Bearer /, ""), token)) return problem(unauthorized("Send the operator token as a bearer token"));
+	const seconds = Number(c.req.query("seconds") ?? 25);
+	if (!Number.isFinite(seconds) || seconds <= 0 || seconds > maxSeconds) return problem(badRequest("invalid-seconds", `seconds must be more than 0 and at most ${maxSeconds}`));
+	if (!runtime.works) return problem(new HttpProblem(409, "jobs-elsewhere", "Conflict", "This process runs no jobs (PROTOBASE_JOBS=off); drain the one that does"));
+	const { drained, remaining, nextDue } = await runtime.drain(seconds);
+	return c.json({
+		drained,
+		remaining,
+		nextDue: nextDue?.toISOString() ?? null
+	});
+});
+//#endregion
+export { drainPath, jobsDrainApp };

package/dist/jobs/failed-jobs.js

@@ -0,0 +1,67 @@
+import { jobsSchema } from "./schema.js";
+import { f, l, resource, view } from "@protobase/schema";
+//#region src/jobs/failed-jobs.ts
+const admins = ({ user }) => user?.roles?.includes("admin") ?? false;
+/**
+* The failed jobs, over the `jobs.failed` view: the admin app lists them with their last error, and retries or deletes
+* one through the view's triggers. Admins only, whatever the project's roles say.
+*/
+const failedJobs = resource("failedJobs").table(`${jobsSchema}.failed`).fields({
+	id: f.uuid().readOnly().filterable(),
+	name: f.text().readOnly().filterable().sortable(),
+	error: f.text().optional().readOnly(),
+	attempts: f.integer().readOnly(),
+	failedAt: f.timestamp().readOnly().filterable().sortable(),
+	createdAt: f.timestamp().readOnly().sortable(),
+	key: f.text().optional().readOnly().filterable(),
+	payload: f.json().optional().readOnly(),
+	output: f.json().optional().readOnly(),
+	retry: f.boolean()
+}).primaryKey((r) => r.id).access({
+	list: admins,
+	read: admins,
+	update: admins,
+	delete: admins,
+	create: () => false
+});
+const failedJobsView = view("failedJobs").names({
+	singular: "Failed job",
+	plural: "Failed jobs"
+}).icon("triangle-alert").title((r) => r.name).help("Jobs that used up their attempts, or failed for good. Retry one once its cause is fixed: it gets one more attempt.").fields((r) => ({
+	name: r.name.label("Job"),
+	error: r.error.label("Last error"),
+	failedAt: r.failedAt.label("Failed"),
+	createdAt: r.createdAt.label("Enqueued"),
+	output: r.output.label("Error details")
+})).list((r) => ({
+	columns: [
+		r.name,
+		r.error,
+		r.attempts,
+		r.failedAt
+	],
+	sort: [[r.failedAt, "desc"]]
+})).layout((r) => [
+	l.section("Failure", [r.error, r.output]),
+	l.section("Job", [
+		r.name,
+		r.key,
+		r.payload
+	]),
+	l.sidebar([
+		r.attempts,
+		r.failedAt,
+		r.createdAt
+	])
+]).actions((a) => [a.update("retry", {
+	label: "Retry",
+	icon: "rotate-ccw",
+	set: { retry: true },
+	confirm: "Run this job once more, from now?"
+}), a.remove("delete", {
+	label: "Delete",
+	icon: "trash-2",
+	confirm: "Delete this job? It will not run again."
+})]);
+//#endregion
+export { failedJobs, failedJobsView };

package/dist/jobs/functions-sql.js

@@ -0,0 +1,126 @@
+/** The comment each Protobase function in the jobs schema carries, which the startup check reads. */
+const functionsComment = `Protobase jobs 2`;
+/** Protobase's own functions beside pg-boss's tables, by name; each carries `functionsComment`. */
+const functionNames = [
+	"enqueue",
+	"next_due",
+	"failed_retry",
+	"failed_delete"
+];
+/**
+* `jobs.enqueue(name, payload, run_at, key, group_key, priority)`: a job from SQL, such as a trigger, in the caller's
+* transaction. It inserts what pg-boss's `send()` does, with the queue's retry settings; it creates a queue that no
+* worker has created yet. A `key` keeps one waiting job, as `mode: 'dedupe'` does, and an ordered job takes its group
+* as `group_key`. Tied to the pg-boss schema version, which the startup check pins.
+*/
+const enqueue = (schema) => `create or replace function ${schema}.enqueue(
+  job_name text,
+  payload jsonb default null,
+  run_at timestamptz default null,
+  key text default null,
+  group_key text default null,
+  priority integer default 0
+) returns uuid
+language plpgsql as $$
+declare
+  queue ${schema}.queue;
+  start timestamptz := coalesce(run_at, ${schema}.job_now());
+  job_id uuid;
+begin
+  if key is not null and group_key is not null then
+    raise exception 'Job "%": an ordered job takes a group_key and no key', job_name;
+  end if;
+  perform ${schema}.create_queue(job_name, jsonb_build_object('policy', case when group_key is null then 'standard' else 'key_strict_fifo' end));
+  select * into queue from ${schema}.queue where name = job_name;
+  if (queue.policy = 'key_strict_fifo') <> (group_key is not null) then
+    raise exception 'Job "%" %', job_name, case when group_key is null then 'is ordered: pass its group_key' else 'is not ordered: it takes no group_key' end;
+  end if;
+  if key is not null then
+    select id into job_id from ${schema}.job
+    where name = job_name and singleton_key = key and state < 'active'
+    order by created_on desc limit 1;
+    if found then
+      return job_id;
+    end if;
+  end if;
+  execute format(
+    'insert into ${schema}.%I (name, data, priority, start_after, singleton_key, retry_limit, retry_delay, retry_backoff,
+       retry_delay_max, expire_seconds, deletion_seconds, keep_until, policy, dead_letter, heartbeat_seconds, upsert_by_key)
+     values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $4 + $12 * interval ''1s'', $13, $14, $15, $16)
+     on conflict do nothing
+     returning id',
+    queue.table_name)
+  using job_name, payload, priority, start, coalesce(key, group_key), queue.retry_limit, queue.retry_delay,
+    queue.retry_backoff, queue.retry_delay_max, queue.expire_seconds, queue.deletion_seconds, queue.retention_seconds,
+    queue.policy, queue.dead_letter, queue.heartbeat_seconds, key is not null
+  into job_id;
+  -- A concurrent enqueue of the same key inserted first: its job is the one waiting.
+  if job_id is null and key is not null then
+    select id into job_id from ${schema}.job
+    where name = job_name and singleton_key = key and state < 'active'
+    order by created_on desc limit 1;
+  end if;
+  return job_id;
+end
+$$;
+comment on function ${schema}.enqueue(text, jsonb, timestamptz, text, text, integer) is '${functionsComment}';`;
+/**
+* `jobs.next_due()`: when the next job is due, or `null` when none waits: what a host that sleeps reads to know when to
+* wake the app, through any pooler. A waiting job counts from its start time, unless a failed job holds back its ordered
+* group; a running one from when its try times out, since its worker may have died with the app. Recurring jobs wait
+* as their next run, so they count too.
+*/
+const nextDue = (schema) => `create or replace function ${schema}.next_due() returns timestamptz
+language sql stable as $$
+  select min(due) from (
+    select j.start_after as due from ${schema}.job j
+    where j.state < 'active' and not j.blocked
+      and not (j.policy = 'key_strict_fifo' and exists (
+        select 1 from ${schema}.job h where h.name = j.name and h.singleton_key = j.singleton_key and h.state = 'failed'))
+    union all
+    select started_on + expire_seconds * interval '1s' from ${schema}.job where state = 'active'
+  ) jobs
+$$;
+comment on function ${schema}.next_due() is '${functionsComment}';`;
+/**
+* `jobs.failed`: the failed jobs, for the admin app's page, with their last error. Updating `retry` to true retries a
+* job, as pg-boss's `retry()` does: one more attempt, from now; deleting a row deletes the job.
+*/
+const failed = (schema) => `create index if not exists job_failed_i on ${schema}.job (completed_on) where state = 'failed';
+drop view if exists ${schema}.failed;
+create view ${schema}.failed as
+select id, name, data as payload, retry_count + 1 as attempts,
+  coalesce(output ->> 'message', output -> 'value' ->> 'message', output ->> 'value') as error,
+  output, completed_on as failed_at, created_on as created_at, singleton_key as key, false as retry
+from ${schema}.job where state = 'failed';
+create or replace function ${schema}.failed_retry() returns trigger
+language plpgsql as $$
+begin
+  if new.retry then
+    update ${schema}.job
+    set state = 'retry', retry_limit = retry_limit + 1, completed_on = null,
+      start_after = greatest(start_after, ${schema}.job_now()), keep_until = greatest(keep_until, ${schema}.job_now() + interval '14 days')
+    where name = old.name and id = old.id and state = 'failed';
+  end if;
+  return new;
+end
+$$;
+comment on function ${schema}.failed_retry() is '${functionsComment}';
+create or replace function ${schema}.failed_delete() returns trigger
+language plpgsql as $$
+begin
+  delete from ${schema}.job where name = old.name and id = old.id and state = 'failed';
+  return old;
+end
+$$;
+comment on function ${schema}.failed_delete() is '${functionsComment}';
+create trigger failed_retry instead of update on ${schema}.failed for each row execute function ${schema}.failed_retry();
+create trigger failed_delete instead of delete on ${schema}.failed for each row execute function ${schema}.failed_delete();`;
+/** The statements that create or replace every Protobase object in `schema`. */
+const functionsSql = (schema) => [
+	enqueue(schema),
+	nextDue(schema),
+	failed(schema)
+].join("\n\n");
+//#endregion
+export { functionNames, functionsComment, functionsSql };

package/dist/jobs/job-types.d.ts

@@ -0,0 +1,73 @@
+import { StandardSchema } from "../functions/read-json.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/job-types.d.ts
+/** What `run` gets beside the payload. */
+export type JobContext = {
+  /** The database pool. The enqueuer's transaction has committed by the time a job runs. */
+  db: Db;
+  job: {
+    id: string;
+    name: string;
+    attempt: number;
+    maxAttempts: number;
+  };
+  /** Aborted when the job's timeout passes, or when the worker shuts down and its grace period ends. */
+  signal: AbortSignal;
+};
+export type JobOptions<Payload> = {
+  /** Letters, digits, `_`, `-`, `.` and `/`; the job's queue in the `jobs` schema. */
+  name: string;
+  /** Checks the payload when it is enqueued and again before it runs, such as a zod schema. */
+  payload?: StandardSchema<Payload>;
+  run: (payload: Payload, context: JobContext) => Promise<void> | void;
+  /** Tries before the job is kept as failed, listed in the admin app. Default 10. */
+  maxAttempts?: number;
+  /** The wait before the first retry, doubling with each one: milliseconds or text such as `30 seconds`. Default `2s`. */
+  retryDelay?: number | string;
+  /** The longest wait between two tries. Default `1 hour`. */
+  maxRetryDelay?: number | string;
+  /** How long one try may run before it counts as failed. Default `15 minutes`. */
+  timeout?: number | string;
+  /** Jobs of this name running at once in each worker process. Default 1. */
+  concurrency?: number;
+  /**
+   * Jobs run one at a time per group, in the order they were enqueued, and every enqueue names its group. A job that is
+   * retrying or failed holds back the rest of its group until it succeeds, or is retried or deleted in the admin app.
+   */
+  ordered?: boolean;
+  /**
+   * Runs the job on this cron schedule, such as `0 7 * * *` (with an optional seconds field first), with no payload: one
+   * run per time, however many workers run. A run missed while no worker ran happens once, as soon as one does.
+   */
+  schedule?: string;
+  /** The time zone `schedule` is read in, such as `Europe/Amsterdam`. Default `UTC`. */
+  timezone?: string;
+};
+/** Where and how one enqueue lands; a job runs as soon as possible by default. */
+export type EnqueueOptions = {
+  /** When the job may run first. */
+  runAt?: Date;
+  /** How long from now before the job may run first: milliseconds or text such as `5 minutes`. */
+  delay?: number | string;
+  /**
+   * Keeps one waiting job per key; a running job does not count. With `mode: 'dedupe'` (the default) an enqueue while
+   * one waits leaves it as it is, with `'replace'` it takes this payload and these options. Not for ordered jobs.
+   */
+  key?: string;
+  mode?: 'dedupe' | 'replace';
+  /** The group of an ordered job; required there, refused elsewhere. */
+  group?: string;
+  /** Higher runs first. Default 0. */
+  priority?: number;
+};
+export type JobDefinition<Payload = any> = {
+  readonly kind: 'job';
+  readonly name: string;
+  readonly options: JobOptions<Payload>;
+  /**
+   * Adds a job and returns its id: in `db`'s own transaction when it is one, so the job commits with the caller's
+   * writes or not at all. With a `key` it returns the waiting job's id when that one is kept.
+   */
+  enqueue(db: Db, ...args: Payload extends undefined ? [payload?: Payload, options?: EnqueueOptions] : [payload: Payload, options?: EnqueueOptions]): Promise<string>;
+};
+//#endregion

package/dist/jobs/options.js

@@ -0,0 +1,44 @@
+import { parseDuration } from "../files/duration.js";
+//#region src/jobs/options.ts
+/** pg-boss counts in whole seconds; a duration rounds up, so a wait is never shorter than asked. */
+const seconds = (value, label) => Math.ceil(parseDuration(value, label) / 1e3);
+const positive = (value, label) => {
+	if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a whole number of at least 1, not ${value}`);
+	return value;
+};
+/** The pg-boss queue a job definition runs in: its policy, retries with backoff and timeout. */
+const queueOptions = (options) => {
+	const label = `Job "${options.name}"`;
+	return {
+		policy: options.ordered ? "key_strict_fifo" : "standard",
+		retryLimit: positive(options.maxAttempts ?? 10, `${label} maxAttempts`) - 1,
+		retryDelay: positive(seconds(options.retryDelay ?? "2s", `${label} retryDelay`), `${label} retryDelay in seconds`),
+		retryBackoff: true,
+		retryDelayMax: positive(seconds(options.maxRetryDelay ?? "1 hour", `${label} maxRetryDelay`), `${label} maxRetryDelay in seconds`),
+		expireInSeconds: positive(seconds(options.timeout ?? "15 minutes", `${label} timeout`), `${label} timeout in seconds`)
+	};
+};
+/** The queue settings `queueOptions` sets, as `getQueue` reads them back, so a changed definition is noticed. */
+const queueSettingsDiffer = (wanted, stored) => Object.keys(wanted).some((key) => key !== "policy" && stored[key] !== wanted[key]);
+/**
+* The pg-boss send options of one enqueue: when, how urgent, and the key a waiting job is found by. An ordered job's
+* group is its key in pg-boss, which keeps one group's jobs in order.
+*/
+const sendOptions = (options, enqueue = {}) => {
+	const label = `Job "${options.name}"`;
+	if (enqueue.runAt && enqueue.delay !== void 0) throw new Error(`${label}: pass runAt or delay, not both`);
+	if (options.ordered && enqueue.group === void 0) throw new Error(`${label} is ordered: name the group each job runs in with { group }`);
+	if (options.ordered && enqueue.key !== void 0) throw new Error(`${label} is ordered: its jobs are found by their group, so it takes no key`);
+	if (!options.ordered && enqueue.group !== void 0) throw new Error(`${label} is not ordered: a group needs ordered: true in defineJob`);
+	if (enqueue.mode && enqueue.key === void 0) throw new Error(`${label}: mode needs a key`);
+	if (enqueue.priority !== void 0 && !Number.isSafeInteger(enqueue.priority)) throw new Error(`${label}: priority must be a whole number`);
+	const startAfter = enqueue.runAt ?? (enqueue.delay === void 0 ? void 0 : seconds(enqueue.delay, `${label} delay`));
+	const singletonKey = options.ordered ? enqueue.group : enqueue.key;
+	return {
+		...startAfter !== void 0 && { startAfter },
+		...enqueue.priority !== void 0 && { priority: enqueue.priority },
+		...singletonKey !== void 0 && { singletonKey }
+	};
+};
+//#endregion
+export { queueOptions, queueSettingsDiffer, sendOptions };

package/dist/jobs/run-job.d.ts

@@ -0,0 +1,9 @@
+import "./job-types.js";
+import { Db } from "@protobase/query";
+import "pg-boss";
+//#region src/jobs/run-job.d.ts
+/** Thrown from a job's `run`, it fails the job now: the attempts it has left are not used. */
+export declare class FatalJobError extends Error {
+  name: string;
+}
+//#endregion

package/dist/jobs/run-job.js

@@ -0,0 +1,78 @@
+//#region src/jobs/run-job.ts
+/** Thrown from a job's `run`, it fails the job now: the attempts it has left are not used. */
+var FatalJobError = class extends Error {
+	name = "FatalJobError";
+};
+/** The payload as the job's `payload` schema returns it, or why it does not pass. */
+const validatePayload = async (definition, payload) => {
+	const schema = definition.options.payload;
+	if (!schema) return { value: payload };
+	const result = await schema["~standard"].validate(payload);
+	if (!result.issues) return { value: result.value };
+	const issues = result.issues.map((issue) => {
+		const field = (issue.path ?? []).map((part) => String(typeof part === "object" ? part.key : part)).join(".");
+		return field ? `${field}: ${issue.message}` : issue.message;
+	});
+	return { invalid: `Job "${definition.name}" has an invalid payload: ${issues.join("; ")}` };
+};
+const checkPayload = async (definition, payload) => {
+	const checked = await validatePayload(definition, payload);
+	if ("invalid" in checked) throw new FatalJobError(checked.invalid);
+	return checked.value;
+};
+const readOnly = (error) => error?.code === "25006";
+const pause = (ms, signal) => new Promise((resolve) => {
+	if (signal.aborted) return resolve();
+	const timer = setTimeout(resolve, ms);
+	signal.addEventListener("abort", () => {
+		clearTimeout(timer);
+		resolve();
+	}, { once: true });
+});
+/** Runs `run` again every `waitMs` while the database is read-only, until the signal ends the try. */
+const throughWriteFreeze = async (run, signal, waitMs = 5e3) => {
+	for (;;) try {
+		return await run();
+	} catch (error) {
+		if (!readOnly(error) || signal.aborted) throw error;
+		await pause(waitMs, signal);
+	}
+};
+/** What a failed try keeps as the job's output, which the admin app shows as its last error. */
+const errorOutput = (error) => error instanceof Error ? {
+	name: error.name,
+	message: error.message,
+	...error.stack && { stack: error.stack }
+} : { message: String(error) };
+/**
+* One try of one job: completed when `run` returns, failed (to be retried while attempts are left) when it throws, and
+* failed for good on a `FatalJobError` or an invalid payload.
+*/
+const runJob = async (definition, job, db) => {
+	try {
+		const payload = await checkPayload(definition, job.data ?? void 0);
+		const context = {
+			db,
+			job: {
+				id: job.id,
+				name: job.name,
+				attempt: job.retryCount + 1,
+				maxAttempts: job.retryLimit + 1
+			},
+			signal: job.signal
+		};
+		await throughWriteFreeze(() => definition.options.run(payload, context), job.signal);
+		return {
+			id: job.id,
+			status: "completed"
+		};
+	} catch (error) {
+		return {
+			id: job.id,
+			status: error instanceof FatalJobError ? "deadletter" : "failed",
+			output: errorOutput(error)
+		};
+	}
+};
+//#endregion
+export { FatalJobError, runJob, throughWriteFreeze, validatePayload };

package/dist/jobs/runtime.d.ts

@@ -0,0 +1,46 @@
+import { JobDefinition } from "./job-types.js";
+import { JobsEnqueuer } from "./current.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/runtime.d.ts
+export type StartJobsOptions = {
+  db: Db;
+  jobs: JobDefinition[];
+  /** Runs the jobs in this process; `false` only enqueues them, for a process whose jobs another one runs. Default true. */
+  work?: boolean;
+  /** Seconds between a worker's checks for due jobs; an enqueue outside a transaction wakes this process's worker at once. Default 2. */
+  pollSeconds?: number;
+  /** Seconds running jobs get to finish when the jobs stop, before they are failed and retried later. Default 30. */
+  shutdownSeconds?: number;
+  /** Gets every error the workers and the queue maintenance meet outside a job's own `run`. */
+  onError: (error: unknown) => void;
+  /** The time recurring jobs are scheduled from; tests pass their own. */
+  now?: () => Date;
+};
+/** What a drain left: whether no job is due or running any more, how many are, and when the next one is due. */
+export type DrainResult = {
+  drained: boolean;
+  remaining: number;
+  nextDue: Date | null;
+};
+export type JobsRuntime = JobsEnqueuer & {
+  /** Whether this process runs the jobs, or only enqueues them. */
+  readonly works: boolean;
+  /** Takes the project's jobs again, as `protobase dev` does when their files change. */
+  update(jobs: JobDefinition[]): Promise<void>;
+  /**
+   * Runs the jobs that are due until none is due or running, or `seconds` have passed: for a host that wakes a sleeping
+   * app to run its jobs. Needs this process to run jobs.
+   */
+  drain(seconds: number): Promise<DrainResult>;
+  /** When the next job is due, or `null` when none waits (`jobs.next_due()`). */
+  nextDue(): Promise<Date | null>;
+  /** Stops taking jobs, lets running ones finish within `shutdownSeconds`, then fails and releases the rest. */
+  stop(): Promise<void>;
+};
+/**
+ * Starts the app's jobs in this process: their queues, a worker per job unless `work` is false, and the maintenance
+ * that retries jobs whose worker died. It reads the jobs schema, never changes it: check it first with
+ * `checkJobsSchema`. Enqueueing through `defineJob(...).enqueue` reaches the runtime started here.
+ */
+export declare const startJobs: ({ db, jobs, work, pollSeconds, shutdownSeconds, onError, now }: StartJobsOptions) => Promise<JobsRuntime>;
+//#endregion

package/dist/jobs/runtime.js

@@ -0,0 +1,156 @@
+import { jobsSchema } from "./schema.js";
+import { clearCurrentJobs, setCurrentJobs } from "./current.js";
+import "./schedule.js";
+import { runJob, validatePayload } from "./run-job.js";
+import { jobsDatabase } from "./database.js";
+import { queueOptions, queueSettingsDiffer, sendOptions } from "./options.js";
+import { createSchedules } from "./schedules.js";
+import { sql } from "kysely";
+import { PgBoss } from "pg-boss";
+//#region src/jobs/runtime.ts
+const createBoss = (db, work) => new PgBoss({
+	db: jobsDatabase(db),
+	schema: jobsSchema,
+	migrate: false,
+	createSchema: false,
+	schedule: false,
+	supervise: work,
+	registerInstance: false,
+	reindex: false,
+	monitorVacuum: false,
+	useListenNotify: false
+});
+/**
+* Starts the app's jobs in this process: their queues, a worker per job unless `work` is false, and the maintenance
+* that retries jobs whose worker died. It reads the jobs schema, never changes it: check it first with
+* `checkJobsSchema`. Enqueueing through `defineJob(...).enqueue` reaches the runtime started here.
+*/
+const startJobs = async ({ db, jobs, work = true, pollSeconds = 2, shutdownSeconds = 30, onError, now = () => /* @__PURE__ */ new Date() }) => {
+	const boss = createBoss(db, work);
+	boss.on("error", onError);
+	await boss.start();
+	const schedules = createSchedules(boss, db, now);
+	const definitions = /* @__PURE__ */ new Map();
+	const workers = /* @__PURE__ */ new Map();
+	const ensureQueue = async (definition) => {
+		const wanted = queueOptions(definition.options);
+		const queue = await boss.getQueue(definition.name);
+		if (!queue) return boss.createQueue(definition.name, wanted);
+		if (queue.policy !== wanted.policy) throw new Error(`Job "${definition.name}" ${definition.options.ordered ? "became" : "is no longer"} ordered, but its queue keeps the old order. Let its jobs finish or delete them, drop the queue with \`select ${jobsSchema}.delete_queue('${definition.name}')\`, then start again.`);
+		if (queueSettingsDiffer(wanted, queue)) {
+			const { policy, ...settings } = wanted;
+			await boss.updateQueue(definition.name, settings);
+		}
+	};
+	const startWorker = async (definition) => {
+		const concurrency = definition.options.concurrency ?? 1;
+		const running = workers.get(definition.name);
+		if (running?.concurrency === concurrency) return;
+		if (running) await boss.offWork(definition.name);
+		const id = await boss.work(definition.name, {
+			batchSize: 1,
+			localConcurrency: concurrency,
+			pollingIntervalSeconds: pollSeconds,
+			includeMetadata: true,
+			perJobResults: true
+		}, async ([job]) => {
+			const current = definitions.get(definition.name) ?? definition;
+			if (current.options.schedule !== void 0 && job.singletonKey === "protobase.schedule") await schedules.following(current, job.startAfter);
+			return [await runJob(current, job, db)];
+		});
+		workers.set(definition.name, {
+			id,
+			concurrency
+		});
+	};
+	const update = async (next) => {
+		for (const definition of next) await ensureQueue(definition);
+		for (const name of definitions.keys()) {
+			if (next.some((definition) => definition.name === name)) continue;
+			definitions.delete(name);
+			if (workers.delete(name)) await boss.offWork(name);
+		}
+		for (const definition of next) definitions.set(definition.name, definition);
+		if (!work) return;
+		for (const definition of next) await startWorker(definition);
+		await schedules.sync(next);
+	};
+	const enqueue = async (definition, target, payload, options = {}) => {
+		const { name } = definition;
+		if (!definitions.has(name)) throw new Error(`Job "${name}" is not among the app's jobs; export it from the module protobase.config.ts passes as \`jobs\``);
+		const checked = await validatePayload(definition, payload);
+		if ("invalid" in checked) throw new Error(checked.invalid);
+		const data = checked.value ?? null;
+		const send = {
+			...sendOptions(definition.options, options),
+			db: jobsDatabase(target)
+		};
+		const keyed = async (key) => {
+			if (options.mode !== "replace") {
+				const [waiting] = await boss.findJobs(name, {
+					key,
+					queued: true,
+					db: send.db
+				});
+				if (waiting) return waiting.id;
+			}
+			return (await boss.upsert(name, data, send)).jobs[0];
+		};
+		const id = options.key === void 0 ? await boss.send(name, data, send) : await keyed(options.key);
+		if (!id) throw new Error(`Job "${name}" was not enqueued`);
+		if (!target.isTransaction) {
+			const worker = workers.get(name);
+			if (worker) boss.notifyWorker(worker.id);
+		}
+		return id;
+	};
+	const nextDue = async () => (await sql`select ${sql.id(jobsSchema, "next_due")}() as due`.execute(db)).rows[0].due;
+	const pending = async () => {
+		const job = sql.id(jobsSchema, "job");
+		const { rows } = await sql`
+      select count(*)::int as count from ${job} j where j.name = any(${[...definitions.keys()]}::text[]) and (
+        (j.state < 'active' and not j.blocked and j.start_after <= now()
+          and not (j.policy = 'key_strict_fifo' and exists (
+            select 1 from ${job} h where h.name = j.name and h.singleton_key = j.singleton_key and h.state = 'failed')))
+        or (j.state = 'active' and j.started_on + j.expire_seconds * interval '1s' > now()))`.execute(db);
+		return rows[0].count;
+	};
+	const drain = async (seconds) => {
+		if (!work) throw new Error("This process runs no jobs, so it cannot drain them");
+		const deadline = Date.now() + seconds * 1e3;
+		for (;;) {
+			for (const worker of workers.values()) boss.notifyWorker(worker.id);
+			const remaining = await pending();
+			if (remaining === 0 || Date.now() >= deadline) return {
+				drained: remaining === 0,
+				remaining,
+				nextDue: await nextDue()
+			};
+			await new Promise((resolve) => setTimeout(resolve, 100));
+		}
+	};
+	const runtime = {
+		works: work,
+		enqueue,
+		update,
+		drain,
+		nextDue,
+		stop: async () => {
+			await boss.stop({
+				graceful: true,
+				timeout: shutdownSeconds * 1e3
+			});
+			clearCurrentJobs(runtime);
+		}
+	};
+	try {
+		await update(jobs);
+		setCurrentJobs(runtime);
+	} catch (error) {
+		await boss.stop({ graceful: false });
+		throw error;
+	}
+	return runtime;
+};
+//#endregion
+export { startJobs };

package/dist/jobs/schedule.js

@@ -0,0 +1,25 @@
+import { createHash } from "node:crypto";
+import { CronExpressionParser } from "cron-parser";
+//#region src/jobs/schedule.ts
+/**
+* The key every run of a recurring job waits under. Each run is a job of its own, enqueued for its time: it is what
+* `jobs.next_due()` sees while the app sleeps, so a host that wakes the app for due jobs wakes it for these too.
+*/
+const scheduleKey = "protobase.schedule";
+/** The first run of `schedule` strictly after `after`, in `timezone`; throws on an expression cron-parser refuses. */
+const nextRun = (schedule, timezone, after) => CronExpressionParser.parse(schedule, {
+	currentDate: after,
+	tz: timezone
+}).next().toDate();
+/** Whether `at` is a run of `schedule`, so a run left from an earlier schedule can be told apart. */
+const isRun = (schedule, timezone, at) => nextRun(schedule, timezone, /* @__PURE__ */ new Date(at.getTime() - 1)).getTime() === at.getTime();
+/**
+* The id of the run of job `name` at `at`: the same in every process, so however many workers enqueue a run, the
+* database keeps one (its primary key), and the run happens once.
+*/
+const runId = (name, at) => {
+	const hex = createHash("sha256").update(`${name}@${at.toISOString()}`).digest("hex");
+	return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-8${hex.slice(13, 16)}-${(parseInt(hex[16], 16) & 3 | 8).toString(16)}${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
+};
+//#endregion
+export { isRun, nextRun, runId, scheduleKey };

package/dist/jobs/schedules.js

@@ -0,0 +1,41 @@
+import { jobsSchema } from "./schema.js";
+import { isRun, nextRun, runId, scheduleKey } from "./schedule.js";
+import { jobsDatabase } from "./database.js";
+import { sql } from "kysely";
+//#region src/jobs/schedules.ts
+const job = sql.id(jobsSchema, "job");
+/** The recurring jobs' runs: enqueued ahead, one per time, and kept in step with their schedules. */
+const createSchedules = (boss, db, now) => {
+	const timezone = (definition) => definition.options.timezone ?? "UTC";
+	const enqueueRun = (definition, at) => boss.send(definition.name, null, {
+		id: runId(definition.name, at),
+		startAfter: at,
+		singletonKey: scheduleKey,
+		db: jobsDatabase(db)
+	});
+	/**
+	* Makes the waiting runs match `definitions`: drops those of jobs without a schedule now, and those at a time the
+	* schedule no longer has, then enqueues each schedule's next run. A run that is due stays, to run as soon as it can.
+	*/
+	const sync = async (definitions) => {
+		const scheduled = definitions.filter((definition) => definition.options.schedule !== void 0);
+		await sql`delete from ${job} where singleton_key = ${scheduleKey} and state < 'active' and not (name = any(${scheduled.map(({ name }) => name)}::text[]))`.execute(db);
+		for (const definition of scheduled) {
+			const stale = (await sql`
+        select id, start_after from ${job} where name = ${definition.name} and singleton_key = ${scheduleKey} and state < 'active'`.execute(db)).rows.filter((run) => !isRun(definition.options.schedule, timezone(definition), new Date(run.start_after))).map((run) => run.id);
+			if (stale.length > 0) await sql`delete from ${job} where name = ${definition.name} and id = any(${stale}::uuid[])`.execute(db);
+			await enqueueRun(definition, nextRun(definition.options.schedule, timezone(definition), now()));
+		}
+	};
+	/** As a run starts, the one after it: from its own time, or from now when it ran late, so missed times are skipped. */
+	const following = async (definition, ranAt) => {
+		const from = new Date(Math.max(ranAt.getTime(), now().getTime()));
+		await enqueueRun(definition, nextRun(definition.options.schedule, timezone(definition), from));
+	};
+	return {
+		sync,
+		following
+	};
+};
+//#endregion
+export { createSchedules };

package/dist/jobs/schema.d.ts

@@ -0,0 +1,15 @@
+import { Db } from "@protobase/query";
+//#region src/jobs/schema.d.ts
+/** The schema the job queue lives in: pg-boss's tables and Protobase's functions. */
+export declare const jobsSchema = "jobs";
+/**
+ * The SQL that brings the jobs schema to what this version of Protobase runs: pg-boss's own construction plan when it
+ * is missing, or its migration plan from the version installed, then Protobase's functions. `undefined` when current.
+ */
+export declare const jobsSchemaMigration: (db: Db) => Promise<string | undefined>;
+/**
+ * Refuses to go on while the jobs schema is missing or not the version this Protobase runs. The server never creates
+ * or migrates it: the project's migrations do, before the new version serves.
+ */
+export declare const checkJobsSchema: (db: Db) => Promise<void>;
+//#endregion

package/dist/jobs/schema.js

@@ -0,0 +1,63 @@
+import { functionNames, functionsComment, functionsSql } from "./functions-sql.js";
+import { CompiledQuery } from "kysely";
+import { getConstructionPlans, getMigrationPlans } from "pg-boss";
+//#region src/jobs/schema.ts
+/** The schema the job queue lives in: pg-boss's tables and Protobase's functions. */
+const jobsSchema = "jobs";
+/** The command that writes what the database lacks as the project's next migration. */
+const migrationCommand = "protobase migration";
+/**
+* A pg-boss plan as a migration the project applies in its own transaction: without its BEGIN and COMMIT, without the
+* timeouts it sets for that transaction, which would hold for the project's later migrations too, and with index
+* builds that are not `CONCURRENTLY`, which Postgres refuses in a transaction. Migrations run while nothing else
+* writes, so a plain index build only costs a lock nobody waits on.
+*/
+const migrationStatements = (plan) => {
+	const match = /^\s*BEGIN;\n\s*SET LOCAL lock_timeout = \d+;\n\s*SET LOCAL idle_in_transaction_session_timeout = \d+;\n([\s\S]*?)\n\s*COMMIT;\n?([\s\S]*)$/.exec(plan);
+	if (!match) throw new Error(`Unexpected pg-boss plan:\n${plan.slice(0, 200)}`);
+	return `${match[1].trim()}\n${match[2]}`.replace(/\b(CREATE (?:UNIQUE )?INDEX|DROP INDEX|REINDEX INDEX) CONCURRENTLY\b/gi, "$1").trim();
+};
+const rows = async (db, text, parameters = []) => (await db.executeQuery(CompiledQuery.raw(text, parameters))).rows;
+/** What the database holds of the jobs schema: pg-boss's version, when installed, and whether Protobase's functions are current. */
+const readJobsSchema = async (db) => {
+	const [{ installed }] = await rows(db, `select to_regclass('${jobsSchema}.version') is not null as installed`);
+	if (!installed) return {
+		pgBoss: void 0,
+		functionsCurrent: false
+	};
+	const [version] = await rows(db, `select version from ${jobsSchema}.version`);
+	const functions = await rows(db, `select p.proname as name, obj_description(p.oid, 'pg_proc') as comment from pg_proc p
+     join pg_namespace n on n.oid = p.pronamespace where n.nspname = $1 and p.proname = any($2)`, [jobsSchema, functionNames]);
+	const functionsCurrent = functionNames.every((name) => functions.some((found) => found.name === name && found.comment === functionsComment));
+	return {
+		pgBoss: version === void 0 ? void 0 : Number(version.version),
+		functionsCurrent
+	};
+};
+const newer = (version) => /* @__PURE__ */ new Error(`The jobs schema is at pg-boss schema version ${version}, newer than the 45 this version of Protobase runs; upgrade Protobase`);
+/**
+* The SQL that brings the jobs schema to what this version of Protobase runs: pg-boss's own construction plan when it
+* is missing, or its migration plan from the version installed, then Protobase's functions. `undefined` when current.
+*/
+const jobsSchemaMigration = async (db) => {
+	const { pgBoss, functionsCurrent } = await readJobsSchema(db);
+	if (pgBoss !== void 0 && pgBoss > 45) throw newer(pgBoss);
+	if (pgBoss === 45 && functionsCurrent) return void 0;
+	const parts = [];
+	if (pgBoss === void 0) parts.push(migrationStatements(getConstructionPlans(jobsSchema)));
+	else if (pgBoss < 45) parts.push(migrationStatements(getMigrationPlans(jobsSchema, pgBoss)));
+	parts.push(functionsSql(jobsSchema));
+	return parts.join("\n\n");
+};
+/**
+* Refuses to go on while the jobs schema is missing or not the version this Protobase runs. The server never creates
+* or migrates it: the project's migrations do, before the new version serves.
+*/
+const checkJobsSchema = async (db) => {
+	const { pgBoss, functionsCurrent } = await readJobsSchema(db);
+	if (pgBoss !== void 0 && pgBoss > 45) throw newer(pgBoss);
+	if (pgBoss === 45 && functionsCurrent) return;
+	throw new Error(`The jobs schema is ${pgBoss === void 0 ? "missing" : "behind this version of Protobase"}. Write it as a migration with \`${migrationCommand}\`, apply it with the project's migrations, then start again.`);
+};
+//#endregion
+export { checkJobsSchema, jobsSchema, jobsSchemaMigration, migrationCommand, migrationStatements, readJobsSchema };

package/dist/project-config.d.ts

@@ -24,13 +24,15 @@ export type ProjectConfig = {
   files?: FilesOptions;
   /** Where the changes that resources' events are raised from are captured (see `EventsOptions`). */
   events?: EventsOptions;
+  /** The module of the app's jobs, e.g. `import * as jobs from './config/jobs'`: every `defineJob` it exports. */
+  jobs?: Record<string, unknown>;
   options?: AdminOptions;
 };
 /** Typed identity, for `export default defineConfig({ extends: [...], config, auth, ... })` in `protobase.config.ts`. */
 export declare const defineConfig: (config: ProjectConfig) => ProjectConfig;
 /**
  * One config from several, each after the ones it `extends`, the last being the app's own. Named things are all kept:
- * resources, views and pages of the config modules, functions, assistant tools, file providers; one name defined by two configs is an error naming
+ * resources, views and pages of the config modules, functions, jobs, assistant tools, file providers; one name defined by two configs is an error naming
  * both. Write hooks run in the same order. Every other value is the last config's that sets it, so the app's own wins.
  */
 export declare const mergeConfig: (...configs: ProjectConfig[]) => ProjectConfig;

package/dist/project-config.js

@@ -1,3 +1,4 @@
+import { jobDefinitions } from "./jobs/define-job.js";
 import { isPageDefinition } from "@protobase/layout";
 //#region src/project-config.ts
 /** Typed identity, for `export default defineConfig({ extends: [...], config, auth, ... })` in `protobase.config.ts`. */
@@ -104,6 +105,16 @@ const mergedFiles = (named) => {
 		...secret !== void 0 && { secret }
 	};
 };
+const mergedJobs = (named) => {
+	const all = named.flatMap(({ config, label }) => jobDefinitions(config.jobs ?? {}).map((value) => ({
+		name: value.name,
+		value,
+		label
+	})));
+	if (all.length === 0) return void 0;
+	checkUnique("The job", all);
+	return Object.fromEntries(all.map(({ name, value }) => [name, value]));
+};
 const mergedOptions = (named) => {
 	const all = named.flatMap(({ config }) => config.options ? [config.options] : []);
 	if (all.length === 0) return void 0;
@@ -117,7 +128,7 @@ const mergedOptions = (named) => {
 };
 /**
 * One config from several, each after the ones it `extends`, the last being the app's own. Named things are all kept:
-* resources, views and pages of the config modules, functions, assistant tools, file providers; one name defined by two configs is an error naming
+* resources, views and pages of the config modules, functions, jobs, assistant tools, file providers; one name defined by two configs is an error naming
 * both. Write hooks run in the same order. Every other value is the last config's that sets it, so the app's own wins.
 */
 const mergeConfig = (...configs) => {
@@ -131,6 +142,7 @@ const mergeConfig = (...configs) => {
 	const module = mergedModule(named);
 	const functions = mergedFunctions(named);
 	const files = mergedFiles(named);
+	const jobs = mergedJobs(named);
 	const options = mergedOptions(named);
 	const name = configs.at(-1)?.name;
 	return {
@@ -142,6 +154,7 @@ const mergeConfig = (...configs) => {
 		...functions && { functions },
 		...files && { files },
 		...events && { events },
+		...jobs && { jobs },
 		...options && { options }
 	};
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/server",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "description": "The Protobase REST API as a Hono app, with Better Auth sign-in.",
   "license": "Apache-2.0",
   "repository": {
@@ -30,18 +30,20 @@
   },
   "dependencies": {
     "@better-auth/passkey": "^1.7.7",
-    "@protobase/layout": "0.8.1",
-    "@protobase/query": "0.8.1",
-    "@protobase/schema": "0.8.1",
+    "@protobase/layout": "0.8.2",
+    "@protobase/query": "0.8.2",
+    "@protobase/schema": "0.8.2",
     "@scalar/hono-api-reference": "^0.12.10",
     "@simplewebauthn/server": "^13.3.1",
     "better-auth": "^1.7.7",
+    "cron-parser": "^5.10.1",
     "file-type": "^22.1.1",
     "hono": "^4.13.13",
     "image-size": "^2.0.4",
     "jose": "^6.2.12",
     "kysely": "^0.29.6",
     "nodemailer": "^10.0.16",
+    "pg-boss": "12.37.1",
     "zod": "^4.6.5"
   }
 }