@protobase/server 0.8.0 → 0.8.2

package/dist/index.d.ts

@@ -6,6 +6,7 @@ import { OrganizationRole, StoredMember, StoredOrganization } from "./better-aut
 import { OrganizationsOptions } from "./better-auth/organizations/options.js";
 import { PlatformSignIn, readPlatformSignIn } from "./better-auth/platform-sign-in.js";
 import { AdminAuth, CreateAuthOptions, UserCreatedHook, createAuth, normalizeRoles } from "./better-auth/create-auth.js";
+import { EventsOptions } from "./changes/events-options.js";
 import { FileProvider, FileStore, StoredObject } from "./files/store.js";
 import { FilesOptions } from "./files/options.js";
 import { Admin, AuditEvent, AuditQueue, Authenticator, PipelineHook, ScanGuardMode, Session, SessionOrganization, TenantValue, User, ViewSource, WriteEvent, WriteOperation } from "./types.js";
@@ -23,6 +24,8 @@ import { AssistantTool, ToolContext, defineTool } from "./assistant/tools.js";
 import { AssistantOptions, AssistantSettings, assistantRoles, assistantSettings, assistantVariables, seesAssistant } from "./assistant/assistant-settings.js";
 import { RuntimeOptions, runtimeUrl, runtimeVariable, seesRuntime } from "./runtime/runtime-settings.js";
 import { AdminOptions } from "./admin-options.js";
+import { StandardSchema, readJson } from "./functions/read-json.js";
+import { EnqueueOptions, JobContext, JobDefinition, JobOptions } from "./jobs/job-types.js";
 import { CreateAdminInput, createAdmin } from "./create-admin.js";
 import { ProjectConfig, defineConfig, mergeConfig } from "./project-config.js";
 import { consoleAuditQueue } from "./audit/console-queue.js";
@@ -33,6 +36,10 @@ import { parseRoles } from "./better-auth/parse-roles.js";
 import { LinkedAccount, NewUser, StoredUser, createUser, deleteUser, hasUsers, listUsers, roleChoices, setUserBanned, setUserRole } from "./better-auth/users.js";
 import { IssueTokenOptions, issueToken, maxTokenTtlSeconds } from "./better-auth/tokens.js";
 import { authSchemaMigration, checkAuthSchema } from "./better-auth/auth-schema.js";
+import { changesMigration, checkChanges, migrationCommand } from "./changes/capture-schema.js";
+import { CapturedRow, CoalescedChange, coalesce } from "./changes/coalesce.js";
+import { ResourceChange, recordOf, resourceChange } from "./changes/resource-change.js";
+import { eventMatcher } from "./changes/match-events.js";
 import { MemberInput, NewOrganization, addAppRoleToOwners, addMember, createOrganization, listOrganizations, removeMember, setMemberRoles } from "./better-auth/organizations/host.js";
 import { AssistantBackend, assistantProtocolRoutes } from "./assistant/protocol-routes.js";
 import { fileConversationStore } from "./assistant/file-conversation-store.js";
@@ -42,11 +49,15 @@ import { WidgetOptions, maxWidgetPropsBytes, widget } from "./assistant/widget.j
 import { QueryOptions, QueryResult, runReadOnlyQuery, runReadWriteQuery } from "./assistant/run-query.js";
 import { readOnlyQueryTool, readWriteQueryTool } from "./assistant/query-tools.js";
 import { FunctionEnv, PublicFunctionEnv, functionApp } from "./functions/function-app.js";
-import { StandardSchema, readJson } from "./functions/read-json.js";
 import { SignatureOptions, StandardWebhookOptions, verifySignature, verifyStandardWebhook } from "./functions/verify-signature.js";
 import { requireEnv } from "./functions/require-env.js";
 import { LocalFilesOptions, localFileStore, localFiles } from "./files/local-files.js";
 import { FileCleanupInput, createFileCleanup } from "./files/file-cleanup.js";
 import { FileObject } from "./files/present.js";
 import { aspectRatio, fileSize, fileType, imageSize, processor } from "./files/processors.js";
-export { type Admin, type AdminAuth, type AdminOptions, type ApiFunction, type ApprovalAnswer, type ApprovalRequest, type AssistantBackend, type AssistantOptions, type AssistantSettings, type AssistantTool, type AuditEvent, type AuditQueue, type Authenticator, type BetterAuthAuthenticatorOptions, type ChatMessage, type ChatTextPart, type CleanupEntry, type CleanupReport, type CleanupSchedule, type Completion, type CompletionOptions, type Conversation, type ConversationRecord, type ConversationStore, type ConversationsOptions, type CorsOptions, type CreateAdminInput, type CreateAuthOptions, type FileCleanupInput, type FileObject, type FileProvider, type FileStore, type FilesOptions, type FunctionApp, type FunctionCaller, type FunctionContext, type FunctionEnv, type FunctionHandler, type FunctionOptions, type FunctionRecords, type FunctionSource, HttpProblem, type IssueTokenOptions, type JwtOptions, type LinkedAccount, type ListParams, type LocalFilesOptions, type MemberInput, ModelError, type ModelSettings, type NewOrganization, type NewUser, type OperatorProvider, type OrganizationRole, type OrganizationsOptions, type PipelineHook, type PlatformSignIn, type ProjectConfig, type ProtectedFunctionOptions, type PublicFunctionContext, type PublicFunctionEnv, type PublicFunctionHandler, type PublicFunctionOptions, type QueryOptions, type QueryResult, type ReasoningDetail, type ReasoningEffort, type RecordKeyInput, type RecordPage, type ResourceSource, type RoleDefinition, type RolesInput, type RuntimeOptions, type ScanGuardMode, type ScheduledDelete, type Session, type SessionOrganization, type SignInPolicy, type SignInRule, type SignatureOptions, type StaffAccessRule, type StaffSignIn, type StandardSchema, type StandardWebhookOptions, type StoredMember, type StoredObject, type StoredOrganization, type StoredUser, type TenantValue, type ToolCall, type ToolContext, type ToolRecord, type ToolRecords, type ToolSpec, type TurnOptions, type User, type UserCreatedHook, type ViewSource, type WidgetOptions, type WriteEvent, type WriteOperation, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, checkAuthSchema, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineTool, deleteUser, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, issueToken, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, removeMember, requestApproval, requireEnv, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };
+import { defineJob, isJobDefinition, jobDefinitions } from "./jobs/define-job.js";
+import { FatalJobError } from "./jobs/run-job.js";
+import { DrainResult, JobsRuntime, StartJobsOptions, startJobs } from "./jobs/runtime.js";
+import { jobsDrainApp } from "./jobs/drain-route.js";
+import { checkJobsSchema, jobsSchema, jobsSchemaMigration } from "./jobs/schema.js";
+export { type Admin, type AdminAuth, type AdminOptions, type ApiFunction, type ApprovalAnswer, type ApprovalRequest, type AssistantBackend, type AssistantOptions, type AssistantSettings, type AssistantTool, type AuditEvent, type AuditQueue, type Authenticator, type BetterAuthAuthenticatorOptions, type CapturedRow, type ChatMessage, type ChatTextPart, type CleanupEntry, type CleanupReport, type CleanupSchedule, type CoalescedChange, type Completion, type CompletionOptions, type Conversation, type ConversationRecord, type ConversationStore, type ConversationsOptions, type CorsOptions, type CreateAdminInput, type CreateAuthOptions, type DrainResult, type EnqueueOptions, type EventsOptions, FatalJobError, type FileCleanupInput, type FileObject, type FileProvider, type FileStore, type FilesOptions, type FunctionApp, type FunctionCaller, type FunctionContext, type FunctionEnv, type FunctionHandler, type FunctionOptions, type FunctionRecords, type FunctionSource, HttpProblem, type IssueTokenOptions, type JobContext, type JobDefinition, type JobOptions, type JobsRuntime, type JwtOptions, type LinkedAccount, type ListParams, type LocalFilesOptions, type MemberInput, ModelError, type ModelSettings, type NewOrganization, type NewUser, type OperatorProvider, type OrganizationRole, type OrganizationsOptions, type PipelineHook, type PlatformSignIn, type ProjectConfig, type ProtectedFunctionOptions, type PublicFunctionContext, type PublicFunctionEnv, type PublicFunctionHandler, type PublicFunctionOptions, type QueryOptions, type QueryResult, type ReasoningDetail, type ReasoningEffort, type RecordKeyInput, type RecordPage, type ResourceChange, type ResourceSource, type RoleDefinition, type RolesInput, type RuntimeOptions, type ScanGuardMode, type ScheduledDelete, type Session, type SessionOrganization, type SignInPolicy, type SignInRule, type SignatureOptions, type StaffAccessRule, type StaffSignIn, type StandardSchema, type StandardWebhookOptions, type StartJobsOptions, type StoredMember, type StoredObject, type StoredOrganization, type StoredUser, type TenantValue, type ToolCall, type ToolContext, type ToolRecord, type ToolRecords, type ToolSpec, type TurnOptions, type User, type UserCreatedHook, type ViewSource, type WidgetOptions, type WriteEvent, type WriteOperation, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, changesMigration, checkAuthSchema, checkChanges, checkJobsSchema, coalesce, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineJob, defineTool, deleteUser, eventMatcher, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, isJobDefinition, issueToken, jobDefinitions, jobsDrainApp, jobsSchema, jobsSchemaMigration, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, migrationCommand, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, recordOf, removeMember, requestApproval, requireEnv, resourceChange, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, startJobs, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };

package/dist/index.js

@@ -22,13 +22,19 @@ import { storageCleanupSchedule } from "./files/cleanup-schedule.js";
 import { localFileStore, localFiles } from "./files/local-files.js";
 import { defineFunction } from "./functions/define-function.js";
 import { runtimeUrl, runtimeVariable, seesRuntime } from "./runtime/runtime-settings.js";
+import { checkJobsSchema, jobsSchema, jobsSchemaMigration } from "./jobs/schema.js";
 import { createAdmin } from "./create-admin.js";
 import { configExports } from "./resource-source.js";
+import { defineJob, isJobDefinition, jobDefinitions } from "./jobs/define-job.js";
 import { defineConfig, mergeConfig } from "./project-config.js";
 import { jwtAuthenticator } from "./auth/jwt.js";
 import { betterAuthAuthenticator } from "./better-auth/authenticator.js";
 import { issueToken, maxTokenTtlSeconds } from "./better-auth/tokens.js";
 import { authSchemaMigration, checkAuthSchema } from "./better-auth/auth-schema.js";
+import { changesMigration, checkChanges, migrationCommand } from "./changes/capture-schema.js";
+import { coalesce } from "./changes/coalesce.js";
+import { recordOf, resourceChange } from "./changes/resource-change.js";
+import { eventMatcher } from "./changes/match-events.js";
 import { maxWidgetPropsBytes, widget } from "./assistant/widget.js";
 import { functionApp } from "./functions/function-app.js";
 import { readJson } from "./functions/read-json.js";
@@ -36,4 +42,7 @@ import { verifySignature, verifyStandardWebhook } from "./functions/verify-signa
 import { requireEnv } from "./functions/require-env.js";
 import { createFileCleanup } from "./files/file-cleanup.js";
 import { aspectRatio, fileSize, fileType, imageSize, processor } from "./files/processors.js";
-export { HttpProblem, ModelError, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, checkAuthSchema, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineTool, deleteUser, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, issueToken, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, removeMember, requestApproval, requireEnv, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };
+import { FatalJobError } from "./jobs/run-job.js";
+import { startJobs } from "./jobs/runtime.js";
+import { jobsDrainApp } from "./jobs/drain-route.js";
+export { FatalJobError, HttpProblem, ModelError, addAppRoleToOwners, addMember, aspectRatio, assistantProtocolRoutes, assistantRoles, assistantSettings, assistantVariables, authSchemaMigration, badRequest, betterAuthAuthenticator, changesMigration, checkAuthSchema, checkChanges, checkJobsSchema, coalesce, configExports, consoleAuditQueue, conversationKey, createAdmin, createAuth, createConversations, createFileCleanup, createOrganization, createUser, defaultSignInPolicy, defineConfig, defineFunction, defineJob, defineTool, deleteUser, eventMatcher, fileConversationStore, fileSize, fileType, forbidden, functionApp, hasUsers, imageSize, isJobDefinition, issueToken, jobDefinitions, jobsDrainApp, jobsSchema, jobsSchemaMigration, jwtAuthenticator, listOrganizations, listUsers, localFileStore, localFiles, maxTokenTtlSeconds, maxWidgetPropsBytes, mergeConfig, migrationCommand, normalizeRoles, notFound, parseRoles, processor, readJson, readOnlyQueryTool, readOperatorSettings, readPlatformSignIn, readWriteQueryTool, recordOf, removeMember, requestApproval, requireEnv, resourceChange, roleChoices, runReadOnlyQuery, runReadWriteQuery, runTurn, runtimeUrl, runtimeVariable, seesAssistant, seesRuntime, setMemberRoles, setUserBanned, setUserRole, signInPolicyMethods, startJobs, storageCleanupSchedule, streamChatCompletion, unauthorized, verifySignature, verifyStandardWebhook, widget, withCacheBreakpoints };

package/dist/jobs/current.d.ts

@@ -0,0 +1,8 @@
+import { EnqueueOptions, JobDefinition } from "./job-types.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/current.d.ts
+/** What `defineJob(...).enqueue` reaches: the jobs runtime this process started. */
+export type JobsEnqueuer = {
+  enqueue(definition: JobDefinition, db: Db, payload: unknown, options?: EnqueueOptions): Promise<string>;
+};
+//#endregion

package/dist/jobs/current.js

@@ -0,0 +1,18 @@
+//#region src/jobs/current.ts
+const key = Symbol.for("protobase.jobs");
+const holder = globalThis;
+/** The running jobs runtime; enqueueing needs one, started by `protobase dev`, `serve` and `worker`, or by `startJobs`. */
+const currentJobs = () => {
+	const runtime = holder[key];
+	if (!runtime) throw new Error("Jobs are not running in this process: start them with startJobs(), as protobase dev, serve and worker do");
+	return runtime;
+};
+const setCurrentJobs = (runtime) => {
+	if (holder[key]) throw new Error("Jobs are already running in this process; stop them before starting them again");
+	holder[key] = runtime;
+};
+const clearCurrentJobs = (runtime) => {
+	if (holder[key] === runtime) delete holder[key];
+};
+//#endregion
+export { clearCurrentJobs, currentJobs, setCurrentJobs };

package/dist/jobs/database.js

@@ -0,0 +1,36 @@
+import { CompiledQuery } from "kysely";
+//#region src/jobs/database.ts
+const block = /^\s*BEGIN;\s*(SET LOCAL lock_timeout = \d+);\s*(SET LOCAL idle_in_transaction_session_timeout = \d+);\s*(SELECT pg_advisory_xact_lock\([^;]*\);\s*)?([\s\S]+?);\s*COMMIT;\s*$/;
+/**
+* The statements of one of pg-boss's transaction texts, in order, or `undefined` for any other text. A text that
+* starts a transaction and is not in that shape is an error, so a pg-boss upgrade that changes it fails loudly.
+*/
+const transactionStatements = (text) => {
+	if (!/^\s*BEGIN;/.test(text)) return void 0;
+	const match = block.exec(text);
+	if (!match) throw new Error(`Unexpected pg-boss transaction:\n${text}`);
+	const [, lockTimeout, idleTimeout, lock, statement] = match;
+	return [
+		lockTimeout,
+		idleTimeout,
+		...lock ? [lock.trim().replace(/;$/, "")] : [],
+		statement
+	];
+};
+/**
+* pg-boss's database over the app's own Kysely pool or transaction: one connection pool for the app and its jobs, the
+* same driver on Node, Bun and PGlite. Every statement runs on its own, or inside one transaction for pg-boss's
+* transaction texts, which works through a pooler in transaction mode: pg-boss takes only transaction-scoped locks.
+*/
+const jobsDatabase = (db) => ({ executeSql: async (text, values) => {
+	if (values && values.length > 0) return { rows: [...(await db.executeQuery(CompiledQuery.raw(text, values))).rows] };
+	const statements = transactionStatements(text);
+	if (!statements) return { rows: [...(await db.executeQuery(CompiledQuery.raw(text))).rows] };
+	return db.transaction().execute(async (trx) => {
+		let rows = [];
+		for (const statement of statements) rows = [...(await trx.executeQuery(CompiledQuery.raw(statement))).rows];
+		return { rows };
+	});
+} });
+//#endregion
+export { jobsDatabase, transactionStatements };

package/dist/jobs/define-job.d.ts

@@ -0,0 +1,8 @@
+import { JobDefinition, JobOptions } from "./job-types.js";
+//#region src/jobs/define-job.d.ts
+/** A job the app's workers run: export it from the module that `protobase.config.ts` passes as `jobs`. */
+export declare const defineJob: <Payload = undefined>(options: JobOptions<Payload>) => JobDefinition<Payload>;
+export declare const isJobDefinition: (value: unknown) => value is JobDefinition;
+/** The job definitions among a module's exports, such as `import * as jobs from './config/jobs'`, each name once. */
+export declare const jobDefinitions: (module: Record<string, unknown>) => JobDefinition[];
+//#endregion

package/dist/jobs/define-job.js

@@ -0,0 +1,38 @@
+import { currentJobs } from "./current.js";
+import { nextRun } from "./schedule.js";
+//#region src/jobs/define-job.ts
+const namePattern = /^[\w.\-/]+$/;
+const checkSchedule = (name, schedule, timezone) => {
+	try {
+		nextRun(schedule, timezone, /* @__PURE__ */ new Date());
+	} catch (error) {
+		if (!(error instanceof Error)) throw error;
+		throw new Error(`Job "${name}" has the schedule "${schedule}" in ${timezone}, which is not a cron schedule there: ${error.message}`, { cause: error });
+	}
+};
+/** A job the app's workers run: export it from the module that `protobase.config.ts` passes as `jobs`. */
+const defineJob = (options) => {
+	if (!namePattern.test(options.name)) throw new Error(`Job name "${options.name}" may only hold letters, digits, "_", "-", "." and "/"`);
+	if (options.schedule !== void 0) checkSchedule(options.name, options.schedule, options.timezone ?? "UTC");
+	const definition = {
+		kind: "job",
+		name: options.name,
+		options,
+		enqueue: async (db, payload, enqueueOptions) => currentJobs().enqueue(definition, db, payload, enqueueOptions)
+	};
+	return definition;
+};
+const isJobDefinition = (value) => typeof value === "object" && value !== null && value.kind === "job" && typeof value.enqueue === "function";
+/** The job definitions among a module's exports, such as `import * as jobs from './config/jobs'`, each name once. */
+const jobDefinitions = (module) => {
+	const byName = /* @__PURE__ */ new Map();
+	for (const value of Object.values(module)) {
+		if (!isJobDefinition(value)) continue;
+		const known = byName.get(value.name);
+		if (known && known !== value) throw new Error(`Two jobs are named "${value.name}"; rename one of them`);
+		byName.set(value.name, value);
+	}
+	return [...byName.values()];
+};
+//#endregion
+export { defineJob, isJobDefinition, jobDefinitions };

package/dist/jobs/drain-route.d.ts

@@ -0,0 +1,21 @@
+import { JobsRuntime } from "./runtime.js";
+//#region src/jobs/drain-route.d.ts
+/**
+ * `POST /api/jobs/drain?seconds=25`: runs the app's due jobs until none is left or `seconds` have passed, and answers
+ * `{ drained, remaining, nextDue }`. For the operator of a host that lets the app sleep and wakes it for its jobs: it
+ * takes the operator's `token` as a bearer token, never a user's.
+ */
+export declare const jobsDrainApp: ({ runtime, token }: {
+  runtime: JobsRuntime;
+  token: string;
+}) => import("hono/hono-base").HonoBase<import("hono/types").BlankEnv, {
+  "/api/jobs/drain": {
+    $post: {
+      input: {};
+      output: {};
+      outputFormat: string;
+      status: import("hono/utils/http-status").StatusCode;
+    };
+  };
+}, "/", "/api/jobs/drain">;
+//#endregion

package/dist/jobs/drain-route.js

@@ -0,0 +1,28 @@
+import { HttpProblem, badRequest, problemResponse, unauthorized } from "../problem.js";
+import { Hono } from "hono";
+import { createHash, timingSafeEqual } from "node:crypto";
+//#region src/jobs/drain-route.ts
+const drainPath = "/api/jobs/drain";
+const digest = (text) => createHash("sha256").update(text).digest();
+const sameToken = (given, token) => given !== void 0 && timingSafeEqual(digest(given), digest(token));
+const maxSeconds = 300;
+/**
+* `POST /api/jobs/drain?seconds=25`: runs the app's due jobs until none is left or `seconds` have passed, and answers
+* `{ drained, remaining, nextDue }`. For the operator of a host that lets the app sleep and wakes it for its jobs: it
+* takes the operator's `token` as a bearer token, never a user's.
+*/
+const jobsDrainApp = ({ runtime, token }) => new Hono().post(drainPath, async (c) => {
+	const problem = (error) => problemResponse(error, c.req.path);
+	if (!sameToken(c.req.header("authorization")?.replace(/^Bearer /, ""), token)) return problem(unauthorized("Send the operator token as a bearer token"));
+	const seconds = Number(c.req.query("seconds") ?? 25);
+	if (!Number.isFinite(seconds) || seconds <= 0 || seconds > maxSeconds) return problem(badRequest("invalid-seconds", `seconds must be more than 0 and at most ${maxSeconds}`));
+	if (!runtime.works) return problem(new HttpProblem(409, "jobs-elsewhere", "Conflict", "This process runs no jobs (PROTOBASE_JOBS=off); drain the one that does"));
+	const { drained, remaining, nextDue } = await runtime.drain(seconds);
+	return c.json({
+		drained,
+		remaining,
+		nextDue: nextDue?.toISOString() ?? null
+	});
+});
+//#endregion
+export { drainPath, jobsDrainApp };

package/dist/jobs/failed-jobs.js

@@ -0,0 +1,67 @@
+import { jobsSchema } from "./schema.js";
+import { f, l, resource, view } from "@protobase/schema";
+//#region src/jobs/failed-jobs.ts
+const admins = ({ user }) => user?.roles?.includes("admin") ?? false;
+/**
+* The failed jobs, over the `jobs.failed` view: the admin app lists them with their last error, and retries or deletes
+* one through the view's triggers. Admins only, whatever the project's roles say.
+*/
+const failedJobs = resource("failedJobs").table(`${jobsSchema}.failed`).fields({
+	id: f.uuid().readOnly().filterable(),
+	name: f.text().readOnly().filterable().sortable(),
+	error: f.text().optional().readOnly(),
+	attempts: f.integer().readOnly(),
+	failedAt: f.timestamp().readOnly().filterable().sortable(),
+	createdAt: f.timestamp().readOnly().sortable(),
+	key: f.text().optional().readOnly().filterable(),
+	payload: f.json().optional().readOnly(),
+	output: f.json().optional().readOnly(),
+	retry: f.boolean()
+}).primaryKey((r) => r.id).access({
+	list: admins,
+	read: admins,
+	update: admins,
+	delete: admins,
+	create: () => false
+});
+const failedJobsView = view("failedJobs").names({
+	singular: "Failed job",
+	plural: "Failed jobs"
+}).icon("triangle-alert").title((r) => r.name).help("Jobs that used up their attempts, or failed for good. Retry one once its cause is fixed: it gets one more attempt.").fields((r) => ({
+	name: r.name.label("Job"),
+	error: r.error.label("Last error"),
+	failedAt: r.failedAt.label("Failed"),
+	createdAt: r.createdAt.label("Enqueued"),
+	output: r.output.label("Error details")
+})).list((r) => ({
+	columns: [
+		r.name,
+		r.error,
+		r.attempts,
+		r.failedAt
+	],
+	sort: [[r.failedAt, "desc"]]
+})).layout((r) => [
+	l.section("Failure", [r.error, r.output]),
+	l.section("Job", [
+		r.name,
+		r.key,
+		r.payload
+	]),
+	l.sidebar([
+		r.attempts,
+		r.failedAt,
+		r.createdAt
+	])
+]).actions((a) => [a.update("retry", {
+	label: "Retry",
+	icon: "rotate-ccw",
+	set: { retry: true },
+	confirm: "Run this job once more, from now?"
+}), a.remove("delete", {
+	label: "Delete",
+	icon: "trash-2",
+	confirm: "Delete this job? It will not run again."
+})]);
+//#endregion
+export { failedJobs, failedJobsView };

package/dist/jobs/functions-sql.js

@@ -0,0 +1,126 @@
+/** The comment each Protobase function in the jobs schema carries, which the startup check reads. */
+const functionsComment = `Protobase jobs 2`;
+/** Protobase's own functions beside pg-boss's tables, by name; each carries `functionsComment`. */
+const functionNames = [
+	"enqueue",
+	"next_due",
+	"failed_retry",
+	"failed_delete"
+];
+/**
+* `jobs.enqueue(name, payload, run_at, key, group_key, priority)`: a job from SQL, such as a trigger, in the caller's
+* transaction. It inserts what pg-boss's `send()` does, with the queue's retry settings; it creates a queue that no
+* worker has created yet. A `key` keeps one waiting job, as `mode: 'dedupe'` does, and an ordered job takes its group
+* as `group_key`. Tied to the pg-boss schema version, which the startup check pins.
+*/
+const enqueue = (schema) => `create or replace function ${schema}.enqueue(
+  job_name text,
+  payload jsonb default null,
+  run_at timestamptz default null,
+  key text default null,
+  group_key text default null,
+  priority integer default 0
+) returns uuid
+language plpgsql as $$
+declare
+  queue ${schema}.queue;
+  start timestamptz := coalesce(run_at, ${schema}.job_now());
+  job_id uuid;
+begin
+  if key is not null and group_key is not null then
+    raise exception 'Job "%": an ordered job takes a group_key and no key', job_name;
+  end if;
+  perform ${schema}.create_queue(job_name, jsonb_build_object('policy', case when group_key is null then 'standard' else 'key_strict_fifo' end));
+  select * into queue from ${schema}.queue where name = job_name;
+  if (queue.policy = 'key_strict_fifo') <> (group_key is not null) then
+    raise exception 'Job "%" %', job_name, case when group_key is null then 'is ordered: pass its group_key' else 'is not ordered: it takes no group_key' end;
+  end if;
+  if key is not null then
+    select id into job_id from ${schema}.job
+    where name = job_name and singleton_key = key and state < 'active'
+    order by created_on desc limit 1;
+    if found then
+      return job_id;
+    end if;
+  end if;
+  execute format(
+    'insert into ${schema}.%I (name, data, priority, start_after, singleton_key, retry_limit, retry_delay, retry_backoff,
+       retry_delay_max, expire_seconds, deletion_seconds, keep_until, policy, dead_letter, heartbeat_seconds, upsert_by_key)
+     values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $4 + $12 * interval ''1s'', $13, $14, $15, $16)
+     on conflict do nothing
+     returning id',
+    queue.table_name)
+  using job_name, payload, priority, start, coalesce(key, group_key), queue.retry_limit, queue.retry_delay,
+    queue.retry_backoff, queue.retry_delay_max, queue.expire_seconds, queue.deletion_seconds, queue.retention_seconds,
+    queue.policy, queue.dead_letter, queue.heartbeat_seconds, key is not null
+  into job_id;
+  -- A concurrent enqueue of the same key inserted first: its job is the one waiting.
+  if job_id is null and key is not null then
+    select id into job_id from ${schema}.job
+    where name = job_name and singleton_key = key and state < 'active'
+    order by created_on desc limit 1;
+  end if;
+  return job_id;
+end
+$$;
+comment on function ${schema}.enqueue(text, jsonb, timestamptz, text, text, integer) is '${functionsComment}';`;
+/**
+* `jobs.next_due()`: when the next job is due, or `null` when none waits: what a host that sleeps reads to know when to
+* wake the app, through any pooler. A waiting job counts from its start time, unless a failed job holds back its ordered
+* group; a running one from when its try times out, since its worker may have died with the app. Recurring jobs wait
+* as their next run, so they count too.
+*/
+const nextDue = (schema) => `create or replace function ${schema}.next_due() returns timestamptz
+language sql stable as $$
+  select min(due) from (
+    select j.start_after as due from ${schema}.job j
+    where j.state < 'active' and not j.blocked
+      and not (j.policy = 'key_strict_fifo' and exists (
+        select 1 from ${schema}.job h where h.name = j.name and h.singleton_key = j.singleton_key and h.state = 'failed'))
+    union all
+    select started_on + expire_seconds * interval '1s' from ${schema}.job where state = 'active'
+  ) jobs
+$$;
+comment on function ${schema}.next_due() is '${functionsComment}';`;
+/**
+* `jobs.failed`: the failed jobs, for the admin app's page, with their last error. Updating `retry` to true retries a
+* job, as pg-boss's `retry()` does: one more attempt, from now; deleting a row deletes the job.
+*/
+const failed = (schema) => `create index if not exists job_failed_i on ${schema}.job (completed_on) where state = 'failed';
+drop view if exists ${schema}.failed;
+create view ${schema}.failed as
+select id, name, data as payload, retry_count + 1 as attempts,
+  coalesce(output ->> 'message', output -> 'value' ->> 'message', output ->> 'value') as error,
+  output, completed_on as failed_at, created_on as created_at, singleton_key as key, false as retry
+from ${schema}.job where state = 'failed';
+create or replace function ${schema}.failed_retry() returns trigger
+language plpgsql as $$
+begin
+  if new.retry then
+    update ${schema}.job
+    set state = 'retry', retry_limit = retry_limit + 1, completed_on = null,
+      start_after = greatest(start_after, ${schema}.job_now()), keep_until = greatest(keep_until, ${schema}.job_now() + interval '14 days')
+    where name = old.name and id = old.id and state = 'failed';
+  end if;
+  return new;
+end
+$$;
+comment on function ${schema}.failed_retry() is '${functionsComment}';
+create or replace function ${schema}.failed_delete() returns trigger
+language plpgsql as $$
+begin
+  delete from ${schema}.job where name = old.name and id = old.id and state = 'failed';
+  return old;
+end
+$$;
+comment on function ${schema}.failed_delete() is '${functionsComment}';
+create trigger failed_retry instead of update on ${schema}.failed for each row execute function ${schema}.failed_retry();
+create trigger failed_delete instead of delete on ${schema}.failed for each row execute function ${schema}.failed_delete();`;
+/** The statements that create or replace every Protobase object in `schema`. */
+const functionsSql = (schema) => [
+	enqueue(schema),
+	nextDue(schema),
+	failed(schema)
+].join("\n\n");
+//#endregion
+export { functionNames, functionsComment, functionsSql };

package/dist/jobs/job-types.d.ts

@@ -0,0 +1,73 @@
+import { StandardSchema } from "../functions/read-json.js";
+import { Db } from "@protobase/query";
+//#region src/jobs/job-types.d.ts
+/** What `run` gets beside the payload. */
+export type JobContext = {
+  /** The database pool. The enqueuer's transaction has committed by the time a job runs. */
+  db: Db;
+  job: {
+    id: string;
+    name: string;
+    attempt: number;
+    maxAttempts: number;
+  };
+  /** Aborted when the job's timeout passes, or when the worker shuts down and its grace period ends. */
+  signal: AbortSignal;
+};
+export type JobOptions<Payload> = {
+  /** Letters, digits, `_`, `-`, `.` and `/`; the job's queue in the `jobs` schema. */
+  name: string;
+  /** Checks the payload when it is enqueued and again before it runs, such as a zod schema. */
+  payload?: StandardSchema<Payload>;
+  run: (payload: Payload, context: JobContext) => Promise<void> | void;
+  /** Tries before the job is kept as failed, listed in the admin app. Default 10. */
+  maxAttempts?: number;
+  /** The wait before the first retry, doubling with each one: milliseconds or text such as `30 seconds`. Default `2s`. */
+  retryDelay?: number | string;
+  /** The longest wait between two tries. Default `1 hour`. */
+  maxRetryDelay?: number | string;
+  /** How long one try may run before it counts as failed. Default `15 minutes`. */
+  timeout?: number | string;
+  /** Jobs of this name running at once in each worker process. Default 1. */
+  concurrency?: number;
+  /**
+   * Jobs run one at a time per group, in the order they were enqueued, and every enqueue names its group. A job that is
+   * retrying or failed holds back the rest of its group until it succeeds, or is retried or deleted in the admin app.
+   */
+  ordered?: boolean;
+  /**
+   * Runs the job on this cron schedule, such as `0 7 * * *` (with an optional seconds field first), with no payload: one
+   * run per time, however many workers run. A run missed while no worker ran happens once, as soon as one does.
+   */
+  schedule?: string;
+  /** The time zone `schedule` is read in, such as `Europe/Amsterdam`. Default `UTC`. */
+  timezone?: string;
+};
+/** Where and how one enqueue lands; a job runs as soon as possible by default. */
+export type EnqueueOptions = {
+  /** When the job may run first. */
+  runAt?: Date;
+  /** How long from now before the job may run first: milliseconds or text such as `5 minutes`. */
+  delay?: number | string;
+  /**
+   * Keeps one waiting job per key; a running job does not count. With `mode: 'dedupe'` (the default) an enqueue while
+   * one waits leaves it as it is, with `'replace'` it takes this payload and these options. Not for ordered jobs.
+   */
+  key?: string;
+  mode?: 'dedupe' | 'replace';
+  /** The group of an ordered job; required there, refused elsewhere. */
+  group?: string;
+  /** Higher runs first. Default 0. */
+  priority?: number;
+};
+export type JobDefinition<Payload = any> = {
+  readonly kind: 'job';
+  readonly name: string;
+  readonly options: JobOptions<Payload>;
+  /**
+   * Adds a job and returns its id: in `db`'s own transaction when it is one, so the job commits with the caller's
+   * writes or not at all. With a `key` it returns the waiting job's id when that one is kept.
+   */
+  enqueue(db: Db, ...args: Payload extends undefined ? [payload?: Payload, options?: EnqueueOptions] : [payload: Payload, options?: EnqueueOptions]): Promise<string>;
+};
+//#endregion

package/dist/jobs/options.js

@@ -0,0 +1,44 @@
+import { parseDuration } from "../files/duration.js";
+//#region src/jobs/options.ts
+/** pg-boss counts in whole seconds; a duration rounds up, so a wait is never shorter than asked. */
+const seconds = (value, label) => Math.ceil(parseDuration(value, label) / 1e3);
+const positive = (value, label) => {
+	if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${label} must be a whole number of at least 1, not ${value}`);
+	return value;
+};
+/** The pg-boss queue a job definition runs in: its policy, retries with backoff and timeout. */
+const queueOptions = (options) => {
+	const label = `Job "${options.name}"`;
+	return {
+		policy: options.ordered ? "key_strict_fifo" : "standard",
+		retryLimit: positive(options.maxAttempts ?? 10, `${label} maxAttempts`) - 1,
+		retryDelay: positive(seconds(options.retryDelay ?? "2s", `${label} retryDelay`), `${label} retryDelay in seconds`),
+		retryBackoff: true,
+		retryDelayMax: positive(seconds(options.maxRetryDelay ?? "1 hour", `${label} maxRetryDelay`), `${label} maxRetryDelay in seconds`),
+		expireInSeconds: positive(seconds(options.timeout ?? "15 minutes", `${label} timeout`), `${label} timeout in seconds`)
+	};
+};
+/** The queue settings `queueOptions` sets, as `getQueue` reads them back, so a changed definition is noticed. */
+const queueSettingsDiffer = (wanted, stored) => Object.keys(wanted).some((key) => key !== "policy" && stored[key] !== wanted[key]);
+/**
+* The pg-boss send options of one enqueue: when, how urgent, and the key a waiting job is found by. An ordered job's
+* group is its key in pg-boss, which keeps one group's jobs in order.
+*/
+const sendOptions = (options, enqueue = {}) => {
+	const label = `Job "${options.name}"`;
+	if (enqueue.runAt && enqueue.delay !== void 0) throw new Error(`${label}: pass runAt or delay, not both`);
+	if (options.ordered && enqueue.group === void 0) throw new Error(`${label} is ordered: name the group each job runs in with { group }`);
+	if (options.ordered && enqueue.key !== void 0) throw new Error(`${label} is ordered: its jobs are found by their group, so it takes no key`);
+	if (!options.ordered && enqueue.group !== void 0) throw new Error(`${label} is not ordered: a group needs ordered: true in defineJob`);
+	if (enqueue.mode && enqueue.key === void 0) throw new Error(`${label}: mode needs a key`);
+	if (enqueue.priority !== void 0 && !Number.isSafeInteger(enqueue.priority)) throw new Error(`${label}: priority must be a whole number`);
+	const startAfter = enqueue.runAt ?? (enqueue.delay === void 0 ? void 0 : seconds(enqueue.delay, `${label} delay`));
+	const singletonKey = options.ordered ? enqueue.group : enqueue.key;
+	return {
+		...startAfter !== void 0 && { startAfter },
+		...enqueue.priority !== void 0 && { priority: enqueue.priority },
+		...singletonKey !== void 0 && { singletonKey }
+	};
+};
+//#endregion
+export { queueOptions, queueSettingsDiffer, sendOptions };

package/dist/jobs/run-job.d.ts

@@ -0,0 +1,9 @@
+import "./job-types.js";
+import { Db } from "@protobase/query";
+import "pg-boss";
+//#region src/jobs/run-job.d.ts
+/** Thrown from a job's `run`, it fails the job now: the attempts it has left are not used. */
+export declare class FatalJobError extends Error {
+  name: string;
+}
+//#endregion