@protobase/server 0.8.2 → 0.9.0

package/dist/webhooks/table/schema-sql.js

@@ -0,0 +1,116 @@
+import { jobsSchema } from "../../jobs/schema.js";
+import { quoteIdent } from "../../changes/capture-sql.js";
+/** The comment the schema's functions carry, which the startup check reads. */
+const webhooksComment = `Protobase webhooks 1`;
+/** The delivery jobs, whose waiting and failed ones the deliveries view lists. */
+const deliveryJobs = ["webhooks.deliver", "webhooks.deliver-ordered"];
+/** The job that sends a subscription a test event. */
+const sendTestJob = "webhooks.send-test";
+const randomSecret = `'whsec_' || encode(decode(replace(gen_random_uuid()::text || gen_random_uuid()::text, '-', ''), 'hex'), 'base64')`;
+/**
+* The subscriptions table. The owner is the API's caller, from the change capture's `actor` setting; the tenant is set by
+* the API like any tenant column. `rotate_secret`, `send_test` and `resend_failed` are commands: an update that sets one
+* runs it and stores false.
+*/
+const subscriptionsSql = (schema, changesSchema) => `create table if not exists ${quoteIdent(schema)}."subscriptions" (
+  "id" uuid primary key default gen_random_uuid(),
+  "name" text not null,
+  "events" text not null,
+  "url" text not null,
+  "filter" text,
+  "headers" jsonb,
+  "payload" text not null default 'full' check ("payload" in ('full', 'thin')),
+  "enabled" boolean not null default true,
+  "secret" text not null default (${randomSecret}),
+  "previous_secret" text,
+  "previous_secret_until" timestamptz,
+  "tenant_id" text,
+  "owner_id" text default (nullif(current_setting('${changesSchema}.actor', true), '')::jsonb ->> 'id'),
+  "disabled_reason" text,
+  "last_success_at" timestamptz,
+  "failing_since" timestamptz,
+  "created_at" timestamptz not null default now(),
+  "updated_at" timestamptz not null default now(),
+  "rotate_secret" boolean not null default false,
+  "send_test" boolean not null default false,
+  "resend_failed" boolean not null default false
+);
+create index if not exists "subscriptions_tenant" on ${quoteIdent(schema)}."subscriptions" ("tenant_id")`;
+const retryJobs = (where) => `update ${jobsSchema}.job
+    set state = 'retry', retry_limit = retry_limit + 1, completed_on = null,
+      start_after = greatest(start_after, ${jobsSchema}.job_now()), keep_until = greatest(keep_until, ${jobsSchema}.job_now() + interval '14 days')
+    where name = any(array['${deliveryJobs.join("', '")}']) and state = 'failed' and ${where}`;
+/** The commands of the subscriptions table, and a re-enabled subscription starting afresh. */
+const commandsSql = (schema) => `create or replace function ${quoteIdent(schema)}."subscription_commands"() returns trigger
+language plpgsql as $$
+begin
+  if new.rotate_secret then
+    new.previous_secret := old.secret;
+    new.previous_secret_until := now() + interval '24 hours';
+    new.secret := ${randomSecret};
+    new.rotate_secret := false;
+  end if;
+  if new.send_test then
+    perform ${jobsSchema}.enqueue('${sendTestJob}', jsonb_build_object('subscription', new.id));
+    new.send_test := false;
+  end if;
+  if new.resend_failed then
+    ${retryJobs(`data ->> 'subscription' = 'ui:' || new.id::text`)};
+    new.resend_failed := false;
+  end if;
+  if new.enabled and not old.enabled then
+    new.disabled_reason := null;
+    new.failing_since := null;
+  end if;
+  new.updated_at := now();
+  return new;
+end
+$$;
+comment on function ${quoteIdent(schema)}."subscription_commands"() is '${webhooksComment}';
+drop trigger if exists "subscription_commands" on ${quoteIdent(schema)}."subscriptions";
+create trigger "subscription_commands" before update on ${quoteIdent(schema)}."subscriptions" for each row execute function ${quoteIdent(schema)}."subscription_commands"()`;
+/**
+* The deliveries of table subscriptions that wait for a retry or failed for good: the delivery jobs, with their
+* subscription's tenant. Updating `resend` to true runs a failed one once more.
+*/
+const deliveriesSql = (schema) => `drop view if exists ${quoteIdent(schema)}."deliveries";
+create view ${quoteIdent(schema)}."deliveries" as
+select j.id, s.id as subscription_id, s.tenant_id,
+  coalesce(j.data -> 'event' ->> 'type', j.data -> 'events' -> 0 ->> 'type') as event_type,
+  coalesce(j.data -> 'event' ->> 'id', j.data -> 'events' -> 0 ->> 'id') as event_id,
+  case when j.state = 'failed' then 'failed' else 'waiting' end as status,
+  coalesce((j.data ->> 'attempt')::integer, j.retry_count + 1) as attempt,
+  case when j.state = 'failed' then null else j.start_after end as next_attempt,
+  coalesce(j.output ->> 'message', j.data ->> 'lastError') as last_error,
+  j.created_on as created_at,
+  false as resend
+from ${jobsSchema}.job j join ${quoteIdent(schema)}."subscriptions" s on j.data ->> 'subscription' = 'ui:' || s.id::text
+where j.name = any(array['${deliveryJobs.join("', '")}']) and j.state in ('created', 'retry', 'failed');
+create or replace function ${quoteIdent(schema)}."deliveries_resend"() returns trigger
+language plpgsql as $$
+begin
+  if new.resend then
+    ${retryJobs("id = old.id")};
+  end if;
+  return new;
+end
+$$;
+comment on function ${quoteIdent(schema)}."deliveries_resend"() is '${webhooksComment}';
+create trigger "deliveries_resend" instead of update on ${quoteIdent(schema)}."deliveries" for each row execute function ${quoteIdent(schema)}."deliveries_resend"()`;
+/** Every try of a table subscription's deliveries, kept 30 days, with `deliveryLog`. */
+const deliveryLogSql = (schema) => `create table if not exists ${quoteIdent(schema)}."delivery_log" (
+  "id" bigint generated always as identity primary key,
+  "subscription_id" uuid not null references ${quoteIdent(schema)}."subscriptions" ("id") on delete cascade,
+  "tenant_id" text,
+  "event_id" text not null,
+  "event_type" text not null,
+  "status" text not null check ("status" in ('delivered', 'failed')),
+  "attempt" integer not null,
+  "status_code" integer,
+  "response" text,
+  "duration_ms" integer not null,
+  "at" timestamptz not null default now()
+);
+create index if not exists "delivery_log_subscription" on ${quoteIdent(schema)}."delivery_log" ("subscription_id", "at")`;
+//#endregion
+export { commandsSql, deliveriesSql, deliveryJobs, deliveryLogSql, sendTestJob, subscriptionsSql, webhooksComment };

package/dist/webhooks/table/schema.d.ts

@@ -0,0 +1,13 @@
+import { EventsOptions } from "../../changes/events-options.js";
+import { WebhooksOptions } from "./options.js";
+import { Db } from "@protobase/query";
+//#region src/webhooks/table/schema.d.ts
+/**
+ * The SQL that brings the `webhooks` schema to what `webhooks()` needs: the subscriptions table, its commands, the
+ * deliveries view and, with `deliveryLog`, the log table. `undefined` when current. Needs the jobs schema, which the
+ * deliveries view reads. Tables are never dropped.
+ */
+export declare const webhooksMigration: (db: Db, options?: WebhooksOptions, events?: EventsOptions) => Promise<string | undefined>;
+/** Refuses to go on while the `webhooks` schema lacks what `webhooks()` needs; the project's migrations add it. */
+export declare const checkWebhooksSchema: (db: Db, options?: WebhooksOptions) => Promise<void>;
+//#endregion

package/dist/webhooks/table/schema.js

@@ -0,0 +1,44 @@
+import { quoteIdent } from "../../changes/capture-sql.js";
+import { changesSchema } from "../../changes/events-options.js";
+import { webhooksSchema } from "./options.js";
+import { commandsSql, deliveriesSql, deliveryLogSql, subscriptionsSql, webhooksComment } from "./schema-sql.js";
+import { migrationCommand } from "../../changes/capture-schema.js";
+import { sql } from "kysely";
+//#region src/webhooks/table/schema.ts
+const functionNames = ["subscription_commands", "deliveries_resend"];
+const readWebhooksSchema = async (db, schema) => {
+	const exists = async (table) => (await sql`select to_regclass(${`${quoteIdent(schema)}.${quoteIdent(table)}`}) is not null as found`.execute(db)).rows[0].found;
+	const { rows } = await sql`select p.proname as name, obj_description(p.oid, 'pg_proc') as comment
+    from pg_proc p join pg_namespace n on n.oid = p.pronamespace where n.nspname = ${schema} and p.proname = any(${functionNames}::text[])`.execute(db);
+	return {
+		subscriptions: await exists("subscriptions"),
+		current: functionNames.every((name) => rows.some((row) => row.name === name && row.comment === webhooksComment)) && await exists("deliveries"),
+		deliveryLog: await exists("delivery_log")
+	};
+};
+/**
+* The SQL that brings the `webhooks` schema to what `webhooks()` needs: the subscriptions table, its commands, the
+* deliveries view and, with `deliveryLog`, the log table. `undefined` when current. Needs the jobs schema, which the
+* deliveries view reads. Tables are never dropped.
+*/
+const webhooksMigration = async (db, options = {}, events) => {
+	const schema = webhooksSchema(options);
+	const found = await readWebhooksSchema(db, schema);
+	const needsLog = Boolean(options.deliveryLog) && !found.deliveryLog;
+	if (found.subscriptions && found.current && !needsLog) return void 0;
+	return `${[
+		`create schema if not exists ${quoteIdent(schema)}`,
+		...found.subscriptions ? [] : [subscriptionsSql(schema, changesSchema(events))],
+		...found.current && found.subscriptions ? [] : [commandsSql(schema), deliveriesSql(schema)],
+		...needsLog ? [deliveryLogSql(schema)] : []
+	].join(";\n\n")};`;
+};
+/** Refuses to go on while the `webhooks` schema lacks what `webhooks()` needs; the project's migrations add it. */
+const checkWebhooksSchema = async (db, options = {}) => {
+	const schema = webhooksSchema(options);
+	const found = await readWebhooksSchema(db, schema);
+	if (found.subscriptions && found.current && (!options.deliveryLog || found.deliveryLog)) return;
+	throw new Error(`The ${schema} schema is missing or behind this version of Protobase. Write it as a migration with \`${migrationCommand}\`, apply it with the project's migrations, then start again.`);
+};
+//#endregion
+export { checkWebhooksSchema, webhooksMigration };

package/dist/webhooks/table/table-subscriptions.js

@@ -0,0 +1,77 @@
+import { quoteIdent } from "../../changes/capture-sql.js";
+import { parseEvents, tableSubscriptionName } from "./options.js";
+import { sql } from "kysely";
+//#region src/webhooks/table/table-subscriptions.ts
+/** A UI subscription is disabled after this long without a success. */
+const failingFor = "3 days";
+/**
+* The subscriptions people made in the admin app, as the dispatcher and the delivery jobs use them: named `ui:<id>`,
+* delivering as their owner with the roles they hold now, signed with their secret and, for 24 hours after a rotation,
+* the previous one too. What happens to their deliveries is kept on their row.
+*/
+const tableSubscriptions = ({ schema, ownerRoles, deliveryLog }) => {
+	const table = sql.raw(`${quoteIdent(schema)}."subscriptions"`);
+	const definition = async (row) => {
+		const tenant = row.tenant_id ?? void 0;
+		const roles = row.owner_id ? await ownerRoles({
+			id: row.owner_id,
+			...tenant !== void 0 && { tenant }
+		}) : [];
+		return {
+			kind: "subscription",
+			name: tableSubscriptionName(row.id),
+			options: {
+				events: parseEvents(row.events),
+				url: row.url,
+				...row.filter && { when: row.filter },
+				...row.headers && { headers: row.headers },
+				payload: row.payload,
+				enabled: row.enabled,
+				secret: row.overlapping && row.previous_secret ? [row.secret, row.previous_secret] : row.secret,
+				...tenant !== void 0 && { tenant },
+				as: { roles }
+			}
+		};
+	};
+	const select = sql`select "id"::text as id, "events", "url", "filter", "headers", "payload", "enabled", "secret", "previous_secret",
+    coalesce("previous_secret_until" > now(), false) as overlapping, "tenant_id", "owner_id" from ${table}`;
+	/** Every enabled one. */
+	const load = async (db) => Promise.all((await sql`${select} where "enabled"`.execute(db)).rows.map(definition));
+	/** One by id, enabled or not; `undefined` once it is deleted. */
+	const find = async (db, id) => {
+		const [row] = (await sql`${select} where "id"::text = ${id}`.execute(db)).rows;
+		return row && definition(row);
+	};
+	/** The row's record of a try: its last success, how long it has been failing, and disabling it when it should stop. */
+	const record = async (db, id, event, tried, result, durationMs) => {
+		if (result.ok) await sql`update ${table} set "last_success_at" = now(), "failing_since" = null where "id"::text = ${id}`.execute(db);
+		else {
+			const gone = result.status === 410;
+			await sql`update ${table} set "failing_since" = coalesce("failing_since", now()),
+          "enabled" = "enabled" and not (${gone} or coalesce("failing_since", now()) < now() - ${failingFor}::interval),
+          "disabled_reason" = case
+            when ${gone} then 'The receiver answered 410 Gone'
+            when coalesce("failing_since", now()) < now() - ${failingFor}::interval then 'No delivery succeeded for ${sql.raw(failingFor)}'
+            else "disabled_reason" end
+        where "id"::text = ${id}`.execute(db);
+		}
+		if (!deliveryLog) return;
+		const status = result.ok ? "delivered" : "failed";
+		const response = result.ok ? null : result.error.slice(0, 2048);
+		await sql`insert into ${sql.raw(`${quoteIdent(schema)}."delivery_log"`)} ("subscription_id", "tenant_id", "event_id", "event_type", "status", "attempt", "status_code", "response", "duration_ms")
+      select "id", "tenant_id", ${event.id}, ${event.type}, ${status}, ${tried}, ${result.status ?? null}, ${response}, ${Math.round(durationMs)} from ${table} where "id"::text = ${id}`.execute(db);
+	};
+	/** Deletes the log's tries older than 30 days. */
+	const prune = async (db) => {
+		await sql`delete from ${sql.raw(`${quoteIdent(schema)}."delivery_log"`)} where "at" < now() - interval '30 days'`.execute(db);
+	};
+	return {
+		load,
+		find,
+		record,
+		prune,
+		deliveryLog
+	};
+};
+//#endregion
+export { tableSubscriptions };

package/dist/webhooks/table/validate-hook.js

@@ -0,0 +1,32 @@
+import { badRequest } from "../../problem.js";
+import { subscriptionProblems } from "../route-events.js";
+import { parseEvents } from "./options.js";
+//#region src/webhooks/table/validate-hook.ts
+const isHeaders = (value) => value === null || value === void 0 || typeof value === "object" && !Array.isArray(value) && Object.values(value).every((item) => typeof item === "string");
+/**
+* Checks a webhook subscription as it is written: events that are declared, a filter that checks against every resource
+* it covers, headers of text, and a URL at a public address. A refusal is a 400 problem, and the write rolls back.
+*/
+const subscriptionWriteCheck = (models, checkUrl) => async (event) => {
+	if (event.resource.name !== "webhookSubscriptions" || !event.after) return;
+	const { events, url, filter, headers } = event.after;
+	const candidate = {
+		kind: "subscription",
+		name: "it",
+		options: {
+			events: parseEvents(events),
+			url,
+			...filter && { when: filter }
+		}
+	};
+	const problems = [
+		...parseEvents(events).length === 0 ? ["names no events"] : [],
+		...subscriptionProblems([candidate], models, []).map((problem) => problem.replace("Subscription \"it\" ", "")),
+		...isHeaders(headers) ? [] : ["headers must be an object of text values"]
+	];
+	const blocked = problems.length === 0 && checkUrl ? await checkUrl(url) : void 0;
+	const all = [...problems, ...blocked ? [blocked] : []];
+	if (all.length > 0) throw badRequest("invalid-webhook-subscription", `The webhook subscription cannot be saved: ${all.join("; ")}`);
+};
+//#endregion
+export { subscriptionWriteCheck };

package/dist/webhooks/table/webhook-table.js

@@ -0,0 +1,23 @@
+import { webhooksSchema } from "./options.js";
+import { checkWebhookUrl } from "./address-check.js";
+import { betterAuthOwnerRoles } from "./owner-roles.js";
+import { tableSubscriptions } from "./table-subscriptions.js";
+//#region src/webhooks/table/webhook-table.ts
+/**
+* What `createAdmin` needs for the subscriptions of `webhooks()`: the table, the owners' current roles (from Better Auth
+* unless the options say otherwise) and, unless allowed, the check that keeps URLs off private addresses.
+*/
+const webhookTable = (options, auth) => {
+	const ownerRoles = options.ownerRoles ?? (auth && betterAuthOwnerRoles(auth));
+	if (!ownerRoles) throw new Error("webhooks() delivers as each subscription's owner, with their current roles: pass `auth` (Better Auth) or `ownerRoles` to webhooks()");
+	return {
+		subscriptions: tableSubscriptions({
+			schema: webhooksSchema(options),
+			ownerRoles,
+			deliveryLog: Boolean(options.deliveryLog)
+		}),
+		...!options.allowPrivateAddresses && { checkUrl: checkWebhookUrl }
+	};
+};
+//#endregion
+export { webhookTable };

package/dist/webhooks/test-event.js

@@ -0,0 +1,48 @@
+import { HttpProblem } from "../problem.js";
+import { subscriberSession } from "./subscriber-view.js";
+import { eventPatternMatches } from "./subscription.js";
+import { encodeKey } from "@protobase/schema";
+//#region src/webhooks/test-event.ts
+/**
+* What Send test event sends: the first declared event the subscription takes, made up from the newest record of its
+* resource that the subscription may read, or with no record when it may read none. It is marked `test`.
+*/
+const testEvent = async (deps, records, subscription) => {
+	const { options } = subscription;
+	const found = deps.registry.entries.flatMap((entry) => (entry.model.events ?? []).map((event) => ({
+		entry,
+		name: event.name
+	}))).find(({ name }) => options.events.some((pattern) => eventPatternMatches(pattern, name)));
+	if (!found) throw new Error(`The subscription "${subscription.name}" takes no declared event`);
+	const { entry, name } = found;
+	const item = (await records(subscriberSession(subscription) ?? { user: {
+		id: `subscription:${subscription.name}`,
+		roles: []
+	} }).list(entry.name, { page_size: 1 }).catch((error) => {
+		if (error instanceof HttpProblem) return void 0;
+		throw error;
+	}))?.items[0];
+	const record = item && Object.fromEntries(Object.entries(item).filter(([field]) => Object.hasOwn(entry.model.fields, field)));
+	const thin = options.payload === "thin";
+	const tenant = entry.model.tenant && record ? record[entry.model.tenant] : void 0;
+	return {
+		id: `test_${crypto.randomUUID()}`,
+		type: name,
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		data: {
+			resource: entry.name,
+			operation: "create",
+			key: record ? encodeKey(entry.model.primaryKey.map((field) => record[field])) : "",
+			sequence: "0",
+			...record && !thin && { after: record },
+			changed: [],
+			...tenant !== void 0 && tenant !== null && { tenant },
+			actor: null,
+			source: "database",
+			transaction: "0"
+		},
+		test: true
+	};
+};
+//#endregion
+export { testEvent };

package/dist/webhooks/webhooks.js

@@ -0,0 +1,233 @@
+import { dispatchJobName } from "../changes/capture-sql.js";
+import { coalesce } from "../changes/coalesce.js";
+import { eventMatcher } from "../changes/match-events.js";
+import { deleteOutbox, readOutbox } from "../changes/outbox.js";
+import { resourceChange } from "../changes/resource-change.js";
+import { callerRecords } from "../functions/records.js";
+import { defineJob } from "../jobs/define-job.js";
+import { FatalJobError } from "../jobs/run-job.js";
+import { maxTries, nextRetryDelay } from "./retry-schedule.js";
+import { subscriberEvents } from "./subscriber-view.js";
+import { attemptDelivery } from "./attempt.js";
+import { subscriptionProblems, subscriptionRouter } from "./route-events.js";
+import { tableSubscriptionName } from "./table/options.js";
+import { sendTestJob } from "./table/schema-sql.js";
+import { testEvent } from "./test-event.js";
+//#region src/webhooks/webhooks.ts
+/** Outbox rows one dispatch reads at a time. */
+const batchSize = 500;
+/** Receivers tried at once by one dispatch. */
+const parallelAttempts = 16;
+const mapLimit = async (items, limit, run) => {
+	const results = new Array(items.length);
+	let next = 0;
+	const lane = async () => {
+		while (next < items.length) {
+			const index = next++;
+			results[index] = await run(items[index]);
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(limit, items.length) }, lane));
+	return results;
+};
+/** The key of an unordered delivery: one waiting retry per event and subscription. */
+const deliveryKey = (payload) => `${payload.event.id}:${payload.subscription}`;
+/** The group of an ordered delivery: one at a time per subscription and record. */
+const deliveryGroup = ({ subscription, change }) => `${subscription.name}:${change.resource}:${change.key}`;
+/**
+* Resource events delivered to the config's subscriptions, as three jobs the app's workers run:
+*
+* - `changes.dispatch`, which `capture()` enqueues once per writing transaction, in one ordered group so one runs at a
+*   time: it turns the committed outbox rows into events, makes the first try of every unordered delivery itself,
+*   queues the failures and the ordered deliveries, and deletes the rows in that same transaction. A process that dies
+*   before that commits leaves the rows, and the job's retry delivers them again, under the same `webhook-id`.
+* - `webhooks.deliver`, one try of an unordered delivery; a failure queues the next try on the Standard Webhooks
+*   schedule, or Retry-After, and the last one is kept as failed.
+* - `webhooks.deliver-ordered`, the ordered deliveries of one dispatch for one subscription and record, retried with
+*   doubling waits up to a day, holding back the record's later deliveries to that subscription meanwhile.
+*
+* With `table`, also `webhooks.send-test`, which Send test event queues, and with its log `webhooks.prune-log`, daily.
+*
+* No jobs without a resource with events; the subscriptions must name declared events.
+*/
+const createWebhooks = ({ deps, subscriptions, functions, schema, log, table }) => {
+	const entries = deps.registry.entries;
+	const models = entries.map((entry) => entry.model);
+	const problems = subscriptionProblems(subscriptions, models, Object.keys(functions));
+	if (problems.length > 0) throw new Error(problems.join("; "));
+	if (!models.some((model) => model.events)) return { jobs: [] };
+	const matchers = new Map(entries.map((entry) => [entry.name, eventMatcher(entry.model, entry.source.eventMatches?.() ?? {})]));
+	const keyColumns = (resource) => {
+		const model = deps.registry.find(resource)?.model;
+		return model ? model.primaryKey.map((name) => model.fields[name].column) : [];
+	};
+	const byName = new Map(subscriptions.map((subscription) => [subscription.name, subscription]));
+	const records = (session) => callerRecords(deps, session);
+	const tableId = (subscription) => subscription.name.startsWith("ui:") ? subscription.name.slice(3) : void 0;
+	const attempt = async (subscription, event, db, tried, signal) => {
+		const id = tableId(subscription);
+		const started = performance.now();
+		const checkUrl = id === void 0 ? void 0 : table?.checkUrl;
+		const result = await attemptDelivery(subscription, event, {
+			db,
+			functions,
+			records,
+			attempt: tried,
+			signal,
+			...checkUrl && { checkUrl }
+		});
+		if (id !== void 0) await table.subscriptions.record(db, id, event, tried, result, performance.now() - started);
+		return result;
+	};
+	const current = async (db) => {
+		if (!table) return subscriptions;
+		const rows = (await table.subscriptions.load(db)).filter((row) => subscriptionProblems([row], models, []).length === 0);
+		return [...subscriptions, ...rows];
+	};
+	const report = (subscription, event, tried, result) => {
+		const what = `webhook ${subscription} ${event.type} ${event.id}`;
+		if (result.ok) log(`${what} delivered${result.status ? ` (${result.status})` : ""}${tried > 1 ? ` on try ${tried}` : ""}`);
+		else log(`${what} failed on try ${tried}: ${result.error}`);
+	};
+	const subscriptionOf = async (name, db) => {
+		const id = name.startsWith("ui:") ? name.slice(3) : void 0;
+		const found = id === void 0 ? byName.get(name) : await table?.subscriptions.find(db, id);
+		if (!found) throw new FatalJobError(`The subscription "${name}" no longer exists`);
+		if (found.options.enabled === false) throw new FatalJobError(`The subscription "${name}" is disabled`);
+		return found;
+	};
+	const deliver = defineJob({
+		name: "webhooks.deliver",
+		maxAttempts: 3,
+		retryDelay: "30 seconds",
+		timeout: "1 minute",
+		concurrency: parallelAttempts,
+		run: async (payload, { db, signal }) => {
+			const subscription = await subscriptionOf(payload.subscription, db);
+			const result = await attempt(subscription, payload.event, db, payload.attempt, signal);
+			report(payload.subscription, payload.event, payload.attempt, result);
+			if (result.ok) return;
+			const delay = result.final ? void 0 : nextRetryDelay(payload.attempt, result.retryAfterMs);
+			if (delay === void 0) throw new FatalJobError(result.error);
+			await deliver.enqueue(db, {
+				...payload,
+				attempt: payload.attempt + 1,
+				lastError: result.error
+			}, {
+				key: deliveryKey(payload),
+				delay
+			});
+		}
+	});
+	const deliverOrdered = defineJob({
+		name: "webhooks.deliver-ordered",
+		ordered: true,
+		maxAttempts: maxTries,
+		retryDelay: "5 seconds",
+		maxRetryDelay: "24 hours",
+		timeout: "5 minutes",
+		concurrency: parallelAttempts,
+		run: async (payload, { db, signal, job }) => {
+			const subscription = await subscriptionOf(payload.subscription, db);
+			for (const event of payload.events) {
+				const result = await attempt(subscription, event, db, job.attempt, signal);
+				report(payload.subscription, event, job.attempt, result);
+				if (result.ok) continue;
+				if (result.final) throw new FatalJobError(result.error);
+				throw new Error(result.error);
+			}
+		}
+	});
+	const deliveriesOf = async (changes, db) => {
+		const route = subscriptionRouter(await current(db), models);
+		const eventOf = subscriberEvents(deps);
+		const deliveries = [];
+		for (const change of changes) for (const { subscription, event } of route(change, matchers.get(change.resource)(change))) {
+			const shaped = await eventOf(subscription, change, event);
+			if (shaped) deliveries.push({
+				subscription,
+				event: shaped,
+				change
+			});
+		}
+		return deliveries;
+	};
+	const dispatchBatch = async (db, signal) => {
+		const rows = await readOutbox(db, schema, models, batchSize);
+		if (rows.length === 0) return 0;
+		const changes = coalesce(rows, keyColumns).flatMap((change) => {
+			const entry = deps.registry.find(change.resource);
+			return entry ? [resourceChange(entry.model, change)] : [];
+		});
+		const deliveries = await deliveriesOf(changes, db);
+		const now = deliveries.filter(({ subscription }) => !subscription.options.ordered);
+		const results = await mapLimit(now, parallelAttempts, ({ subscription, event }) => attempt(subscription, event, db, 1, signal));
+		await db.transaction().execute(async (trx) => {
+			for (const [index, delivery] of now.entries()) {
+				const result = results[index];
+				if (result.ok) continue;
+				const delay = result.final ? void 0 : nextRetryDelay(1, result.retryAfterMs);
+				if (delay === void 0) continue;
+				const payload = {
+					subscription: delivery.subscription.name,
+					event: delivery.event,
+					attempt: 2,
+					lastError: result.error
+				};
+				await deliver.enqueue(trx, payload, {
+					key: deliveryKey(payload),
+					delay
+				});
+			}
+			const groups = /* @__PURE__ */ new Map();
+			for (const delivery of deliveries.filter(({ subscription }) => subscription.options.ordered)) {
+				const group = deliveryGroup(delivery);
+				const payload = groups.get(group) ?? {
+					subscription: delivery.subscription.name,
+					events: []
+				};
+				payload.events.push(delivery.event);
+				groups.set(group, payload);
+			}
+			for (const [group, payload] of groups) await deliverOrdered.enqueue(trx, payload, { group });
+			await deleteOutbox(trx, schema, rows.map((row) => row.id));
+		});
+		for (const [index, { subscription, event }] of now.entries()) report(subscription.name, event, 1, results[index]);
+		return rows.length;
+	};
+	const dispatch = defineJob({
+		name: dispatchJobName,
+		ordered: true,
+		maxAttempts: 1e6,
+		retryDelay: "1 second",
+		maxRetryDelay: "1 minute",
+		timeout: "5 minutes",
+		run: async (_payload, { db, signal }) => {
+			while (await dispatchBatch(db, signal) === batchSize && !signal.aborted);
+		}
+	});
+	const sendTest = defineJob({
+		name: sendTestJob,
+		maxAttempts: 1,
+		timeout: "1 minute",
+		run: async (payload, { db, signal }) => {
+			const subscription = await subscriptionOf(tableSubscriptionName(payload.subscription), db);
+			const event = await testEvent(deps, records, subscription);
+			report(subscription.name, event, 1, await attempt(subscription, event, db, 1, signal));
+		}
+	});
+	const pruneLog = defineJob({
+		name: "webhooks.prune-log",
+		schedule: "17 3 * * *",
+		run: async (_payload, { db }) => table.subscriptions.prune(db)
+	});
+	return { jobs: [
+		dispatch,
+		deliver,
+		deliverOrdered,
+		...table ? [sendTest] : [],
+		...table?.subscriptions.deliveryLog ? [pruneLog] : []
+	] };
+};
+//#endregion
+export { createWebhooks };

package/dist/write-pipeline.js

@@ -4,7 +4,7 @@ import { tenantScope } from "./tenant.js";
 import { requestAccess, requireOperation, targetFilter, writableFields } from "./request-access.js";
 import { writeTransaction } from "./transactions.js";
 import { attachFiles } from "./files/attach.js";
-import { actorSettings } from "./changes/actor.js";
+import { actorSettings, wakeDispatch } from "./changes/actor.js";
 import { validateBody } from "./validate-body.js";
 import { assertWritten, checkPrecondition, decide, denied, firstKeyField, missing, notifier, shown } from "./write-shared.js";
 import { applyUpdate, planUpdate } from "./write-update.js";
@@ -90,7 +90,9 @@ const writeIn = async (trx, deps, access, request) => {
 const runWrite = async (deps, entry, session, request) => {
 	const access = await requestAccess(deps, entry, session);
 	requireOperation(access, request.operation === "undelete" ? "update" : request.operation);
-	return writeTransaction(deps.db, deps.statementTimeoutMs, (trx) => writeIn(trx, deps, access, request), actorSettings(deps, session));
+	const result = await writeTransaction(deps.db, deps.statementTimeoutMs, (trx) => writeIn(trx, deps, access, request), actorSettings(deps, session));
+	wakeDispatch(deps);
+	return result;
 };
 //#endregion
 export { runWrite, writeIn };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/server",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "The Protobase REST API as a Hono app, with Better Auth sign-in.",
   "license": "Apache-2.0",
   "repository": {
@@ -30,9 +30,9 @@
   },
   "dependencies": {
     "@better-auth/passkey": "^1.7.7",
-    "@protobase/layout": "0.8.2",
-    "@protobase/query": "0.8.2",
-    "@protobase/schema": "0.8.2",
+    "@protobase/layout": "0.9.0",
+    "@protobase/query": "0.9.0",
+    "@protobase/schema": "0.9.0",
     "@scalar/hono-api-reference": "^0.12.10",
     "@simplewebauthn/server": "^13.3.1",
     "better-auth": "^1.7.7",