@protobase/server 0.68.5 → 0.69.0

package/dist/workflows/plugin-records.js

@@ -3,8 +3,8 @@ import { tenantScope } from "../tenant.js";
 import { issueTicket } from "../files/ticket.js";
 import { runBatch } from "../batch/run-batch.js";
 import { assistantRecords, toolEntry } from "../assistant/records.js";
-import { effectsStore } from "./effects-store.js";
 import { runFileRef } from "./run-files.js";
+import { effectsStore } from "./effects-store.js";
 import { sql } from "kysely";
 import { StepError } from "@protobase/workflows";
 //#region src/workflows/plugin-records.ts

package/dist/workflows/plugin-services.js

@@ -1,6 +1,6 @@
+import { pluginFetch } from "./plugin-fetch.js";
 import { readRunFile, storeRunFile } from "./run-files.js";
 import { pluginRecords } from "./plugin-records.js";
-import { pluginFetch } from "./plugin-fetch.js";
 //#region src/workflows/plugin-services.ts
 /** The plugin context's `files`: those of the run alone, in the app's `private` file provider. */
 const runFiles = (files, runId) => ({

package/dist/workflows/ready/ready-start.js

@@ -1,7 +1,7 @@
 import { HttpProblem } from "../../problem.js";
 import { assistantRecords } from "../../assistant/records.js";
-import { saverSession } from "../plugin-records.js";
 import { startWorkflow } from "../start-workflow.js";
+import { saverSession } from "../plugin-records.js";
 import { triggerFailed, triggerRecovered } from "../triggers/trigger-row.js";
 import { encodeKey } from "@protobase/schema/internal";
 import { checkFilter, evaluateFilter, parseFilter } from "@protobase/schema";

package/dist/workflows/run-files.js

@@ -62,5 +62,11 @@ const readRunFile = async (files, runId, value, field) => {
 		bytes
 	};
 };
+/** Removes a file a run stored, as when the run it was kept for turned out to be started already. */
+const removeRunFile = async (files, value) => {
+	const ref = parseUri(value.uri);
+	if (!ref) throw new Error(`${value.uri} is not a file URI`);
+	await runStore(files).delete(ref.path);
+};
 //#endregion
-export { isFileValue, readRunFile, runFileLimit, runFileRef, storeRunFile };
+export { isFileValue, readRunFile, removeRunFile, runFileLimit, runFileRef, storeRunFile };

package/dist/workflows/save-hook.js

@@ -1,5 +1,6 @@
 import { badRequest } from "../problem.js";
 import { syncTrigger } from "./triggers/trigger-sync.js";
+import { saveConsent } from "./connections/connection-consent.js";
 import { sql } from "kysely";
 import { checkDefinition, problemText } from "@protobase/workflows";
 //#region src/workflows/save-hook.ts
@@ -11,7 +12,7 @@ import { checkDefinition, problemText } from "@protobase/workflows";
 * `definition` field's `errors` with the state and field in their `params`, for a form to show each on its step; the
 * write rolls back.
 */
-const workflowSaveHook = (schema, types, jobs) => async (event, trx) => {
+const workflowSaveHook = (schema, types, jobs, consent) => async (event, trx) => {
 	if (event.resource.name !== "workflows" || !event.after) return;
 	if (event.operation !== "create" && event.operation !== "update") return;
 	const changed = event.operation === "create" || event.after.version !== event.before?.version;
@@ -35,7 +36,11 @@ const workflowSaveHook = (schema, types, jobs) => async (event, trx) => {
 	}
 	if (changed) await sql`insert into ${sql.id(schema, "versions")} (workflow_id, version, definition, saved_by, tenant_id)
       select w.id, w.version, w.definition, ${String(event.user.id)}, w.tenant_id from ${sql.id(schema, "workflows")} w where w.id = ${String(event.after.id)}`.execute(trx);
-	await syncTrigger(trx, schema, jobs, event.after);
+	const id = String(event.after.id);
+	const enabled = event.after.enabled_version;
+	const enabledBy = enabled === null ? null : await consent(event.session, event.after.definition);
+	await saveConsent(trx, schema, id, enabled, enabledBy);
+	await syncTrigger(trx, schema, jobs, event.after, types.connections);
 };
 //#endregion
 export { workflowSaveHook };

package/dist/workflows/schema-sql.js

@@ -1,6 +1,6 @@
 import { quoteIdent } from "../changes/capture-sql.js";
 /** The comment the schema's function carries, which the startup check reads. */
-const workflowsComment = `Protobase workflows 6`;
+const workflowsComment = `Protobase workflows 8`;
 const commentVersion = (comment) => {
 	const match = comment?.match(/^Protobase workflows (\d+)$/);
 	return match ? Number(match[1]) : void 0;
@@ -83,7 +83,9 @@ const triggerStateBytes = 65536;
 /**
 * Which version of a workflow is on (`enabled_version`, null when off), and one row per trigger of an enabled version:
 * its type and input, and the state its kind keeps. A change raises `generation`, so a job queued for an older one
-* ends without acting. A trigger being stopped stays `stopping` until its stop job removes it. Written whole each time:
+* ends without acting. A poll or renewal acts only as the job `poll_job_id` names, on the try `poll_attempt` claimed,
+* so a try that ran past its timeout writes nothing once its retry claimed the trigger. A deleted workflow leaves its
+* trigger without a workflow, which its delete hook stops. A trigger being stopped stays `stopping` until its stop job removes it. Written whole each time:
 * every statement is idempotent.
 */
 const triggersSql = (schema) => {
@@ -94,9 +96,11 @@ do $$ begin
     alter table ${s}."workflows" add constraint "workflows_enabled_version" foreign key ("id", "enabled_version") references ${s}."versions" ("workflow_id", "version") deferrable initially deferred;
   end if;
 end $$;
+-- Who turned the version on, signed in as themselves: the consent its runs sign in through their connections with.
+alter table ${s}."versions" add column if not exists "enabled_by" text;
 create table if not exists ${s}."triggers" (
   "id" uuid primary key default gen_random_uuid(),
-  "workflow_id" uuid not null references ${s}."workflows" ("id") on delete cascade,
+  "workflow_id" uuid references ${s}."workflows" ("id") on delete set null,
   "version" integer not null,
   "tenant_id" text,
   "type" text not null,
@@ -107,12 +111,22 @@ create table if not exists ${s}."triggers" (
   "cursor" jsonb check (octet_length("cursor"::text) <= ${triggerStateBytes}),
   "store" jsonb check (octet_length("store"::text) <= ${triggerStateBytes}),
   "poll_job_id" text,
+  "poll_attempt" integer,
   "consecutive_failures" integer not null,
   "last_error" jsonb,
   "next_poll_at" timestamptz,
   "created_at" timestamptz not null default now(),
   "updated_at" timestamptz not null default now()
 );
+alter table ${s}."triggers" add column if not exists "poll_attempt" integer;
+-- A deleted workflow's trigger stays, without its workflow, until its stop job unsubscribed it at its provider.
+alter table ${s}."triggers" alter column "workflow_id" drop not null;
+do $$ begin
+  if exists (select 1 from pg_constraint where conname = 'triggers_workflow_id_fkey' and confdeltype = 'c' and connamespace = ${`'${schema.replaceAll("'", "''")}'`}::regnamespace) then
+    alter table ${s}."triggers" drop constraint "triggers_workflow_id_fkey";
+    alter table ${s}."triggers" add constraint "triggers_workflow_id_fkey" foreign key ("workflow_id") references ${s}."workflows" ("id") on delete set null;
+  end if;
+end $$;
 create unique index if not exists "triggers_workflow" on ${s}."triggers" ("workflow_id") where "status" <> 'stopping';
 create index if not exists "triggers_workflow_id" on ${s}."triggers" ("workflow_id");
 create index if not exists "triggers_updated_at" on ${s}."triggers" ("updated_at");

package/dist/workflows/schema.js

@@ -2,6 +2,7 @@ import { quoteIdent } from "../changes/capture-sql.js";
 import { migrationCommand } from "../changes/capture-schema.js";
 import { workflowsSchema } from "./options.js";
 import { commentVersion, effectsSql, functionsSql, indexesSql, tablesSql, triggersSql, workflowsTriggers } from "./schema-sql.js";
+import { hooksSql } from "./hooks/hooks-sql.js";
 import { sql } from "kysely";
 //#region src/workflows/schema.ts
 const readWorkflowsSchema = async (db, schema) => {
@@ -15,7 +16,7 @@ const readWorkflowsSchema = async (db, schema) => {
 	const row = rows[0];
 	return {
 		tables: row.found,
-		current: row.found && row.trigger_rows && commentVersion(row.comment) === 6 && row.triggers === workflowsTriggers.length
+		current: row.found && row.trigger_rows && commentVersion(row.comment) === 8 && row.triggers === workflowsTriggers.length
 	};
 };
 /**
@@ -31,6 +32,7 @@ const workflowsMigration = async (db, options = {}) => {
 		...found.tables ? [] : [tablesSql(schema)],
 		triggersSql(schema),
 		effectsSql(schema),
+		hooksSql(schema),
 		indexesSql(schema),
 		functionsSql(schema)
 	].join(";\n\n")};`;

package/dist/workflows/start-workflow.d.ts

@@ -19,10 +19,12 @@ export type StartWorkflowOptions = {
   schema?: string;
   /** The version to run, as a trigger runs the enabled one; default the workflow's current version. */
   version?: number;
+  /** The run's id, when the caller kept something under it first, such as a webhook's large body; default a new one. */
+  id?: string;
 };
 /**
  * Starts a run of a workflow from the app's own code, such as a job or a write hook, and returns the run's id. In
  * `db`'s own transaction when it is one, so the run starts when the caller's writes commit, or not at all.
  */
-export declare const startWorkflow: (db: Db, { workflow, input, tenant, startedBy, key, schema, version }: StartWorkflowOptions) => Promise<string>;
+export declare const startWorkflow: (db: Db, { workflow, input, tenant, startedBy, key, schema, version, id: runId }: StartWorkflowOptions) => Promise<string>;
 //#endregion

package/dist/workflows/start-workflow.js

@@ -7,13 +7,13 @@ import { sql } from "kysely";
 * Starts a run of a workflow from the app's own code, such as a job or a write hook, and returns the run's id. In
 * `db`'s own transaction when it is one, so the run starts when the caller's writes commit, or not at all.
 */
-const startWorkflow = async (db, { workflow, input, tenant, startedBy, key, schema = "workflows", version }) => {
+const startWorkflow = async (db, { workflow, input, tenant, startedBy, key, schema = "workflows", version, id: runId }) => {
 	checkRunInput(input);
 	const job = currentJobs().definition(segmentJobName);
 	if (!job) throw new Error("Workflows are not running in this app: add `workflows: {}` to protobase.config.ts");
 	const runs = sql.id(schema, "runs");
-	const { rows } = await sql`insert into ${runs} (workflow_id, version, input, tenant_id, started_by, key)
-    values (${workflow}, ${version ?? null}, ${input === void 0 ? null : JSON.stringify(input)}::jsonb, ${tenant === void 0 ? null : String(tenant)}, ${startedBy ?? null}, ${key ?? null})
+	const { rows } = await sql`insert into ${runs} (id, workflow_id, version, input, tenant_id, started_by, key)
+    values (${runId === void 0 ? sql`gen_random_uuid()` : sql`${runId}::uuid`}, ${workflow}, ${version ?? null}, ${input === void 0 ? null : JSON.stringify(input)}::jsonb, ${tenant === void 0 ? null : String(tenant)}, ${startedBy ?? null}, ${key ?? null})
     on conflict (workflow_id, key) do nothing returning id`.execute(db);
 	if (rows.length === 0) {
 		const existing = (await sql`select id, status from ${runs} where workflow_id = ${workflow} and key = ${key}`.execute(db)).rows[0];
@@ -24,5 +24,11 @@ const startWorkflow = async (db, { workflow, input, tenant, startedBy, key, sche
 	await job.enqueue(db, { run: id }, { key: id });
 	return id;
 };
+/** Queues the first segment of runs a trigger inserted itself, in `db`'s transaction. */
+const queueRuns = async (db, ids) => {
+	const job = currentJobs().definition(segmentJobName);
+	if (!job) throw new Error("Workflows are not running in this app: add `workflows: {}` to protobase.config.ts");
+	for (const id of ids) await job.enqueue(db, { run: id }, { key: id });
+};
 //#endregion
-export { startWorkflow };
+export { queueRuns, startWorkflow };

package/dist/workflows/triggers/delete-hook.js

@@ -0,0 +1,15 @@
+import { sql } from "kysely";
+//#region src/workflows/triggers/delete-hook.ts
+/**
+* As a workflow is deleted, in the delete's transaction: its trigger, which the delete left without a workflow, stops at
+* once, so no poll or call starts a run of it, and its stop job, queued in the same write, ends any subscription at its
+* provider and then removes it. The delete removed its webhook URL already.
+*/
+const workflowDeleteHook = (schema, jobs) => async (event, trx) => {
+	if (event.resource.name !== "workflows" || event.operation !== "delete") return;
+	const { rows } = await sql`update ${sql.id(schema, "triggers")} set status = 'stopping', generation = generation + 1, updated_at = now()
+    where workflow_id is null and status <> 'stopping' returning id`.execute(trx);
+	for (const { id } of rows) await jobs.stop.enqueue(trx, { trigger: id });
+};
+//#endregion
+export { workflowDeleteHook };

package/dist/workflows/triggers/plugin-call.js

@@ -0,0 +1,45 @@
+import { pluginFetch } from "../plugin-fetch.js";
+import { ConnectionError } from "@protobase/workflows/plugin";
+//#region src/workflows/triggers/plugin-call.ts
+/** The longest a plugin's error message is kept on its trigger, which the workflow's readers see. */
+const messageLength = 1e3;
+/**
+* Runs a plugin's trigger code: any error it throws is the plugin's, which the trigger records rather than the job
+* retrying it; a `ConnectionError('link_expired')` is a sign-in to renew. The message goes through `redact` and is cut
+* to 1 000 characters, since the trigger's readers see it.
+*/
+const callPlugin = (work, redact) => Promise.resolve().then(work).then((value) => ({ value }), (error) => {
+	const message = redact(error instanceof Error ? error.message : String(error)).slice(0, messageLength);
+	if (error instanceof ConnectionError && error.code === "link_expired") return { expired: message };
+	if (error instanceof ConnectionError && error.code === "not_connected") return {
+		failed: message,
+		unconnected: true
+	};
+	return { failed: message };
+});
+/**
+* `redact` after blanking out `values` that only this call knew, such as a webhook's URL key and secret, as written and
+* URL-encoded, so a message that quotes them keeps neither.
+*/
+const redactAlso = (values, redact) => {
+	const blanked = [...new Set(values.flatMap((value) => [value, encodeURIComponent(value)]))].filter((value) => value.length > 0);
+	return (text) => redact(blanked.reduce((kept, value) => kept.replaceAll(value, "[redacted]"), text));
+};
+/**
+* What one call of a trigger's hook gets: its input, its connection, the guarded `fetch`, and the job's signal, which
+* aborts at its timeout; and `redact` for its outcome, which also blanks out each token its connection gave this call.
+*/
+const triggerCall = ({ allowPrivateAddresses, connect }, trigger, signal, redact) => {
+	const issued = [];
+	return {
+		context: {
+			input: trigger.input,
+			connection: connect?.(trigger, (token) => void issued.push(token)),
+			fetch: pluginFetch({ allowPrivateAddresses }),
+			signal
+		},
+		redact: (text) => redactAlso(issued, redact)(text)
+	};
+};
+//#endregion
+export { callPlugin, redactAlso, triggerCall };

package/dist/workflows/triggers/poll-job.js

@@ -0,0 +1,120 @@
+import { defineJob } from "../../jobs/define-job.js";
+import { callPlugin, triggerCall } from "./plugin-call.js";
+import { pollProblem, stateProblem } from "./poll-result.js";
+import { queueRuns } from "../start-workflow.js";
+import { backoffPollAt, nextPollAt, pollEvery } from "./poll-schedule.js";
+import { claimTrigger, saveNeedsSignIn, savePluginFailure, savePoll, saveRenewal, setNextJob } from "./trigger-claim.js";
+//#region src/workflows/triggers/poll-job.ts
+/** The plugin's most items a poll, as it declares them, never more than 100. */
+const pollLimit = (type) => Math.min(type.limit ?? 100, 100);
+/** How often a webhook trigger renews its subscription at its provider. */
+const renewEvery = (type) => type.renewEvery ?? "1 day";
+/**
+* The jobs that poll a polling trigger and renew a webhook trigger's subscription at its provider, each on the trigger's
+* claim (`claimTrigger`): only the job the trigger names runs it, so a stale or duplicate job does nothing. The plugin's
+* error marks the trigger failing and backs off; an expired sign-in stops it, with every trigger on that connection,
+* until its owner signs in again; any other error is thrown, and the job retries under its own id.
+*/
+const pollJobs = (input) => {
+	const { schema, redact } = input;
+	const queueNext = async (trx, job, claim, due) => {
+		const id = await job.enqueue(trx, {
+			trigger: claim.trigger,
+			generation: claim.generation,
+			due: due.toISOString()
+		}, { runAt: due });
+		await setNextJob(trx, schema, claim.trigger, id);
+	};
+	const failed = (db, job, claim, message, nextAt) => db.transaction().execute(async (trx) => {
+		if (await savePluginFailure(trx, schema, claim, message, nextAt)) await queueNext(trx, job, claim, nextAt);
+	});
+	const poll = defineJob({
+		name: "workflows.poll",
+		timeout: "5 minutes",
+		retryDelay: "5 seconds",
+		run: async ({ trigger, generation, due }, { db, job, signal }) => {
+			const claim = {
+				trigger,
+				generation,
+				job: job.id,
+				attempt: job.attempt
+			};
+			const claimed = await claimTrigger(db, schema, claim);
+			const type = claimed && input.polling(claimed.type);
+			if (!claimed || !type) return;
+			const interval = await callPlugin(async () => pollEvery(type, claimed.input), redact);
+			if (!("value" in interval)) return failed(db, poll, claim, "expired" in interval ? interval.expired : interval.failed, backoffPollAt("1 minute", claimed.consecutive_failures + 1, /* @__PURE__ */ new Date()));
+			const every = interval.value;
+			const limit = pollLimit(type);
+			const call = triggerCall(input, claimed, signal, redact);
+			const outcome = await callPlugin(() => type.poll({
+				...call.context,
+				cursor: claimed.cursor,
+				store: claimed.store,
+				limit,
+				due: new Date(due)
+			}), call.redact);
+			const now = /* @__PURE__ */ new Date();
+			if ("expired" in outcome) return saveNeedsSignIn(db, schema, claim, claimed, outcome.expired);
+			const backoff = backoffPollAt(every, claimed.consecutive_failures + 1, now);
+			if ("failed" in outcome) return failed(db, poll, claim, outcome.failed, backoff);
+			const problem = pollProblem(outcome.value, limit);
+			if (problem) return failed(db, poll, claim, `The poll ${problem}`, backoff);
+			const { items, cursor, store } = outcome.value;
+			const dueAt = new Date(due);
+			const nextAt = nextPollAt(every, dueAt.getTime() > now.getTime() - 1e3 ? dueAt : now);
+			await db.transaction().execute(async (trx) => {
+				const runs = await savePoll(trx, schema, claim, items, {
+					cursor,
+					store: store === void 0 ? claimed.store : store,
+					nextAt
+				});
+				if (!runs) return;
+				await queueRuns(trx, runs);
+				await queueNext(trx, poll, claim, nextAt);
+			});
+		}
+	});
+	const renew = defineJob({
+		name: "workflows.renew",
+		timeout: "5 minutes",
+		retryDelay: "5 seconds",
+		run: async ({ trigger, generation }, { db, job, signal }) => {
+			const claim = {
+				trigger,
+				generation,
+				job: job.id,
+				attempt: job.attempt
+			};
+			const claimed = await claimTrigger(db, schema, claim);
+			const type = claimed && input.webhooks(claimed.type);
+			if (!claimed || !type?.renew) return;
+			const renewal = type.renew;
+			const call = triggerCall(input, claimed, signal, redact);
+			const outcome = await callPlugin(() => renewal({
+				...call.context,
+				state: claimed.store
+			}), call.redact);
+			const now = /* @__PURE__ */ new Date();
+			if ("expired" in outcome) return saveNeedsSignIn(db, schema, claim, claimed, outcome.expired);
+			const backoff = backoffPollAt("1 minute", claimed.consecutive_failures + 1, now);
+			if ("failed" in outcome) return failed(db, renew, claim, outcome.failed, backoff);
+			const problem = stateProblem(outcome.value);
+			if (problem) return failed(db, renew, claim, `The renewal ${problem}`, backoff);
+			const nextAt = nextPollAt(renewEvery(type), now);
+			await db.transaction().execute(async (trx) => {
+				if (await saveRenewal(trx, schema, claim, {
+					store: outcome.value,
+					nextAt
+				})) await queueNext(trx, renew, claim, nextAt);
+			});
+		}
+	});
+	return {
+		poll,
+		renew,
+		jobs: [poll, renew]
+	};
+};
+//#endregion
+export { pollJobs, pollLimit, renewEvery };

package/dist/workflows/triggers/poll-result.js

@@ -0,0 +1,32 @@
+import { triggerStateBytes } from "../schema-sql.js";
+import { limits, nestingDepth } from "@protobase/workflows";
+const jsonBytes = (value) => new TextEncoder().encode(JSON.stringify(value ?? null)).length;
+/** What is wrong with one item's data as a run's input, as the run's own limits say. */
+const inputProblem = (data) => {
+	if (nestingDepth(data, limits.inputDepth) > limits.inputDepth) return `nests deeper than ${limits.inputDepth} levels`;
+	if (jsonBytes(data) > limits.inputBytes) return `is over ${limits.inputBytes / 1024 / 1024} MiB as JSON`;
+};
+/** What is wrong with the items a plugin gave, each needing a key when `keyed`; `undefined` when they all fit. */
+const itemsProblem = (items, most, keyed) => {
+	if (!Array.isArray(items)) return "gave no items array";
+	if (items.length > most) return `gave ${items.length} items, more than its limit of ${most}`;
+	for (const [index, item] of items.entries()) {
+		const { key, data } = item ?? {};
+		if ((keyed || key !== void 0) && (typeof key !== "string" || key.length === 0 || key.length > 200)) return `item ${index} has no key of 1 to 200 characters`;
+		const problem = inputProblem(data);
+		if (problem) return `item ${index}'s data ${problem}`;
+	}
+};
+/** What is wrong with a subscription's state a webhook plugin gave: over 64 KiB as JSON. */
+const stateProblem = (state) => jsonBytes(state) > 65536 ? `gave a state over ${triggerStateBytes / 1024} KiB as JSON` : void 0;
+/** What is wrong with a poll's result: its items, and its cursor and store over 64 KiB as JSON. */
+const pollProblem = (result, limit) => {
+	if (typeof result !== "object" || result === null) return "gave no result";
+	const { items, cursor, store } = result;
+	const problem = itemsProblem(items, limit, true);
+	if (problem) return problem;
+	if (jsonBytes(cursor) > 65536) return `gave a cursor over ${triggerStateBytes / 1024} KiB as JSON`;
+	if (store !== void 0 && jsonBytes(store) > 65536) return `gave a store over ${triggerStateBytes / 1024} KiB as JSON`;
+};
+//#endregion
+export { itemsProblem, pollProblem, stateProblem };

package/dist/workflows/triggers/poll-schedule.js

@@ -0,0 +1,32 @@
+import { nextRun } from "../../jobs/schedule.js";
+import { parseDuration } from "../../files/duration.js";
+//#region src/workflows/triggers/poll-schedule.ts
+/** The shortest time between two polls, whatever a plugin's `every` says. */
+const shortestMs = 1e3;
+/** The longest wait a failing trigger's backoff reaches. */
+const longestBackoffMs = 36e5;
+/** The base of a cron trigger's backoff, which has no interval of its own. */
+const cronBackoffMs = 6e4;
+/** How often a polling trigger polls for this input. */
+const pollEvery = (type, input) => typeof type.every === "function" ? type.every(input) : type.every;
+/** The time of the poll after one due at `after`: an interval later, or the cron's next time. */
+const nextPollAt = (every, after) => {
+	if (typeof every === "string") return new Date(after.getTime() + Math.max(parseDuration(every, "every"), shortestMs));
+	return nextRun(every.cron, every.timezone ?? "UTC", after);
+};
+/** The first poll of a trigger that starts at `now`: at once on an interval, at the cron's next time otherwise. */
+const firstPollAt = (every, now) => typeof every === "string" ? now : nextRun(every.cron, every.timezone ?? "UTC", now);
+/**
+* When a trigger that failed `failures` times in a row polls again: `every × 2^failures`, at most an hour and never
+* sooner than its interval; a cron trigger at its first time after a minute × 2^failures, at most an hour, from now.
+*/
+const backoffPollAt = (every, failures, now) => {
+	const factor = 2 ** Math.min(failures, 20);
+	if (typeof every === "string") {
+		const interval = Math.max(parseDuration(every, "every"), shortestMs);
+		return new Date(now.getTime() + Math.max(interval, Math.min(interval * factor, longestBackoffMs)));
+	}
+	return nextRun(every.cron, every.timezone ?? "UTC", new Date(now.getTime() + Math.min(cronBackoffMs * factor, longestBackoffMs)));
+};
+//#endregion
+export { backoffPollAt, firstPollAt, nextPollAt, pollEvery };

package/dist/workflows/triggers/trigger-claim.js

@@ -0,0 +1,81 @@
+import { sql } from "kysely";
+//#region src/workflows/triggers/trigger-claim.ts
+/**
+* Claims the trigger for this try of the job its row names, in one statement: only while it runs on this generation,
+* and only for a later try than the last that claimed it, so a retry of a job whose worker died takes its claim back,
+* and a try that ran past its timeout no longer holds it. `undefined` when the trigger was stopped, changed or moved
+* to another job.
+*/
+const claimTrigger = async (db, schema, { trigger, generation, job, attempt }) => {
+	const { rows } = await sql`update ${sql.id(schema, "triggers")} set poll_attempt = ${attempt}
+    where id = ${trigger} and generation = ${generation} and poll_job_id = ${job} and status in ('active', 'failing')
+      and (poll_attempt is null or poll_attempt < ${attempt})
+    returning id, workflow_id, version, tenant_id, type, input, cursor, store, connection_id, consecutive_failures`.execute(db);
+	return rows[0];
+};
+const held = (schema, { trigger, generation, job, attempt }) => sql`select id, workflow_id, version, tenant_id from ${sql.id(schema, "triggers")}
+    where id = ${trigger} and generation = ${generation} and poll_job_id = ${job} and poll_attempt = ${attempt} and status in ('active', 'failing') for update`;
+/**
+* Saves a successful poll in one statement that locks the trigger row while this try holds its claim: the runs of its
+* items, each once per workflow and key, and the new cursor and store, the trigger active again. The ids of the runs it
+* started, or `undefined` when the claim was gone and nothing was written.
+*/
+const savePoll = async (trx, schema, claim, items, { cursor, store, nextAt }) => {
+	const triggers = sql.id(schema, "triggers");
+	const { rows } = await sql`with t as (${held(schema, claim)}),
+    started as (insert into ${sql.id(schema, "runs")} (workflow_id, version, input, key, tenant_id, started_by)
+      select t.workflow_id, t.version, i.data, i.key, t.tenant_id, 'trigger:' || t.id from t cross join jsonb_to_recordset(${JSON.stringify(items)}::jsonb) as i(key text, data jsonb)
+      on conflict (workflow_id, key) do nothing returning id),
+    saved as (update ${triggers} set cursor = ${JSON.stringify(cursor ?? null)}::jsonb, store = ${JSON.stringify(store ?? null)}::jsonb, status = 'active',
+      consecutive_failures = 0, last_error = null, poll_attempt = null, next_poll_at = ${nextAt}, updated_at = now() from t where ${triggers}.id = t.id returning 1)
+    select (select count(*) from saved)::int as saved, coalesce((select json_agg(id) from started), '[]'::json) as runs`.execute(trx);
+	const row = rows[0];
+	return row.saved > 0 ? row.runs : void 0;
+};
+/** Saves a renewal's state while this try holds its claim; whether it did. */
+const saveRenewal = async (trx, schema, claim, { store, nextAt }) => {
+	const triggers = sql.id(schema, "triggers");
+	const { rows } = await sql`with t as (${held(schema, claim)})
+    update ${triggers} set store = ${JSON.stringify(store ?? null)}::jsonb, status = 'active', consecutive_failures = 0, last_error = null, poll_attempt = null,
+      next_poll_at = ${nextAt}, updated_at = now() from t where ${triggers}.id = t.id returning 1`.execute(trx);
+	return rows.length > 0;
+};
+/** Marks the trigger failing with the plugin's error, while this try holds its claim; whether it did. */
+const savePluginFailure = async (trx, schema, claim, message, nextAt) => {
+	const triggers = sql.id(schema, "triggers");
+	const error = JSON.stringify({
+		code: "plugin_error",
+		message,
+		at: (/* @__PURE__ */ new Date()).toISOString()
+	});
+	const { rows } = await sql`with t as (${held(schema, claim)})
+    update ${triggers} set status = 'failing', consecutive_failures = consecutive_failures + 1, last_error = ${error}::jsonb, poll_attempt = null,
+      next_poll_at = ${nextAt}, updated_at = now() from t where ${triggers}.id = t.id returning 1`.execute(trx);
+	return rows.length > 0;
+};
+/**
+* Names the job that polls or renews the trigger next, on the row the transaction holds locked. The new job's tries
+* count from one, so a claim an older job's try still holds is dropped with it.
+*/
+const setNextJob = async (trx, schema, trigger, job) => {
+	await sql`update ${sql.id(schema, "triggers")} set poll_job_id = ${job}, poll_attempt = null where id = ${trigger}`.execute(trx);
+};
+/**
+* The connection's sign-in expired: this trigger, while this try holds its claim, and every running trigger of the same
+* tenant on the same connection wait for a new sign-in, with no next poll. A trigger without a connection is the only
+* one it moves.
+*/
+const saveNeedsSignIn = async (db, schema, claim, trigger, message) => {
+	const error = JSON.stringify({
+		code: "link_expired",
+		message,
+		at: (/* @__PURE__ */ new Date()).toISOString()
+	});
+	const { trigger: id, generation, job, attempt } = claim;
+	await sql`update ${sql.id(schema, "triggers")} set status = 'needs_sign_in', last_error = ${error}::jsonb, poll_job_id = null, poll_attempt = null, next_poll_at = null, updated_at = now()
+    where status in ('active', 'failing') and (
+      (id = ${id} and generation = ${generation} and poll_job_id = ${job} and poll_attempt = ${attempt})
+      or (${trigger.connection_id}::text is not null and connection_id = ${trigger.connection_id} and tenant_id is not distinct from ${trigger.tenant_id}))`.execute(db);
+};
+//#endregion
+export { claimTrigger, saveNeedsSignIn, savePluginFailure, savePoll, saveRenewal, setNextJob };

package/dist/workflows/triggers/trigger-jobs.js

@@ -1,7 +1,10 @@
 import { defineJob } from "../../jobs/define-job.js";
+import { callPlugin, triggerCall } from "./plugin-call.js";
 import { readyAt, readyWhen, resolveTimeTerms } from "../ready/ready-when.js";
 import { activeTriggers, currentTrigger, triggerFailed } from "./trigger-row.js";
 import { orRefusal, readyContext, startIfReady } from "../ready/ready-start.js";
+import { pollJobs } from "./poll-job.js";
+import { triggerStarts } from "./trigger-start.js";
 import { encodeKey } from "@protobase/schema/internal";
 import { printFilter } from "@protobase/schema";
 import { sql } from "kysely";
@@ -10,12 +13,15 @@ import { sql } from "kysely";
 const passSize = 100;
 /**
 * The jobs that run triggers on every server process, so turning a workflow on, changing or turning it off needs no
-* restart: `workflows.trigger-start` (for a ready trigger, one pass over the records that already match, 100 per job),
-* `workflows.trigger-stop` (removes a stopped trigger), and `workflows.ready-check` (starts a run of one record when it
-* is ready, at the time its time terms say). Each acts only on the generation it was queued for.
+* restart: `workflows.trigger-start` (for a ready trigger, one pass over the records that already match, 100 per job;
+* for a polling one its first poll; for a subscribing webhook one its subscription), `workflows.trigger-stop`
+* (unsubscribes a stopped webhook trigger at its provider and removes it), `workflows.ready-check` (starts a run of one
+* record when it is ready, at the time its time terms say), and the poll and renewal jobs. Each acts only on the
+* generation it was queued for.
 */
 const triggerJobs = (input) => {
 	const { schema } = input;
+	const polls = pollJobs(input);
 	const readyCheck = defineJob({
 		name: "workflows.ready-check",
 		timeout: "5 minutes",
@@ -42,12 +48,20 @@ const triggerJobs = (input) => {
 			key: `ready:${trigger.id}:${trigger.generation}:${recordKey}:${at.toISOString()}`
 		});
 	};
+	const starts = triggerStarts(input, polls);
 	const start = defineJob({
 		name: "workflows.trigger-start",
 		timeout: "5 minutes",
-		run: async ({ trigger: id, generation, page_token }, { db }) => {
+		run: async ({ trigger: id, generation, page_token }, { db, signal }) => {
 			const trigger = await currentTrigger(db, schema, id, generation);
-			if (!trigger || !input.declared(trigger.type)) return;
+			if (!trigger) return;
+			const restart = async (later, at) => void await start.enqueue(later, {
+				trigger: id,
+				generation
+			}, { runAt: at });
+			if (input.polling(trigger.type)) return starts.startPolling(db, trigger, signal, restart);
+			if (input.webhooks(trigger.type)) return starts.startWebhook(db, trigger, signal, restart);
+			if (!input.declared(trigger.type)) return;
 			const found = await orRefusal(readyContext(input, trigger));
 			if ("refused" in found) return triggerFailed(db, schema, trigger, "refused", found.refused);
 			if (!found.value) return;
@@ -71,9 +85,40 @@ const triggerJobs = (input) => {
 			});
 		}
 	});
+	/**
+	* Ends a stopped webhook trigger's subscription at its provider: whether nothing is left there, so the trigger can go.
+	* Any error is tried again, as the job's, until the provider ends it. A sign-in the unsubscribe can no longer use, as
+	* once the workflow was turned off and so its connection with it, or once the provider refused it, cannot end it:
+	* the trigger then stays `stopping` with that error, as the record of a subscription the provider may still hold.
+	*/
+	const unsubscribe = async (db, trigger, signal) => {
+		const { rows } = await sql`select type, input, workflow_id, version, store, store is not null as subscribed
+      from ${sql.id(schema, "triggers")} where id = ${trigger} and status = 'stopping'`.execute(db);
+		const row = rows[0];
+		const type = row?.subscribed ? input.webhooks(row.type) : void 0;
+		if (!row || !type?.unsubscribe) return true;
+		const end = type.unsubscribe;
+		const call = triggerCall(input, row, signal, input.redact);
+		const outcome = await callPlugin(() => end({
+			...call.context,
+			state: row.store
+		}), call.redact);
+		if ("value" in outcome) return true;
+		if ("failed" in outcome && !outcome.unconnected) throw new Error(`The trigger's unsubscribe failed: ${outcome.failed}`);
+		const reason = "expired" in outcome ? outcome.expired : outcome.failed;
+		const error = JSON.stringify({
+			code: "expired" in outcome ? "link_expired" : "not_connected",
+			message: `The subscription could not be ended at the provider: ${reason}`,
+			at: (/* @__PURE__ */ new Date()).toISOString()
+		});
+		await sql`update ${sql.id(schema, "triggers")} set last_error = ${error}::jsonb, updated_at = now() where id = ${trigger} and status = 'stopping'`.execute(db);
+		return false;
+	};
 	const stop = defineJob({
 		name: "workflows.trigger-stop",
-		run: async ({ trigger }, { db }) => {
+		timeout: "5 minutes",
+		run: async ({ trigger }, { db, signal }) => {
+			if (!await unsubscribe(db, trigger, signal)) return;
 			await sql`delete from ${sql.id(schema, "triggers")} where id = ${trigger} and status = 'stopping'`.execute(db);
 		}
 	});
@@ -103,7 +148,8 @@ const triggerJobs = (input) => {
 		jobs: [
 			start,
 			stop,
-			readyCheck
+			readyCheck,
+			...polls.jobs
 		],
 		start,
 		stop,