@plitzi/sdk-server 0.37.1 → 0.37.2

package/dist/modules/actions/jobs/worker.js

@@ -0,0 +1,262 @@
+import { ActionRunError } from "../runtime/errors.js";
+import { randomUUID } from "node:crypto";
+//#region src/modules/actions/jobs/worker.ts
+/**
+* Refusals that mean "not now" rather than "not ever".
+*
+* Both are the cluster protecting itself — another run holds the single-flight key, or this replica is already
+* carrying as much as it will — and neither says anything about the job. So the job goes back to the queue with
+* its attempt REFUNDED: burning one would let a busy afternoon retire a nightly digest as dead.
+*/
+var TRY_AGAIN = /* @__PURE__ */ new Set(["duplicate", "over_capacity"]);
+var messageOf = (error) => error instanceof Error ? error.message : String(error);
+/**
+* Why a run that ended badly did, in the words an operator can act on: the step that failed and what it said.
+*
+* Read from `steps` — the outline, whose errors are already redacted of every credential the run resolved — so
+* nothing lands on the job that the run history would not show. A run that failed without any step failing (a
+* timeout between steps, a budget spent) has no step to name, and says how it ended instead.
+*/
+var failureOf = (result) => {
+	const failed = result.steps.find((step) => step.phase === "flow" && step.status === "failed");
+	return failed ? `step "${failed.id}" failed: ${failed.error ?? "no reason given"}` : `the flow ended ${result.status}`;
+};
+/**
+* Refusals no number of attempts will change: the action was deleted, its schedule was switched off, or the
+* document refuses this trigger. Retrying is noise, and the job stops where an operator can see why.
+*/
+var GIVE_UP = /* @__PURE__ */ new Set([
+	"not_found",
+	"disabled",
+	"forbidden",
+	"recursion"
+]);
+var createJobWorker = ({ queue, lookups, module, workers = 4, pollMs = 1e3, leaseMs = 3e4, backoff = {}, workerId = `${process.pid}-${randomUUID().slice(0, 8)}`, onError = (error) => console.error("[Actions] job worker failed:", error) }) => {
+	const { baseMs = 3e4, maxMs = 9e5 } = backoff;
+	const active = /* @__PURE__ */ new Map();
+	/** Every job this worker started, until it has settled — what `stop()` waits for. */
+	const inFlight = /* @__PURE__ */ new Set();
+	let timer;
+	let heart;
+	let stopping = false;
+	let pass;
+	const isStopping = () => stopping;
+	/** Doubles per attempt, with a jitter so a batch that failed together does not retry together. */
+	const retryDelay = (attempts) => {
+		const step = Math.min(maxMs, baseMs * 2 ** Math.max(0, attempts - 1));
+		return step + Math.floor(Math.random() * Math.min(step, 3e4));
+	};
+	const settle = async (settlement) => {
+		try {
+			await queue.settle(settlement);
+		} catch (error) {
+			onError(error);
+		}
+	};
+	/** Back to the queue with nothing held against it — see {@link TRY_AGAIN}. */
+	const release = (job, delayMs, error) => settle({
+		jobId: job.id,
+		workerId,
+		status: "pending",
+		delayMs,
+		error
+	});
+	const fail = (job, runId, error) => {
+		const exhausted = job.attempts >= job.maxAttempts;
+		return settle({
+			jobId: job.id,
+			workerId,
+			status: exhausted ? "dead" : "pending",
+			...exhausted ? {} : { delayMs: retryDelay(job.attempts) },
+			...runId ? { runId } : {},
+			error,
+			attempt: {
+				status: "failed",
+				workerId,
+				...runId ? { runId } : {},
+				error
+			}
+		});
+	};
+	const execute = async (job) => {
+		const entry = await lookups.getAction(job.spaceId, job.actionId);
+		if (!entry) {
+			await settle({
+				jobId: job.id,
+				workerId,
+				status: "dead",
+				error: "the action this job belongs to no longer exists",
+				attempt: {
+					status: "failed",
+					workerId,
+					error: "action not found"
+				}
+			});
+			return;
+		}
+		let run;
+		try {
+			run = await module.guards.begin({
+				spaceId: job.spaceId,
+				actionId: job.actionId,
+				callerId: "schedule",
+				input: job.input,
+				idempotencyKey: job.id,
+				sharedKey: true,
+				ttlMs: module.limitsFor(entry.document).timeoutMs
+			});
+		} catch (error) {
+			const refusal = error instanceof ActionRunError ? error.reason : void 0;
+			/**
+			* Two different things, and telling them apart is what decides whether the job survives.
+			*
+			* A REFUSAL is the cluster answering: another run holds the key, this replica is full. A throw that is not a
+			* refusal means the guards could not be consulted at all — the shared store is unreachable — and that is
+			* infrastructure rather than anything about this job. Both go back with the attempt REFUNDED.
+			*
+			* Counting an outage as attempts is how a queue loses work: with three attempts and a widening backoff, a
+			* Redis blip of two minutes retires every scheduled job in the cluster as `dead`, permanently, and the digest
+			* nobody got is indistinguishable from one nobody asked for. The job comes back instead, carrying why, and an
+			* operator sees a queue that is waiting rather than a queue that has been emptied.
+			*/
+			if (!refusal || TRY_AGAIN.has(refusal)) {
+				await release(job, retryDelay(1), `not started: ${refusal ?? messageOf(error)}`);
+				return;
+			}
+			await fail(job, void 0, `could not start: ${refusal}`);
+			return;
+		}
+		active.set(job.id, run.controller);
+		try {
+			const result = await module.runAction({
+				entry,
+				input: job.input,
+				spaceId: job.spaceId,
+				environment: job.environment,
+				trigger: job.trigger,
+				callerId: "schedule",
+				runId: run.runId,
+				signal: run.controller.signal
+			});
+			if (result.status === "completed") {
+				await settle({
+					jobId: job.id,
+					workerId,
+					status: "succeeded",
+					runId: run.runId,
+					attempt: {
+						status: "succeeded",
+						workerId,
+						runId: run.runId
+					}
+				});
+				return;
+			}
+			if (result.status === "aborted" && run.controller.signal.aborted) {
+				await settle({
+					jobId: job.id,
+					workerId,
+					status: "cancelled",
+					runId: run.runId,
+					attempt: {
+						status: "cancelled",
+						workerId,
+						runId: run.runId
+					}
+				});
+				return;
+			}
+			await fail(job, run.runId, failureOf(result));
+		} catch (error) {
+			const reason = error instanceof ActionRunError ? error.reason : void 0;
+			if (reason && GIVE_UP.has(reason)) {
+				await settle({
+					jobId: job.id,
+					workerId,
+					status: "dead",
+					runId: run.runId,
+					error: reason,
+					attempt: {
+						status: "failed",
+						workerId,
+						runId: run.runId,
+						error: reason
+					}
+				});
+				return;
+			}
+			await fail(job, run.runId, messageOf(error));
+		} finally {
+			active.delete(job.id);
+			await module.guards.end(run);
+		}
+	};
+	const poll = async () => {
+		const free = workers - active.size;
+		if (stopping || free <= 0) return 0;
+		const claimed = await queue.claim({
+			workerId,
+			leaseMs,
+			limit: free
+		});
+		if (isStopping()) {
+			await Promise.all(claimed.map((job) => release(job, 0, "not started: the replica was shutting down")));
+			return 0;
+		}
+		for (const job of claimed) {
+			const running = execute(job).catch(onError);
+			inFlight.add(running);
+			running.finally(() => inFlight.delete(running));
+		}
+		return claimed.length;
+	};
+	/** Renews every claim this worker holds, and stops the ones an operator asked to cancel. */
+	const beat = async () => {
+		const held = [...active.keys()];
+		if (held.length === 0) return;
+		for (const jobId of await queue.heartbeat({
+			jobIds: held,
+			workerId,
+			leaseMs
+		})) active.get(jobId)?.abort();
+	};
+	/** One pass at a time, and the one under way is kept so `stop()` can wait for the claim it may be making. */
+	const guarded = async () => {
+		if (pass) return;
+		pass = poll().then(() => void 0).catch(onError).finally(() => {
+			pass = void 0;
+		});
+		await pass;
+	};
+	return {
+		poll,
+		beat,
+		inFlight: () => active.size,
+		start: () => {
+			if (timer || workers <= 0) return;
+			timer = setInterval(() => void guarded(), pollMs);
+			timer.unref();
+			heart = setInterval(() => void beat().catch(onError), Math.max(1e3, Math.floor(leaseMs / 3)));
+			heart.unref();
+		},
+		stop: async () => {
+			stopping = true;
+			clearInterval(timer);
+			timer = void 0;
+			await pass;
+			if (inFlight.size > 0) console.info(`[Actions] waiting for ${inFlight.size} running job(s) to finish before stopping`);
+			/**
+			* Drained rather than abandoned: a deploy finishes the jobs it started instead of leaving them to be retried.
+			*
+			* The heartbeat is stopped only AFTER, and that is the half that makes waiting safe. A job longer than its
+			* lease used to lose its claim the moment the drain began — another replica took it over and ran it again
+			* while this one was still finishing it, and the drain gave up at the lease regardless.
+			*/
+			await Promise.allSettled([...inFlight]);
+			clearInterval(heart);
+			heart = void 0;
+		}
+	};
+};
+//#endregion
+export { createJobWorker };

package/dist/modules/actions/jobs/worker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/modules/actions/moduleFor.js

@@ -23,6 +23,7 @@ var actionsModuleFor = (config) => {
 		limits: config.action?.limits,
 		concurrency: config.action?.concurrency,
 		kv: config.action?.kv,
+		jobs: config.action?.jobs,
 		rateLimit: config.action?.rateLimit,
 		idempotency: config.action?.idempotency,
 		dbDrivers: config.action?.dbDrivers,

package/dist/modules/actions/runtime/precheck.js

@@ -1,6 +1,6 @@
+import { findTriggerNode, triggerParams } from "./triggers.js";
 import { ActionRunError } from "./errors.js";
 import { applyFields } from "./scope.js";
-import { findTriggerNode, triggerParams } from "./triggers.js";
 import { triggerAccess, triggerInput } from "@plitzi/sdk-shared/actions";
 //#region src/modules/actions/runtime/precheck.ts
 var authorize = (access, kind, user) => {

package/dist/modules/actions/runtime/renderResolver.js

@@ -1,9 +1,9 @@
-import { ActionRunError } from "./errors.js";
 import { findTriggerNode, triggerParams } from "./triggers.js";
+import { ActionRunError } from "./errors.js";
 import onAbort from "../../../helpers/onAbort.js";
 import { createRenderShare } from "./renderShare.js";
-import { createHash, randomUUID } from "node:crypto";
 import { triggerCacheMs } from "@plitzi/sdk-shared/actions";
+import { createHash, randomUUID } from "node:crypto";
 //#region src/modules/actions/runtime/renderResolver.ts
 /**
 * A render run that resolved nothing, carrying what it did.

package/dist/modules/actions/transport/callHandler.js

@@ -1,5 +1,5 @@
-import { ActionRunError } from "../runtime/errors.js";
 import { triggerParams } from "../runtime/triggers.js";
+import { ActionRunError } from "../runtime/errors.js";
 import { precheckRun } from "../runtime/precheck.js";
 import onAbort from "../../../helpers/onAbort.js";
 import { openStream, wantsStream } from "./stream.js";

package/dist/modules/actions/transport/webhookHandler.js

@@ -1,5 +1,5 @@
-import { ActionRunError } from "../runtime/errors.js";
 import { findTriggerNode, triggerParams } from "../runtime/triggers.js";
+import { ActionRunError } from "../runtime/errors.js";
 import { precheckRun } from "../runtime/precheck.js";
 import onAbort from "../../../helpers/onAbort.js";
 import { reportReject } from "../runtime/report.js";

package/dist/modules/actions/types.d.ts

@@ -1,5 +1,5 @@
 import { ActionEmailSender } from './runtime/email';
-import { ActionEmailConfig, ActionEntry, ActionLimits, ActionRejectRecord, ActionRunRecord, ActionRunStatus, ActionRunStep, ActionTriggerType, ConnectorManifest, Environment, InteractionCallbackParam, InteractionNode, InteractionNodeStatus, SpaceRevision, SSRUser } from '@plitzi/sdk-shared';
+import { ActionEmailConfig, ActionJobQueue, ActionEntry, ActionLimits, ActionRejectRecord, ActionRunRecord, ActionRunStatus, ActionRunStep, ActionTriggerType, ConnectorManifest, Environment, InteractionCallbackParam, InteractionNode, InteractionNodeStatus, SpaceRevision, SSRUser } from '@plitzi/sdk-shared';
 /** Resolved secret material, as the connector engine already models it. */
 export type ActionCredential = Record<string, string>;
 /**
@@ -71,6 +71,15 @@ export type ActionLookups = {
     getAction: (spaceId: number, actionId: string, at?: SpaceRevision) => Promise<ActionEntry | undefined>;
     /** Only the builder's catalog, the MCP and the scheduler need the list; a page call never asks for it. */
     listActions?: (spaceId: number, at?: SpaceRevision) => Promise<ActionEntry[]>;
+    /**
+     * Which spaces have something scheduled, for the periodic reconcile.
+     *
+     * Optional, and a safety net rather than the mechanism: a deployment keeps the schedule rows current by calling
+     * `reconcile` whenever a space's actions change, and this is what catches the write that did not happen — a
+     * restore from backup, a row edited by hand, a deploy that missed a save. Absent, nothing reconciles on a timer
+     * and the rows are exactly as correct as the calls that wrote them.
+     */
+    listScheduledSpaces?: () => Promise<number[]>;
     getCredential?: (spaceId: number, identifier: string) => Promise<ActionCredential | undefined>;
     getConnector?: (spaceId: number, connectorId: string, at?: SpaceRevision) => Promise<ConnectorManifest | undefined>;
 };
@@ -159,6 +168,15 @@ export type ActionsConfig = {
     };
     /** Where the `kv` tasks keep things. Omitted → an in-process Map, which is per-replica by definition. */
     kv?: ActionKvAdapter;
+    /**
+     * Scheduled runs, and the durable queue they go through. See {@link ActionJobsConfig}.
+     *
+     * On by default wherever `lookups.listActions` is answerable, because a deployment that can list its actions can
+     * schedule them and the old behaviour — a `schedule` trigger that silently did nothing unless the deployment
+     * built a timer and a lock of its own — was a trigger the editor offered and the server did not honour. `false`
+     * turns it off for a replica that must only serve pages.
+     */
+    jobs?: ActionJobsConfig | false;
     /** Inbound webhooks are public, so they are counted per caller per minute. Default 60. */
     rateLimit?: {
         webhookPerMinute?: number;
@@ -200,6 +218,63 @@ export type ActionsConfig = {
     };
     fetchImpl?: typeof fetch;
 };
+/**
+ * Scheduled runs: how they are produced, how many run at once, and where they are kept while they wait.
+ *
+ * Supplying nothing still schedules — over an in-process queue, which is correct for one replica and silently
+ * wrong for two, since each gets its own copy of every schedule. A deployment with more than one supplies a
+ * {@link ActionJobQueue} over a store they share.
+ */
+export type ActionJobsConfig = {
+    /**
+     * Where jobs and schedules live. Omitted → this process's memory: nothing survives a restart, and a second
+     * replica produces the same fires all over again.
+     */
+    queue?: ActionJobQueue;
+    /**
+     * How many jobs this replica runs at once. Default 4; `0` makes it a producer that never consumes.
+     *
+     * The number that decides whether a backlog drains in minutes or hours, and it belongs to the deployment because
+     * only it knows what its flows do — a queue of five hundred HTTP calls wants a far higher number than a queue of
+     * five hundred renders. It never widens the run guards: `concurrency.perSpace` still caps one space's runs per
+     * replica, and jobs over that ceiling go back to the queue rather than failing.
+     */
+    workers?: number;
+    /** Whether this replica sweeps schedules into the queue at all. Default true. */
+    produce?: boolean;
+    /**
+     * The spaces this server schedules for, when it serves a known few.
+     *
+     * The self-hosted shape: one space, named once. A multi-tenant deployment answers the same question with
+     * {@link ActionLookups.listScheduledSpaces} instead, which wins when both are given. With neither, nothing tells
+     * the scheduler where to look and it says so at boot rather than sitting silent — a schedule that never fires is
+     * indistinguishable from a cron expression that is wrong.
+     */
+    spaces?: number[];
+    /** How often a free worker looks for a job. Default 1s. */
+    pollMs?: number;
+    /** How often this replica looks for due schedules. Default 15s. */
+    schedulePollMs?: number;
+    /**
+     * How long a claim is held before another replica may take the job over — the failover window. Default 30s,
+     * renewed every third of it while a job runs.
+     */
+    leaseMs?: number;
+    /** Attempts a scheduled job gets before it is left for an operator as `dead`. Default 3. */
+    maxAttempts?: number;
+    /** First retry delay, doubling per attempt, up to `maxMs`. Defaults to 30s and 15 minutes. */
+    backoff?: {
+        baseMs?: number;
+        maxMs?: number;
+    };
+    /** Which environment scheduled runs execute in. Default `main`. */
+    environment?: Environment;
+    /** How often schedules are re-derived from documents, where `listScheduledSpaces` allows it. Default 15 min. */
+    reconcileMs?: number;
+    /** Names this replica in the job history. Defaults to the pid and a random suffix. */
+    workerId?: string;
+    onError?: (error: unknown) => void;
+};
 /** Re-exported so the module's own files import one place, and a deployment writing an `onRun` or an `onReject`
  *  sees the same shapes the module emits. */
 export type { ActionEmailConfig, ActionEmailDelivery, ActionEmailMessage, ActionEmailTransport, ActionRejectRecord, ActionRunRecord } from '@plitzi/sdk-shared';

package/dist/modules/mongo/index.d.ts

@@ -0,0 +1,4 @@
+export { createMongoJobQueue } from './jobQueue';
+export { createMongoKv } from './kv';
+export type { MongoDbSource, MongoJobQueue, MongoJobQueueOptions } from './jobQueue';
+export type { MongoKvOptions } from './kv';

package/dist/modules/mongo/jobQueue.d.ts

@@ -0,0 +1,38 @@
+import { ActionJobQueue } from '@plitzi/sdk-shared';
+import { Db } from 'mongodb';
+/** The database, or how to reach it — a getter for a deployment whose client is replaced when it reconnects. */
+export type MongoDbSource = Db | (() => Db);
+export declare const resolveDb: (source: MongoDbSource) => Db;
+export type MongoJobQueueOptions = {
+    db: MongoDbSource;
+    /** Collection names, for a deployment with a naming scheme of its own. */
+    collections?: {
+        jobs?: string;
+        schedules?: string;
+    };
+    /** How long a finished job stays readable. A waiting or running one never expires. Default 90 days. */
+    retentionDays?: number;
+};
+export type MongoJobQueue = ActionJobQueue & {
+    /** Takes the store's clock again on the next call — after a failover to another primary. */
+    resyncClock: () => void;
+};
+/**
+ * An {@link ActionJobQueue} over two collections of a Mongo database the deployment owns.
+ *
+ * The helper for a deployment that already runs Mongo: it holds no connection of its own and creates nothing but the
+ * indexes its queries need, on first use. Every replica pointed at the same database shares the jobs and the
+ * schedules, which is what running more than one needs.
+ *
+ * ```ts
+ * import { createMongoJobQueue } from '@plitzi/sdk-server/mongo';
+ *
+ * const queue = createMongoJobQueue({ db: client.db('app') });
+ * createServer({ action: { lookups, kv, jobs: { queue } } });
+ * ```
+ *
+ * The contract's rules, as Mongo keeps them: every instant is the SERVER's (`hello.localTime`), `enqueue` is idempotent
+ * by the job's `_id`, `claim` takes one document per `findOneAndUpdate` — the atomic unit — and reaps a lapsed lease in
+ * the same operation, and a schedule advances by compare-and-set on the instant that was read.
+ */
+export declare const createMongoJobQueue: ({ db, collections: names, retentionDays }: MongoJobQueueOptions) => MongoJobQueue;