@kitr/jobs 0.0.0-stage → 0.3.0

package/dist/index.js

@@ -0,0 +1,350 @@
+import { KitrError, createDefineConfig, validateSync } from "@kitr/core";
+// src/index.ts
+/** Declare a background job. List it in `config/jobs.ts`, then queue it with `useJobs(app).dispatch(job, payload)`. */
+function defineJob(name, options) {
+	return {
+		name,
+		...options
+	};
+}
+/** Types `config/jobs.ts`. Returns its argument unchanged. */
+const defineConfig = createDefineConfig();
+/**
+* The default driver: an in-process queue sorted by `runAt`. Lost on restart and not shared between
+* processes; use `createRedisJobsDriver` for that. A pulled job is reserved, not removed: it
+* counts in `size()` until the worker acks it, and returns to the queue if that takes longer than
+* `visibilityTimeout`.
+*/
+function createMemoryJobsDriver(options = {}) {
+	const visibility = options.visibilityTimeout ?? 3e4;
+	const queue = [];
+	const reserved = /* @__PURE__ */ new Map();
+	const key = (job) => `${job.id}:${job.attempt}`;
+	const sort = () => queue.sort((a, b) => a.runAt - b.runAt);
+	return {
+		name: "memory",
+		async push(job) {
+			queue.push(job);
+			sort();
+		},
+		async pull(now, limit) {
+			let returned = false;
+			for (const [k, entry] of reserved) if (entry.until <= now) {
+				reserved.delete(k);
+				queue.push(entry.job);
+				returned = true;
+			}
+			if (returned) sort();
+			const due = [];
+			while (queue.length && queue[0].runAt <= now && due.length < limit) due.push(queue.shift());
+			for (const job of due) reserved.set(key(job), {
+				job,
+				until: now + visibility
+			});
+			return due;
+		},
+		async ack(job) {
+			reserved.delete(key(job));
+		},
+		async size() {
+			return queue.length + reserved.size;
+		},
+		async clear() {
+			queue.length = 0;
+			reserved.clear();
+		}
+	};
+}
+/**
+* A durable queue over Redis, shared by every worker. Each due job goes to exactly one worker:
+* pulling moves it from the queue to a reserved set, and only the worker whose `ZREM` succeeds
+* runs it. A job whose worker dies returns to the queue after `visibilityTimeout`. Scheduled
+* jobs queue once per run however many workers poll, through `claim`.
+*
+*   driver: createRedisJobsDriver(new Redis(process.env.REDIS_URL))
+*/
+function createRedisJobsDriver(redis, options = {}) {
+	const prefix = options.prefix ?? "kitr:jobs:";
+	const visibility = options.visibilityTimeout ?? 3e5;
+	const queue = `${prefix}queue`;
+	const reserved = `${prefix}reserved`;
+	const data = `${prefix}data`;
+	const member = (job) => `${job.id}:${job.attempt}`;
+	return {
+		name: "redis",
+		async push(job) {
+			await redis.hset(data, member(job), JSON.stringify(job));
+			await redis.zadd(queue, job.runAt, member(job));
+		},
+		async pull(now, limit) {
+			for (const m of await redis.zrangebyscore(reserved, "-inf", now, "LIMIT", 0, 100)) if (await redis.zrem(reserved, m) === 1) await redis.zadd(queue, now, m);
+			const due = [];
+			for (const m of await redis.zrangebyscore(queue, "-inf", now, "LIMIT", 0, limit)) {
+				if (await redis.zrem(queue, m) !== 1) continue;
+				await redis.zadd(reserved, now + visibility, m);
+				const raw = await redis.hget(data, m);
+				if (raw) due.push(JSON.parse(raw));
+				else await redis.zrem(reserved, m);
+			}
+			return due;
+		},
+		async ack(job) {
+			await redis.zrem(reserved, member(job));
+			await redis.hdel(data, member(job));
+		},
+		async size() {
+			return await redis.zcard(queue) + await redis.zcard(reserved);
+		},
+		async clear() {
+			await redis.del(queue, reserved, data);
+		},
+		async claim(key, ttlMs) {
+			return await redis.set(`${prefix}lock:${key}`, "1", "PX", ttlMs, "NX") === "OK";
+		}
+	};
+}
+const units = {
+	ms: 1,
+	s: 1e3,
+	m: 6e4,
+	h: 36e5,
+	d: 864e5
+};
+/** Parse an interval such as `'30s'`, `'5m'` or `'1.5h'` into milliseconds (`'5m'` → 300000). Throws on anything else. */
+function parseInterval(text) {
+	const m = /^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$/.exec(text.trim());
+	if (!m) throw new Error(`[@kitr/jobs] Bad interval "${text}" (use 30s, 5m, 1h, 1d)`);
+	return Number(m[1]) * units[m[2]];
+}
+function cronField(field, min, max, expression) {
+	const bad = () => /* @__PURE__ */ new Error(`[@kitr/jobs] Bad cron "${expression}" (field "${field}")`);
+	const int = (text) => {
+		if (!text || !/^\d+$/.test(text)) throw bad();
+		return Number(text);
+	};
+	const sets = field.split(",").map((part) => {
+		const [range, stepText, extra] = part.split("/");
+		if (extra !== void 0) throw bad();
+		const step = stepText === void 0 ? 1 : int(stepText);
+		let lo = min;
+		let hi = max;
+		if (range !== "*") {
+			const [a, b, more] = range.split("-");
+			if (more !== void 0) throw bad();
+			lo = int(a);
+			hi = b !== void 0 ? int(b) : stepText !== void 0 ? max : lo;
+		}
+		if (step < 1 || lo < min || hi > max || lo > hi) throw bad();
+		return (v) => v >= lo && v <= hi && (v - lo) % step === 0;
+	});
+	return (v) => sets.some((s) => s(v));
+}
+/** Does a 5-field cron expression (minute hour day month weekday) match this minute? */
+function cronMatches(expression, date) {
+	const fields = expression.trim().split(/\s+/);
+	if (fields.length !== 5) throw new Error(`[@kitr/jobs] Bad cron "${expression}" (5 fields)`);
+	const [mi, h, d, mo, w] = fields;
+	const minute = cronField(mi, 0, 59, expression);
+	const hour = cronField(h, 0, 23, expression);
+	const dayOfMonth = cronField(d, 1, 31, expression);
+	const month = cronField(mo, 1, 12, expression);
+	const weekday = cronField(w, 0, 7, expression);
+	const dayOk = dayOfMonth(date.getDate());
+	const weekdayOk = weekday(date.getDay()) || date.getDay() === 0 && weekday(7);
+	const day = d !== "*" && w !== "*" ? dayOk || weekdayOk : dayOk && weekdayOk;
+	return minute(date.getMinutes()) && hour(date.getHours()) && day && month(date.getMonth() + 1);
+}
+/** Create the jobs service without the provider, for scripts and tests. Takes the same options as `config/jobs.ts`. */
+function createJobs(app, config = {}) {
+	const driver = config.driver ?? createMemoryJobsDriver();
+	const registry = /* @__PURE__ */ new Map();
+	const concurrency = config.concurrency ?? 5;
+	const active = /* @__PURE__ */ new Set();
+	let timer;
+	let running = false;
+	let lastCronMinute = -1;
+	const intervals = /* @__PURE__ */ new Map();
+	const lastRun = /* @__PURE__ */ new Map();
+	const lastSlot = /* @__PURE__ */ new Map();
+	const log = (level, msg, data) => {
+		const logger = app.container.has("logger") ? app.container.use("logger") : void 0;
+		const fields = data instanceof Error ? { error: {
+			name: data.name,
+			message: data.message,
+			stack: data.stack
+		} } : data === void 0 || typeof data === "object" && data !== null ? data : { data };
+		logger?.[level]?.(msg, fields);
+	};
+	const validate = (job, payload) => {
+		if (!job.schema) return payload;
+		const result = validateSync(job.schema, payload);
+		if (!result.success) throw KitrError.validation(result.fieldErrors, `[jobs] ${job.name}: bad payload`);
+		return result.data;
+	};
+	async function runOnce(job, payload, attempt, id) {
+		const controller = new AbortController();
+		const ms = job.timeout ?? 6e4;
+		let timeout;
+		const expired = new Promise((_, reject) => {
+			timeout = setTimeout(() => {
+				const error = /* @__PURE__ */ new Error(`[jobs] ${job.name} timed out after ${ms}ms`);
+				controller.abort(error);
+				reject(error);
+			}, ms);
+		});
+		try {
+			return await Promise.race([Promise.resolve().then(() => job.run(payload, {
+				app,
+				attempt,
+				jobId: id,
+				signal: controller.signal
+			})), expired]);
+		} finally {
+			clearTimeout(timeout);
+		}
+	}
+	async function execute(queued) {
+		try {
+			await handle(queued);
+		} finally {
+			await driver.ack?.(queued);
+		}
+	}
+	async function handle(queued) {
+		const job = registry.get(queued.name);
+		if (!job) {
+			log("warn", `[jobs] unknown job "${queued.name}" dropped`);
+			return;
+		}
+		let payload;
+		try {
+			payload = validate(job, queued.payload);
+		} catch (error) {
+			log("error", `[jobs] ${job.name} dropped: bad payload`, error);
+			config.onFailed?.(queued, error);
+			return;
+		}
+		try {
+			await runOnce(job, payload, queued.attempt, queued.id);
+		} catch (error) {
+			const retries = job.retries ?? 3;
+			if (queued.attempt <= retries) {
+				const wait = (job.backoff ?? 1e3) * 2 ** (queued.attempt - 1);
+				log("warn", `[jobs] ${job.name} failed (attempt ${queued.attempt}), retry in ${wait}ms`, error);
+				await driver.push({
+					...queued,
+					attempt: queued.attempt + 1,
+					runAt: Date.now() + wait
+				});
+			} else {
+				log("error", `[jobs] ${job.name} failed after ${queued.attempt} attempts`, error);
+				config.onFailed?.(queued, error);
+			}
+		}
+	}
+	const dispatchScheduled = (job) => jobs.dispatch(job, void 0).catch((error) => {
+		log("error", `[jobs] could not queue scheduled ${job.name}`, error);
+	});
+	const dispatchOnce = async (job, key, ttlMs) => {
+		try {
+			if (driver.claim && !await driver.claim(key, ttlMs)) return;
+		} catch (error) {
+			log("error", `[jobs] could not claim scheduled ${job.name}`, error);
+			return;
+		}
+		await dispatchScheduled(job);
+	};
+	async function tick() {
+		if (!running) return;
+		try {
+			await poll();
+		} catch (error) {
+			log("error", "[jobs] worker poll failed", error);
+		}
+		if (running) timer = setTimeout(tick, config.pollInterval ?? 500);
+	}
+	async function poll() {
+		const now = Date.now();
+		const minute = Math.floor(now / 6e4);
+		if (minute !== lastCronMinute) {
+			lastCronMinute = minute;
+			for (const job of registry.values()) if (typeof job.schedule === "string" && cronMatches(job.schedule, new Date(now))) dispatchOnce(job, `cron:${job.name}:${minute}`, 12e4);
+		}
+		for (const [name, every] of intervals) if (driver.claim) {
+			const slot = Math.floor(now / every);
+			if (lastSlot.get(name) === slot) continue;
+			lastSlot.set(name, slot);
+			dispatchOnce(registry.get(name), `every:${name}:${slot}`, every + 6e4);
+		} else if (now - (lastRun.get(name) ?? 0) >= every) {
+			lastRun.set(name, now);
+			dispatchScheduled(registry.get(name));
+		}
+		const room = concurrency - active.size;
+		if (room > 0) for (const queued of await driver.pull(now, room)) {
+			const p = execute(queued).finally(() => active.delete(p));
+			active.add(p);
+		}
+	}
+	for (const job of config.jobs ?? []) {
+		registry.set(job.name, job);
+		if (job.schedule && typeof job.schedule === "object") intervals.set(job.name, parseInterval(job.schedule.every));
+		else if (typeof job.schedule === "string") cronMatches(job.schedule, /* @__PURE__ */ new Date());
+	}
+	const jobs = {
+		driver,
+		jobs: registry,
+		get running() {
+			return running;
+		},
+		register(...list) {
+			for (const job of list) {
+				registry.set(job.name, job);
+				if (job.schedule && typeof job.schedule === "object") intervals.set(job.name, parseInterval(job.schedule.every));
+				else if (typeof job.schedule === "string") cronMatches(job.schedule, /* @__PURE__ */ new Date());
+			}
+			return jobs;
+		},
+		async dispatch(job, payload, options = {}) {
+			if (!registry.has(job.name)) registry.set(job.name, job);
+			const id = crypto.randomUUID();
+			const runAt = options.at ? options.at.getTime() : Date.now() + (options.delay ?? 0);
+			await driver.push({
+				id,
+				name: job.name,
+				payload: validate(job, payload),
+				attempt: 1,
+				runAt
+			});
+			return id;
+		},
+		async run(job, payload, attempt = 1) {
+			const retries = job.retries ?? 3;
+			const valid = validate(job, payload);
+			let error;
+			for (let a = attempt; a <= retries + 1; a++) try {
+				return await runOnce(job, valid, a, crypto.randomUUID());
+			} catch (e) {
+				error = e;
+				if (a <= retries) await new Promise((r) => setTimeout(r, (job.backoff ?? 1e3) * 2 ** (a - 1)));
+			}
+			throw error;
+		},
+		start() {
+			if (running) return;
+			running = true;
+			lastCronMinute = Math.floor(Date.now() / 6e4);
+			timer = setTimeout(tick, 0);
+		},
+		async stop() {
+			running = false;
+			if (timer) clearTimeout(timer);
+			await Promise.allSettled([...active]);
+		}
+	};
+	return jobs;
+}
+/** The jobs service the provider bound in this app's container. Throws in the browser, where nothing is bound. */
+function useJobs(app) {
+	return app.container.use("jobs");
+}
+export { createJobs, createMemoryJobsDriver, createRedisJobsDriver, cronMatches, defineConfig, defineJob, parseInterval, useJobs };

package/dist/kitr.d.ts

@@ -0,0 +1,41 @@
+import { n as commands_d_exports } from "./commands-Bo7c-TIC.js";
+/**
+ * `kitr add jobs`: config, an example job, the provider, and the AGENTS.md notes. One question:
+ * whether the worker runs inside the server process or a separate one (`--no-worker`).
+ */
+declare const _default: {
+  readonly name: "jobs";
+  readonly requires: ["server"];
+  readonly questions: [{
+    readonly name: "worker";
+    readonly type: "confirm";
+    readonly message: "Run the job worker inside the server process?";
+    readonly default: true;
+  }];
+  readonly add: (ctx: import("@kitr/core/package").AddContext) => void;
+  readonly remove: (ctx: import("@kitr/core/package").AddContext) => void;
+  readonly commands: () => Promise<typeof commands_d_exports>;
+  readonly generators: {
+    readonly job: {
+      readonly description: "Create a background job in server/jobs";
+      readonly definition: "A background unit of work with a payload schema, retries and an optional schedule.";
+      readonly target: (name: string, project: import("@kitr/core/package").ProjectInfo) => {
+        file: string;
+        vars: Record<string, string>;
+      };
+      readonly stub: ({ name }: import("@kitr/core/package").GeneratorVars) => string;
+    };
+  };
+  readonly doctor: (project: import("@kitr/core/package").ProjectInfo) => {
+    level: "info";
+    message: string;
+    fix: string;
+  }[];
+  readonly vite: {
+    readonly serverOnlyConfig: true;
+    readonly provider: {
+      readonly client: false;
+    };
+  };
+};
+export { _default as default };

package/dist/kitr.js

@@ -0,0 +1,73 @@
+import { definePackage, generatorFile } from "@kitr/core/package";
+// src/kitr.ts
+const pascal = (s) => s.replace(/(^|[-_/\s])(\w)/g, (_, __, c) => c.toUpperCase()).replace(/[^\w]/g, "");
+const snake = (s) => s.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/[-\s/]+/g, "_").toLowerCase();
+const camel = (s) => {
+	const p = pascal(s);
+	return p.charAt(0).toLowerCase() + p.slice(1);
+};
+/**
+* `kitr add jobs`: config, an example job, the provider, and the AGENTS.md notes. One question:
+* whether the worker runs inside the server process or a separate one (`--no-worker`).
+*/
+var kitr_default = definePackage({
+	name: "jobs",
+	requires: ["server"],
+	questions: [{
+		name: "worker",
+		type: "confirm",
+		message: "Run the job worker inside the server process?",
+		default: true
+	}],
+	add(ctx) {
+		const worker = ctx.answers.worker !== false;
+		ctx.provider.add("@kitr/jobs/provider");
+		ctx.config("jobs", `import { defineConfig } from '@kitr/jobs'\nimport { exampleJob } from '#server/jobs/example'\n\nexport default defineConfig({\n  // Every job the worker runs (kitr make:job <name> writes one in server/jobs/).\n  jobs: [exampleJob],\n  // In-memory queue by default: fine for one process. For several, pass a shared driver.\n  concurrency: 5,\n  worker: ${worker}, // ${worker ? "false when a separate worker process runs the jobs" : "the worker runs in its own process: `kitr jobs:work`"}\n})\n`);
+		ctx.write("server/jobs/example.ts", `import { defineJob } from '@kitr/jobs'\nimport { z } from 'zod'\n\n/** Dispatch with \`useJobs(app).dispatch(exampleJob, { id })\`; it runs on the worker with retries. */\nexport const exampleJob = defineJob('example', {\n  schema: z.object({ id: z.string() }),\n  retries: 3,\n  // schedule: { every: '1h' },\n  async run({ id }, ctx) {\n    console.info(\`example job \${id} (attempt \${ctx.attempt})\`)\n  },\n})\n`);
+		if (!worker) ctx.scripts.add("worker", "kitr jobs:work");
+		ctx.agents.add("jobs", `- Background work is a job in \`server/jobs/*.ts\` (\`kitr make:job <name>\`, \`defineJob\` with a schema, retries and an optional schedule), listed in \`config/jobs.ts\`. Dispatch from controllers and actions with \`ctx.app.container.use('jobs').dispatch(job, payload)\`; never \`setTimeout\` work in a request.${worker ? "" : "\n- The worker is a separate process: `kitr jobs:work` (the `worker` script). The server only dispatches."}`);
+	},
+	remove(ctx) {
+		ctx.provider.remove("@kitr/jobs/provider");
+		ctx.scripts.remove("worker", "kitr jobs:work");
+		ctx.agents.remove("jobs");
+	},
+	commands: () => import("./commands.js"),
+	generators: { job: {
+		description: "Create a background job in server/jobs",
+		definition: "A background unit of work with a payload schema, retries and an optional schedule.",
+		target: (name, project) => generatorFile(project, "server", "jobs", name),
+		stub: ({ name }) => `import { defineJob } from '@kitr/jobs'
+import { z } from 'zod'
+
+export const ${camel(name)}Input = z.object({
+  id: z.string(),
+})
+
+/** List it in config/jobs.ts, then dispatch with \`useJobs(app).dispatch(${camel(name)}, { id })\`; it runs on the worker with retries. */
+export const ${camel(name)} = defineJob('${snake(name)}', {
+  schema: ${camel(name)}Input,
+  retries: 3,
+  // schedule: { every: '1h' },
+  async run({ id }, ctx) {
+    console.info(\`${snake(name)} \${id} (attempt \${ctx.attempt})\`)
+  },
+})
+`
+	} },
+	doctor(project) {
+		if (!project.exists("config/jobs.ts")) return [];
+		const src = project.read("config/jobs.ts");
+		if (/jobs\s*:\s*\[\s*\]/.test(src)) return [{
+			level: "info",
+			message: "config/jobs.ts lists no jobs",
+			fix: "kitr make:job <name>, then add it to `jobs: [...]`"
+		}];
+		return [];
+	},
+	vite: {
+		serverOnlyConfig: true,
+		provider: { client: false }
+	}
+});
+export { kitr_default as default };

package/dist/provider.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Binds `jobs` from `config/jobs.ts`. The worker starts when the server is ready and drains
+ * on shutdown; `worker: false` leaves starting to a separate process (`jobs.start()`).
+ */
+export declare const jobsProvider: import("@kitr/core").KitrProvider;
+export { jobsProvider as default };

package/dist/provider.js

@@ -0,0 +1,26 @@
+import { createJobs } from "./index.js";
+import { defineProvider } from "@kitr/core";
+// src/provider.ts
+/**
+* Binds `jobs` from `config/jobs.ts`. The worker starts when the server is ready and drains
+* on shutdown; `worker: false` leaves starting to a separate process (`jobs.start()`).
+*/
+const jobsProvider = defineProvider({
+	name: "jobs",
+	register(app) {
+		if (app.isClient) return;
+		const config = app.config.get("jobs", {});
+		app.container.singleton("jobs", () => createJobs(app, config));
+	},
+	boot(app) {
+		if (!app.isClient) app.container.use("jobs");
+	},
+	ready(app) {
+		if (app.isClient) return;
+		if (app.config.get("jobs", {}).worker !== false) app.container.use("jobs").start();
+	},
+	async shutdown(app) {
+		if (app.container.has("jobs")) await app.container.use("jobs").stop();
+	}
+});
+export { jobsProvider as default, jobsProvider };