alepha 0.19.3 → 0.19.5

package/dist/api/jobs/index.js

@@ -1,6 +1,5 @@
 import { $atom, $hook, $inject, $module, $state, Alepha, AlephaError, KIND, PipelinePrimitive, createPrimitive, t } from "alepha";
-import { AlephaBatch } from "alepha/batch";
-import { AlephaLock } from "alepha/lock";
+import { AlephaLock, LockProvider } from "alepha/lock";
 import { $queue, AlephaQueue } from "alepha/queue";
 import { AlephaScheduler, CronProvider } from "alepha/scheduler";
 import { $secure } from "alepha/security";
@@ -32,6 +31,7 @@ const jobCronInfoSchema = t.object({
 	]),
 	concurrency: t.integer(),
 	hasSchema: t.boolean(),
+	paused: t.boolean(),
 	lastExecution: t.optional(t.object({
 		id: t.uuid(),
 		status: t.text(),
@@ -57,7 +57,6 @@ const jobExecutionEntity = $entity({
 			"retrying",
 			"running",
 			"completed",
-			"failed",
 			"dead",
 			"cancelled"
 		]), "pending"),
@@ -126,7 +125,6 @@ const jobExecutionQuerySchema = t.extend(pageQuerySchema, {
 		"retrying",
 		"running",
 		"completed",
-		"failed",
 		"dead",
 		"cancelled"
 	])),
@@ -155,7 +153,8 @@ const jobQueueDepthSchema = t.object({
 	scheduled: t.integer(),
 	retrying: t.integer(),
 	dead: t.integer(),
-	concurrency: t.integer()
+	concurrency: t.integer(),
+	paused: t.boolean()
 });
 //#endregion
 //#region ../../src/api/jobs/schemas/jobRegistrationSchema.ts
@@ -180,10 +179,7 @@ const jobRegistrationSchema = t.object({
 		retries: t.integer(),
 		hasBackoff: t.boolean()
 	})),
-	batch: t.optional(t.object({
-		size: t.integer(),
-		window: t.text()
-	}))
+	paused: t.boolean()
 });
 //#endregion
 //#region ../../src/api/jobs/schemas/jobStatsSchema.ts
@@ -225,7 +221,8 @@ const jobConfig = $atom({
 		}),
 		delayed: t.object({ interval: t.integer({ description: "Sweep interval (ms)." }) }),
 		logRetentionDays: t.integer({ description: "Days to keep completed/dead executions." }),
-		logMaxEntries: t.integer({ description: "Max log entries captured per execution." })
+		logMaxEntries: t.integer({ description: "Max log entries captured per execution." }),
+		drainTimeout: t.integer({ description: "Max time (ms) to wait for in-flight jobs during shutdown." })
 	}),
 	default: {
 		recovery: {
@@ -235,7 +232,8 @@ const jobConfig = $atom({
 		},
 		delayed: { interval: 3e5 },
 		logRetentionDays: 30,
-		logMaxEntries: 100
+		logMaxEntries: 100,
+		drainTimeout: 3e4
 	}
 });
 //#endregion
@@ -256,11 +254,14 @@ var JobProvider = class JobProvider {
 	alepha = $inject(Alepha);
 	dt = $inject(DateTimeProvider);
 	cronProvider = $inject(CronProvider);
+	lockProvider = $inject(LockProvider);
 	config = $state(jobConfig);
 	log = $logger();
 	executions = $repository(jobExecutionEntity);
 	executionLogs = $repository(jobExecutionLogEntity);
 	jobs = /* @__PURE__ */ new Map();
+	pausedJobs = /* @__PURE__ */ new Set();
+	inFlight = /* @__PURE__ */ new Set();
 	/**
 	* When set, job executions are dispatched through a queue (e.g. `JobQueueProvider`).
 	* When null, jobs execute inline (fire-and-forget). Useful for serverless environments.
@@ -346,8 +347,35 @@ var JobProvider = class JobProvider {
 		return execution.id;
 	}
 	async pushMany(name, items) {
-		const ids = [];
+		if (items.length === 0) return [];
+		const opts = this.getRegistration(name).options;
+		if (!opts.schema) throw new AlephaError(`Cannot push to job '${name}': no schema defined. Use trigger() for cron-only jobs.`);
+		const maxAttempts = (opts.retry?.retries ?? 0) + 1;
+		const keyed = [];
+		const bulkRows = [];
 		for (const item of items) {
+			const validated = this.alepha.codec.validate(opts.schema, item.payload);
+			if (item.key) keyed.push({
+				...item,
+				payload: validated
+			});
+			else {
+				const status = item.delay || item.scheduledAt ? "scheduled" : "pending";
+				let scheduledAt;
+				if (item.scheduledAt) scheduledAt = item.scheduledAt.toISOString();
+				else if (item.delay) scheduledAt = this.dt.now().add(this.dt.duration(item.delay)).toISOString();
+				bulkRows.push({
+					jobName: name,
+					payload: validated,
+					status,
+					priority: PRIORITY_MAP[item.priority ?? opts.priority ?? "normal"],
+					maxAttempts,
+					scheduledAt
+				});
+			}
+		}
+		const ids = [];
+		for (const item of keyed) {
 			const id = await this.push(name, item.payload, {
 				key: item.key,
 				delay: item.delay,
@@ -356,6 +384,17 @@ var JobProvider = class JobProvider {
 			});
 			ids.push(id);
 		}
+		if (bulkRows.length > 0) {
+			const created = await this.executions.createMany(bulkRows);
+			for (const exec of created) {
+				ids.push(exec.id);
+				if (exec.status === "pending" && !this.stopping) await this.scheduleProcessing(name, exec.id);
+			}
+		}
+		this.log.debug(`pushMany '${name}': ${ids.length} jobs created`, {
+			bulk: bulkRows.length,
+			keyed: keyed.length
+		});
 		return ids;
 	}
 	async trigger(name, context) {
@@ -403,6 +442,19 @@ var JobProvider = class JobProvider {
 		});
 	}
 	async scheduleProcessing(jobName, executionId) {
+		if (this.pausedJobs.has(jobName)) {
+			this.log.debug(`Job '${jobName}' is paused, deferring`, { executionId });
+			return;
+		}
+		const maxConcurrency = this.getRegistration(jobName).options.concurrency ?? 1;
+		const runningCount = await this.executions.count({
+			jobName: { eq: jobName },
+			status: { eq: "running" }
+		});
+		if (runningCount >= maxConcurrency) {
+			this.log.debug(`Job '${jobName}' at concurrency limit (${runningCount}/${maxConcurrency}), deferring`, { executionId });
+			return;
+		}
 		if (this.queueDispatch) {
 			this.log.debug(`Dispatching job '${jobName}' via queue`, { executionId });
 			await this.queueDispatch(jobName, executionId);
@@ -412,6 +464,15 @@ var JobProvider = class JobProvider {
 		}
 	}
 	async processExecution(jobName, executionId) {
+		const promise = this.processExecutionInner(jobName, executionId);
+		this.inFlight.add(promise);
+		try {
+			await promise;
+		} finally {
+			this.inFlight.delete(promise);
+		}
+	}
+	async processExecutionInner(jobName, executionId) {
 		const registration = this.getRegistration(jobName);
 		if (!await this.claim(executionId)) {
 			this.log.debug(`Execution ${executionId} already claimed, skipping`);
@@ -483,12 +544,39 @@ var JobProvider = class JobProvider {
 						name: jobName,
 						executionId
 					}, { catch: true });
+					await this.dispatchNextPending(jobName);
 				}
 			}, { context });
 		} finally {
 			this.logs.delete(context);
 		}
 	}
+	/**
+	* After a job finishes (success, failure, or cancel), dispatch any pending
+	* jobs that were deferred due to the concurrency limit.
+	*/
+	async dispatchNextPending(jobName) {
+		if (this.stopping || this.pausedJobs.has(jobName)) return;
+		const registration = this.jobs.get(jobName);
+		if (!registration) return;
+		const available = (registration.options.concurrency ?? 1) - await this.executions.count({
+			jobName: { eq: jobName },
+			status: { eq: "running" }
+		});
+		if (available <= 0) return;
+		const pending = await this.executions.findMany({
+			where: {
+				jobName: { eq: jobName },
+				status: { eq: "pending" }
+			},
+			orderBy: {
+				column: "priority",
+				direction: "asc"
+			},
+			limit: available
+		});
+		for (const exec of pending) await this.scheduleProcessing(jobName, exec.id);
+	}
 	async claim(executionId) {
 		const execution = await this.executions.findById(executionId);
 		if (!execution) return false;
@@ -606,13 +694,20 @@ var JobProvider = class JobProvider {
 	async recoverySweep() {
 		this.log.trace("Starting recovery sweep");
 		if (this.stopping) return;
+		if (!await this.tryLock("_alepha:jobs:recovery-lock", 3e5)) return;
 		try {
 			const now = this.dt.now();
 			const staleThreshold = now.subtract(this.config.recovery.staleThreshold, "millisecond").toISOString();
 			const pendingWhere = this.executions.createQueryWhere();
 			pendingWhere.status = { eq: "pending" };
 			pendingWhere.createdAt = { lte: staleThreshold };
-			const stalePending = await this.executions.findMany({ where: pendingWhere });
+			const stalePending = await this.executions.findMany({
+				where: pendingWhere,
+				orderBy: {
+					column: "priority",
+					direction: "asc"
+				}
+			});
 			for (const exec of stalePending) {
 				if (!this.jobs.has(exec.jobName)) continue;
 				this.log.debug(`Recovery sweep: re-dispatching stale pending job ${exec.jobName} (${exec.id})`);
@@ -639,6 +734,8 @@ var JobProvider = class JobProvider {
 			}
 		} catch (e) {
 			this.log.error("Recovery sweep failed", { error: e });
+		} finally {
+			await this.releaseLock("_alepha:jobs:recovery-lock");
 		}
 	}
 	/**
@@ -651,12 +748,19 @@ var JobProvider = class JobProvider {
 	async delayedDispatchSweep() {
 		this.log.trace("Starting delayed dispatch sweep");
 		if (this.stopping) return;
+		if (!await this.tryLock("_alepha:jobs:dispatch-lock", 6e4)) return;
 		try {
 			const now = this.dt.nowISOString();
 			const where = this.executions.createQueryWhere();
 			where.status = { inArray: ["scheduled", "retrying"] };
 			where.scheduledAt = { lte: now };
-			const ready = await this.executions.findMany({ where });
+			const ready = await this.executions.findMany({
+				where,
+				orderBy: {
+					column: "priority",
+					direction: "asc"
+				}
+			});
 			for (const exec of ready) {
 				if (!this.jobs.has(exec.jobName)) continue;
 				await this.executions.updateById(exec.id, { status: "pending" });
@@ -664,6 +768,8 @@ var JobProvider = class JobProvider {
 			}
 		} catch (e) {
 			this.log.error("Delayed dispatch sweep failed", { error: e });
+		} finally {
+			await this.releaseLock("_alepha:jobs:dispatch-lock");
 		}
 	}
 	/**
@@ -683,18 +789,55 @@ var JobProvider = class JobProvider {
 				"cancelled"
 			] };
 			where.completedAt = { lte: cutoff };
-			const old = await this.executions.findMany({ where });
-			for (const exec of old) {
-				try {
-					await this.executionLogs.deleteById(exec.id);
-				} catch {}
-				await this.executions.deleteById(exec.id);
+			const expiredIds = await this.executions.findMany({
+				where,
+				columns: ["id"]
+			});
+			if (expiredIds.length > 0) {
+				const ids = expiredIds.map((e) => e.id);
+				await this.executionLogs.deleteMany({ id: { inArray: ids } });
+				await this.executions.deleteMany({ id: { inArray: ids } });
+				this.log.info(`Log purge: deleted ${ids.length} old execution records`);
 			}
-			if (old.length > 0) this.log.info(`Log purge: deleted ${old.length} old execution records`);
 		} catch (e) {
 			this.log.error("Log purge failed", { error: e });
 		}
 	}
+	pauseJob(name) {
+		this.getRegistration(name);
+		this.pausedJobs.add(name);
+		this.log.info(`Paused job '${name}'`);
+	}
+	async resumeJob(name) {
+		this.getRegistration(name);
+		this.pausedJobs.delete(name);
+		this.log.info(`Resumed job '${name}'`);
+		const pending = await this.executions.findMany({
+			where: {
+				jobName: { eq: name },
+				status: { eq: "pending" }
+			},
+			orderBy: {
+				column: "priority",
+				direction: "asc"
+			}
+		});
+		for (const exec of pending) await this.scheduleProcessing(name, exec.id);
+	}
+	isJobPaused(name) {
+		return this.pausedJobs.has(name);
+	}
+	getPausedJobs() {
+		return [...this.pausedJobs];
+	}
+	async tryLock(key, ttlMs) {
+		const lockValue = `${this.workerId},${this.dt.nowISOString()}`;
+		const [lockId] = (await this.lockProvider.set(key, lockValue, true, ttlMs)).split(",");
+		return lockId === this.workerId;
+	}
+	async releaseLock(key) {
+		await this.lockProvider.del(key);
+	}
 	onStart = $hook({
 		on: "start",
 		handler: async () => {
@@ -729,7 +872,14 @@ var JobProvider = class JobProvider {
 		on: "stop",
 		handler: async () => {
 			this.stopping = true;
-			for (const controller of this.abortControllers.values()) controller.abort();
+			if (this.inFlight.size > 0) {
+				this.log.info(`Draining ${this.inFlight.size} in-flight job(s)...`);
+				await Promise.race([Promise.allSettled([...this.inFlight]), this.dt.wait([this.config.drainTimeout, "millisecond"])]);
+			}
+			if (this.abortControllers.size > 0) {
+				this.log.warn(`Aborting ${this.abortControllers.size} remaining job(s) after drain timeout`);
+				for (const controller of this.abortControllers.values()) controller.abort();
+			}
 		}
 	});
 	getRegistration(name) {
@@ -741,7 +891,7 @@ var JobProvider = class JobProvider {
 //#endregion
 //#region ../../src/api/jobs/primitives/$job.ts
 /**
-* Job primitive for defining scheduled and on-demand tasks with payload validation, retry policies, and batching.
+* Job primitive for defining scheduled and on-demand tasks with payload validation and retry policies.
 */
 const $job = (options) => {
 	return createPrimitive(JobPrimitive, options);
@@ -783,6 +933,24 @@ var JobPrimitive = class extends PipelinePrimitive {
 	async trigger(context) {
 		return this.jobProvider.trigger(this.name, context);
 	}
+	/**
+	* Pause this job. Pushed items are still accepted but processing is held.
+	*/
+	pause() {
+		this.jobProvider.pauseJob(this.name);
+	}
+	/**
+	* Resume a paused job and dispatch any pending items.
+	*/
+	async resume() {
+		return this.jobProvider.resumeJob(this.name);
+	}
+	/**
+	* Whether this job is currently paused.
+	*/
+	get paused() {
+		return this.jobProvider.isJobPaused(this.name);
+	}
 };
 $job[KIND] = JobPrimitive;
 //#endregion
@@ -822,7 +990,7 @@ var JobService = class {
           COUNT(*) FILTER (WHERE ${e.status} = 'retrying') AS retrying,
           COUNT(*) FILTER (WHERE ${e.status} = 'dead') AS dead,
           COUNT(*) FILTER (WHERE ${e.status} = 'completed' AND ${e.completedAt} >= ${periodAgo}) AS completed_24h,
-          COUNT(*) FILTER (WHERE ${e.status} IN ('dead', 'failed') AND ${e.completedAt} >= ${periodAgo}) AS failed_24h
+          COUNT(*) FILTER (WHERE ${e.status} = 'dead' AND ${e.completedAt} >= ${periodAgo}) AS failed_24h
         FROM ${e}
       `, t.object({
 			running: t.string(),
@@ -867,10 +1035,7 @@ var JobService = class {
 					retries: opts.retry.retries,
 					hasBackoff: Boolean(opts.retry.backoff)
 				} : void 0,
-				batch: opts.batch ? {
-					size: opts.batch.size,
-					window: String(opts.batch.window)
-				} : void 0
+				paused: this.jobProvider.isJobPaused(name)
 			};
 			result.push(registration);
 		}
@@ -944,6 +1109,23 @@ var JobService = class {
 		});
 		return { ok: true };
 	}
+	pauseJob(name, context) {
+		const job = this.alepha.primitives($job).find((j) => j.name === name);
+		if (!job) throw new NotFoundError(`Job not found: ${name}`);
+		this.log.info(`Pausing job '${name}'`, { pausedBy: context?.pausedByName ?? context?.pausedBy });
+		job.pause();
+		return { ok: true };
+	}
+	async resumeJob(name, context) {
+		const job = this.alepha.primitives($job).find((j) => j.name === name);
+		if (!job) throw new NotFoundError(`Job not found: ${name}`);
+		this.log.info(`Resuming job '${name}'`, { resumedBy: context?.resumedByName ?? context?.resumedBy });
+		await job.resume();
+		return { ok: true };
+	}
+	getPausedJobs() {
+		return this.jobProvider.getPausedJobs();
+	}
 	async getCronJobs() {
 		const jobs = this.jobProvider.getRegisteredJobs();
 		const cronJobNames = [];
@@ -960,6 +1142,7 @@ var JobService = class {
 				priority: opts.priority ?? "normal",
 				concurrency: opts.concurrency ?? 1,
 				hasSchema: Boolean(opts.schema),
+				paused: this.jobProvider.isJobPaused(name),
 				lastExecution: last ? {
 					id: last.id,
 					status: last.status,
@@ -1003,7 +1186,8 @@ var JobService = class {
 				scheduled: Number(row?.scheduled ?? 0),
 				retrying: Number(row?.retrying ?? 0),
 				dead: Number(row?.dead ?? 0),
-				concurrency: reg.options.concurrency ?? 1
+				concurrency: reg.options.concurrency ?? 1,
+				paused: this.jobProvider.isJobPaused(name)
 			});
 		}
 		return result;
@@ -1021,10 +1205,10 @@ var JobService = class {
         SELECT
           ds.date::text AS date,
           COALESCE(COUNT(*) FILTER (WHERE ${e.status} = 'completed'), 0) AS completed,
-          COALESCE(COUNT(*) FILTER (WHERE ${e.status} IN ('dead', 'failed')), 0) AS failed
+          COALESCE(COUNT(*) FILTER (WHERE ${e.status} = 'dead'), 0) AS failed
         FROM date_series ds
         LEFT JOIN ${e} ON DATE(${e.completedAt}) = ds.date
-          AND ${e.status} IN ('completed', 'dead', 'failed')
+          AND ${e.status} IN ('completed', 'dead')
         GROUP BY ds.date
         ORDER BY ds.date ASC
       `, t.object({
@@ -1040,11 +1224,7 @@ var JobService = class {
 	async getActivitySqlite(days = 14) {
 		const startDate = this.dt.now().subtract(days - 1, "day");
 		const where = this.executions.createQueryWhere();
-		where.status = { inArray: [
-			"completed",
-			"dead",
-			"failed"
-		] };
+		where.status = { inArray: ["completed", "dead"] };
 		where.completedAt = { gte: startDate.startOf("day").toISOString() };
 		const executions = await this.executions.findMany({ where });
 		const byDate = /* @__PURE__ */ new Map();
@@ -1077,7 +1257,7 @@ var JobService = class {
           COUNT(*) AS failures,
           (ARRAY_AGG(${e.error} ORDER BY ${e.completedAt} DESC))[1] AS last_error
         FROM ${e}
-        WHERE ${e.status} IN ('dead', 'failed')
+        WHERE ${e.status} = 'dead'
           AND ${e.completedAt} >= ${periodAgoIso}
         GROUP BY ${e.jobName}
         ORDER BY failures DESC
@@ -1093,7 +1273,7 @@ var JobService = class {
 	}
 	async getTopFailuresSqlite(periodAgoIso) {
 		const where = this.executions.createQueryWhere();
-		where.status = { inArray: ["dead", "failed"] };
+		where.status = { eq: "dead" };
 		where.completedAt = { gte: periodAgoIso };
 		const failures = await this.executions.findMany({
 			where,
@@ -1290,6 +1470,45 @@ var AdminJobController = class {
 		},
 		handler: ({ query }) => this.jobService.getTopFailures(query.days)
 	});
+	pauseJob = $action({
+		method: "POST",
+		path: `${this.url}/pause`,
+		group: this.group,
+		use: [$secure({ permissions: ["admin:job:trigger"] })],
+		schema: {
+			body: t.object({ name: t.text() }),
+			response: okSchema
+		},
+		handler: ({ body, user }) => {
+			return this.jobService.pauseJob(body.name, {
+				pausedBy: user?.id,
+				pausedByName: user?.name
+			});
+		}
+	});
+	resumeJob = $action({
+		method: "POST",
+		path: `${this.url}/resume`,
+		group: this.group,
+		use: [$secure({ permissions: ["admin:job:trigger"] })],
+		schema: {
+			body: t.object({ name: t.text() }),
+			response: okSchema
+		},
+		handler: async ({ body, user }) => {
+			return this.jobService.resumeJob(body.name, {
+				resumedBy: user?.id,
+				resumedByName: user?.name
+			});
+		}
+	});
+	getPausedJobs = $action({
+		path: `${this.url}/paused`,
+		group: this.group,
+		use: [$secure({ permissions: ["admin:job:read"] })],
+		schema: { response: t.array(t.text()) },
+		handler: () => this.jobService.getPausedJobs()
+	});
 };
 //#endregion
 //#region ../../src/api/jobs/providers/JobQueueProvider.ts
@@ -1351,7 +1570,6 @@ const AlephaApiJobs = $module({
 		AlephaQueue,
 		AlephaScheduler,
 		AlephaLock,
-		AlephaBatch,
 		JobProvider,
 		JobQueueProvider,
 		JobService,
@@ -1362,7 +1580,6 @@ const AlephaApiJobs = $module({
 		const useQueue = env.ALEPHA_JOBS_QUEUE === 1 ? true : env.ALEPHA_JOBS_QUEUE === 0 ? false : !alepha.isServerless();
 		alepha.with(AlephaScheduler);
 		alepha.with(AlephaLock);
-		alepha.with(AlephaBatch);
 		alepha.with(JobProvider);
 		alepha.with(JobService);
 		alepha.with(AdminJobController);