alepha 0.19.2 → 0.19.4

package/src/api/jobs/controllers/AdminJobController.ts

@@ -163,4 +163,48 @@ export class AdminJobController {
     },
     handler: ({ query }) => this.jobService.getTopFailures(query.days),
   });
+
+  public readonly pauseJob = $action({
+    method: "POST",
+    path: `${this.url}/pause`,
+    group: this.group,
+    use: [$secure({ permissions: ["admin:job:trigger"] })],
+    schema: {
+      body: t.object({ name: t.text() }),
+      response: okSchema,
+    },
+    handler: ({ body, user }) => {
+      return this.jobService.pauseJob(body.name, {
+        pausedBy: user?.id,
+        pausedByName: user?.name,
+      });
+    },
+  });
+
+  public readonly resumeJob = $action({
+    method: "POST",
+    path: `${this.url}/resume`,
+    group: this.group,
+    use: [$secure({ permissions: ["admin:job:trigger"] })],
+    schema: {
+      body: t.object({ name: t.text() }),
+      response: okSchema,
+    },
+    handler: async ({ body, user }) => {
+      return this.jobService.resumeJob(body.name, {
+        resumedBy: user?.id,
+        resumedByName: user?.name,
+      });
+    },
+  });
+
+  public readonly getPausedJobs = $action({
+    path: `${this.url}/paused`,
+    group: this.group,
+    use: [$secure({ permissions: ["admin:job:read"] })],
+    schema: {
+      response: t.array(t.text()),
+    },
+    handler: () => this.jobService.getPausedJobs(),
+  });
 }

package/src/api/jobs/entities/jobExecutionEntity.ts

@@ -19,7 +19,6 @@ export const jobExecutionEntity = $entity({
         "retrying",
         "running",
         "completed",
-        "failed",
         "dead",
         "cancelled",
       ]),
@@ -59,6 +58,5 @@ export type JobStatus =
   | "retrying"
   | "running"
   | "completed"
-  | "failed"
   | "dead"
   | "cancelled";

package/src/api/jobs/index.ts

@@ -1,5 +1,4 @@
 import { $module, type Alepha, type Static, t } from "alepha";
-import { AlephaBatch } from "alepha/batch";
 import type { DateTime } from "alepha/datetime";
 import { AlephaLock } from "alepha/lock";
 import { AlephaQueue } from "alepha/queue";
@@ -83,7 +82,6 @@ export const AlephaApiJobs = $module({
     AlephaQueue,
     AlephaScheduler,
     AlephaLock,
-    AlephaBatch,
     JobProvider,
     JobQueueProvider,
     JobService,
@@ -100,7 +98,6 @@ export const AlephaApiJobs = $module({
 
     alepha.with(AlephaScheduler);
     alepha.with(AlephaLock);
-    alepha.with(AlephaBatch);
     alepha.with(JobProvider);
     alepha.with(JobService);
     alepha.with(AdminJobController);

package/src/api/jobs/primitives/$job.ts

@@ -17,7 +17,7 @@ import {
 } from "../providers/JobProvider.ts";
 
 /**
- * Job primitive for defining scheduled and on-demand tasks with payload validation, retry policies, and batching.
+ * Job primitive for defining scheduled and on-demand tasks with payload validation and retry policies.
  */
 export const $job = <T extends TSchema = TSchema>(
   options: JobPrimitiveOptions<T>,
@@ -52,11 +52,6 @@ export interface JobRetryOptions {
   when?: (error: Error) => boolean;
 }
 
-export interface JobBatchOptions {
-  size: number;
-  window: DurationLike;
-}
-
 export type JobPriority = "critical" | "high" | "normal" | "low";
 
 export interface JobPrimitiveOptions<T extends TSchema = TSchema>
@@ -93,11 +88,6 @@ export interface JobPrimitiveOptions<T extends TSchema = TSchema>
    */
   concurrency?: number;
 
-  /**
-   * Consumer batching configuration.
-   */
-  batch?: JobBatchOptions;
-
   /**
    * Default priority for pushed jobs.
    * @default "normal"
@@ -162,6 +152,27 @@ export class JobPrimitive<
   public async trigger(context?: JobTriggerContext): Promise<void> {
     return this.jobProvider.trigger(this.name, context);
   }
+
+  /**
+   * Pause this job. Pushed items are still accepted but processing is held.
+   */
+  public pause(): void {
+    this.jobProvider.pauseJob(this.name);
+  }
+
+  /**
+   * Resume a paused job and dispatch any pending items.
+   */
+  public async resume(): Promise<void> {
+    return this.jobProvider.resumeJob(this.name);
+  }
+
+  /**
+   * Whether this job is currently paused.
+   */
+  public get paused(): boolean {
+    return this.jobProvider.isJobPaused(this.name);
+  }
 }
 
 $job[KIND] = JobPrimitive;

package/src/api/jobs/providers/JobProvider.ts

@@ -8,6 +8,7 @@ import {
   type TSchema,
 } from "alepha";
 import { DateTimeProvider, type DurationLike } from "alepha/datetime";
+import { LockProvider } from "alepha/lock";
 import type { LogEntry } from "alepha/logger";
 import { $logger } from "alepha/logger";
 import { $repository } from "alepha/orm";
@@ -81,12 +82,15 @@ export class JobProvider {
   protected readonly alepha = $inject(Alepha);
   protected readonly dt = $inject(DateTimeProvider);
   protected readonly cronProvider = $inject(CronProvider);
+  protected readonly lockProvider = $inject(LockProvider);
   protected readonly config = $state(jobConfig);
   protected readonly log = $logger();
   protected readonly executions = $repository(jobExecutionEntity);
   protected readonly executionLogs = $repository(jobExecutionLogEntity);
 
   protected readonly jobs = new Map<string, JobRegistration>();
+  protected readonly pausedJobs = new Set<string>();
+  protected readonly inFlight = new Set<Promise<void>>();
 
   /**
    * When set, job executions are dispatched through a queue (e.g. `JobQueueProvider`).
@@ -116,12 +120,16 @@ export class JobProvider {
     });
 
     if (options.cron) {
-      this.cronProvider.createCronJob(name, options.cron, () =>
-        this.trigger(name, {
-          triggeredBy: "system",
-          triggeredByName: "system (cron)",
-        }),
-      );
+      this.cronProvider.createCronJob(name, options.cron, async () => {
+        try {
+          await this.trigger(name, {
+            triggeredBy: "system",
+            triggeredByName: "system (cron)",
+          });
+        } catch (error) {
+          this.log.error(`Cron trigger failed for job '${name}'`, error);
+        }
+      });
     }
   }
 
@@ -223,8 +231,61 @@ export class JobProvider {
     name: string,
     items: Array<PushManyItem>,
   ): Promise<string[]> {
-    const ids: string[] = [];
+    if (items.length === 0) return [];
+
+    const registration = this.getRegistration(name);
+    const opts = registration.options;
+
+    if (!opts.schema) {
+      throw new AlephaError(
+        `Cannot push to job '${name}': no schema defined. Use trigger() for cron-only jobs.`,
+      );
+    }
+
+    const maxAttempts = (opts.retry?.retries ?? 0) + 1;
+
+    // Keyed items need upsert logic — fall back to individual push
+    const keyed: PushManyItem[] = [];
+    const bulkRows: Array<{
+      jobName: string;
+      payload: Record<string, unknown>;
+      status: JobStatus;
+      priority: number;
+      maxAttempts: number;
+      scheduledAt?: string;
+    }> = [];
+
     for (const item of items) {
+      const validated = this.alepha.codec.validate(opts.schema, item.payload);
+      if (item.key) {
+        keyed.push({ ...item, payload: validated as Static<TSchema> });
+      } else {
+        const isDelayed = item.delay || item.scheduledAt;
+        const status: JobStatus = isDelayed ? "scheduled" : "pending";
+        let scheduledAt: string | undefined;
+        if (item.scheduledAt) {
+          scheduledAt = item.scheduledAt.toISOString();
+        } else if (item.delay) {
+          scheduledAt = this.dt
+            .now()
+            .add(this.dt.duration(item.delay))
+            .toISOString();
+        }
+        bulkRows.push({
+          jobName: name,
+          payload: validated as Record<string, unknown>,
+          status,
+          priority: PRIORITY_MAP[item.priority ?? opts.priority ?? "normal"],
+          maxAttempts,
+          scheduledAt,
+        });
+      }
+    }
+
+    const ids: string[] = [];
+
+    // Keyed: sequential upserts
+    for (const item of keyed) {
       const id = await this.push(name, item.payload, {
         key: item.key,
         delay: item.delay,
@@ -233,6 +294,23 @@ export class JobProvider {
       });
       ids.push(id);
     }
+
+    // Non-keyed: single bulk insert
+    if (bulkRows.length > 0) {
+      const created = await this.executions.createMany(bulkRows);
+      for (const exec of created) {
+        ids.push(exec.id);
+        if (exec.status === "pending" && !this.stopping) {
+          await this.scheduleProcessing(name, exec.id);
+        }
+      }
+    }
+
+    this.log.debug(`pushMany '${name}': ${ids.length} jobs created`, {
+      bulk: bulkRows.length,
+      keyed: keyed.length,
+    });
+
     return ids;
   }
 
@@ -326,6 +404,25 @@ export class JobProvider {
     jobName: string,
     executionId: string,
   ): Promise<void> {
+    if (this.pausedJobs.has(jobName)) {
+      this.log.debug(`Job '${jobName}' is paused, deferring`, { executionId });
+      return;
+    }
+
+    const registration = this.getRegistration(jobName);
+    const maxConcurrency = registration.options.concurrency ?? 1;
+    const runningCount = await this.executions.count({
+      jobName: { eq: jobName },
+      status: { eq: "running" },
+    });
+    if (runningCount >= maxConcurrency) {
+      this.log.debug(
+        `Job '${jobName}' at concurrency limit (${runningCount}/${maxConcurrency}), deferring`,
+        { executionId },
+      );
+      return;
+    }
+
     if (this.queueDispatch) {
       this.log.debug(`Dispatching job '${jobName}' via queue`, { executionId });
       await this.queueDispatch(jobName, executionId);
@@ -338,6 +435,19 @@ export class JobProvider {
   public async processExecution(
     jobName: string,
     executionId: string,
+  ): Promise<void> {
+    const promise = this.processExecutionInner(jobName, executionId);
+    this.inFlight.add(promise);
+    try {
+      await promise;
+    } finally {
+      this.inFlight.delete(promise);
+    }
+  }
+
+  protected async processExecutionInner(
+    jobName: string,
+    executionId: string,
   ): Promise<void> {
     const registration = this.getRegistration(jobName);
 
@@ -451,6 +561,9 @@ export class JobProvider {
               { name: jobName, executionId },
               { catch: true },
             );
+
+            // A slot just opened — dispatch next pending job if any
+            await this.dispatchNextPending(jobName);
           }
         },
         { context },
@@ -460,6 +573,36 @@ export class JobProvider {
     }
   }
 
+  /**
+   * After a job finishes (success, failure, or cancel), dispatch any pending
+   * jobs that were deferred due to the concurrency limit.
+   */
+  protected async dispatchNextPending(jobName: string): Promise<void> {
+    if (this.stopping || this.pausedJobs.has(jobName)) return;
+
+    const registration = this.jobs.get(jobName);
+    if (!registration) return;
+
+    const maxConcurrency = registration.options.concurrency ?? 1;
+    const runningCount = await this.executions.count({
+      jobName: { eq: jobName },
+      status: { eq: "running" },
+    });
+
+    const available = maxConcurrency - runningCount;
+    if (available <= 0) return;
+
+    const pending = await this.executions.findMany({
+      where: { jobName: { eq: jobName }, status: { eq: "pending" } },
+      orderBy: { column: "priority", direction: "asc" },
+      limit: available,
+    });
+
+    for (const exec of pending) {
+      await this.scheduleProcessing(jobName, exec.id);
+    }
+  }
+
   protected async claim(executionId: string): Promise<boolean> {
     const execution = await this.executions.findById(executionId);
     if (!execution) return false;
@@ -646,10 +789,14 @@ export class JobProvider {
   protected async recoverySweep(): Promise<void> {
     this.log.trace("Starting recovery sweep");
     if (this.stopping) return;
+
+    const acquired = await this.tryLock("_alepha:jobs:recovery-lock", 300_000);
+    if (!acquired) return;
+
     try {
       const now = this.dt.now();
 
-      // 1. Stale pending jobs
+      // 1. Stale pending jobs (priority-ordered)
       const staleThreshold = now
         .subtract(this.config.recovery.staleThreshold, "millisecond")
         .toISOString();
@@ -660,6 +807,7 @@ export class JobProvider {
 
       const stalePending = await this.executions.findMany({
         where: pendingWhere,
+        orderBy: { column: "priority", direction: "asc" },
       });
 
       for (const exec of stalePending) {
@@ -708,6 +856,8 @@ export class JobProvider {
       }
     } catch (e) {
       this.log.error("Recovery sweep failed", { error: e });
+    } finally {
+      await this.releaseLock("_alepha:jobs:recovery-lock");
     }
   }
 
@@ -721,6 +871,10 @@ export class JobProvider {
   protected async delayedDispatchSweep(): Promise<void> {
     this.log.trace("Starting delayed dispatch sweep");
     if (this.stopping) return;
+
+    const acquired = await this.tryLock("_alepha:jobs:dispatch-lock", 60_000);
+    if (!acquired) return;
+
     try {
       const now = this.dt.nowISOString();
 
@@ -728,7 +882,10 @@ export class JobProvider {
       where.status = { inArray: ["scheduled", "retrying"] };
       where.scheduledAt = { lte: now };
 
-      const ready = await this.executions.findMany({ where });
+      const ready = await this.executions.findMany({
+        where,
+        orderBy: { column: "priority", direction: "asc" },
+      });
 
       for (const exec of ready) {
         if (!this.jobs.has(exec.jobName)) continue;
@@ -737,6 +894,8 @@ export class JobProvider {
       }
     } catch (e) {
       this.log.error("Delayed dispatch sweep failed", { error: e });
+    } finally {
+      await this.releaseLock("_alepha:jobs:dispatch-lock");
     }
   }
 
@@ -758,25 +917,66 @@ export class JobProvider {
       where.status = { inArray: ["completed", "dead", "cancelled"] };
       where.completedAt = { lte: cutoff };
 
-      const old = await this.executions.findMany({ where });
-
-      for (const exec of old) {
-        try {
-          await this.executionLogs.deleteById(exec.id);
-        } catch {
-          // Log record may not exist
-        }
-        await this.executions.deleteById(exec.id);
-      }
-
-      if (old.length > 0) {
-        this.log.info(`Log purge: deleted ${old.length} old execution records`);
+      // Bulk-delete logs first (FK-safe), then executions
+      const expiredIds = await this.executions.findMany({
+        where,
+        columns: ["id"] as any,
+      });
+      if (expiredIds.length > 0) {
+        const ids = expiredIds.map((e) => e.id);
+        await this.executionLogs.deleteMany({ id: { inArray: ids } });
+        await this.executions.deleteMany({ id: { inArray: ids } });
+        this.log.info(`Log purge: deleted ${ids.length} old execution records`);
       }
     } catch (e) {
       this.log.error("Log purge failed", { error: e });
     }
   }
 
+  // --- Pause / Resume ---
+
+  public pauseJob(name: string): void {
+    this.getRegistration(name);
+    this.pausedJobs.add(name);
+    this.log.info(`Paused job '${name}'`);
+  }
+
+  public async resumeJob(name: string): Promise<void> {
+    this.getRegistration(name);
+    this.pausedJobs.delete(name);
+    this.log.info(`Resumed job '${name}'`);
+
+    // Dispatch any pending items for this job
+    const pending = await this.executions.findMany({
+      where: { jobName: { eq: name }, status: { eq: "pending" } },
+      orderBy: { column: "priority", direction: "asc" },
+    });
+    for (const exec of pending) {
+      await this.scheduleProcessing(name, exec.id);
+    }
+  }
+
+  public isJobPaused(name: string): boolean {
+    return this.pausedJobs.has(name);
+  }
+
+  public getPausedJobs(): string[] {
+    return [...this.pausedJobs];
+  }
+
+  // --- Lock helpers ---
+
+  protected async tryLock(key: string, ttlMs: number): Promise<boolean> {
+    const lockValue = `${this.workerId},${this.dt.nowISOString()}`;
+    const result = await this.lockProvider.set(key, lockValue, true, ttlMs);
+    const [lockId] = result.split(",");
+    return lockId === this.workerId;
+  }
+
+  protected async releaseLock(key: string): Promise<void> {
+    await this.lockProvider.del(key);
+  }
+
   // --- Lifecycle hooks ---
 
   protected readonly onStart = $hook({
@@ -839,9 +1039,23 @@ export class JobProvider {
     handler: async () => {
       this.stopping = true;
 
-      // Abort any running executions
-      for (const controller of this.abortControllers.values()) {
-        controller.abort();
+      // Drain: wait for in-flight jobs to finish before aborting
+      if (this.inFlight.size > 0) {
+        this.log.info(`Draining ${this.inFlight.size} in-flight job(s)...`);
+        await Promise.race([
+          Promise.allSettled([...this.inFlight]),
+          this.dt.wait([this.config.drainTimeout, "millisecond"]),
+        ]);
+      }
+
+      // Abort any still-running executions after drain timeout
+      if (this.abortControllers.size > 0) {
+        this.log.warn(
+          `Aborting ${this.abortControllers.size} remaining job(s) after drain timeout`,
+        );
+        for (const controller of this.abortControllers.values()) {
+          controller.abort();
+        }
       }
     },
   });

package/src/api/jobs/schemas/jobConfigAtom.ts

@@ -23,6 +23,9 @@ export const jobConfig = $atom({
     logMaxEntries: t.integer({
       description: "Max log entries captured per execution.",
     }),
+    drainTimeout: t.integer({
+      description: "Max time (ms) to wait for in-flight jobs during shutdown.",
+    }),
   }),
   default: {
     recovery: {
@@ -35,6 +38,7 @@ export const jobConfig = $atom({
     },
     logRetentionDays: 30,
     logMaxEntries: 100,
+    drainTimeout: 30_000,
   },
 });
 

package/src/api/jobs/schemas/jobCronInfoSchema.ts

@@ -7,6 +7,7 @@ export const jobCronInfoSchema = t.object({
   priority: t.enum(["critical", "high", "normal", "low"]),
   concurrency: t.integer(),
   hasSchema: t.boolean(),
+  paused: t.boolean(),
   lastExecution: t.optional(
     t.object({
       id: t.uuid(),

package/src/api/jobs/schemas/jobExecutionQuerySchema.ts

@@ -14,7 +14,6 @@ export const jobExecutionQuerySchema = t.extend(pageQuerySchema, {
       "retrying",
       "running",
       "completed",
-      "failed",
       "dead",
       "cancelled",
     ]),

package/src/api/jobs/schemas/jobQueueDepthSchema.ts

@@ -8,6 +8,7 @@ export const jobQueueDepthSchema = t.object({
   retrying: t.integer(),
   dead: t.integer(),
   concurrency: t.integer(),
+  paused: t.boolean(),
 });
 
 export type JobQueueDepth = Static<typeof jobQueueDepthSchema>;

package/src/api/jobs/schemas/jobRegistrationSchema.ts

@@ -14,12 +14,7 @@ export const jobRegistrationSchema = t.object({
       hasBackoff: t.boolean(),
     }),
   ),
-  batch: t.optional(
-    t.object({
-      size: t.integer(),
-      window: t.text(),
-    }),
-  ),
+  paused: t.boolean(),
 });
 
 export type JobRegistration = Static<typeof jobRegistrationSchema>;