@falcondev-oss/workflow 0.6.2 → 0.7.0

package/dist/index.d.mts

@@ -1,30 +1,26 @@
-import { ConnectionOptions, Job, JobSchedulerTemplateOptions, JobsOptions, Queue, QueueEvents, QueueEventsOptions, QueueOptions, UnrecoverableError, WorkerOptions } from "bullmq";
 import "@antfu/utils";
 import IORedis, { RedisOptions } from "ioredis";
-import { Span } from "@opentelemetry/api";
+import { Meter, Span } from "@opentelemetry/api";
+import { AddOptions, Job, Queue, QueueOptions, WorkerOptions } from "groupmq";
 import { SuperJSONResult } from "superjson";
 import { Options } from "p-retry";
 import { StandardSchemaV1 } from "@standard-schema/spec";
 import { Except, IsUnknown, SetOptional, Tagged } from "type-fest";
 
-//#region src/errors.d.ts
-declare class WorkflowInputError extends UnrecoverableError {
-  issues: readonly StandardSchemaV1.Issue[];
-  constructor(message: string, issues: readonly StandardSchemaV1.Issue[]);
-}
-//#endregion
 //#region src/settings.d.ts
 type WorkflowLogger = {
   info?: (...data: any[]) => void;
   success?: (...data: any[]) => void;
 };
 declare const Settings: {
-  defaultPrefix: string;
   defaultConnection: (() => Promise<IORedis> | IORedis) | undefined;
-  defaultCronTimezone: string | undefined;
   logger: WorkflowLogger | undefined;
+  metrics: {
+    meter: Meter;
+    prefix: string;
+  } | undefined;
 };
-declare function createRedisConnection(opts: RedisOptions): Promise<IORedis>;
+declare function createRedis(opts: RedisOptions): Promise<IORedis>;
 //#endregion
 //#region src/serializer.d.ts
 type Serialized<T> = Tagged<SuperJSONResult, 'data', T>;
@@ -45,7 +41,7 @@ declare class WorkflowStep {
   private workflowJobId;
   private stepNamePrefix;
   constructor(opts: {
-    queue: WorkflowQueueInternal<any, any>;
+    queue: WorkflowQueueInternal<unknown>;
     workflowJobId: string;
     workflowId: string;
     stepNamePrefix?: string;
@@ -66,22 +62,22 @@ interface WorkflowStepOptions {
 }
 //#endregion
 //#region src/types.d.ts
-type WorkflowJobInternal<Input, Output> = Job<Serialized<{
+type WorkflowJobPayloadInternal<Input> = Serialized<{
   input: Input | undefined;
   stepData: Record<string, WorkflowStepData>;
   tracingHeaders: unknown;
-}>, Serialized<Output>, string>;
-type WorkflowQueueInternal<Input, Output> = Queue<WorkflowJobInternal<Input, Output>>;
+}>;
+type WorkflowQueueInternal<Input> = Queue<WorkflowJobPayloadInternal<Input>>;
 //#endregion
 //#region src/job.d.ts
 declare class WorkflowJob<Output> {
   private job;
-  private queueEvents;
+  groupId: string;
+  id: string;
   constructor(opts: {
-    job: WorkflowJobInternal<unknown, Output>;
-    queueEvents: QueueEvents;
+    job: Job<unknown>;
   });
-  wait(timeoutMs?: number): Promise<Output>;
+  wait(timeoutMs?: number): Promise<Output | undefined>;
 }
 //#endregion
 //#region src/workflow.d.ts
@@ -89,27 +85,34 @@ interface WorkflowOptions<RunInput, Input, Output> {
   id: string;
   schema?: StandardSchemaV1<RunInput, Input>;
   run: (ctx: WorkflowRunContext<Input>) => Promise<Output>;
-  queueOptions?: SetOptional<QueueOptions, 'connection'>;
-  workerOptions?: SetOptional<WorkerOptions, 'connection'>;
-  queueEventsOptions?: SetOptional<QueueEventsOptions, 'connection'>;
-  connection?: ConnectionOptions;
+  getGroupId?: (input: IsUnknown<Input> extends true ? undefined : Input) => string | undefined | Promise<string | undefined>;
+  queueOptions?: WorkflowQueueOptions;
+  workerOptions?: WorkflowWorkerOptions<Input>;
+  redis?: IORedis;
 }
+type WorkflowJobRunOptions<Input> = SetOptional<Except<AddOptions<WorkflowJobPayloadInternal<Input>>, 'data'>, 'groupId'> & {
+  priority?: 'high' | 'normal';
+};
+type WorkflowQueueOptions = SetOptional<Except<QueueOptions, 'namespace'>, 'redis'>;
+type WorkflowWorkerOptions<Input> = Except<WorkerOptions<WorkflowJobPayloadInternal<Input>>, 'queue' | 'handler' | 'name'> & {
+  metrics?: {
+    meter: Meter;
+    prefix: string;
+  };
+};
 declare class Workflow<RunInput, Input, Output> {
   id: string;
   private opts;
   private queue?;
-  private queueEvents?;
   constructor(opts: WorkflowOptions<RunInput, Input, Output>);
-  work(opts?: Omit<SetOptional<WorkerOptions, 'connection'>, 'autorun'>): Promise<this>;
-  run(input: RunInput, opts?: JobsOptions): Promise<WorkflowJob<Output>>;
-  runIn(input: RunInput, delayMs: number, opts?: Except<JobsOptions, 'delay'>): Promise<WorkflowJob<Output>>;
-  runAt(input: RunInput, date: Date, opts?: Except<JobsOptions, 'delay'>): Promise<WorkflowJob<Output>>;
-  private runSchedule;
-  runCron(schedulerId: string, cron: string, input: RunInput, opts?: JobSchedulerTemplateOptions): Promise<void>;
-  runEvery(schedulerId: string, everyMs: number, input: RunInput, opts?: JobSchedulerTemplateOptions): Promise<void>;
-  exportPrometheusMetrics(globalVariables?: Record<string, string>): Promise<string>;
+  work(opts?: WorkflowWorkerOptions<Input>): Promise<this>;
+  run(input: RunInput, opts?: WorkflowJobRunOptions<Input>): Promise<WorkflowJob<Output>>;
+  runIn(input: RunInput, delayMs: number, opts?: WorkflowJobRunOptions<Input>): Promise<WorkflowJob<Output>>;
+  runAt(input: RunInput, date: Date, opts?: WorkflowJobRunOptions<Input>): Promise<WorkflowJob<Output>>;
+  runCron(scheduleId: string, cron: string, input: RunInput, opts?: WorkflowJobRunOptions<Input>): Promise<WorkflowJob<Output>>;
+  runEvery(scheduleId: string, everyMs: number, input: RunInput, opts?: WorkflowJobRunOptions<Input>): Promise<WorkflowJob<Output>>;
   private getOrCreateQueue;
-  private getOrCreateQueueEvents;
+  private setupMetrics;
 }
 interface WorkflowRunContext<Input> {
   input: IsUnknown<Input> extends true ? undefined : Input;
@@ -117,4 +120,4 @@ interface WorkflowRunContext<Input> {
   span: Span;
 }
 //#endregion
-export { Settings, Workflow, WorkflowInputError, WorkflowOptions, WorkflowRunContext, createRedisConnection };
+export { Settings, Workflow, WorkflowJobRunOptions, WorkflowOptions, WorkflowQueueOptions, WorkflowRunContext, WorkflowWorkerOptions, createRedis };

package/dist/index.mjs

@@ -1,28 +1,18 @@
-import { Queue, QueueEvents, UnrecoverableError, Worker } from "bullmq";
 import { createSingletonPromise } from "@antfu/utils";
 import IORedis from "ioredis";
+import { randomUUID } from "node:crypto";
 import { ROOT_CONTEXT, SpanKind, SpanStatusCode, context, propagation, trace } from "@opentelemetry/api";
 import { asyncExitHook } from "exit-hook";
+import { Queue, Worker } from "groupmq";
+import { setInterval, setTimeout } from "node:timers/promises";
 import { deserialize, serialize } from "superjson";
-import { setTimeout } from "node:timers/promises";
 import pRetry from "p-retry";
 
-//#region src/errors.ts
-var WorkflowInputError = class extends UnrecoverableError {
-	issues;
-	constructor(message, issues) {
-		super(message);
-		this.issues = issues;
-	}
-};
-
-//#endregion
 //#region src/settings.ts
 const Settings = {
-	defaultPrefix: "falcondev-oss-workflow",
 	defaultConnection: void 0,
-	defaultCronTimezone: void 0,
-	logger: void 0
+	logger: void 0,
+	metrics: void 0
 };
 const defaultRedisOptions = {
 	lazyConnect: true,
@@ -36,7 +26,7 @@ const defaultRedisConnection = createSingletonPromise(async () => {
 	await redis.connect();
 	return redis;
 });
-async function createRedisConnection(opts) {
+async function createRedis(opts) {
 	const redis = new IORedis({
 		...defaultRedisOptions,
 		...opts
@@ -58,13 +48,26 @@ function deserialize$1(data) {
 //#region src/job.ts
 var WorkflowJob = class {
 	job;
-	queueEvents;
+	groupId;
+	id;
 	constructor(opts) {
 		this.job = opts.job;
-		this.queueEvents = opts.queueEvents;
+		this.groupId = opts.job.groupId;
+		this.id = opts.job.id;
 	}
 	async wait(timeoutMs) {
-		return deserialize$1(await this.job.waitUntilFinished(this.queueEvents, timeoutMs));
+		if (this.job.finishedOn) {
+			const returnValue = this.job.returnvalue;
+			return returnValue && deserialize$1(returnValue);
+		}
+		for await (const _ of setInterval(1e3, void 0, { signal: timeoutMs ? AbortSignal.timeout(timeoutMs) : void 0 })) {
+			const updatedJob = await this.job.queue.getJob(this.job.id).catch(() => null);
+			if (!updatedJob) return;
+			if (updatedJob.finishedOn) {
+				const returnValue = updatedJob.returnvalue;
+				return returnValue && deserialize$1(returnValue);
+			}
+		}
 	}
 };
 
@@ -115,7 +118,7 @@ var WorkflowStep = class WorkflowStep {
 	async do(stepName, run, options) {
 		const name = this.addNamePrefix(stepName);
 		const stepData = await this.getStepData("do", name);
-		if (stepData?.result) return stepData.result;
+		if (stepData && "result" in stepData) return stepData.result;
 		const initialAttempt = stepData?.attempt ?? 0;
 		await this.updateStepData(name, {
 			type: "do",
@@ -144,7 +147,7 @@ var WorkflowStep = class WorkflowStep {
 			return result;
 		}, {
 			...options?.retry,
-			retries: (options?.retry?.retries ?? 0) - initialAttempt,
+			retries: Math.max((options?.retry?.retries ?? 0) - initialAttempt, 0),
 			onFailedAttempt: async (ctx) => {
 				await this.updateStepData(name, {
 					type: "do",
@@ -156,7 +159,6 @@ var WorkflowStep = class WorkflowStep {
 	}
 	async wait(stepName, durationMs) {
 		const name = this.addNamePrefix(stepName);
-		const job = await this.getWorkflowJob();
 		const existingStepData = await this.getStepData("wait", name);
 		const now = Date.now();
 		const stepData = existingStepData ?? {
@@ -170,12 +172,7 @@ var WorkflowStep = class WorkflowStep {
 			"workflow.job_id": this.workflowJobId,
 			"workflow.step_name": name
 		} }, async () => {
-			const remainingMs = Math.max(0, stepData.startedAt + stepData.durationMs - now);
-			const interval = setInterval(() => {
-				job.updateProgress(name);
-			}, 15e3);
-			await setTimeout(remainingMs);
-			clearInterval(interval);
+			await setTimeout(Math.max(0, stepData.startedAt + stepData.durationMs - now));
 		});
 	}
 	async waitUntil(stepName, date) {
@@ -193,11 +190,11 @@ var WorkflowStep = class WorkflowStep {
 		const job = await this.getWorkflowJob();
 		const jobData = deserialize$1(job.data);
 		jobData.stepData[stepName] = data;
-		await Promise.all([job.updateData(serialize$1(jobData)), job.updateProgress(stepName)]);
+		await job.updateData(serialize$1(jobData));
 	}
 	async getWorkflowJob() {
 		const job = await this.queue.getJob(this.workflowJobId);
-		if (!job) throw new UnrecoverableError(`Could not find workflow job with ID ${this.workflowJobId}`);
+		if (!job) throw new Error(`Could not find workflow job with ID ${this.workflowJobId}`);
 		return job;
 	}
 };
@@ -208,49 +205,54 @@ var Workflow = class {
 	id;
 	opts;
 	queue;
-	queueEvents;
 	constructor(opts) {
 		this.id = opts.id;
 		this.opts = opts;
 	}
 	async work(opts) {
 		const queue = await this.getOrCreateQueue();
-		const worker = new Worker(this.opts.id, async (job) => {
-			Settings.logger?.info?.(`Processing workflow job ${job.id} of workflow ${this.opts.id}`);
-			const jobId = job.id;
-			if (!jobId) throw new Error("Job ID is missing");
-			const deserializedData = deserialize$1(job.data);
-			const parsedData = this.opts.schema && await this.opts.schema["~standard"].validate(deserializedData.input);
-			if (parsedData?.issues) throw new WorkflowInputError("Invalid workflow input", parsedData.issues);
-			return runWithTracing(`workflow-worker/${this.opts.id}`, {
-				attributes: {
-					"workflow.id": this.opts.id,
-					"workflow.job_id": jobId
-				},
-				kind: SpanKind.CONSUMER
-			}, async (span) => {
-				const start = performance.now();
-				const result = await this.opts.run({
-					input: parsedData?.value,
-					step: new WorkflowStep({
-						queue,
-						workflowJobId: jobId,
-						workflowId: this.opts.id
-					}),
-					span
-				});
-				const end = performance.now();
-				Settings.logger?.success?.(`Completed workflow job ${job.id} of workflow ${this.opts.id} in ${(end - start).toFixed(2)} ms`);
-				return serialize$1(result);
-			}, propagation.extract(ROOT_CONTEXT, deserializedData.tracingHeaders));
-		}, {
-			connection: this.opts.connection ?? await defaultRedisConnection(),
-			prefix: Settings.defaultPrefix,
+		const worker = new Worker({
+			handler: async (job) => {
+				Settings.logger?.info?.(`Processing workflow job ${job.id} of workflow ${this.opts.id}`);
+				const jobId = job.id;
+				if (!jobId) throw new Error("Job ID is missing");
+				const deserializedData = deserialize$1(job.data);
+				const parsedData = this.opts.schema && await this.opts.schema["~standard"].validate(deserializedData.input);
+				if (parsedData?.issues) throw new Error(`Invalid workflow input`);
+				return runWithTracing(`workflow-worker/${this.opts.id}`, {
+					attributes: {
+						"workflow.id": this.opts.id,
+						"workflow.job_id": jobId
+					},
+					kind: SpanKind.CONSUMER
+				}, async (span) => {
+					const start = performance.now();
+					const result = await this.opts.run({
+						input: parsedData?.value,
+						step: new WorkflowStep({
+							queue,
+							workflowJobId: jobId,
+							workflowId: this.opts.id
+						}),
+						span
+					});
+					const end = performance.now();
+					Settings.logger?.success?.(`Completed workflow job ${job.id} of workflow ${this.opts.id} in ${(end - start).toFixed(2)} ms`);
+					return serialize$1(result);
+				}, propagation.extract(ROOT_CONTEXT, deserializedData.tracingHeaders));
+			},
+			queue,
 			...this.opts.workerOptions,
 			...opts
 		});
-		await worker.waitUntilReady();
-		Settings.logger?.info?.(`Worker started for workflow ${this.opts.id}`);
+		worker.on("ready", () => {
+			Settings.logger?.info?.(`Worker started for workflow ${this.opts.id}`);
+		});
+		worker.on("failed", (job) => {
+			Settings.logger?.info?.(`Workflow job ${job.id} of workflow ${this.opts.id} failed`);
+		});
+		const metricsOpts = opts?.metrics ?? this.opts.workerOptions?.metrics ?? Settings.metrics;
+		if (metricsOpts) await this.setupMetrics(metricsOpts);
 		asyncExitHook(async (signal) => {
 			Settings.logger?.info?.(`Received ${signal}, shutting down worker for workflow ${this.opts.id}...`);
 			await worker.close();
@@ -259,7 +261,7 @@ var Workflow = class {
 	}
 	async run(input, opts) {
 		const parsedInput = this.opts.schema && await this.opts.schema["~standard"].validate(input);
-		if (parsedInput?.issues) throw new WorkflowInputError("Invalid workflow input", parsedInput.issues);
+		if (parsedInput?.issues) throw new Error("Invalid workflow input");
 		const queue = await this.getOrCreateQueue();
 		return runWithTracing(`workflow-producer/${this.opts.id}`, {
 			attributes: { "workflow.id": this.opts.id },
@@ -267,14 +269,16 @@ var Workflow = class {
 		}, async () => {
 			const tracingHeaders = {};
 			propagation.inject(context.active(), tracingHeaders);
-			return new WorkflowJob({
-				job: await queue.add("workflow-job", serialize$1({
+			return new WorkflowJob({ job: await queue.add({
+				groupId: await this.opts.getGroupId?.(parsedInput?.value) ?? randomUUID(),
+				data: serialize$1({
 					input: parsedInput?.value,
 					stepData: {},
 					tracingHeaders
-				}), opts),
-				queueEvents: await this.getOrCreateQueueEvents()
-			});
+				}),
+				orderMs: opts?.priority === "high" ? 0 : void 0,
+				...opts
+			}) });
 		});
 	}
 	async runIn(input, delayMs, opts) {
@@ -284,59 +288,65 @@ var Workflow = class {
 		});
 	}
 	async runAt(input, date, opts) {
-		const now = Date.now();
-		return date.getTime() < now ? this.run(input, opts) : this.runIn(input, date.getTime() - Date.now(), opts);
-	}
-	async runSchedule(schedulerId, repeatOpts, input, opts) {
-		const parsedInput = this.opts.schema && await this.opts.schema["~standard"].validate(input);
-		if (parsedInput?.issues) throw new WorkflowInputError("Invalid workflow input", parsedInput.issues);
-		await (await this.getOrCreateQueue()).upsertJobScheduler(schedulerId, repeatOpts, {
-			name: "workflow-job",
-			data: serialize$1({
-				input: parsedInput?.value,
-				stepData: {},
-				tracingHeaders: {}
-			}),
-			opts
+		return this.run(input, {
+			runAt: date,
+			...opts
 		});
 	}
-	async runCron(schedulerId, cron, input, opts) {
-		return this.runSchedule(schedulerId, { pattern: cron }, input, opts);
-	}
-	async runEvery(schedulerId, everyMs, input, opts) {
-		return this.runSchedule(schedulerId, { every: everyMs }, input, opts);
+	async runCron(scheduleId, cron, input, opts) {
+		return this.run(input, {
+			groupId: scheduleId,
+			repeat: { pattern: cron },
+			...opts
+		});
 	}
-	async exportPrometheusMetrics(globalVariables) {
-		return (await this.getOrCreateQueue()).exportPrometheusMetrics({
-			workflowId: this.id,
-			workflowPrefix: Settings.defaultPrefix,
-			...globalVariables
+	async runEvery(scheduleId, everyMs, input, opts) {
+		return this.run(input, {
+			groupId: scheduleId,
+			repeat: { every: everyMs },
+			...opts
 		});
 	}
 	async getOrCreateQueue() {
-		if (!this.queue) this.queue = new Queue(this.opts.id, {
-			prefix: Settings.defaultPrefix,
-			connection: this.opts.connection ?? await defaultRedisConnection(),
-			defaultJobOptions: {
-				removeOnComplete: true,
-				removeOnFail: { age: 1440 * 60 },
-				...this.opts.queueOptions?.defaultJobOptions
-			},
+		if (!this.queue) this.queue = new Queue({
+			namespace: this.opts.id,
+			redis: this.opts.redis ?? await defaultRedisConnection(),
+			keepFailed: 100,
 			...this.opts.queueOptions
 		});
-		await this.queue.waitUntilReady();
 		return this.queue;
 	}
-	async getOrCreateQueueEvents() {
-		if (!this.queueEvents) this.queueEvents = new QueueEvents(this.opts.id, {
-			prefix: Settings.defaultPrefix,
-			connection: this.opts.connection ?? await defaultRedisConnection(),
-			...this.opts.queueEventsOptions
-		});
-		await this.queueEvents.waitUntilReady();
-		return this.queueEvents;
+	async setupMetrics({ meter, prefix }) {
+		const attributes = { workflow_id: this.opts.id };
+		const queue = await this.getOrCreateQueue();
+		const completedJobsGauge = meter.createObservableGauge(`${prefix}_workflow_completed_jobs`, { description: "Number of completed workflow jobs" });
+		const activeJobsGauge = meter.createObservableGauge(`${prefix}_workflow_active_jobs`, { description: "Number of active workflow jobs" });
+		const failedJobsGauge = meter.createObservableGauge(`${prefix}_workflow_failed_jobs`, { description: "Number of failed workflow jobs" });
+		const waitingJobsGauge = meter.createObservableGauge(`${prefix}_workflow_waiting_jobs`, { description: "Number of waiting workflow jobs" });
+		const delayedJobsGauge = meter.createObservableGauge(`${prefix}_workflow_delayed_jobs`, { description: "Number of delayed workflow jobs" });
+		const groupCountGauge = meter.createObservableGauge(`${prefix}_workflow_groups`, { description: "Number of workflow job groups" });
+		meter.addBatchObservableCallback(async (observableResult) => {
+			try {
+				const [counts, groupCount] = await Promise.all([queue.getJobCounts(), queue.getUniqueGroupsCount()]);
+				observableResult.observe(completedJobsGauge, counts.completed, attributes);
+				observableResult.observe(activeJobsGauge, counts.active, attributes);
+				observableResult.observe(failedJobsGauge, counts.failed, attributes);
+				observableResult.observe(waitingJobsGauge, counts.waiting, attributes);
+				observableResult.observe(delayedJobsGauge, counts.delayed, attributes);
+				observableResult.observe(groupCountGauge, groupCount, attributes);
+			} catch (err) {
+				console.error("Error collecting workflow metrics:", err);
+			}
+		}, [
+			completedJobsGauge,
+			activeJobsGauge,
+			failedJobsGauge,
+			waitingJobsGauge,
+			delayedJobsGauge,
+			groupCountGauge
+		]);
 	}
 };
 
 //#endregion
-export { Settings, Workflow, WorkflowInputError, createRedisConnection };
+export { Settings, Workflow, createRedis };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@falcondev-oss/workflow",
   "type": "module",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "Simple type-safe queue worker with durable execution based on BullMQ.",
   "license": "MIT",
   "repository": "github:falcondev-oss/workflow",
@@ -31,28 +31,29 @@
     "node": "24",
     "pnpm": "10"
   },
-  "peerDependencies": {
-    "@opentelemetry/api": "^1.9.0"
-  },
   "dependencies": {
     "@antfu/utils": "^9.3.0",
+    "@opentelemetry/api": "^1.9.0",
     "@standard-schema/spec": "^1.1.0",
     "@types/node": "^25.0.3",
-    "bullmq": "^5.66.4",
     "exit-hook": "^5.0.1",
+    "groupmq": "^1.1.0",
     "ioredis": "^5.8.2",
     "p-retry": "^7.1.1",
     "superjson": "^2.2.6",
-    "tsx": "^4.21.0",
-    "type-fest": "^5.3.1",
-    "zod": "^4.3.4"
+    "type-fest": "^5.3.1"
   },
   "devDependencies": {
     "@falcondev-oss/configs": "^5.0.2",
+    "@testcontainers/redis": "^11.11.0",
+    "arktype": "^2.1.29",
     "eslint": "^9.39.2",
     "prettier": "^3.7.4",
     "tsdown": "0.19.0-beta.5",
-    "typescript": "^5.9.3"
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.18",
+    "zod": "^4.3.4"
   },
   "scripts": {
     "build": "tsdown",