@falcondev-oss/workflow 0.1.0

package/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2019 GitHub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,65 @@
+# @falcondev-oss/workflow
+
+Simple type-safe queue worker with durable execution based on BullMQ.
+
+## Installation
+
+```bash
+npm install @falcondev-oss/workflow
+```
+
+## Usage
+
+```ts
+const workflow = new Workflow({
+  id: 'example-workflow',
+  input: z.object({
+    timezone: z.string().default('UTC'),
+    name: z.string(),
+  }),
+  async run({ input, step }) {
+    await step.do('send welcome', () => {
+      console.log(`Welcome, ${input.name}! Timezone: ${input.timezone}`)
+    })
+
+    await step.wait('wait a lil', 60_000)
+
+    const isEngaged = await step.do('check engagement', () => {
+      return Math.random() > 0.5
+    })
+
+    if (!isEngaged) return { engagementLevel: 'low' }
+
+    await step.do('send tips', () => {
+      console.log(`Here are some tips to get started, ${input.name}!`)
+    })
+
+    await step.wait('wait feedback', 3000)
+
+    await step.do('send survey', () => {
+      console.log(`Hi ${input.name}, please take our survey!`)
+    })
+
+    return {
+      engagementLevel: 'high',
+    }
+  },
+})
+
+// Start worker
+await workflow.work()
+
+// Run workflow
+const job = await workflow.run({
+  name: 'John Doe',
+  timezone: 'America/New_York',
+})
+
+// Wait for completion
+const result = await job.wait()
+console.log(result.engagementLevel)
+```
+
+## Inspiration
+
+- https://x.com/imsh4yy/status/1984073526605967785?s=46

package/dist/index.d.mts

@@ -0,0 +1,96 @@
+import { ConnectionOptions, Job, JobsOptions, Queue, QueueEvents, QueueEventsOptions, QueueOptions, UnrecoverableError, WorkerOptions } from "bullmq";
+import "@antfu/utils";
+import IORedis from "ioredis";
+import { SuperJSONResult } from "superjson";
+import { Options } from "p-retry";
+import { StandardSchemaV1 } from "@standard-schema/spec";
+import { SetOptional, Tagged } from "type-fest";
+
+//#region src/errors.d.ts
+declare class WorkflowInputError extends UnrecoverableError {
+  issues: readonly StandardSchemaV1.Issue[];
+  constructor(message: string, issues: readonly StandardSchemaV1.Issue[]);
+}
+//#endregion
+//#region src/settings.d.ts
+declare const Settings: {
+  defaultPrefix: string;
+  defaultConnection: IORedis | undefined;
+};
+//#endregion
+//#region src/serializer.d.ts
+type Serialized<T> = Tagged<SuperJSONResult, 'data', T>;
+//#endregion
+//#region src/step.d.ts
+type WorkflowStepData = {
+  type: 'do';
+  result?: unknown;
+  attempt: number;
+} | {
+  type: 'wait';
+  durationMs: number;
+  startedAt: number;
+};
+declare class WorkflowStep {
+  private workflowId;
+  private queue;
+  private workflowJobId;
+  constructor(opts: {
+    queue: WorkflowQueueInternal<any, any>;
+    workflowJobId: string;
+    workflowId: string;
+  });
+  do<R>(name: string, run: () => R, options?: WorkflowStepOptions): Promise<R>;
+  wait(name: string, durationMs: number): Promise<void>;
+  waitUntil(name: string, date: Date): Promise<void>;
+  private getStepData;
+  private updateStepData;
+  private getWorkflowJob;
+}
+interface WorkflowStepOptions {
+  retry?: Options;
+}
+//#endregion
+//#region src/types.d.ts
+type WorkflowJobInternal<Input, Output> = Job<Serialized<{
+  input: Input;
+  stepData: Record<string, WorkflowStepData>;
+}>, Serialized<Output>, string>;
+type WorkflowQueueInternal<Input, Output> = Queue<WorkflowJobInternal<Input, Output>>;
+//#endregion
+//#region src/job.d.ts
+declare class WorkflowJob<Output> {
+  private job;
+  private queueEvents;
+  constructor(opts: {
+    job: WorkflowJobInternal<unknown, Output>;
+    queueEvents: QueueEvents;
+  });
+  wait(timeoutMs?: number): Promise<Output>;
+}
+//#endregion
+//#region src/workflow.d.ts
+interface WorkflowOptions<RunInput, Input, Output> {
+  id: string;
+  input: StandardSchemaV1<RunInput, Input>;
+  run: (context: WorkflowRunContext<Input>) => Promise<Output>;
+  queueOptions?: SetOptional<QueueOptions, 'connection'>;
+  queueEventsOptions?: SetOptional<QueueEventsOptions, 'connection'>;
+  connection?: ConnectionOptions;
+}
+declare class Workflow<RunInput, Input, Output> {
+  private opts;
+  private queue?;
+  private queueEvents?;
+  constructor(opts: WorkflowOptions<RunInput, Input, Output>);
+  work(opts?: Omit<SetOptional<WorkerOptions, 'connection'>, 'autorun'>): Promise<this>;
+  run(input: RunInput, opts?: JobsOptions): Promise<WorkflowJob<Output>>;
+  private getOrCreateQueue;
+  private getOrCreateQueueEvents;
+}
+interface WorkflowRunContext<Input> {
+  input: Input;
+  step: WorkflowStep;
+}
+//#endregion
+export { Settings, Workflow, WorkflowInputError, WorkflowOptions, WorkflowRunContext };

package/dist/index.mjs

@@ -0,0 +1,243 @@
+import { Queue, QueueEvents, UnrecoverableError, Worker } from "bullmq";
+import { createSingletonPromise } from "@antfu/utils";
+import IORedis from "ioredis";
+import { deserialize, serialize } from "superjson";
+import { setTimeout } from "node:timers/promises";
+import pRetry from "p-retry";
+import { SpanStatusCode, trace } from "@opentelemetry/api";
+
+//#region src/errors.ts
+var WorkflowInputError = class extends UnrecoverableError {
+	issues;
+	constructor(message, issues) {
+		super(message);
+		this.issues = issues;
+	}
+};
+
+//#endregion
+//#region src/settings.ts
+const Settings = {
+	defaultPrefix: "falcondev-oss-workflow",
+	defaultConnection: void 0
+};
+const defaultRedisConnection = createSingletonPromise(async () => {
+	if (Settings.defaultConnection) return Settings.defaultConnection;
+	const redis = new IORedis({
+		lazyConnect: true,
+		maxRetriesPerRequest: null
+	});
+	await redis.connect();
+	return redis;
+});
+
+//#endregion
+//#region src/serializer.ts
+function serialize$1(data) {
+	return serialize(data);
+}
+function deserialize$1(data) {
+	return deserialize(data);
+}
+
+//#endregion
+//#region src/job.ts
+var WorkflowJob = class {
+	job;
+	queueEvents;
+	constructor(opts) {
+		this.job = opts.job;
+		this.queueEvents = opts.queueEvents;
+	}
+	async wait(timeoutMs) {
+		return deserialize$1(await this.job.waitUntilFinished(this.queueEvents, timeoutMs));
+	}
+};
+
+//#endregion
+//#region src/tracer.ts
+function getTracer() {
+	return trace.getTracer("falcondev-oss-workflow");
+}
+async function runWithTracing(spanName, attributes, fn) {
+	return getTracer().startActiveSpan(spanName, async (span) => {
+		try {
+			span.setAttributes(attributes);
+			const result = await fn();
+			span.setStatus({ code: SpanStatusCode.OK });
+			return result;
+		} catch (err_) {
+			const err = err_;
+			span.recordException(err);
+			span.setStatus({
+				code: SpanStatusCode.ERROR,
+				message: err.message
+			});
+			throw err_;
+		} finally {
+			span.end();
+		}
+	});
+}
+
+//#endregion
+//#region src/step.ts
+var WorkflowStep = class {
+	workflowId;
+	queue;
+	workflowJobId;
+	constructor(opts) {
+		this.queue = opts.queue;
+		this.workflowJobId = opts.workflowJobId;
+		this.workflowId = opts.workflowId;
+	}
+	async do(name, run, options) {
+		const stepData = await this.getStepData("do", "name");
+		if (stepData && "result" in stepData) return stepData.result;
+		const initialAttempt = stepData?.attempt ?? 0;
+		await this.updateStepData(name, {
+			type: "do",
+			attempt: initialAttempt
+		});
+		return pRetry(async (attempt) => {
+			const result = await runWithTracing(`step:${name}`, {
+				"workflow.id": this.workflowId,
+				"workflow.job_id": this.workflowJobId,
+				"workflow.step_name": name,
+				"workflow.step.attempt": attempt
+			}, run);
+			await this.updateStepData(name, {
+				type: "do",
+				result,
+				attempt: initialAttempt + attempt
+			});
+			return result;
+		}, {
+			...options?.retry,
+			retries: (options?.retry?.retries ?? 0) - initialAttempt,
+			onFailedAttempt: async (context) => {
+				await this.updateStepData(name, {
+					type: "do",
+					attempt: initialAttempt + context.attemptNumber
+				});
+				return options?.retry?.onFailedAttempt?.(context);
+			}
+		});
+	}
+	async wait(name, durationMs) {
+		const job = await this.getWorkflowJob();
+		const existingStepData = await this.getStepData("wait", name);
+		const now = Date.now();
+		const stepData = existingStepData ?? {
+			type: "wait",
+			durationMs,
+			startedAt: now
+		};
+		await this.updateStepData(name, stepData);
+		await runWithTracing(`step:${name}`, {
+			"workflow.id": this.workflowId,
+			"workflow.job_id": this.workflowJobId,
+			"workflow.step_name": name
+		}, async () => {
+			const remainingMs = Math.max(0, stepData.startedAt + stepData.durationMs - now);
+			const interval = setInterval(() => {
+				job.updateProgress(name);
+			}, 15e3);
+			await setTimeout(remainingMs);
+			clearInterval(interval);
+		});
+	}
+	async waitUntil(name, date) {
+		const now = Date.now();
+		const targetTime = date.getTime();
+		const durationMs = Math.max(0, targetTime - now);
+		return this.wait(name, durationMs);
+	}
+	async getStepData(type, stepName) {
+		const stepData = deserialize$1((await this.getWorkflowJob()).data).stepData[stepName];
+		if (stepData && stepData.type !== type) throw new Error(`Step "${stepName}" is of type "${stepData.type}", expected "${type}"`);
+		return stepData;
+	}
+	async updateStepData(stepName, data) {
+		const job = await this.getWorkflowJob();
+		const jobData = deserialize$1(job.data);
+		jobData.stepData[stepName] = data;
+		await Promise.all([job.updateData(serialize$1(jobData)), job.updateProgress(stepName)]);
+	}
+	async getWorkflowJob() {
+		const job = await this.queue.getJob(this.workflowJobId);
+		if (!job) throw new UnrecoverableError(`Could not find workflow job with ID ${this.workflowJobId}`);
+		return job;
+	}
+};
+
+//#endregion
+//#region src/workflow.ts
+var Workflow = class {
+	opts;
+	queue;
+	queueEvents;
+	constructor(opts) {
+		this.opts = opts;
+	}
+	async work(opts) {
+		const queue = await this.getOrCreateQueue();
+		await new Worker(this.opts.id, async (job) => {
+			const jobId = job.id;
+			if (!jobId) throw new Error("Job ID is missing");
+			const deserializedData = deserialize$1(job.data);
+			const parsedData = await this.opts.input["~standard"].validate(deserializedData.input);
+			if (parsedData.issues) throw new WorkflowInputError("Invalid workflow input", parsedData.issues);
+			return runWithTracing(`workflow:${this.opts.id}`, {
+				"workflow.id": this.opts.id,
+				"workflow.job_id": jobId
+			}, async () => {
+				return serialize$1(await this.opts.run({
+					input: parsedData.value,
+					step: new WorkflowStep({
+						queue,
+						workflowJobId: jobId,
+						workflowId: this.opts.id
+					})
+				}));
+			});
+		}, {
+			connection: this.opts.connection ?? await defaultRedisConnection(),
+			prefix: Settings.defaultPrefix,
+			...opts
+		}).waitUntilReady();
+		return this;
+	}
+	async run(input, opts) {
+		const parsedInput = await this.opts.input["~standard"].validate(input);
+		if (parsedInput.issues) throw new WorkflowInputError("Invalid workflow input", parsedInput.issues);
+		return new WorkflowJob({
+			job: await (await this.getOrCreateQueue()).add("workflow-job", serialize$1({
+				input: parsedInput.value,
+				stepData: {}
+			}), opts),
+			queueEvents: await this.getOrCreateQueueEvents()
+		});
+	}
+	async getOrCreateQueue() {
+		if (!this.queue) this.queue = new Queue(this.opts.id, {
+			prefix: Settings.defaultPrefix,
+			connection: this.opts.connection ?? await defaultRedisConnection(),
+			...this.opts.queueOptions
+		});
+		await this.queue.waitUntilReady();
+		return this.queue;
+	}
+	async getOrCreateQueueEvents() {
+		if (!this.queueEvents) this.queueEvents = new QueueEvents(this.opts.id, {
+			prefix: Settings.defaultPrefix,
+			connection: this.opts.connection ?? await defaultRedisConnection(),
+			...this.opts.queueEventsOptions
+		});
+		await this.queueEvents.waitUntilReady();
+		return this.queueEvents;
+	}
+};
+
+//#endregion
+export { Settings, Workflow, WorkflowInputError };

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@falcondev-oss/workflow",
+  "type": "module",
+  "version": "0.1.0",
+  "description": "Simple type-safe queue worker with durable execution based on BullMQ.",
+  "license": "MIT",
+  "repository": "github:falcondev-oss/workflow",
+  "bugs": {
+    "url": "https://github.com/falcondev-oss/workflow/issues"
+  },
+  "keywords": [
+    "queue",
+    "worker",
+    "bullmq",
+    "durable-execution",
+    "tracing",
+    "typescript"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": "24",
+    "pnpm": "10"
+  },
+  "peerDependencies": {
+    "@opentelemetry/api": "^1.9.0"
+  },
+  "dependencies": {
+    "@antfu/utils": "^9.3.0",
+    "@standard-schema/spec": "^1.1.0",
+    "@types/node": "^25.0.3",
+    "bullmq": "^5.66.4",
+    "ioredis": "^5.8.2",
+    "p-retry": "^7.1.1",
+    "superjson": "^2.2.6",
+    "tsx": "^4.21.0",
+    "type-fest": "^5.3.1",
+    "zod": "^4.3.4"
+  },
+  "devDependencies": {
+    "@falcondev-oss/configs": "^5.0.2",
+    "eslint": "^9.39.2",
+    "prettier": "^3.7.4",
+    "tsdown": "0.19.0-beta.5",
+    "typescript": "^5.9.3"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "type-check": "tsc --noEmit",
+    "lint": "eslint --cache . && prettier --check --cache .",
+    "lint:ci": "eslint --cache --cache-strategy content . && prettier --check --cache --cache-strategy content .",
+    "lint:fix": "eslint --fix --cache . && prettier --write --cache ."
+  },
+  "types": "./dist/index.d.mts"
+}