@falcondev-oss/workflow 0.9.0 → 0.10.0

package/dist/index.d.mts

@@ -13,6 +13,7 @@ type WorkflowLogger = {
   success?: (...data: any[]) => void;
   error?: (...data: any[]) => void;
   debug?: (...data: any[]) => void;
+  warn?: (...data: any[]) => void;
 };
 declare const Settings: {
   defaultConnection: (() => Promise<IORedis> | IORedis) | undefined;
@@ -45,11 +46,16 @@ declare class WorkflowStep {
   private workflowJobId;
   private stepNamePrefix;
   private updateStepDataMutex;
+  private meta;
   constructor(opts: {
     queue: WorkflowQueueInternal<unknown>;
     workflowJobId: string;
     workflowId: string;
     stepNamePrefix?: string;
+    meta: {
+      stepPromises: Set<Promise<any>>;
+      isCanceled: boolean;
+    };
   });
   private addNamePrefix;
   do<R>(stepName: string, run: (ctx: {

package/dist/index.mjs

@@ -7296,7 +7296,7 @@ var require_parser = /* @__PURE__ */ __commonJSMin(((exports, module) => {
 }));
 
 //#endregion
-//#region node_modules/.pnpm/groupmq@1.1.1-next.2_patch_hash=0f055dcc6e397c02b849be5416debe7d948309b119c1f5282e0cfc6f2a063543_ioredis@5.9.2/node_modules/groupmq/dist/index.js
+//#region node_modules/.pnpm/groupmq@1.1.1-next.2_patch_hash=1f311e9b6cc731809ddc67335576540abb3c2e90f98c33be84e8bb3c8113391c_ioredis@5.9.2/node_modules/groupmq/dist/index.js
 var import_parser = /* @__PURE__ */ __toESM$1(require_parser(), 1);
 const __INLINED_LUA_SCRIPTS__ = {
 	"change-delay": "-- argv: ns, jobId, newDelayUntil, now\nlocal ns = KEYS[1]\nlocal jobId = ARGV[1]\nlocal newDelayUntil = tonumber(ARGV[2])\nlocal now = tonumber(ARGV[3])\n\nlocal jobKey = ns .. \":job:\" .. jobId\nlocal delayedKey = ns .. \":delayed\"\nlocal readyKey = ns .. \":ready\"\n\n-- Check if job exists\nlocal exists = redis.call(\"EXISTS\", jobKey)\nif exists == 0 then\n  return 0\nend\n\nlocal groupId = redis.call(\"HGET\", jobKey, \"groupId\")\nif not groupId then\n  return 0\nend\n\nlocal gZ = ns .. \":g:\" .. groupId\n\n-- Update job's delayUntil field\nredis.call(\"HSET\", jobKey, \"delayUntil\", tostring(newDelayUntil))\n\n-- Check if job is currently in delayed set\nlocal inDelayed = redis.call(\"ZSCORE\", delayedKey, jobId)\n\nif newDelayUntil > 0 and newDelayUntil > now then\n  -- Job should be delayed\n  redis.call(\"HSET\", jobKey, \"status\", \"delayed\")\n  if inDelayed then\n    -- Update existing delay\n    redis.call(\"ZADD\", delayedKey, newDelayUntil, jobId)\n  else\n    -- Move to delayed\n    redis.call(\"ZADD\", delayedKey, newDelayUntil, jobId)\n    -- If this is the head job, remove group from ready\n    local head = redis.call(\"ZRANGE\", gZ, 0, 0)\n    if head and #head > 0 and head[1] == jobId then\n      redis.call(\"ZREM\", readyKey, groupId)\n    end\n  end\nelse\n  -- Job should be ready immediately\n  redis.call(\"HSET\", jobKey, \"status\", \"waiting\")\n  if inDelayed then\n    -- Remove from delayed\n    redis.call(\"ZREM\", delayedKey, jobId)\n    -- If this is the head job, ensure group is in ready\n    local head = redis.call(\"ZRANGE\", gZ, 0, 0, \"WITHSCORES\")\n    if head and #head >= 2 and head[1] == jobId then\n      local headScore = tonumber(head[2])\n      redis.call(\"ZADD\", readyKey, headScore, groupId)\n    end\n  end\nend\n\nreturn 1\n\n\n",
@@ -8782,7 +8782,7 @@ var Queue = class {
 	async addRepeatingJob(opts) {
 		if (!opts.repeat) throw new Error("Repeat options are required for repeating jobs");
 		const now = Date.now();
-		const repeatKey = `${opts.groupId}:${JSON.stringify(opts.repeat)}:${now}:${Math.random().toString(36).slice(2)}`;
+		const repeatKey = `${opts.groupId}:${now}:${Math.random().toString(36).slice(2)}`;
 		let nextRunTime;
 		if ("every" in opts.repeat) nextRunTime = now + opts.repeat.every;
 		else nextRunTime = this.getNextCronTime(opts.repeat.pattern, now);
@@ -8799,7 +8799,7 @@ var Queue = class {
 		const repeatJobKey = `${this.ns}:repeat:${repeatKey}`;
 		await this.r.set(repeatJobKey, JSON.stringify(repeatJobData));
 		await this.r.zadd(`${this.ns}:repeat:schedule`, nextRunTime, repeatKey);
-		const lookupKey = `${this.ns}:repeat:lookup:${opts.groupId}:${JSON.stringify(opts.repeat)}`;
+		const lookupKey = `${this.ns}:repeat:lookup:${opts.groupId}`;
 		await this.r.set(lookupKey, repeatKey);
 		const repeatId = `repeat:${repeatKey}`;
 		const jobHashKey = `${this.ns}:job:${repeatId}`;
@@ -8821,9 +8821,9 @@ var Queue = class {
 	/**
 	* Remove a repeating job
 	*/
-	async removeRepeatingJob(groupId, repeat) {
+	async removeRepeatingJob(groupId) {
 		try {
-			const lookupKey = `${this.ns}:repeat:lookup:${groupId}:${JSON.stringify(repeat)}`;
+			const lookupKey = `${this.ns}:repeat:lookup:${groupId}`;
 			const repeatKey = await this.r.get(lookupKey);
 			if (!repeatKey) return false;
 			const repeatJobKey = `${this.ns}:repeat:${repeatKey}`;
@@ -9628,11 +9628,13 @@ var WorkflowStep = class WorkflowStep {
 	workflowJobId;
 	stepNamePrefix;
 	updateStepDataMutex = new Mutex();
+	meta;
 	constructor(opts) {
 		this.queue = opts.queue;
 		this.workflowJobId = opts.workflowJobId;
 		this.workflowId = opts.workflowId;
 		this.stepNamePrefix = opts.stepNamePrefix ? `${opts.stepNamePrefix}|` : "";
+		this.meta = opts.meta;
 	}
 	addNamePrefix(name) {
 		return `${this.stepNamePrefix}${name}`;
@@ -9650,7 +9652,7 @@ var WorkflowStep = class WorkflowStep {
 			attempt: initialAttempt
 		});
 		Settings.logger?.debug?.(`[${this.workflowId}/${this.workflowJobId}] Running step '${name}' (attempt ${initialAttempt + 1})`);
-		return pRetry(async (attempt) => {
+		const promise = pRetry(async (attempt) => {
 			const result = await runWithTracing(`workflow-worker/${this.workflowId}/step/${name}`, { attributes: {
 				"workflow.id": this.workflowId,
 				"workflow.job_id": this.workflowJobId,
@@ -9661,7 +9663,8 @@ var WorkflowStep = class WorkflowStep {
 					queue: this.queue,
 					workflowId: this.workflowId,
 					workflowJobId: this.workflowJobId,
-					stepNamePrefix: name
+					stepNamePrefix: name,
+					meta: this.meta
 				}),
 				span
 			}));
@@ -9682,8 +9685,14 @@ var WorkflowStep = class WorkflowStep {
 					attempt: initialAttempt + ctx.attemptNumber
 				});
 				return options?.retry?.onFailedAttempt?.(ctx);
+			},
+			shouldRetry: async (context$1) => {
+				if (this.meta.isCanceled) return false;
+				return options?.retry?.shouldRetry?.(context$1) ?? true;
 			}
 		});
+		this.meta.stepPromises.add(promise);
+		return promise.finally(() => this.meta.stepPromises.delete(promise));
 	}
 	async wait(stepName, durationMs) {
 		const name = this.addNamePrefix(stepName);
@@ -9758,19 +9767,33 @@ var Workflow = class {
 					},
 					kind: SpanKind.CONSUMER
 				}, async (span) => {
+					const stepMeta = {
+						stepPromises: /* @__PURE__ */ new Set(),
+						isCanceled: false
+					};
 					const start = performance.now();
-					const result = await this.opts.run({
-						input: parsedData?.value,
-						step: new WorkflowStep({
-							queue,
-							workflowJobId: jobId,
-							workflowId: this.opts.id
-						}),
-						span
-					});
-					const end = performance.now();
-					Settings.logger?.success?.(`[${this.opts.id}] Completed job ${job.id} in ${(end - start).toFixed(2)} ms`);
-					return serialize$1(result);
+					try {
+						const result = await this.opts.run({
+							input: parsedData?.value,
+							step: new WorkflowStep({
+								queue,
+								workflowJobId: jobId,
+								workflowId: this.opts.id,
+								meta: stepMeta
+							}),
+							span
+						});
+						const end = performance.now();
+						Settings.logger?.success?.(`[${this.opts.id}] Completed job ${job.id} in ${(end - start).toFixed(2)} ms`);
+						return serialize$1(result);
+					} catch (err) {
+						stepMeta.isCanceled = true;
+						if (stepMeta.stepPromises.size > 0) {
+							Settings.logger?.warn?.(`[${this.opts.id}] Job failed but there are still ${stepMeta.stepPromises.size} running step(s), waiting for them to finish. Be careful when using 'Promise.all([step0, step1, ...])', as running steps are not canceled when one of them fails.`);
+							await Promise.allSettled(stepMeta.stepPromises);
+						}
+						throw err;
+					}
 				}, propagation.extract(ROOT_CONTEXT, deserializedData.tracingHeaders));
 			},
 			queue,
@@ -9796,6 +9819,10 @@ var Workflow = class {
 		const parsedInput = this.opts.schema && await this.opts.schema["~standard"].validate(input);
 		if (parsedInput?.issues) throw new Error("Invalid workflow input");
 		const queue = await this.getOrCreateQueue();
+		const groupId = opts?.groupId ?? await this.opts.getGroupId?.(parsedInput?.value) ?? randomUUID();
+		if (opts?.repeat) {
+			if (await queue.removeRepeatingJob(groupId)) Settings.logger?.debug?.(`[${this.opts.id}] Removed existing repeating job with groupId '${groupId}' before adding new one with schedule ${JSON.stringify(opts.repeat)}`);
+		}
 		return runWithTracing(`workflow-producer/${this.opts.id}`, {
 			attributes: { "workflow.id": this.opts.id },
 			kind: SpanKind.PRODUCER
@@ -9804,7 +9831,6 @@ var Workflow = class {
 			propagation.inject(context.active(), tracingHeaders);
 			const orderMs = opts?.orderMs ?? (opts?.priority === "high" ? 0 : void 0) ?? this.opts.jobOptions?.orderMs ?? (this.opts.jobOptions?.priority === "high" ? 0 : void 0);
 			return new WorkflowJob({ job: await queue.add({
-				groupId: await this.opts.getGroupId?.(parsedInput?.value) ?? randomUUID(),
 				data: serialize$1({
 					input: parsedInput?.value,
 					stepData: {},
@@ -9812,6 +9838,7 @@ var Workflow = class {
 				}),
 				...this.opts.jobOptions,
 				...opts,
+				groupId,
 				orderMs
 			}) });
 		});
@@ -9831,6 +9858,7 @@ var Workflow = class {
 	async "runCron"(scheduleId, cron, input, opts) {
 		return this.run(input, {
 			groupId: scheduleId,
+			jobId: scheduleId,
 			repeat: { pattern: cron },
 			...opts
 		});
@@ -9838,6 +9866,7 @@ var Workflow = class {
 	async "runEvery"(scheduleId, everyMs, input, opts) {
 		return this.run(input, {
 			groupId: scheduleId,
+			jobId: scheduleId,
 			repeat: { every: everyMs },
 			...opts
 		});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@falcondev-oss/workflow",
   "type": "module",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Simple type-safe queue worker with durable execution using Redis.",
   "license": "MIT",
   "repository": "github:falcondev-oss/workflow",