@blokjs/trigger-worker 0.2.1 → 0.4.0

package/src/WorkerTrigger.test.ts

@@ -8,6 +8,7 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "./WorkerTrigger";
 import { InMemoryAdapter } from "./adapters/InMemoryAdapter";
+import { computeXDelayHoldMs } from "./adapters/NATSAdapter";
 
 // ============================================================================
 // WorkerJob Interface Tests
@@ -407,23 +408,22 @@ describe("BullMQAdapter", () => {
 	});
 
 	it("should use default values when env vars not set", () => {
-		const originalHost = process.env.REDIS_HOST;
-		const originalPort = process.env.REDIS_PORT;
-
-		delete process.env.REDIS_HOST;
-		delete process.env.REDIS_PORT;
+		// Pure: don't mutate process.env (races with other parallel workers
+		// when running via `nx run-many -t test` and pollutes other tests).
+		// Simulate "env unset" by reading from an explicit snapshot rather
+		// than the live process.env.
+		const fakeEnv: Record<string, string | undefined> = {
+			REDIS_HOST: undefined,
+			REDIS_PORT: undefined,
+		};
 
 		const config = {
-			host: process.env.REDIS_HOST || "localhost",
-			port: Number.parseInt(process.env.REDIS_PORT || "6379", 10),
+			host: fakeEnv.REDIS_HOST || "localhost",
+			port: Number.parseInt(fakeEnv.REDIS_PORT || "6379", 10),
 		};
 
 		expect(config.host).toBe("localhost");
 		expect(config.port).toBe(6379);
-
-		// Restore
-		process.env.REDIS_HOST = originalHost;
-		process.env.REDIS_PORT = originalPort;
 	});
 });
 
@@ -482,7 +482,7 @@ describe("Exponential Backoff", () => {
 		const maxDelay = 30000;
 
 		const delays = [0, 1, 2, 3, 4, 5].map((attempt) => {
-			const exponential = Math.min(base * Math.pow(2, attempt), maxDelay);
+			const exponential = Math.min(base * 2 ** attempt, maxDelay);
 			return exponential;
 		});
 
@@ -498,13 +498,43 @@ describe("Exponential Backoff", () => {
 		const base = 1000;
 		const maxDelay = 30000;
 
-		const delay = Math.min(base * Math.pow(2, 10), maxDelay);
+		const delay = Math.min(base * 2 ** 10, maxDelay);
 		expect(delay).toBe(30000);
 	});
 
 	it("should support custom base delay", () => {
 		const base = 500;
-		const exponential = base * Math.pow(2, 2);
+		const exponential = base * 2 ** 2;
 		expect(exponential).toBe(2000);
 	});
 });
+
+// ============================================================================
+// NATSAdapter — computeXDelayHoldMs (Tier 2 polish: x-delay enforcement)
+// ============================================================================
+
+describe("NATSAdapter — computeXDelayHoldMs", () => {
+	it("returns 0 when no delay was set", () => {
+		expect(computeXDelayHoldMs(0, 1_000_000, 1_000_000)).toBe(0);
+		expect(computeXDelayHoldMs(-50, 1_000_000, 1_000_000)).toBe(0);
+	});
+
+	it("returns the full delay when the message just arrived", () => {
+		// createdMs == nowMs (just published), delay 5s → wait 5s.
+		expect(computeXDelayHoldMs(5000, 2_000_000, 2_000_000)).toBe(5000);
+	});
+
+	it("returns the remaining delay when partially elapsed", () => {
+		// Published 2s ago, delay 5s → wait 3s remaining.
+		expect(computeXDelayHoldMs(5000, 1_000_000, 1_002_000)).toBe(3000);
+	});
+
+	it("returns 0 when the delay has already elapsed", () => {
+		// Published 10s ago, delay 5s → fire immediately.
+		expect(computeXDelayHoldMs(5000, 1_000_000, 1_010_000)).toBe(0);
+	});
+
+	it("clamps to 0 when nowMs is far in the future", () => {
+		expect(computeXDelayHoldMs(5000, 1_000_000, 9_999_999)).toBe(0);
+	});
+});

package/src/WorkerTrigger.ts

@@ -19,14 +19,21 @@
  *    - Ack on success, retry or DLQ on failure
  */
 
-import type { HelperResponse, WorkerTriggerOpts } from "@blokjs/helper";
+import { type HelperResponse, type WorkerTriggerOpts, tryParseDuration } from "@blokjs/helper";
 import {
+	type BlokService,
+	ConcurrencyLimitError,
+	ConcurrencyMetrics,
 	DefaultLogger,
+	DeferredDispatchSignal,
 	type GlobalOptions,
-	type BlokService,
+	Janitor,
 	NodeMap,
+	QueueExpiredError,
+	RunTracker,
 	TriggerBase,
 	type TriggerResponse,
+	createConcurrencyBackend,
 } from "@blokjs/runner";
 import type { Context, RequestContext } from "@blokjs/shared";
 import { type Span, SpanStatusCode, metrics, trace } from "@opentelemetry/api";
@@ -205,6 +212,63 @@ export abstract class WorkerTrigger extends TriggerBase {
 		const startTime = this.startCounter();
 
 		try {
+			// Tier 2 #6 follow-up · install the cross-process concurrency
+			// backend (NATS KV) when the operator opted in via
+			// `BLOK_CONCURRENCY_BACKEND=nats-kv`. Default null preserves the
+			// existing in-process behavior.
+			//
+			// PR 3 D1 — record install attempts via OTel counter.
+			try {
+				const backend = createConcurrencyBackend();
+				if (backend) {
+					await backend.connect();
+					RunTracker.getInstance().setConcurrencyBackend(backend);
+					ConcurrencyMetrics.getInstance().recordBackendInstall({
+						backend: backend.name,
+						status: "success",
+					});
+					this.logger.log(`[concurrency] backend installed: ${backend.name}`);
+				}
+			} catch (err) {
+				ConcurrencyMetrics.getInstance().recordBackendInstall({
+					backend: "unknown",
+					status: "failure",
+				});
+				this.logger.error(
+					`[concurrency] backend install failed: ${err instanceof Error ? err.message : String(err)}; falling back to in-process behavior`,
+				);
+			}
+
+			// Tier 2 quick-wins follow-up · install crash handlers + recover
+			// orphaned runs from a previous (dead) process. Idempotent + opt-out
+			// via `BLOK_CRASH_AUTOFLIP_DISABLED=1`.
+			try {
+				WorkerTrigger.installCrashHandlers(this.logger);
+				const orphaned = WorkerTrigger.recoverOrphanedRuns(undefined, this.logger);
+				if (orphaned > 0) {
+					this.logger.log(`[crash-autoflip] flipped ${orphaned} orphaned run(s) to crashed on boot`);
+				}
+			} catch (err) {
+				this.logger.error(`[crash-autoflip] setup failed: ${err instanceof Error ? err.message : String(err)}`);
+			}
+
+			// Tier 2 follow-up · start the periodic storage janitor.
+			// Idempotent (singleton); opt-out via `BLOK_JANITOR_DISABLED=1`.
+			try {
+				Janitor.getInstance(RunTracker.getInstance().getStore(), this.logger).start();
+			} catch (err) {
+				this.logger.error(`[janitor] setup failed: ${err instanceof Error ? err.message : String(err)}`);
+			}
+
+			// Tier 2 follow-up · install graceful shutdown handlers
+			// (SIGTERM / SIGINT). Idempotent; opt-out via
+			// `BLOK_GRACEFUL_SHUTDOWN_DISABLED=1`.
+			try {
+				WorkerTrigger.installShutdownHandlers(this, this.logger);
+			} catch (err) {
+				this.logger.error(`[shutdown] setup failed: ${err instanceof Error ? err.message : String(err)}`);
+			}
+
 			// Connect to job backend
 			await this.adapter.connect();
 			this.logger.log(`Connected to ${this.adapter.provider} worker system`);
@@ -373,7 +437,7 @@ export abstract class WorkerTrigger extends TriggerBase {
 
 				// Store worker metadata in context
 				if (!ctx.vars) ctx.vars = {};
-				ctx.vars["_worker_job"] = {
+				ctx.vars._worker_job = {
 					id: job.id,
 					queue: job.queue,
 					attempts: String(job.attempts),
@@ -420,6 +484,74 @@ export abstract class WorkerTrigger extends TriggerBase {
 				await job.complete();
 			} catch (error) {
 				const errorMessage = (error as Error).message;
+
+				// Tier 2 #5 + #7 — deferred dispatch (delay/TTL/debounce).
+				// The run was deferred to a future timer; ACK without retry.
+				// The in-process scheduler owns the eventual dispatch, NOT
+				// the broker — re-queueing here would create a duplicate.
+				if (error instanceof DeferredDispatchSignal) {
+					span.setAttribute("success", false);
+					span.setAttribute("deferred", true);
+					span.setAttribute("deferred_status", error.info.status);
+					span.setStatus({ code: SpanStatusCode.OK, message: `deferred:${error.info.status}` });
+
+					this.logger.log(
+						`[scheduling] job ${jobId} runId=${error.info.runId} status=${error.info.status} ` +
+							`scheduledAt=${error.info.scheduledAt} pingCount=${error.info.pingCount} → ACK (no requeue)`,
+					);
+
+					await job.complete();
+					return;
+				}
+
+				// PR 1-5 polish — queue-mode TTL elapsed. The run is already
+				// flipped to `expired` (see TriggerBase queue branch); ACK
+				// without retry so the broker doesn't redeliver — the run
+				// will never succeed (timer won't re-fire). Distinct from
+				// the throttled NACK below.
+				if (error instanceof QueueExpiredError) {
+					span.setAttribute("success", false);
+					span.setAttribute("queue_expired", true);
+					span.setStatus({ code: SpanStatusCode.OK, message: "queue_expired" });
+
+					this.logger.log(
+						`[concurrency] job ${jobId} runId=${error.info.runId} key='${error.info.concurrencyKey}' ` +
+							`queueExpiredAt=${error.info.queueExpiredAt} → ACK (no requeue, run expired)`,
+					);
+
+					await job.complete();
+					return;
+				}
+
+				// Tier 2 #6 — concurrency gate denial. Distinct from a normal
+				// failure: NACK with redelivery so the broker re-queues the
+				// job with its existing back-off semantics. Doesn't count
+				// against the workflow's retry budget (different invariant).
+				// We always pass `willRetry: true` regardless of `job.attempts`
+				// because throttling is a transient resource state, not a
+				// permanent failure.
+				if (error instanceof ConcurrencyLimitError) {
+					span.setAttribute("success", false);
+					span.setAttribute("will_retry", true);
+					span.setAttribute("throttled", true);
+					span.setStatus({ code: SpanStatusCode.OK, message: "concurrency_limit_reached" });
+
+					this.logger.log(
+						`[concurrency] job ${jobId} key='${error.info.concurrencyKey}' ` +
+							`limit=${error.info.concurrencyLimit} inFlight=${error.info.currentInFlight} → NACK + redelivery`,
+					);
+
+					workerRetries.add(1, {
+						env: process.env.NODE_ENV,
+						queue: config.queue,
+						workflow_name: this.configuration?.name || "unknown",
+						reason: "throttled",
+					});
+
+					await job.fail(error as Error, true);
+					return;
+				}
+
 				const shouldRetry = job.attempts < job.maxRetries;
 
 				// Set span error
@@ -429,8 +561,17 @@ export abstract class WorkerTrigger extends TriggerBase {
 				span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage });
 
 				if (shouldRetry) {
-					// Retry with exponential backoff
-					const backoffMs = this.calculateBackoff(job.attempts, config.delay);
+					// Retry with exponential backoff. `config.delay` widened to
+					// `string | number | undefined` in Tier 2 #5 (duration strings).
+					// `calculateBackoff` only handles numbers; normalize via
+					// tryParseDuration. Fail-open to undefined → default backoff.
+					const delayMs =
+						typeof config.delay === "number"
+							? config.delay
+							: typeof config.delay === "string"
+								? (tryParseDuration(config.delay) ?? undefined)
+								: undefined;
+					const backoffMs = this.calculateBackoff(job.attempts, delayMs);
 					workerRetries.add(1, {
 						env: process.env.NODE_ENV,
 						queue: config.queue,
@@ -492,7 +633,7 @@ export abstract class WorkerTrigger extends TriggerBase {
 	protected calculateBackoff(attempt: number, baseDelay?: number): number {
 		const base = baseDelay ?? 1000;
 		const maxDelay = 30000; // 30 seconds max
-		const exponential = Math.min(base * Math.pow(2, attempt), maxDelay);
+		const exponential = Math.min(base * 2 ** attempt, maxDelay);
 		const jitter = Math.random() * exponential * 0.1; // 10% jitter
 		return Math.floor(exponential + jitter);
 	}