@techstream/quark-core 1.1.0

package/src/queue/index.js

@@ -0,0 +1,214 @@
+/**
+ * @techstream/quark-core - Job Queue Module
+ * Provides BullMQ queue initialization and management
+ */
+
+import { Queue, QueueEvents, Worker } from "bullmq";
+import { ServiceError } from "../errors.js";
+
+/**
+ * Default Redis configuration
+ */
+const DEFAULT_REDIS_CONFIG = {
+	host: process.env.REDIS_HOST || "localhost",
+	port: parseInt(process.env.REDIS_PORT || "6379", 10),
+	db: parseInt(process.env.REDIS_DB || "0", 10),
+	retryStrategy: (times) => {
+		const delay = Math.min(times * 50, 2000);
+		return delay;
+	},
+};
+
+/**
+ * Map to store singleton queue instances
+ */
+const queues = new Map();
+
+/**
+ * Creates or retrieves a singleton BullMQ queue
+ * @param {string} name - Queue name
+ * @param {Object} options - Queue options
+ * @param {Object} options.redis - Redis connection config
+ * @param {Object} options.defaultJobOptions - Default job options
+ * @returns {Queue} BullMQ Queue instance
+ */
+export const createQueue = (name, options = {}) => {
+	if (queues.has(name)) {
+		return queues.get(name);
+	}
+
+	const {
+		redis = DEFAULT_REDIS_CONFIG,
+		defaultJobOptions = {
+			attempts: 3,
+			backoff: {
+				type: "exponential",
+				delay: 2000,
+			},
+			removeOnComplete: true,
+		},
+		...queueOptions
+	} = options;
+
+	const queue = new Queue(name, {
+		connection: redis,
+		defaultJobOptions,
+		...queueOptions,
+	});
+
+	queues.set(name, queue);
+
+	// Clean up on graceful shutdown
+	queue.on("error", (error) => {
+		console.error(`Queue "${name}" error:`, error);
+	});
+
+	return queue;
+};
+
+/**
+ * Creates a job processor worker
+ * @param {string} queueName - Queue name
+ * @param {Function} handler - Job handler function(job) => Promise<any>
+ * @param {Object} options - Worker options
+ * @param {Object} options.redis - Redis connection config
+ * @param {number} options.concurrency - Concurrency level
+ * @returns {Worker} BullMQ Worker instance
+ */
+export const createWorker = (queueName, handler, options = {}) => {
+	const {
+		redis = DEFAULT_REDIS_CONFIG,
+		concurrency = 1,
+		...workerOptions
+	} = options;
+
+	const worker = new Worker(queueName, handler, {
+		connection: redis,
+		concurrency,
+		...workerOptions,
+	});
+
+	worker.on("error", (error) => {
+		console.error(`Worker for queue "${queueName}" error:`, error);
+	});
+
+	worker.on("failed", (job, error) => {
+		console.error(
+			`Job ${job.id} in queue "${queueName}" failed:`,
+			error.message,
+		);
+	});
+
+	return worker;
+};
+
+/**
+ * Gets or creates queue events listener
+ * @param {string} queueName - Queue name
+ * @param {Object} options - QueueEvents options
+ * @returns {QueueEvents} BullMQ QueueEvents instance
+ */
+export const createQueueEvents = (queueName, options = {}) => {
+	const { redis = DEFAULT_REDIS_CONFIG, ...eventsOptions } = options;
+
+	return new QueueEvents(queueName, {
+		connection: redis,
+		...eventsOptions,
+	});
+};
+
+/**
+ * Utility to add a job to a queue with error handling
+ * @param {Queue} queue - BullMQ Queue instance
+ * @param {string} jobName - Job name/type (e.g., 'send-welcome-email')
+ * @param {Object} data - Job data
+ * @param {Object} jobOptions - Job-specific options
+ * @returns {Promise<Job>} Queued job
+ */
+export const addJob = async (queue, jobName, data, jobOptions = {}) => {
+	try {
+		const job = await queue.add(jobName, data, jobOptions);
+		return job;
+	} catch (error) {
+		throw new ServiceError(
+			"BullMQ",
+			`Failed to add job "${jobName}" to queue "${queue.name}": ${error.message}`,
+			500,
+		);
+	}
+};
+
+/**
+ * Utility to get job status and progress
+ * @param {Job} job - BullMQ Job instance
+ * @returns {Promise<Object>} Job status information
+ */
+export const getJobStatus = async (job) => {
+	return {
+		id: job.id,
+		state: await job.getState(),
+		progress: job.progress(),
+		attempts: job.attemptsMade,
+		maxAttempts: job.opts.attempts,
+		data: job.data,
+	};
+};
+
+/**
+ * Clears all jobs from a queue (use with caution!)
+ * @param {Queue} queue - BullMQ Queue instance
+ * @param {Object} options - Clear options
+ * @returns {Promise<void>}
+ */
+export const clearQueue = async (queue, options = {}) => {
+	const { grace = 5000 } = options;
+	try {
+		await queue.clean(grace, 100);
+	} catch (error) {
+		console.error(`Failed to clear queue "${queue.name}":`, error);
+		throw error;
+	}
+};
+
+/**
+ * Gracefully closes all queues, workers, and listeners
+ * Useful for process shutdown
+ * @returns {Promise<void>}
+ */
+export const closeAllQueues = async () => {
+	try {
+		for (const [name, queue] of queues) {
+			await queue.close();
+			console.log(`Queue "${name}" closed`);
+		}
+		queues.clear();
+	} catch (error) {
+		console.error("Failed to close queues:", error);
+		throw error;
+	}
+};
+
+/**
+ * Health check for Redis connectivity
+ * @returns {Promise<boolean>} True if Redis is accessible
+ */
+export const checkQueueHealth = async () => {
+	try {
+		if (queues.size === 0) {
+			// Create a temporary queue to test connectivity
+			const testQueue = new Queue("_health_check", {
+				connection: DEFAULT_REDIS_CONFIG,
+			});
+			await testQueue.client.ping();
+			await testQueue.close();
+			return true;
+		}
+
+		const [firstQueue] = queues.values();
+		await firstQueue.client.ping();
+		return true;
+	} catch (error) {
+		console.error("Queue health check failed:", error);
+		return false;
+	}
+};

package/src/rate-limiter.js

@@ -0,0 +1,253 @@
+/**
+ * @techstream/quark-core - Rate Limiting Module
+ * Provides both in-memory and Redis-based rate limiting
+ */
+
+/**
+ * In-memory rate limiter (for single-instance deployments)
+ */
+class MemoryRateLimiter {
+	constructor() {
+		this.store = new Map();
+		this.cleanupInterval = null;
+	}
+
+	/**
+	 * Start periodic cleanup of expired records
+	 */
+	startCleanup(intervalMs = 60000) {
+		if (this.cleanupInterval) return;
+
+		this.cleanupInterval = setInterval(() => {
+			const now = Date.now();
+			for (const [key, record] of this.store.entries()) {
+				if (now > record.resetTime) {
+					this.store.delete(key);
+				}
+			}
+		}, intervalMs);
+
+		// Don't keep the process alive just for cleanup
+		this.cleanupInterval.unref();
+	}
+
+	/**
+	 * Stop cleanup interval
+	 */
+	stopCleanup() {
+		if (this.cleanupInterval) {
+			clearInterval(this.cleanupInterval);
+			this.cleanupInterval = null;
+		}
+	}
+
+	/**
+	 * Check and increment rate limit
+	 */
+	async checkLimit(key, maxRequests, windowMs) {
+		const now = Date.now();
+		const record = this.store.get(key) || {
+			count: 0,
+			resetTime: now + windowMs,
+		};
+
+		// Reset if window has passed
+		if (now > record.resetTime) {
+			record.count = 0;
+			record.resetTime = now + windowMs;
+		}
+
+		// Check if limit exceeded
+		if (record.count >= maxRequests) {
+			return {
+				limited: true,
+				remaining: 0,
+				resetTime: record.resetTime,
+			};
+		}
+
+		// Increment counter
+		record.count++;
+		this.store.set(key, record);
+
+		return {
+			limited: false,
+			remaining: maxRequests - record.count,
+			resetTime: record.resetTime,
+		};
+	}
+
+	/**
+	 * Reset a specific key
+	 */
+	async reset(key) {
+		this.store.delete(key);
+	}
+
+	/**
+	 * Clear all records
+	 */
+	async clear() {
+		this.store.clear();
+	}
+}
+
+/**
+ * Redis-based rate limiter (for multi-instance deployments)
+ * Uses Lua scripting for atomic check-and-increment.
+ */
+class RedisRateLimiter {
+	constructor(redisClient, options = {}) {
+		this.redis = redisClient;
+		this.failOpen = options.failOpen ?? true;
+	}
+
+	/**
+	 * Check and increment rate limit using an atomic Lua script.
+	 * Prevents TOCTOU race conditions by doing INCR + PEXPIRE in one round-trip.
+	 */
+	async checkLimit(key, maxRequests, windowMs) {
+		const now = Date.now();
+		const windowKey = `ratelimit:${key}`;
+
+		try {
+			// Atomic Lua: INCR the key, set PEXPIRE on first request, return [count, pttl]
+			const luaScript = `
+				local count = redis.call('INCR', KEYS[1])
+				if count == 1 then
+					redis.call('PEXPIRE', KEYS[1], ARGV[1])
+				end
+				local ttl = redis.call('PTTL', KEYS[1])
+				return {count, ttl}
+			`;
+
+			const [currentCount, ttl] = await this.redis.eval(
+				luaScript,
+				1,
+				windowKey,
+				windowMs,
+			);
+
+			const resetTime = ttl > 0 ? now + ttl : now + windowMs;
+
+			if (currentCount > maxRequests) {
+				return {
+					limited: true,
+					remaining: 0,
+					resetTime,
+				};
+			}
+
+			return {
+				limited: false,
+				remaining: maxRequests - currentCount,
+				resetTime,
+			};
+		} catch (error) {
+			// Configurable fail-open / fail-closed behaviour
+			if (this.failOpen) {
+				return {
+					limited: false,
+					remaining: maxRequests,
+					resetTime: now + windowMs,
+				};
+			}
+			return {
+				limited: true,
+				remaining: 0,
+				resetTime: now + windowMs,
+			};
+		}
+	}
+
+	/**
+	 * Reset a specific key
+	 */
+	async reset(key) {
+		await this.redis.del(`ratelimit:${key}`);
+	}
+
+	/**
+	 * Clear all rate limit keys using SCAN (non-blocking, production-safe)
+	 */
+	async clear() {
+		let cursor = "0";
+		do {
+			const [nextCursor, keys] = await this.redis.scan(
+				cursor,
+				"MATCH",
+				"ratelimit:*",
+				"COUNT",
+				100,
+			);
+			cursor = nextCursor;
+			if (keys.length > 0) {
+				await this.redis.del(...keys);
+			}
+		} while (cursor !== "0");
+	}
+}
+
+/**
+ * Create a rate limiter instance
+ * @param {Object} options - Configuration options
+ * @param {string} options.type - "memory" or "redis"
+ * @param {Object} options.redisClient - Redis client instance (required if type is "redis")
+ * @returns {MemoryRateLimiter|RedisRateLimiter}
+ */
+export function createRateLimiter(options = {}) {
+	const { type = "memory", redisClient = null } = options;
+
+	if (type === "redis") {
+		if (!redisClient) {
+			throw new Error("Redis client is required for Redis-based rate limiting");
+		}
+		return new RedisRateLimiter(redisClient, { failOpen: options.failOpen });
+	}
+
+	const limiter = new MemoryRateLimiter();
+	limiter.startCleanup();
+	return limiter;
+}
+
+/**
+ * Rate limiter configuration
+ */
+export const RATE_LIMIT_PRESETS = {
+	strict: {
+		windowMs: 15 * 60 * 1000, // 15 minutes
+		maxRequests: 5,
+	},
+	moderate: {
+		windowMs: 15 * 60 * 1000, // 15 minutes
+		maxRequests: 50,
+	},
+	relaxed: {
+		windowMs: 15 * 60 * 1000, // 15 minutes
+		maxRequests: 100,
+	},
+	auth: {
+		windowMs: 15 * 60 * 1000, // 15 minutes
+		maxRequests: 5,
+	},
+	api: {
+		windowMs: 15 * 60 * 1000, // 15 minutes
+		maxRequests: 100,
+	},
+};
+
+/**
+ * Helper function to create rate limit middleware
+ * @param {Object} limiter - Rate limiter instance
+ * @param {Object} config - Rate limit configuration
+ * @returns {Function} Middleware function
+ */
+export function createRateLimitMiddleware(
+	limiter,
+	config = RATE_LIMIT_PRESETS.api,
+) {
+	return async (ip, path) => {
+		const key = `${ip}:${path}`;
+		return limiter.checkLimit(key, config.maxRequests, config.windowMs);
+	};
+}

package/src/rate-limiter.test.js

@@ -0,0 +1,130 @@
+import assert from "node:assert";
+import { test } from "node:test";
+import {
+	createRateLimiter,
+	createRateLimitMiddleware,
+	RATE_LIMIT_PRESETS,
+} from "../src/rate-limiter.js";
+
+test("Rate Limiter Module", async (t) => {
+	await t.test("createRateLimiter creates memory limiter by default", () => {
+		const limiter = createRateLimiter();
+		assert(limiter !== null);
+	});
+
+	await t.test(
+		"createRateLimiter throws error for Redis without client",
+		() => {
+			assert.throws(
+				() => createRateLimiter({ type: "redis" }),
+				/Redis client is required/,
+			);
+		},
+	);
+
+	await t.test("Memory rate limiter allows requests under limit", async () => {
+		const limiter = createRateLimiter({ type: "memory" });
+		const result = await limiter.checkLimit("test-key", 5, 60000);
+
+		assert.strictEqual(result.limited, false);
+		assert.strictEqual(result.remaining, 4);
+		assert(result.resetTime > Date.now());
+	});
+
+	await t.test("Memory rate limiter blocks requests over limit", async () => {
+		const limiter = createRateLimiter({ type: "memory" });
+
+		// Make 5 requests (limit is 5)
+		for (let i = 0; i < 5; i++) {
+			await limiter.checkLimit("test-key-2", 5, 60000);
+		}
+
+		// 6th request should be blocked
+		const result = await limiter.checkLimit("test-key-2", 5, 60000);
+		assert.strictEqual(result.limited, true);
+		assert.strictEqual(result.remaining, 0);
+	});
+
+	await t.test("Memory rate limiter resets after window", async () => {
+		const limiter = createRateLimiter({ type: "memory" });
+
+		// Make a request with very short window
+		const result1 = await limiter.checkLimit("test-key-3", 1, 10);
+		assert.strictEqual(result1.limited, false);
+
+		// Second request should be blocked
+		const result2 = await limiter.checkLimit("test-key-3", 1, 10);
+		assert.strictEqual(result2.limited, true);
+
+		// Wait for window to expire
+		await new Promise((resolve) => setTimeout(resolve, 20));
+
+		// Should be allowed again
+		const result3 = await limiter.checkLimit("test-key-3", 1, 10);
+		assert.strictEqual(result3.limited, false);
+	});
+
+	await t.test("Memory rate limiter can reset a key", async () => {
+		const limiter = createRateLimiter({ type: "memory" });
+
+		await limiter.checkLimit("test-key-4", 1, 60000);
+		await limiter.reset("test-key-4");
+
+		const result = await limiter.checkLimit("test-key-4", 1, 60000);
+		assert.strictEqual(result.limited, false);
+	});
+
+	await t.test("Memory rate limiter can clear all keys", async () => {
+		const limiter = createRateLimiter({ type: "memory" });
+
+		await limiter.checkLimit("key-1", 1, 60000);
+		await limiter.checkLimit("key-2", 1, 60000);
+		await limiter.clear();
+
+		const result1 = await limiter.checkLimit("key-1", 1, 60000);
+		const result2 = await limiter.checkLimit("key-2", 1, 60000);
+
+		assert.strictEqual(result1.limited, false);
+		assert.strictEqual(result2.limited, false);
+	});
+
+	await t.test("RATE_LIMIT_PRESETS have correct structure", () => {
+		assert(RATE_LIMIT_PRESETS.strict);
+		assert(RATE_LIMIT_PRESETS.moderate);
+		assert(RATE_LIMIT_PRESETS.relaxed);
+		assert(RATE_LIMIT_PRESETS.auth);
+		assert(RATE_LIMIT_PRESETS.api);
+
+		assert.strictEqual(RATE_LIMIT_PRESETS.auth.maxRequests, 5);
+		assert.strictEqual(RATE_LIMIT_PRESETS.api.maxRequests, 100);
+	});
+
+	await t.test("createRateLimitMiddleware returns a function", () => {
+		const limiter = createRateLimiter({ type: "memory" });
+		const middleware = createRateLimitMiddleware(limiter);
+
+		assert.strictEqual(typeof middleware, "function");
+	});
+
+	await t.test("createRateLimitMiddleware works with API preset", async () => {
+		const limiter = createRateLimiter({ type: "memory" });
+		const middleware = createRateLimitMiddleware(
+			limiter,
+			RATE_LIMIT_PRESETS.api,
+		);
+
+		const result = await middleware("192.168.1.1", "/api/posts");
+
+		assert.strictEqual(result.limited, false);
+		assert(result.remaining > 0);
+	});
+
+	await t.test("Cleanup interval is started for memory limiter", () => {
+		const limiter = createRateLimiter({ type: "memory" });
+		assert(limiter.cleanupInterval !== null);
+
+		// Clean up
+		limiter.stopCleanup();
+		assert(limiter.cleanupInterval === null);
+	});
+});

package/src/redis.js

@@ -0,0 +1,96 @@
+/**
+ * Builds REDIS_URL from individual environment variables if not explicitly provided.
+ * Allows configuration via individual REDIS_* vars instead of a single REDIS_URL.
+ */
+function getRedisUrl() {
+	if (process.env.REDIS_URL) {
+		return process.env.REDIS_URL;
+	}
+
+	const host = process.env.REDIS_HOST || "localhost";
+	const port = process.env.REDIS_PORT || "6379";
+
+	return `redis://${host}:${port}`;
+}
+
+/**
+ * Creates a Redis configuration object from environment variables.
+ * Returns the URL and any additional options — does not create an actual connection.
+ * Pass the URL to your Redis library of choice (e.g., ioredis).
+ */
+export const createRedisConfig = (options = {}) => {
+	const redisUrl = getRedisUrl();
+
+	return {
+		url: redisUrl,
+		...options,
+	};
+};
+
+/**
+ * @deprecated Use `createRedisConfig` instead.
+ */
+export const createRedisClient = createRedisConfig;
+
+/**
+ * Pings Redis to verify connectivity.
+ * Dynamically imports ioredis so it doesn't fail at load time if the package is not installed.
+ * Creates a temporary connection, sends PING, expects PONG, then disconnects.
+ *
+ * @param {object} [options]
+ * @param {number} [options.timeout=3000] - Connection/ping timeout in milliseconds.
+ * @returns {Promise<{ status: "ok", latencyMs: number } | { status: "error", message: string }>}
+ */
+export async function pingRedis({ timeout = 3000 } = {}) {
+	/** @type {import("ioredis").default | null} */
+	let client = null;
+
+	try {
+		const { default: Redis } = await import("ioredis");
+
+		const url = getRedisUrl();
+
+		client = new Redis(url, {
+			lazyConnect: true,
+			connectTimeout: timeout,
+			maxRetriesPerRequest: 0,
+			enableReadyCheck: false,
+		});
+
+		await client.connect();
+
+		const start = performance.now();
+		const result = await Promise.race([
+			client.ping(),
+			new Promise((_, reject) =>
+				setTimeout(() => reject(new Error("PING timed out")), timeout),
+			),
+		]);
+
+		const latencyMs = Math.round(performance.now() - start);
+
+		if (result !== "PONG") {
+			return {
+				status: "error",
+				message: `Unexpected PING response: ${result}`,
+			};
+		}
+
+		return { status: "ok", latencyMs };
+	} catch (/** @type {any} */ error) {
+		const message =
+			error?.code === "MODULE_NOT_FOUND" ||
+			error?.code === "ERR_MODULE_NOT_FOUND"
+				? "ioredis is not installed"
+				: (error?.message ?? String(error));
+		return { status: "error", message };
+	} finally {
+		try {
+			await client?.disconnect();
+		} catch {
+			// ignore disconnect errors
+		}
+	}
+}
+
+export { getRedisUrl };