@sockethub/data-layer 1.0.0-alpha.12 → 1.0.0-alpha.14

package/src/job-base.ts

@@ -1,125 +0,0 @@
-import EventEmitter from "node:events";
-import { type Crypto, crypto } from "@sockethub/crypto";
-import type { ActivityStream } from "@sockethub/schemas";
-import IORedis, { type Redis } from "ioredis";
-
-import type { JobDataDecrypted, JobEncrypted, RedisConfig } from "./types.js";
-
-let sharedRedisConnection: Redis | null = null;
-
-/**
- * Creates or returns a shared Redis connection to enable connection pooling.
- * This prevents connection exhaustion under high load by reusing a single
- * connection across all JobQueue and JobWorker instances.
- *
- * @param config - Redis configuration with optional timeout and retry settings
- * @returns Shared Redis connection instance
- */
-export function createIORedisConnection(config: RedisConfig): Redis {
-    if (!sharedRedisConnection) {
-        sharedRedisConnection = new IORedis(config.url, {
-            connectionName: config.connectionName,
-            enableOfflineQueue: false,
-            maxRetriesPerRequest: config.maxRetriesPerRequest ?? null,
-            connectTimeout: config.connectTimeout ?? 10000,
-            disconnectTimeout: config.disconnectTimeout ?? 5000,
-            lazyConnect: false,
-            retryStrategy: (times: number) => {
-                // Stop retrying after 3 attempts to fail fast
-                if (times > 3) return null;
-                // Exponential backoff: 200ms, 400ms, 800ms
-                return Math.min(2 ** (times - 1) * 200, 2000);
-            },
-        });
-    }
-    return sharedRedisConnection;
-}
-
-/**
- * Resets the shared Redis connection. Used primarily for testing.
- * Disconnects the current connection before resetting.
- */
-export async function resetSharedRedisConnection(): Promise<void> {
-    if (sharedRedisConnection) {
-        try {
-            sharedRedisConnection.disconnect(false);
-        } catch (_err) {
-            // Ignore disconnect errors during cleanup
-        }
-        sharedRedisConnection = null;
-    }
-}
-
-/**
- * Gets the total number of active Redis client connections.
- *
- * Note: This queries the Redis server directly using CLIENT LIST and reports
- * ALL active connections to the Redis instance. This includes connections from
- * Sockethub (BullMQ queues, workers, and the shared connection) as well as any
- * other applications or services connected to the same Redis server.
- *
- * @returns Number of active Redis connections, or 0 if no connection exists
- */
-export async function getRedisConnectionCount(): Promise<number> {
-    if (!sharedRedisConnection) {
-        return 0;
-    }
-
-    try {
-        const clientList = (await sharedRedisConnection.client(
-            "LIST",
-        )) as string;
-        // CLIENT LIST returns one line per connection, filter out empty lines
-        const connections = clientList
-            .split("\n")
-            .filter((line: string) => line.trim());
-        return connections.length;
-    } catch (_err) {
-        // Return 0 if Redis query fails (connection issues, etc.)
-        return 0;
-    }
-}
-
-export class JobBase extends EventEmitter {
-    protected crypto: Crypto;
-    private readonly secret: string;
-
-    constructor(secret: string) {
-        super();
-        if (secret.length !== 32) {
-            throw new Error(
-                `secret must be a 32 char string, length: ${secret.length}`,
-            );
-        }
-        this.secret = secret;
-        this.initCrypto();
-    }
-
-    initCrypto() {
-        this.crypto = crypto;
-    }
-
-    disconnectBase() {
-        this.removeAllListeners();
-    }
-
-    /**
-     * @param job
-     * @private
-     */
-    protected decryptJobData(job: JobEncrypted): JobDataDecrypted {
-        return {
-            title: job.data.title,
-            msg: this.decryptActivityStream(job.data.msg) as ActivityStream,
-            sessionId: job.data.sessionId,
-        };
-    }
-
-    protected decryptActivityStream(msg: string): ActivityStream {
-        return this.crypto.decrypt(msg, this.secret);
-    }
-
-    protected encryptActivityStream(msg: ActivityStream): string {
-        return this.crypto.encrypt(msg, this.secret);
-    }
-}

package/src/job-queue.test.ts

@@ -1,295 +0,0 @@
-import { expect } from "chai";
-import * as sinon from "sinon";
-
-import type { Logger } from "@sockethub/schemas";
-
-import { JobQueue } from "./index";
-
-const mockLogger: Logger = {
-    error: () => {},
-    warn: () => {},
-    info: () => {},
-    debug: () => {},
-};
-
-describe("JobQueue", () => {
-    let MockBull, jobQueue, cryptoMocks, sandbox;
-
-    beforeEach(() => {
-        sandbox = sinon.createSandbox();
-        cryptoMocks = {
-            objectHash: sandbox.stub(),
-            decrypt: sandbox.stub(),
-            encrypt: sandbox.stub(),
-            hash: sandbox.stub(),
-        };
-        MockBull = sandbox.stub().returns({
-            add: sandbox.stub(),
-            getJob: sandbox.stub(),
-            removeAllListeners: sandbox.stub(),
-            pause: sandbox.stub(),
-            resume: sandbox.stub(),
-            isPaused: sandbox.stub(),
-            obliterate: sandbox.stub(),
-            isRunning: sandbox.stub(),
-            close: sandbox.stub(),
-            emit: sandbox.stub(),
-            disconnect: sandbox.stub(),
-            on: sandbox.stub().callsArgWith(1, "a job id", "a result string"),
-        });
-
-        class TestJobQueue extends JobQueue {
-            init() {
-                // BullMQ v5+ prohibits colons in queue names, so replace with dashes
-                const queueName = this.queueId.replace(/:/g, "-");
-                this.redisConnection = MockBull();
-                this.queue = MockBull(queueName, {
-                    connection: this.redisConnection,
-                });
-                this.events = MockBull(queueName, {
-                    connection: this.redisConnection,
-                });
-            }
-            initCrypto() {
-                this.crypto = cryptoMocks;
-            }
-            getQueueId() {
-                return this.queueId;
-            }
-        }
-        jobQueue = new TestJobQueue(
-            "a parent id",
-            "a session id",
-            "secret is 32 char long like this",
-            {
-                url: "redis config",
-            },
-            mockLogger,
-        );
-        jobQueue.emit = sandbox.stub();
-    });
-
-    afterEach(() => {
-        sinon.restore();
-        sandbox.reset();
-    });
-
-    it("returns a valid JobQueue object", () => {
-        sinon.assert.calledThrice(MockBull);
-        sinon.assert.calledWith(
-            MockBull,
-            "sockethub-a parent id-data-layer-queue-a session id",
-            {
-                connection: MockBull(),
-            },
-        );
-        expect(typeof jobQueue).to.equal("object");
-        expect(jobQueue.getQueueId()).to.equal(
-            `sockethub:a parent id:data-layer:queue:a session id`,
-        );
-        expect(typeof jobQueue.add).to.equal("function");
-        expect(typeof jobQueue.getJob).to.equal("function");
-        expect(typeof jobQueue.shutdown).to.equal("function");
-    });
-
-    // describe("initResultEvents", () => {
-    //     it("registers handlers when called", () => {
-    //         bullMocks.on.reset();
-    //         // jobQueue.initResultEvents();
-    //         expect(bullMocks.on.callCount).to.eql(4);
-    //         sinon.assert.calledWith(bullMocks.on, "global:completed");
-    //         sinon.assert.calledWith(bullMocks.on, "global:error");
-    //         sinon.assert.calledWith(bullMocks.on, "global:failed");
-    //         sinon.assert.calledWith(bullMocks.on, "failed");
-    //     });
-    // });
-
-    describe("createJob", () => {
-        it("returns expected job format", () => {
-            cryptoMocks.encrypt.returns("an encrypted message");
-            const job = jobQueue.createJob("a socket id", {
-                platform: "some context",
-                id: "an identifier",
-            });
-            expect(job).to.eql({
-                title: "some context-an identifier",
-                msg: "an encrypted message",
-                sessionId: "a socket id",
-            });
-        });
-
-        it("uses counter when no id provided", () => {
-            cryptoMocks.encrypt.returns("an encrypted message");
-            let job = jobQueue.createJob("a socket id", {
-                platform: "some context",
-            });
-            expect(job).to.eql({
-                title: "some context-0",
-                msg: "an encrypted message",
-                sessionId: "a socket id",
-            });
-            job = jobQueue.createJob("a socket id", {
-                platform: "some context",
-            });
-            expect(job).to.eql({
-                title: "some context-1",
-                msg: "an encrypted message",
-                sessionId: "a socket id",
-            });
-        });
-    });
-
-    describe("getJob", () => {
-        const encryptedJob = {
-            data: {
-                title: "a title",
-                msg: "an encrypted msg",
-                sessionId: "a socket id",
-            },
-        };
-
-        it("handles fetching a valid job", async () => {
-            jobQueue.queue.getJob.returns(encryptedJob);
-            cryptoMocks.decrypt.returns("an unencrypted message");
-            const job = await jobQueue.getJob("a valid job");
-            sinon.assert.calledOnceWithExactly(
-                jobQueue.queue.getJob,
-                "a valid job",
-            );
-            encryptedJob.data.msg = "an unencrypted message";
-            expect(job).to.eql(encryptedJob);
-        });
-
-        it("handles fetching an invalid job", async () => {
-            jobQueue.queue.getJob.returns(undefined);
-            const job = await jobQueue.getJob("an invalid job");
-            expect(job).to.eql(undefined);
-            sinon.assert.calledOnceWithExactly(
-                jobQueue.queue.getJob,
-                "an invalid job",
-            );
-            sinon.assert.notCalled(cryptoMocks.decrypt);
-        });
-
-        it("removes sessionSecret", async () => {
-            jobQueue.queue.getJob.returns(encryptedJob);
-            cryptoMocks.decrypt.returns({
-                foo: "bar",
-                sessionSecret: "yarg",
-            });
-            const job = await jobQueue.getJob("a valid job");
-            sinon.assert.calledOnceWithExactly(
-                jobQueue.queue.getJob,
-                "a valid job",
-            );
-            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-            // @ts-ignore
-            encryptedJob.data.msg = {
-                foo: "bar",
-            };
-            expect(job).to.eql(encryptedJob);
-        });
-    });
-
-    describe("add", () => {
-        it("stores encrypted job", async () => {
-            cryptoMocks.encrypt.returns("encrypted foo");
-            jobQueue.queue.isPaused.returns(false);
-            const resultJob = {
-                title: "a platform-an identifier",
-                sessionId: "a socket id",
-                msg: "encrypted foo",
-            };
-            const res = await jobQueue.add("a socket id", {
-                platform: "a platform",
-                id: "an identifier",
-            });
-            sinon.assert.calledOnce(jobQueue.queue.isPaused);
-            sinon.assert.notCalled(jobQueue.queue.emit);
-            sinon.assert.calledOnceWithExactly(
-                jobQueue.queue.add,
-                "a platform-an identifier",
-                resultJob,
-                {
-                    removeOnComplete: { age: 300 },
-                    removeOnFail: { age: 300 },
-                },
-            );
-            expect(res).to.eql(resultJob);
-        });
-        it("fails job if queue paused", async () => {
-            cryptoMocks.encrypt.returns("encrypted foo");
-            jobQueue.queue.isPaused.returns(true);
-            try {
-                await jobQueue.add("a socket id", {
-                    platform: "a platform",
-                    id: "an identifier",
-                });
-            } catch (err) {
-                expect(err.toString()).to.eql("Error: queue closed");
-            }
-            sinon.assert.calledOnce(jobQueue.queue.isPaused);
-            sinon.assert.notCalled(jobQueue.queue.add);
-        });
-    });
-
-    it("pause", async () => {
-        await jobQueue.pause();
-        sinon.assert.calledOnce(jobQueue.queue.pause);
-    });
-
-    it("resume", async () => {
-        await jobQueue.resume();
-        sinon.assert.calledOnce(jobQueue.queue.resume);
-    });
-
-    describe("shutdown", () => {
-        // it("is sure to pause when not already paused", async () => {
-        //     sinon.assert.notCalled(jobQueue.queue.pause);
-        //     jobQueue.initWorker();
-        //     sinon.assert.notCalled(jobQueue.queue.pause);
-        //     jobQueue.queue.isPaused.returns(false);
-        //     sinon.assert.notCalled(jobQueue.queue.pause);
-        //     await jobQueue.shutdown();
-        //     sinon.assert.calledOnce(jobQueue.queue.pause);
-        //     sinon.assert.calledOnce(jobQueue.queue.removeAllListeners);
-        //     sinon.assert.calledOnce(jobQueue.queue.obliterate);
-        // });
-        it("skips pausing when already paused", async () => {
-            jobQueue.queue.isPaused.returns(true);
-            sinon.assert.notCalled(jobQueue.queue.pause);
-            await jobQueue.shutdown();
-            sinon.assert.notCalled(jobQueue.queue.pause);
-            sinon.assert.calledOnce(jobQueue.queue.removeAllListeners);
-            sinon.assert.calledOnce(jobQueue.queue.obliterate);
-        });
-    });
-
-    describe("decryptJobData", () => {
-        it("decrypts and returns expected object", () => {
-            cryptoMocks.decrypt.returnsArg(0);
-            const jobData = {
-                data: {
-                    title: "foo",
-                    msg: "encryptedjobdata",
-                    sessionId: "foobar",
-                },
-            };
-            const secret = "secretstring";
-            expect(jobQueue.decryptJobData(jobData, secret)).to.be.eql(
-                jobData.data,
-            );
-        });
-    });
-
-    describe("decryptActivityStream", () => {
-        it("decrypts and returns expected object", () => {
-            cryptoMocks.decrypt.returnsArg(0);
-            const jobData = "encryptedjobdata";
-            const secret = "secretstring";
-            expect(jobQueue.decryptActivityStream(jobData, secret)).to.be.eql(
-                jobData,
-            );
-        });
-    });
-});

package/src/job-queue.ts

@@ -1,247 +0,0 @@
-import {
-    createLogger,
-    getLoggerNamespace,
-    type Logger,
-} from "@sockethub/logger";
-import { type ActivityStream, resolvePlatformId } from "@sockethub/schemas";
-import { type Job, Queue, QueueEvents, Worker } from "bullmq";
-
-import { JobBase } from "./job-base.js";
-import { buildQueueId } from "./queue-id.js";
-import type { JobDataEncrypted, JobDecrypted, RedisConfig } from "./types.js";
-
-export async function verifyJobQueue(config: RedisConfig): Promise<void> {
-    const log = createLogger("data-layer:verify-job-queue");
-
-    return new Promise((resolve, reject) => {
-        const worker = new Worker(
-            "connectiontest",
-            async (job) => {
-                if (job.name !== "foo" || job.data?.foo !== "bar") {
-                    reject(
-                        "Worker received invalid job data during JobQueue connection test",
-                    );
-                }
-                job.data.test = "touched by worker";
-            },
-            {
-                connection: config,
-            },
-        );
-        worker.on("completed", async (job: Job) => {
-            if (job.name !== "foo" || job.data?.test !== "touched by worker") {
-                reject(
-                    "Worker job completed unsuccessfully during JobQueue connection test",
-                );
-            }
-            log.info("job queue connection verified");
-            await queue.close();
-            await worker.close();
-            resolve();
-        });
-        worker.on("error", (err) => {
-            log.warn(
-                `connection verification worker error received ${err.toString()}`,
-            );
-            reject(err);
-        });
-        const queue = new Queue("connectiontest", {
-            connection: config,
-        });
-        queue.on("error", (err) => {
-            log.warn(
-                `connection verification queue error received ${err.toString()}`,
-            );
-            reject(err);
-        });
-        queue.add(
-            "foo",
-            { foo: "bar" },
-            { removeOnComplete: true, removeOnFail: true },
-        );
-    });
-}
-
-/**
- * Redis-backed job queue for managing ActivityStreams message processing.
- *
- * Creates isolated queues per platform instance and session, providing reliable
- * message delivery and processing coordination between Sockethub server and
- * platform workers.
- *
- * @example
- * ```typescript
- * const queue = new JobQueue('irc-platform', 'session123', secret, redisConfig);
- * await queue.add('socket-id', activityStreamMessage);
- * ```
- */
-export class JobQueue extends JobBase {
-    private readonly connectionName: string;
-    protected readonly queueId: string;
-    protected queue: Queue;
-    protected events: QueueEvents;
-    private readonly log: Logger;
-    private counter = 0;
-    private initialized = false;
-
-    /**
-     * Creates a new JobQueue instance.
-     *
-     * @param parentId - Sockethub instance identifier for queue isolation
-     * @param instanceId - Unique identifier for the platform instance
-     * @param secret - 32-character encryption secret for message security
-     * @param redisConfig - Redis connection configuration
-     */
-    constructor(
-        parentId: string,
-        instanceId: string,
-        secret: string,
-        redisConfig: RedisConfig,
-    ) {
-        super(secret);
-        // Create logger with full namespace (context will be prepended automatically)
-        this.log = createLogger(`data-layer:queue:${parentId}:${instanceId}`);
-
-        this.queueId = buildQueueId(parentId, instanceId);
-        // Use logger's full namespace (includes context) for Redis connection name
-        this.connectionName = getLoggerNamespace(this.log);
-        redisConfig.connectionName = this.connectionName;
-        this.init(redisConfig);
-    }
-
-    protected init(redisConfig: RedisConfig) {
-        if (this.initialized) {
-            throw new Error(`JobQueue already initialized for ${this.queueId}`);
-        }
-        this.initialized = true;
-
-        // BullMQ v5+ prohibits colons in queue names; derive the queue name
-        // from the canonical queue id by replacing ':' with '-'.
-        const queueName = this.queueId.replace(/:/g, "-");
-        // Let BullMQ create its own connections (it duplicates them internally anyway)
-        this.queue = new Queue(queueName, {
-            connection: redisConfig,
-        });
-        this.events = new QueueEvents(queueName, {
-            connection: redisConfig,
-        });
-
-        // Handle Redis contention errors (e.g., BUSY from Lua scripts)
-        this.queue.on("error", (err) => {
-            this.log.warn(`queue error: ${err.message}`);
-        });
-
-        this.events.on("error", (err) => {
-            this.log.warn(`events error: ${err.message}`);
-        });
-
-        this.events.on("completed", async ({ jobId, returnvalue }) => {
-            const job = await this.getJob(jobId);
-            if (!job) {
-                this.log.debug(`completed job ${jobId} (already removed)`);
-                return;
-            }
-            this.log.debug(`completed ${job.data.title} ${job.data.msg.type}`);
-            this.emit("completed", job.data, returnvalue);
-        });
-
-        this.events.on("failed", async ({ jobId, failedReason }) => {
-            const job = await this.getJob(jobId);
-            if (!job) {
-                this.log.debug(
-                    `failed job ${jobId} (already removed): ${failedReason}`,
-                );
-                return;
-            }
-            this.log.warn(
-                `failed ${job.data.title} ${job.data.msg.type}: ${failedReason}`,
-            );
-            this.emit("failed", job.data, failedReason);
-        });
-        this.log.info("initialized");
-    }
-
-    /**
-     * Adds an ActivityStreams message to the job queue for processing.
-     *
-     * @param socketId - Socket.IO connection identifier for response routing
-     * @param msg - ActivityStreams message to be processed by platform worker
-     * @returns Promise resolving to encrypted job data
-     * @throws Error if queue is closed or Redis connection fails
-     */
-    async add(
-        socketId: string,
-        msg: ActivityStream,
-    ): Promise<JobDataEncrypted> {
-        const job = this.createJob(socketId, msg);
-        if (await this.queue.isPaused()) {
-            // this.queue.emit("error", new Error("queue closed"));
-            this.log.debug(
-                `failed to add ${job.title} ${msg.type} to queue: queue closed`,
-            );
-            throw new Error("queue closed");
-        }
-        await this.queue.add(job.title, job, {
-            // Auto-remove jobs after 5 minutes to prevent Redis memory buildup.
-            // Jobs only need to exist long enough for event handlers to look them up
-            // by jobId and send results to clients (typically < 1 second).
-            removeOnComplete: { age: 300 }, // 5 minutes in seconds
-            removeOnFail: { age: 300 },
-        });
-        this.log.debug(`added ${job.title} ${msg.type} to queue`);
-        return job;
-    }
-
-    /**
-     * Pauses job processing. New jobs can still be added but won't be processed.
-     */
-    async pause() {
-        await this.queue.pause();
-        this.log.debug("paused");
-    }
-
-    /**
-     * Resumes job processing after being paused.
-     */
-    async resume() {
-        await this.queue.resume();
-        this.log.debug("resumed");
-    }
-
-    /**
-     * Gracefully shuts down the queue, cleaning up all resources and connections.
-     */
-    async shutdown() {
-        this.removeAllListeners();
-        this.queue.removeAllListeners();
-        if (!(await this.queue.isPaused())) {
-            await this.queue.pause();
-        }
-        await this.queue.obliterate({ force: true });
-        await this.queue.close();
-        await this.events.close();
-    }
-
-    private async getJob(jobId: string): Promise<JobDecrypted> {
-        const job = await this.queue.getJob(jobId);
-        if (job) {
-            job.data = this.decryptJobData(job);
-            try {
-                delete job.data.msg.sessionSecret;
-            } catch (_e) {
-                // this property should never be exposed externally
-            }
-        }
-        return job;
-    }
-
-    private createJob(socketId: string, msg: ActivityStream): JobDataEncrypted {
-        const platformId = resolvePlatformId(msg) || "unknown";
-        const title = `${platformId}-${msg.id ? msg.id : this.counter++}`;
-        return {
-            title: title,
-            sessionId: socketId,
-            msg: this.encryptActivityStream(msg),
-        };
-    }
-}