@sockethub/data-layer 1.0.0-alpha.10

package/src/job-queue.test.ts

@@ -0,0 +1,280 @@
+import { expect } from "chai";
+import * as sinon from "sinon";
+
+import { JobQueue } from "./index";
+
+describe("JobQueue", () => {
+    let MockBull, jobQueue, cryptoMocks, sandbox;
+
+    beforeEach(() => {
+        sandbox = sinon.createSandbox();
+        cryptoMocks = {
+            objectHash: sandbox.stub(),
+            decrypt: sandbox.stub(),
+            encrypt: sandbox.stub(),
+            hash: sandbox.stub(),
+        };
+        MockBull = sandbox.stub().returns({
+            add: sandbox.stub(),
+            getJob: sandbox.stub(),
+            removeAllListeners: sandbox.stub(),
+            pause: sandbox.stub(),
+            resume: sandbox.stub(),
+            isPaused: sandbox.stub(),
+            obliterate: sandbox.stub(),
+            isRunning: sandbox.stub(),
+            close: sandbox.stub(),
+            emit: sandbox.stub(),
+            disconnect: sandbox.stub(),
+            on: sandbox.stub().callsArgWith(1, "a job id", "a result string"),
+        });
+
+        class TestJobQueue extends JobQueue {
+            init() {
+                this.redisConnection = MockBull();
+                this.queue = MockBull(this.uid, {
+                    connection: this.redisConnection,
+                });
+                this.events = MockBull(this.uid, {
+                    connection: this.redisConnection,
+                });
+            }
+            initCrypto() {
+                this.crypto = cryptoMocks;
+            }
+        }
+        jobQueue = new TestJobQueue(
+            "a parent id",
+            "a session id",
+            "secret is 32 char long like this",
+            {
+                url: "redis config",
+            },
+        );
+        jobQueue.emit = sandbox.stub();
+    });
+
+    afterEach(() => {
+        sinon.restore();
+        sandbox.reset();
+    });
+
+    it("returns a valid JobQueue object", () => {
+        sinon.assert.calledThrice(MockBull);
+        sinon.assert.calledWith(
+            MockBull,
+            "sockethub:data-layer:queue:a parent id:a session id",
+            {
+                connection: MockBull(),
+            },
+        );
+        expect(typeof jobQueue).to.equal("object");
+        expect(jobQueue.uid).to.equal(
+            `sockethub:data-layer:queue:a parent id:a session id`,
+        );
+        expect(typeof jobQueue.add).to.equal("function");
+        expect(typeof jobQueue.getJob).to.equal("function");
+        expect(typeof jobQueue.shutdown).to.equal("function");
+    });
+
+    // describe("initResultEvents", () => {
+    //     it("registers handlers when called", () => {
+    //         bullMocks.on.reset();
+    //         // jobQueue.initResultEvents();
+    //         expect(bullMocks.on.callCount).to.eql(4);
+    //         sinon.assert.calledWith(bullMocks.on, "global:completed");
+    //         sinon.assert.calledWith(bullMocks.on, "global:error");
+    //         sinon.assert.calledWith(bullMocks.on, "global:failed");
+    //         sinon.assert.calledWith(bullMocks.on, "failed");
+    //     });
+    // });
+
+    describe("createJob", () => {
+        it("returns expected job format", () => {
+            cryptoMocks.encrypt.returns("an encrypted message");
+            const job = jobQueue.createJob("a socket id", {
+                context: "some context",
+                id: "an identifier",
+            });
+            expect(job).to.eql({
+                title: "some context-an identifier",
+                msg: "an encrypted message",
+                sessionId: "a socket id",
+            });
+        });
+
+        it("uses counter when no id provided", () => {
+            cryptoMocks.encrypt.returns("an encrypted message");
+            let job = jobQueue.createJob("a socket id", {
+                context: "some context",
+            });
+            expect(job).to.eql({
+                title: "some context-0",
+                msg: "an encrypted message",
+                sessionId: "a socket id",
+            });
+            job = jobQueue.createJob("a socket id", {
+                context: "some context",
+            });
+            expect(job).to.eql({
+                title: "some context-1",
+                msg: "an encrypted message",
+                sessionId: "a socket id",
+            });
+        });
+    });
+
+    describe("getJob", () => {
+        const encryptedJob = {
+            data: {
+                title: "a title",
+                msg: "an encrypted msg",
+                sessionId: "a socket id",
+            },
+        };
+
+        it("handles fetching a valid job", async () => {
+            jobQueue.queue.getJob.returns(encryptedJob);
+            cryptoMocks.decrypt.returns("an unencrypted message");
+            const job = await jobQueue.getJob("a valid job");
+            sinon.assert.calledOnceWithExactly(
+                jobQueue.queue.getJob,
+                "a valid job",
+            );
+            encryptedJob.data.msg = "an unencrypted message";
+            expect(job).to.eql(encryptedJob);
+        });
+
+        it("handles fetching an invalid job", async () => {
+            jobQueue.queue.getJob.returns(undefined);
+            const job = await jobQueue.getJob("an invalid job");
+            expect(job).to.eql(undefined);
+            sinon.assert.calledOnceWithExactly(
+                jobQueue.queue.getJob,
+                "an invalid job",
+            );
+            sinon.assert.notCalled(cryptoMocks.decrypt);
+        });
+
+        it("removes sessionSecret", async () => {
+            jobQueue.queue.getJob.returns(encryptedJob);
+            cryptoMocks.decrypt.returns({
+                foo: "bar",
+                sessionSecret: "yarg",
+            });
+            const job = await jobQueue.getJob("a valid job");
+            sinon.assert.calledOnceWithExactly(
+                jobQueue.queue.getJob,
+                "a valid job",
+            );
+            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+            // @ts-ignore
+            encryptedJob.data.msg = {
+                foo: "bar",
+            };
+            expect(job).to.eql(encryptedJob);
+        });
+    });
+
+    describe("add", () => {
+        it("stores encrypted job", async () => {
+            cryptoMocks.encrypt.returns("encrypted foo");
+            jobQueue.queue.isPaused.returns(false);
+            const resultJob = {
+                title: "a platform-an identifier",
+                sessionId: "a socket id",
+                msg: "encrypted foo",
+            };
+            const res = await jobQueue.add("a socket id", {
+                context: "a platform",
+                id: "an identifier",
+            });
+            sinon.assert.calledOnce(jobQueue.queue.isPaused);
+            sinon.assert.notCalled(jobQueue.queue.emit);
+            sinon.assert.calledOnceWithExactly(
+                jobQueue.queue.add,
+                "a platform-an identifier",
+                resultJob,
+                {
+                    removeOnComplete: { age: 300 },
+                    removeOnFail: { age: 300 },
+                },
+            );
+            expect(res).to.eql(resultJob);
+        });
+        it("fails job if queue paused", async () => {
+            cryptoMocks.encrypt.returns("encrypted foo");
+            jobQueue.queue.isPaused.returns(true);
+            try {
+                await jobQueue.add("a socket id", {
+                    context: "a platform",
+                    id: "an identifier",
+                });
+            } catch (err) {
+                expect(err.toString()).to.eql("Error: queue closed");
+            }
+            sinon.assert.calledOnce(jobQueue.queue.isPaused);
+            sinon.assert.notCalled(jobQueue.queue.add);
+        });
+    });
+
+    it("pause", async () => {
+        await jobQueue.pause();
+        sinon.assert.calledOnce(jobQueue.queue.pause);
+    });
+
+    it("resume", async () => {
+        await jobQueue.resume();
+        sinon.assert.calledOnce(jobQueue.queue.resume);
+    });
+
+    describe("shutdown", () => {
+        // it("is sure to pause when not already paused", async () => {
+        //     sinon.assert.notCalled(jobQueue.queue.pause);
+        //     jobQueue.initWorker();
+        //     sinon.assert.notCalled(jobQueue.queue.pause);
+        //     jobQueue.queue.isPaused.returns(false);
+        //     sinon.assert.notCalled(jobQueue.queue.pause);
+        //     await jobQueue.shutdown();
+        //     sinon.assert.calledOnce(jobQueue.queue.pause);
+        //     sinon.assert.calledOnce(jobQueue.queue.removeAllListeners);
+        //     sinon.assert.calledOnce(jobQueue.queue.obliterate);
+        // });
+        it("skips pausing when already paused", async () => {
+            jobQueue.queue.isPaused.returns(true);
+            sinon.assert.notCalled(jobQueue.queue.pause);
+            await jobQueue.shutdown();
+            sinon.assert.notCalled(jobQueue.queue.pause);
+            sinon.assert.calledOnce(jobQueue.queue.removeAllListeners);
+            sinon.assert.calledOnce(jobQueue.queue.obliterate);
+        });
+    });
+
+    describe("decryptJobData", () => {
+        it("decrypts and returns expected object", () => {
+            cryptoMocks.decrypt.returnsArg(0);
+            const jobData = {
+                data: {
+                    title: "foo",
+                    msg: "encryptedjobdata",
+                    sessionId: "foobar",
+                },
+            };
+            const secret = "secretstring";
+            expect(jobQueue.decryptJobData(jobData, secret)).to.be.eql(
+                jobData.data,
+            );
+        });
+    });
+
+    describe("decryptActivityStream", () => {
+        it("decrypts and returns expected object", () => {
+            cryptoMocks.decrypt.returnsArg(0);
+            const jobData = "encryptedjobdata";
+            const secret = "secretstring";
+            expect(jobQueue.decryptActivityStream(jobData, secret)).to.be.eql(
+                jobData,
+            );
+        });
+    });
+});

package/src/job-queue.ts

@@ -0,0 +1,227 @@
+import { type Job, Queue, QueueEvents, Worker } from "bullmq";
+import debug, { type Debugger } from "debug";
+
+import type { ActivityStream } from "@sockethub/schemas";
+
+import { JobBase, createIORedisConnection } from "./job-base.js";
+import type { JobDataEncrypted, JobDecrypted, RedisConfig } from "./types.js";
+
+export async function verifyJobQueue(config: RedisConfig): Promise<void> {
+    const log = debug("sockethub:data-layer:queue");
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(
+            "connectiontest",
+            async (job) => {
+                if (job.name !== "foo" || job.data?.foo !== "bar") {
+                    reject(
+                        "Worker received invalid job data during JobQueue connection test",
+                    );
+                }
+                job.data.test = "touched by worker";
+            },
+            {
+                connection: createIORedisConnection(config),
+            },
+        );
+        worker.on("completed", async (job: Job) => {
+            if (job.name !== "foo" || job.data?.test !== "touched by worker") {
+                reject(
+                    "Worker job completed unsuccessfully during JobQueue connection test",
+                );
+            }
+            log("connection verified");
+            await queue.close();
+            await worker.close();
+            resolve();
+        });
+        worker.on("error", (err) => {
+            log(
+                `connection verification worker error received ${err.toString()}`,
+            );
+            reject(err);
+        });
+        const queue = new Queue("connectiontest", {
+            connection: createIORedisConnection(config),
+        });
+        queue.on("error", (err) => {
+            log(
+                `connection verification queue error received ${err.toString()}`,
+            );
+            reject(err);
+        });
+        queue.add(
+            "foo",
+            { foo: "bar" },
+            { removeOnComplete: true, removeOnFail: true },
+        );
+    });
+}
+
+/**
+ * Redis-backed job queue for managing ActivityStreams message processing.
+ *
+ * Creates isolated queues per platform instance and session, providing reliable
+ * message delivery and processing coordination between Sockethub server and
+ * platform workers.
+ *
+ * @example
+ * ```typescript
+ * const queue = new JobQueue('irc-platform', 'session123', secret, redisConfig);
+ * await queue.add('socket-id', activityStreamMessage);
+ * ```
+ */
+export class JobQueue extends JobBase {
+    readonly uid: string;
+    protected queue: Queue;
+    protected events: QueueEvents;
+    private readonly debug: Debugger;
+    private counter = 0;
+    private initialized = false;
+
+    /**
+     * Creates a new JobQueue instance.
+     *
+     * @param instanceId - Unique identifier for the platform instance
+     * @param sessionId - Client session identifier for queue isolation
+     * @param secret - 32-character encryption secret for message security
+     * @param redisConfig - Redis connection configuration
+     */
+    constructor(
+        instanceId: string,
+        sessionId: string,
+        secret: string,
+        redisConfig: RedisConfig,
+    ) {
+        super(secret);
+        this.uid = `sockethub:data-layer:queue:${instanceId}:${sessionId}`;
+        this.debug = debug(this.uid);
+        this.init(redisConfig);
+    }
+
+    protected init(redisConfig: RedisConfig) {
+        if (this.initialized) {
+            throw new Error(`JobQueue already initialized for ${this.uid}`);
+        }
+        this.initialized = true;
+
+        // BullMQ v5+ prohibits colons in queue names, so replace with dashes
+        // while keeping uid with colons for debug namespace convention
+        const queueName = this.uid.replace(/:/g, "-");
+        // Let BullMQ create its own connections for better lifecycle management
+        this.queue = new Queue(queueName, {
+            connection: redisConfig,
+        });
+        this.events = new QueueEvents(queueName, {
+            connection: redisConfig,
+        });
+
+        this.events.on("completed", async ({ jobId, returnvalue }) => {
+            const job = await this.getJob(jobId);
+            if (!job) {
+                this.debug(`completed job ${jobId} (already removed)`);
+                return;
+            }
+            this.debug(`completed ${job.data.title} ${job.data.msg.type}`);
+            this.emit("completed", job.data, returnvalue);
+        });
+
+        this.events.on("failed", async ({ jobId, failedReason }) => {
+            const job = await this.getJob(jobId);
+            if (!job) {
+                this.debug(
+                    `failed job ${jobId} (already removed): ${failedReason}`,
+                );
+                return;
+            }
+            this.debug(
+                `failed ${job.data.title} ${job.data.msg.type}: ${failedReason}`,
+            );
+            this.emit("failed", job.data, failedReason);
+        });
+        this.debug("initialized");
+    }
+
+    /**
+     * Adds an ActivityStreams message to the job queue for processing.
+     *
+     * @param socketId - Socket.IO connection identifier for response routing
+     * @param msg - ActivityStreams message to be processed by platform worker
+     * @returns Promise resolving to encrypted job data
+     * @throws Error if queue is closed or Redis connection fails
+     */
+    async add(
+        socketId: string,
+        msg: ActivityStream,
+    ): Promise<JobDataEncrypted> {
+        const job = this.createJob(socketId, msg);
+        if (await this.queue.isPaused()) {
+            // this.queue.emit("error", new Error("queue closed"));
+            this.debug(
+                `failed to add ${job.title} ${msg.type} to queue: queue closed`,
+            );
+            throw new Error("queue closed");
+        }
+        await this.queue.add(job.title, job, {
+            // Auto-remove jobs after 5 minutes to prevent Redis memory buildup.
+            // Jobs only need to exist long enough for event handlers to look them up
+            // by jobId and send results to clients (typically < 1 second).
+            removeOnComplete: { age: 300 }, // 5 minutes in seconds
+            removeOnFail: { age: 300 },
+        });
+        this.debug(`added ${job.title} ${msg.type} to queue`);
+        return job;
+    }
+
+    /**
+     * Pauses job processing. New jobs can still be added but won't be processed.
+     */
+    async pause() {
+        await this.queue.pause();
+        this.debug("paused");
+    }
+
+    /**
+     * Resumes job processing after being paused.
+     */
+    async resume() {
+        await this.queue.resume();
+        this.debug("resumed");
+    }
+
+    /**
+     * Gracefully shuts down the queue, cleaning up all resources and connections.
+     */
+    async shutdown() {
+        this.removeAllListeners();
+        this.queue.removeAllListeners();
+        if (!(await this.queue.isPaused())) {
+            await this.queue.pause();
+        }
+        await this.queue.obliterate({ force: true });
+        await this.queue.close();
+        await this.events.close();
+    }
+
+    private async getJob(jobId: string): Promise<JobDecrypted> {
+        const job = await this.queue.getJob(jobId);
+        if (job) {
+            job.data = this.decryptJobData(job);
+            try {
+                // biome-ignore lint/performance/noDelete: <explanation>
+                delete job.data.msg.sessionSecret;
+            } catch (e) {
+                // this property should never be exposed externally
+            }
+        }
+        return job;
+    }
+
+    private createJob(socketId: string, msg: ActivityStream): JobDataEncrypted {
+        const title = `${msg.context}-${msg.id ? msg.id : this.counter++}`;
+        return {
+            title: title,
+            sessionId: socketId,
+            msg: this.encryptActivityStream(msg),
+        };
+    }
+}

package/src/job-worker.test.ts

@@ -0,0 +1,128 @@
+import { expect } from "chai";
+import * as sinon from "sinon";
+
+import { JobWorker } from "./index";
+
+describe("JobWorker", () => {
+    let MockBull, jobWorker, cryptoMocks, sandbox;
+
+    beforeEach(() => {
+        sandbox = sinon.createSandbox();
+        cryptoMocks = {
+            objectHash: sandbox.stub(),
+            decrypt: sandbox.stub(),
+            encrypt: sandbox.stub(),
+            hash: sandbox.stub(),
+        };
+        MockBull = sandbox.stub().returns({
+            add: sandbox.stub(),
+            getJob: sandbox.stub(),
+            removeAllListeners: sandbox.stub(),
+            pause: sandbox.stub(),
+            resume: sandbox.stub(),
+            isPaused: sandbox.stub(),
+            obliterate: sandbox.stub(),
+            isRunning: sandbox.stub(),
+            close: sandbox.stub(),
+            emit: sandbox.stub(),
+            disconnect: sandbox.stub(),
+            on: sandbox.stub().callsArgWith(1, "a job id", "a result string"),
+        });
+
+        class TestJobWorker extends JobWorker {
+            init() {
+                this.redisConnection = MockBull();
+                this.worker = MockBull(
+                    this.uid,
+                    this.jobHandler.bind(this),
+                    this.redisConnection,
+                );
+            }
+            initCrypto() {
+                this.crypto = cryptoMocks;
+            }
+        }
+        jobWorker = new TestJobWorker(
+            "a parent id",
+            "a session id",
+            "secret is 32 char long like this",
+            {
+                url: "redis config",
+            },
+        );
+        jobWorker.emit = sandbox.stub();
+    });
+
+    afterEach(() => {
+        sinon.restore();
+        sandbox.reset();
+    });
+
+    it("returns a valid JobWorker object", () => {
+        expect(typeof jobWorker).to.equal("object");
+        expect(jobWorker.uid).to.equal(
+            `sockethub:data-layer:worker:a parent id:a session id`,
+        );
+        expect(typeof jobWorker.onJob).to.equal("function");
+        expect(typeof jobWorker.shutdown).to.equal("function");
+    });
+
+    describe("onJob", () => {
+        it("queues the handler", () => {
+            jobWorker.onJob(() => {
+                throw new Error("This handler should never be called");
+            });
+            sinon.assert.notCalled(jobWorker.worker.on);
+        });
+    });
+
+    describe("jobHandler", () => {
+        it("calls handler as expected", async () => {
+            cryptoMocks.decrypt.returns("an unencrypted message");
+            const encryptedJob = {
+                data: {
+                    title: "a title",
+                    msg: "an encrypted message",
+                    sessionId: "a socket id",
+                },
+            };
+            jobWorker.onJob((job) => {
+                const decryptedData = encryptedJob.data;
+                decryptedData.msg = "an unencrypted message";
+                expect(job).to.eql(decryptedData);
+                decryptedData.msg += " handled";
+                return decryptedData;
+            });
+            const result = await jobWorker.jobHandler(encryptedJob);
+            expect(result.msg).to.eql("an unencrypted message handled");
+        });
+    });
+
+    describe("decryptJobData", () => {
+        it("decrypts and returns expected object", () => {
+            cryptoMocks.decrypt.returnsArg(0);
+            const jobData = {
+                data: {
+                    title: "foo",
+                    msg: "encryptedjobdata",
+                    sessionId: "foobar",
+                },
+            };
+            const secret = "secretstring";
+            expect(jobWorker.decryptJobData(jobData, secret)).to.be.eql(
+                jobData.data,
+            );
+        });
+    });
+
+    describe("decryptActivityStream", () => {
+        it("decrypts and returns expected object", () => {
+            cryptoMocks.decrypt.returnsArg(0);
+            const jobData = "encryptedjobdata";
+            const secret = "secretstring";
+            expect(jobWorker.decryptActivityStream(jobData, secret)).to.be.eql(
+                jobData,
+            );
+        });
+    });
+});