@sockethub/data-layer 1.0.0-alpha.4 → 1.0.0-alpha.5

package/src/job-queue.ts

@@ -1,148 +1,227 @@
-import Queue from 'bull';
-import crypto from '@sockethub/crypto';
-import {
-  JobDataDecrypted,
-  JobDataEncrypted, JobDecrypted,
-  JobEncrypted,
-  RedisConfig
-} from "./types";
-import debug, {Debugger} from 'debug';
-import EventEmitter from "events";
-import {IActivityStream} from "@sockethub/schemas";
+import { type Job, Queue, QueueEvents, Worker } from "bullmq";
+import debug, { type Debugger } from "debug";
 
-interface JobHandler {
-  (job: JobDataDecrypted, done: CallableFunction)
-}
+import type { ActivityStream } from "@sockethub/schemas";
 
-export default class JobQueue extends EventEmitter {
-  readonly uid: string;
-  private readonly bull: Queue;
-  private readonly debug: Debugger;
-  private readonly secret: string;
-  private handler: JobHandler;
-  private counter = 0;
+import { JobBase, createIORedisConnection } from "./job-base.js";
+import type { JobDataEncrypted, JobDecrypted, RedisConfig } from "./types.js";
 
-  constructor(instanceId: string, sessionId: string, secret: string, redisConfig: RedisConfig) {
-    super();
-    this.bull = new Queue(instanceId + sessionId, { redis: redisConfig });
-    this.uid = `sockethub:data-layer:job-queue:${instanceId}:${sessionId}`;
-    this.secret = secret;
-    this.debug = debug(this.uid);
+export async function verifyJobQueue(config: RedisConfig): Promise<void> {
+    const log = debug("sockethub:data-layer:queue");
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(
+            "connectiontest",
+            async (job) => {
+                if (job.name !== "foo" || job.data?.foo !== "bar") {
+                    reject(
+                        "Worker received invalid job data during JobQueue connection test",
+                    );
+                }
+                job.data.test = "touched by worker";
+            },
+            {
+                connection: createIORedisConnection(config),
+            },
+        );
+        worker.on("completed", async (job: Job) => {
+            if (job.name !== "foo" || job.data?.test !== "touched by worker") {
+                reject(
+                    "Worker job completed unsuccessfully during JobQueue connection test",
+                );
+            }
+            log("connection verified");
+            await queue.close();
+            await worker.close();
+            resolve();
+        });
+        worker.on("error", (err) => {
+            log(
+                `connection verification worker error received ${err.toString()}`,
+            );
+            reject(err);
+        });
+        const queue = new Queue("connectiontest", {
+            connection: createIORedisConnection(config),
+        });
+        queue.on("error", (err) => {
+            log(
+                `connection verification queue error received ${err.toString()}`,
+            );
+            reject(err);
+        });
+        queue.add(
+            "foo",
+            { foo: "bar" },
+            { removeOnComplete: true, removeOnFail: true },
+        );
+    });
+}
 
-    this.debug('initialized');
-  }
+/**
+ * Redis-backed job queue for managing ActivityStreams message processing.
+ *
+ * Creates isolated queues per platform instance and session, providing reliable
+ * message delivery and processing coordination between Sockethub server and
+ * platform workers.
+ *
+ * @example
+ * ```typescript
+ * const queue = new JobQueue('irc-platform', 'session123', secret, redisConfig);
+ * await queue.add('socket-id', activityStreamMessage);
+ * ```
+ */
+export class JobQueue extends JobBase {
+    readonly uid: string;
+    protected queue: Queue;
+    protected events: QueueEvents;
+    private readonly debug: Debugger;
+    private counter = 0;
+    private initialized = false;
 
-  async add(socketId: string, msg: IActivityStream): Promise<JobDataEncrypted> {
-    const job = this.createJob(socketId, msg);
-    const isPaused = await this.bull.isPaused();
-    if (isPaused) {
-      this.bull.emit('failed', job, 'queue closed');
-      return undefined;
+    /**
+     * Creates a new JobQueue instance.
+     *
+     * @param instanceId - Unique identifier for the platform instance
+     * @param sessionId - Client session identifier for queue isolation
+     * @param secret - 32-character encryption secret for message security
+     * @param redisConfig - Redis connection configuration
+     */
+    constructor(
+        instanceId: string,
+        sessionId: string,
+        secret: string,
+        redisConfig: RedisConfig,
+    ) {
+        super(secret);
+        this.uid = `sockethub:data-layer:queue:${instanceId}:${sessionId}`;
+        this.debug = debug(this.uid);
+        this.init(redisConfig);
     }
-    this.debug(`adding ${job.title} ${msg.type}`);
-    this.bull.add(job);
-    return job;
-  }
 
-  initResultEvents() {
-    this.bull.on('global:completed', async (jobId: string, result: string) => {
-      const r = result ? JSON.parse(result) : "";
-      const job = await this.getJob(jobId);
-      if (job) {
-        this.debug(`completed ${job.data.title} ${job.data.msg.type}`);
-        this.emit('global:completed', job.data, r);
-        await job.remove();
-      }
-    });
-    this.bull.on('global:error', async (jobId: string, result: string) => {
-      this.debug("unknown queue error", jobId, result);
-    });
-    this.bull.on('global:failed', async (jobId, result: string) => {
-      const job = await this.getJob(jobId);
-      if (job) {
-        this.debug(`failed ${job.data.title} ${job.data.msg.type}`);
-        this.emit('global:failed', job.data, result);
-        await job.remove();
-      }
-    });
-    this.bull.on('failed', (job: JobDataEncrypted, result: string) => {
-      // locally failed jobs (eg. due to paused queue)
-      const unencryptedJobData: JobDataDecrypted = {
-        title: job.title,
-        msg: this.decryptActivityStream(job.msg),
-        sessionId: job.sessionId
-      };
-      this.debug(`failed ${unencryptedJobData.title} ${unencryptedJobData.msg.type}`);
-      this.emit('global:failed', unencryptedJobData, result);
-    });
-  }
+    protected init(redisConfig: RedisConfig) {
+        if (this.initialized) {
+            throw new Error(`JobQueue already initialized for ${this.uid}`);
+        }
+        this.initialized = true;
 
-  async getJob(jobId: string): Promise<JobDecrypted> {
-    const job = await this.bull.getJob(jobId);
-    if (job) {
-      job.data = this.decryptJobData(job);
-      try {
-        delete job.data.msg.sessionSecret;
-      } catch (e) {
-        // this property should never be exposed externally
-      }
-    }
-    return job;
-  }
+        // BullMQ v5+ prohibits colons in queue names, so replace with dashes
+        // while keeping uid with colons for debug namespace convention
+        const queueName = this.uid.replace(/:/g, "-");
+        // Let BullMQ create its own connections for better lifecycle management
+        this.queue = new Queue(queueName, {
+            connection: redisConfig,
+        });
+        this.events = new QueueEvents(queueName, {
+            connection: redisConfig,
+        });
 
-  onJob(handler: JobHandler): void {
-    this.handler = handler;
-    this.bull.process(this.jobHandler.bind(this));
-  }
+        this.events.on("completed", async ({ jobId, returnvalue }) => {
+            const job = await this.getJob(jobId);
+            if (!job) {
+                this.debug(`completed job ${jobId} (already removed)`);
+                return;
+            }
+            this.debug(`completed ${job.data.title} ${job.data.msg.type}`);
+            this.emit("completed", job.data, returnvalue);
+        });
 
-  async pause() {
-    await this.bull.pause();
-    this.debug('paused');
-  }
+        this.events.on("failed", async ({ jobId, failedReason }) => {
+            const job = await this.getJob(jobId);
+            if (!job) {
+                this.debug(
+                    `failed job ${jobId} (already removed): ${failedReason}`,
+                );
+                return;
+            }
+            this.debug(
+                `failed ${job.data.title} ${job.data.msg.type}: ${failedReason}`,
+            );
+            this.emit("failed", job.data, failedReason);
+        });
+        this.debug("initialized");
+    }
 
-  async resume() {
-    await this.bull.resume();
-    this.debug('resumed');
-  }
+    /**
+     * Adds an ActivityStreams message to the job queue for processing.
+     *
+     * @param socketId - Socket.IO connection identifier for response routing
+     * @param msg - ActivityStreams message to be processed by platform worker
+     * @returns Promise resolving to encrypted job data
+     * @throws Error if queue is closed or Redis connection fails
+     */
+    async add(
+        socketId: string,
+        msg: ActivityStream,
+    ): Promise<JobDataEncrypted> {
+        const job = this.createJob(socketId, msg);
+        if (await this.queue.isPaused()) {
+            // this.queue.emit("error", new Error("queue closed"));
+            this.debug(
+                `failed to add ${job.title} ${msg.type} to queue: queue closed`,
+            );
+            throw new Error("queue closed");
+        }
+        await this.queue.add(job.title, job, {
+            // Auto-remove jobs after 5 minutes to prevent Redis memory buildup.
+            // Jobs only need to exist long enough for event handlers to look them up
+            // by jobId and send results to clients (typically < 1 second).
+            removeOnComplete: { age: 300 }, // 5 minutes in seconds
+            removeOnFail: { age: 300 },
+        });
+        this.debug(`added ${job.title} ${msg.type} to queue`);
+        return job;
+    }
 
-  async shutdown() {
-    this.debug('shutdown');
-    const isPaused = await this.bull.isPaused(true);
-    if (!isPaused) {
-      await this.bull.pause();
+    /**
+     * Pauses job processing. New jobs can still be added but won't be processed.
+     */
+    async pause() {
+        await this.queue.pause();
+        this.debug("paused");
     }
-    await this.bull.obliterate({ force: true });
-    await this.bull.removeAllListeners();
-  }
 
-  private createJob(socketId: string, msg): JobDataEncrypted {
-    const title = `${msg.context}-${(msg.id) ? msg.id : this.counter++}`;
-    return {
-      title: title,
-      sessionId: socketId,
-      msg: crypto.encrypt(msg, this.secret)
-    };
-  }
+    /**
+     * Resumes job processing after being paused.
+     */
+    async resume() {
+        await this.queue.resume();
+        this.debug("resumed");
+    }
 
-  private jobHandler(encryptedJob: JobEncrypted, done: CallableFunction): void {
-    const job = this.decryptJobData(encryptedJob);
-    this.debug(`handling ${job.title} ${job.msg.type}`);
-    this.handler(job, done);
-  }
+    /**
+     * Gracefully shuts down the queue, cleaning up all resources and connections.
+     */
+    async shutdown() {
+        this.removeAllListeners();
+        this.queue.removeAllListeners();
+        if (!(await this.queue.isPaused())) {
+            await this.queue.pause();
+        }
+        await this.queue.obliterate({ force: true });
+        await this.queue.close();
+        await this.events.close();
+    }
 
-  /**
-   * @param job
-   * @private
-   */
-  private decryptJobData(job: JobEncrypted): JobDataDecrypted {
-    return {
-      title: job.data.title,
-      msg: this.decryptActivityStream(job.data.msg),
-      sessionId: job.data.sessionId
-    };
-  }
+    private async getJob(jobId: string): Promise<JobDecrypted> {
+        const job = await this.queue.getJob(jobId);
+        if (job) {
+            job.data = this.decryptJobData(job);
+            try {
+                // biome-ignore lint/performance/noDelete: <explanation>
+                delete job.data.msg.sessionSecret;
+            } catch (e) {
+                // this property should never be exposed externally
+            }
+        }
+        return job;
+    }
 
-  private decryptActivityStream(msg: string): IActivityStream {
-    return crypto.decrypt(msg, this.secret);
-  }
-}
+    private createJob(socketId: string, msg: ActivityStream): JobDataEncrypted {
+        const title = `${msg.context}-${msg.id ? msg.id : this.counter++}`;
+        return {
+            title: title,
+            sessionId: socketId,
+            msg: this.encryptActivityStream(msg),
+        };
+    }
+}

package/src/job-worker.test.ts

@@ -0,0 +1,128 @@
+import { expect } from "chai";
+import * as sinon from "sinon";
+
+import { JobWorker } from "./index";
+
+describe("JobWorker", () => {
+    let MockBull, jobWorker, cryptoMocks, sandbox;
+
+    beforeEach(() => {
+        sandbox = sinon.createSandbox();
+        cryptoMocks = {
+            objectHash: sandbox.stub(),
+            decrypt: sandbox.stub(),
+            encrypt: sandbox.stub(),
+            hash: sandbox.stub(),
+        };
+        MockBull = sandbox.stub().returns({
+            add: sandbox.stub(),
+            getJob: sandbox.stub(),
+            removeAllListeners: sandbox.stub(),
+            pause: sandbox.stub(),
+            resume: sandbox.stub(),
+            isPaused: sandbox.stub(),
+            obliterate: sandbox.stub(),
+            isRunning: sandbox.stub(),
+            close: sandbox.stub(),
+            emit: sandbox.stub(),
+            disconnect: sandbox.stub(),
+            on: sandbox.stub().callsArgWith(1, "a job id", "a result string"),
+        });
+
+        class TestJobWorker extends JobWorker {
+            init() {
+                this.redisConnection = MockBull();
+                this.worker = MockBull(
+                    this.uid,
+                    this.jobHandler.bind(this),
+                    this.redisConnection,
+                );
+            }
+            initCrypto() {
+                this.crypto = cryptoMocks;
+            }
+        }
+        jobWorker = new TestJobWorker(
+            "a parent id",
+            "a session id",
+            "secret is 32 char long like this",
+            {
+                url: "redis config",
+            },
+        );
+        jobWorker.emit = sandbox.stub();
+    });
+
+    afterEach(() => {
+        sinon.restore();
+        sandbox.reset();
+    });
+
+    it("returns a valid JobWorker object", () => {
+        expect(typeof jobWorker).to.equal("object");
+        expect(jobWorker.uid).to.equal(
+            `sockethub:data-layer:worker:a parent id:a session id`,
+        );
+        expect(typeof jobWorker.onJob).to.equal("function");
+        expect(typeof jobWorker.shutdown).to.equal("function");
+    });
+
+    describe("onJob", () => {
+        it("queues the handler", () => {
+            jobWorker.onJob(() => {
+                throw new Error("This handler should never be called");
+            });
+            sinon.assert.notCalled(jobWorker.worker.on);
+        });
+    });
+
+    describe("jobHandler", () => {
+        it("calls handler as expected", async () => {
+            cryptoMocks.decrypt.returns("an unencrypted message");
+            const encryptedJob = {
+                data: {
+                    title: "a title",
+                    msg: "an encrypted message",
+                    sessionId: "a socket id",
+                },
+            };
+            jobWorker.onJob((job) => {
+                const decryptedData = encryptedJob.data;
+                decryptedData.msg = "an unencrypted message";
+                expect(job).to.eql(decryptedData);
+                decryptedData.msg += " handled";
+                return decryptedData;
+            });
+            const result = await jobWorker.jobHandler(encryptedJob);
+            expect(result.msg).to.eql("an unencrypted message handled");
+        });
+    });
+
+    describe("decryptJobData", () => {
+        it("decrypts and returns expected object", () => {
+            cryptoMocks.decrypt.returnsArg(0);
+            const jobData = {
+                data: {
+                    title: "foo",
+                    msg: "encryptedjobdata",
+                    sessionId: "foobar",
+                },
+            };
+            const secret = "secretstring";
+            expect(jobWorker.decryptJobData(jobData, secret)).to.be.eql(
+                jobData.data,
+            );
+        });
+    });
+
+    describe("decryptActivityStream", () => {
+        it("decrypts and returns expected object", () => {
+            cryptoMocks.decrypt.returnsArg(0);
+            const jobData = "encryptedjobdata";
+            const secret = "secretstring";
+            expect(jobWorker.decryptActivityStream(jobData, secret)).to.be.eql(
+                jobData,
+            );
+        });
+    });
+});

package/src/job-worker.ts

@@ -0,0 +1,97 @@
+import { Worker } from "bullmq";
+import debug, { type Debugger } from "debug";
+
+import { JobBase, createIORedisConnection } from "./job-base.js";
+import type { JobEncrypted, JobHandler, RedisConfig } from "./types.js";
+
+/**
+ * Worker for processing jobs from a Redis queue within platform child processes.
+ *
+ * Connects to the same queue as its corresponding JobQueue instance and processes
+ * jobs using a platform-specific handler function. Provides automatic decryption
+ * of job data and error handling.
+ *
+ * @example
+ * ```typescript
+ * const worker = new JobWorker('irc-platform', 'session123', secret, redisConfig);
+ * worker.onJob(async (job) => {
+ *   // Process the decrypted ActivityStreams message
+ *   return await processMessage(job.msg);
+ * });
+ * ```
+ */
+export class JobWorker extends JobBase {
+    readonly uid: string;
+    protected worker: Worker;
+    protected handler: JobHandler;
+    private readonly debug: Debugger;
+    private readonly redisConfig: RedisConfig;
+    private readonly queueId: string;
+    private initialized = false;
+
+    /**
+     * Creates a new JobWorker instance.
+     *
+     * @param instanceId - Must match the instanceId of the corresponding JobQueue
+     * @param sessionId - Must match the sessionId of the corresponding JobQueue
+     * @param secret - 32-character encryption secret, must match JobQueue secret
+     * @param redisConfig - Redis connection configuration
+     */
+    constructor(
+        instanceId: string,
+        sessionId: string,
+        secret: string,
+        redisConfig: RedisConfig,
+    ) {
+        super(secret);
+        this.uid = `sockethub:data-layer:worker:${instanceId}:${sessionId}`;
+        this.queueId = `sockethub:data-layer:queue:${instanceId}:${sessionId}`;
+        this.debug = debug(this.uid);
+        this.redisConfig = redisConfig;
+    }
+
+    protected init() {
+        if (this.initialized) {
+            throw new Error(`JobWorker already initialized for ${this.uid}`);
+        }
+        this.initialized = true;
+        // BullMQ v5+ prohibits colons in queue names, so replace with dashes
+        // while keeping queueId with colons for debug namespace convention
+        const queueName = this.queueId.replace(/:/g, "-");
+        // Let BullMQ create its own connection for better lifecycle management
+        this.worker = new Worker(queueName, this.jobHandler.bind(this), {
+            connection: this.redisConfig,
+            // Prevent infinite retry loops when platform child process crashes mid-job.
+            // If worker disappears (crash/disconnect), job becomes "stalled" and retries
+            // up to maxStalledCount times (with default 30s interval) before failing permanently.
+            maxStalledCount: 3,
+        });
+        this.debug("initialized");
+    }
+
+    /**
+     * Registers a job handler function and starts processing jobs from the queue.
+     *
+     * @param handler - Function that processes decrypted job data and returns results
+     */
+    onJob(handler: JobHandler): void {
+        this.handler = handler;
+        this.init();
+    }
+
+    /**
+     * Gracefully shuts down the worker, stopping job processing and cleaning up connections.
+     */
+    async shutdown() {
+        await this.worker.pause();
+        this.removeAllListeners();
+        this.worker.removeAllListeners();
+        await this.worker.close();
+    }
+
+    protected async jobHandler(encryptedJob: JobEncrypted) {
+        const job = this.decryptJobData(encryptedJob);
+        this.debug(`handling ${job.title} ${job.msg.type}`);
+        return await this.handler(job);
+    }
+}

package/src/types.ts

@@ -1,37 +1,35 @@
-import {IActivityStream} from "@sockethub/schemas";
+import type {
+    ActivityStream,
+    InternalActivityStream,
+} from "@sockethub/schemas";
 
-export type RedisConfigUrl = string;
-
-export interface RedisConfigProps {
-  host: string,
-  port: string
-}
-
-export type RedisConfig = RedisConfigProps | RedisConfigUrl;
+export type RedisConfig = {
+    url: string;
+};
 
 export interface JobDataEncrypted {
-  title?: string;
-  msg: string;
-  sessionId: string;
+    title?: string;
+    msg: string;
+    sessionId: string;
 }
 
 export interface JobDataDecrypted {
-  title?: string;
-  msg: IActivityStream;
-  sessionId: string;
+    title?: string;
+    msg: InternalActivityStream;
+    sessionId: string;
 }
 
 export interface JobEncrypted {
-  data: JobDataEncrypted,
-  remove?: {
-    (): void;
-  };
+    data: JobDataEncrypted;
+    remove?: () => void;
 }
 
 export interface JobDecrypted {
-  data: JobDataDecrypted,
-  remove?: {
-    (): void;
-  };
+    data: JobDataDecrypted;
+    remove?: () => void;
+    returnvalue: unknown;
 }
 
+export type JobHandler = (
+    job: JobDataDecrypted,
+) => Promise<string | undefined | ActivityStream>;

package/typedoc.json

@@ -0,0 +1,11 @@
+{
+  "entryPoints": ["./src/index.ts"],
+  "out": "./docs",
+  "readme": "none",
+  "includeVersion": true,
+  "exclude": ["**/*.test.ts", "**/*.spec.ts", "**/node_modules/**"],
+  "skipErrorChecking": true,
+  "compilerOptions": {
+    "skipLibCheck": true
+  }
+}