@sockethub/data-layer 1.0.0-alpha.10 → 1.0.0-alpha.12

package/src/job-queue.ts

@@ -1,13 +1,18 @@
+import {
+    createLogger,
+    getLoggerNamespace,
+    type Logger,
+} from "@sockethub/logger";
+import { type ActivityStream, resolvePlatformId } from "@sockethub/schemas";
 import { type Job, Queue, QueueEvents, Worker } from "bullmq";
-import debug, { type Debugger } from "debug";
 
-import type { ActivityStream } from "@sockethub/schemas";
-
-import { JobBase, createIORedisConnection } from "./job-base.js";
+import { JobBase } from "./job-base.js";
+import { buildQueueId } from "./queue-id.js";
 import type { JobDataEncrypted, JobDecrypted, RedisConfig } from "./types.js";
 
 export async function verifyJobQueue(config: RedisConfig): Promise<void> {
-    const log = debug("sockethub:data-layer:queue");
+    const log = createLogger("data-layer:verify-job-queue");
+
     return new Promise((resolve, reject) => {
         const worker = new Worker(
             "connectiontest",
@@ -20,7 +25,7 @@ export async function verifyJobQueue(config: RedisConfig): Promise<void> {
                 job.data.test = "touched by worker";
             },
             {
-                connection: createIORedisConnection(config),
+                connection: config,
             },
         );
         worker.on("completed", async (job: Job) => {
@@ -29,22 +34,22 @@ export async function verifyJobQueue(config: RedisConfig): Promise<void> {
                     "Worker job completed unsuccessfully during JobQueue connection test",
                 );
             }
-            log("connection verified");
+            log.info("job queue connection verified");
             await queue.close();
             await worker.close();
             resolve();
         });
         worker.on("error", (err) => {
-            log(
+            log.warn(
                 `connection verification worker error received ${err.toString()}`,
             );
             reject(err);
         });
         const queue = new Queue("connectiontest", {
-            connection: createIORedisConnection(config),
+            connection: config,
         });
         queue.on("error", (err) => {
-            log(
+            log.warn(
                 `connection verification queue error received ${err.toString()}`,
             );
             reject(err);
@@ -71,43 +76,49 @@ export async function verifyJobQueue(config: RedisConfig): Promise<void> {
  * ```
  */
 export class JobQueue extends JobBase {
-    readonly uid: string;
+    private readonly connectionName: string;
+    protected readonly queueId: string;
     protected queue: Queue;
     protected events: QueueEvents;
-    private readonly debug: Debugger;
+    private readonly log: Logger;
     private counter = 0;
     private initialized = false;
 
     /**
      * Creates a new JobQueue instance.
      *
+     * @param parentId - Sockethub instance identifier for queue isolation
      * @param instanceId - Unique identifier for the platform instance
-     * @param sessionId - Client session identifier for queue isolation
      * @param secret - 32-character encryption secret for message security
      * @param redisConfig - Redis connection configuration
      */
     constructor(
+        parentId: string,
         instanceId: string,
-        sessionId: string,
         secret: string,
         redisConfig: RedisConfig,
     ) {
         super(secret);
-        this.uid = `sockethub:data-layer:queue:${instanceId}:${sessionId}`;
-        this.debug = debug(this.uid);
+        // Create logger with full namespace (context will be prepended automatically)
+        this.log = createLogger(`data-layer:queue:${parentId}:${instanceId}`);
+
+        this.queueId = buildQueueId(parentId, instanceId);
+        // Use logger's full namespace (includes context) for Redis connection name
+        this.connectionName = getLoggerNamespace(this.log);
+        redisConfig.connectionName = this.connectionName;
         this.init(redisConfig);
     }
 
     protected init(redisConfig: RedisConfig) {
         if (this.initialized) {
-            throw new Error(`JobQueue already initialized for ${this.uid}`);
+            throw new Error(`JobQueue already initialized for ${this.queueId}`);
         }
         this.initialized = true;
 
-        // BullMQ v5+ prohibits colons in queue names, so replace with dashes
-        // while keeping uid with colons for debug namespace convention
-        const queueName = this.uid.replace(/:/g, "-");
-        // Let BullMQ create its own connections for better lifecycle management
+        // BullMQ v5+ prohibits colons in queue names; derive the queue name
+        // from the canonical queue id by replacing ':' with '-'.
+        const queueName = this.queueId.replace(/:/g, "-");
+        // Let BullMQ create its own connections (it duplicates them internally anyway)
         this.queue = new Queue(queueName, {
             connection: redisConfig,
         });
@@ -115,30 +126,39 @@ export class JobQueue extends JobBase {
             connection: redisConfig,
         });
 
+        // Handle Redis contention errors (e.g., BUSY from Lua scripts)
+        this.queue.on("error", (err) => {
+            this.log.warn(`queue error: ${err.message}`);
+        });
+
+        this.events.on("error", (err) => {
+            this.log.warn(`events error: ${err.message}`);
+        });
+
         this.events.on("completed", async ({ jobId, returnvalue }) => {
             const job = await this.getJob(jobId);
             if (!job) {
-                this.debug(`completed job ${jobId} (already removed)`);
+                this.log.debug(`completed job ${jobId} (already removed)`);
                 return;
             }
-            this.debug(`completed ${job.data.title} ${job.data.msg.type}`);
+            this.log.debug(`completed ${job.data.title} ${job.data.msg.type}`);
             this.emit("completed", job.data, returnvalue);
         });
 
         this.events.on("failed", async ({ jobId, failedReason }) => {
             const job = await this.getJob(jobId);
             if (!job) {
-                this.debug(
+                this.log.debug(
                     `failed job ${jobId} (already removed): ${failedReason}`,
                 );
                 return;
             }
-            this.debug(
+            this.log.warn(
                 `failed ${job.data.title} ${job.data.msg.type}: ${failedReason}`,
             );
             this.emit("failed", job.data, failedReason);
         });
-        this.debug("initialized");
+        this.log.info("initialized");
     }
 
     /**
@@ -156,7 +176,7 @@ export class JobQueue extends JobBase {
         const job = this.createJob(socketId, msg);
         if (await this.queue.isPaused()) {
             // this.queue.emit("error", new Error("queue closed"));
-            this.debug(
+            this.log.debug(
                 `failed to add ${job.title} ${msg.type} to queue: queue closed`,
             );
             throw new Error("queue closed");
@@ -168,7 +188,7 @@ export class JobQueue extends JobBase {
             removeOnComplete: { age: 300 }, // 5 minutes in seconds
             removeOnFail: { age: 300 },
         });
-        this.debug(`added ${job.title} ${msg.type} to queue`);
+        this.log.debug(`added ${job.title} ${msg.type} to queue`);
         return job;
     }
 
@@ -177,7 +197,7 @@ export class JobQueue extends JobBase {
      */
     async pause() {
         await this.queue.pause();
-        this.debug("paused");
+        this.log.debug("paused");
     }
 
     /**
@@ -185,7 +205,7 @@ export class JobQueue extends JobBase {
      */
     async resume() {
         await this.queue.resume();
-        this.debug("resumed");
+        this.log.debug("resumed");
     }
 
     /**
@@ -207,9 +227,8 @@ export class JobQueue extends JobBase {
         if (job) {
             job.data = this.decryptJobData(job);
             try {
-                // biome-ignore lint/performance/noDelete: <explanation>
                 delete job.data.msg.sessionSecret;
-            } catch (e) {
+            } catch (_e) {
                 // this property should never be exposed externally
             }
         }
@@ -217,7 +236,8 @@ export class JobQueue extends JobBase {
     }
 
     private createJob(socketId: string, msg: ActivityStream): JobDataEncrypted {
-        const title = `${msg.context}-${msg.id ? msg.id : this.counter++}`;
+        const platformId = resolvePlatformId(msg) || "unknown";
+        const title = `${platformId}-${msg.id ? msg.id : this.counter++}`;
         return {
             title: title,
             sessionId: socketId,

package/src/job-worker.test.ts

@@ -1,8 +1,17 @@
 import { expect } from "chai";
 import * as sinon from "sinon";
 
+import type { Logger } from "@sockethub/schemas";
+
 import { JobWorker } from "./index";
 
+const mockLogger: Logger = {
+    error: () => {},
+    warn: () => {},
+    info: () => {},
+    debug: () => {},
+};
+
 describe("JobWorker", () => {
     let MockBull, jobWorker, cryptoMocks, sandbox;
 
@@ -32,8 +41,9 @@ describe("JobWorker", () => {
         class TestJobWorker extends JobWorker {
             init() {
                 this.redisConnection = MockBull();
+                const queueName = this.queueId.replace(/:/g, "-");
                 this.worker = MockBull(
-                    this.uid,
+                    queueName,
                     this.jobHandler.bind(this),
                     this.redisConnection,
                 );
@@ -41,6 +51,9 @@ describe("JobWorker", () => {
             initCrypto() {
                 this.crypto = cryptoMocks;
             }
+            getQueueId() {
+                return this.queueId;
+            }
         }
         jobWorker = new TestJobWorker(
             "a parent id",
@@ -49,6 +62,7 @@ describe("JobWorker", () => {
             {
                 url: "redis config",
             },
+            mockLogger,
         );
         jobWorker.emit = sandbox.stub();
     });
@@ -60,8 +74,8 @@ describe("JobWorker", () => {
 
     it("returns a valid JobWorker object", () => {
         expect(typeof jobWorker).to.equal("object");
-        expect(jobWorker.uid).to.equal(
-            `sockethub:data-layer:worker:a parent id:a session id`,
+        expect(jobWorker.getQueueId()).to.equal(
+            `sockethub:a parent id:data-layer:queue:a session id`,
         );
         expect(typeof jobWorker.onJob).to.equal("function");
         expect(typeof jobWorker.shutdown).to.equal("function");

package/src/job-worker.ts

@@ -1,7 +1,12 @@
+import {
+    createLogger,
+    getLoggerNamespace,
+    type Logger,
+} from "@sockethub/logger";
 import { Worker } from "bullmq";
-import debug, { type Debugger } from "debug";
 
-import { JobBase, createIORedisConnection } from "./job-base.js";
+import { JobBase } from "./job-base.js";
+import { buildQueueId } from "./queue-id.js";
 import type { JobEncrypted, JobHandler, RedisConfig } from "./types.js";
 
 /**
@@ -21,44 +26,52 @@ import type { JobEncrypted, JobHandler, RedisConfig } from "./types.js";
  * ```
  */
 export class JobWorker extends JobBase {
-    readonly uid: string;
+    private readonly connectionName: string;
     protected worker: Worker;
     protected handler: JobHandler;
-    private readonly debug: Debugger;
+    private readonly log: Logger;
     private readonly redisConfig: RedisConfig;
-    private readonly queueId: string;
+    protected readonly queueId: string;
     private initialized = false;
 
     /**
      * Creates a new JobWorker instance.
      *
+     * @param parentId - Must match the parentId of the corresponding JobQueue
      * @param instanceId - Must match the instanceId of the corresponding JobQueue
-     * @param sessionId - Must match the sessionId of the corresponding JobQueue
      * @param secret - 32-character encryption secret, must match JobQueue secret
      * @param redisConfig - Redis connection configuration
      */
     constructor(
+        parentId: string,
         instanceId: string,
-        sessionId: string,
         secret: string,
         redisConfig: RedisConfig,
     ) {
         super(secret);
-        this.uid = `sockethub:data-layer:worker:${instanceId}:${sessionId}`;
-        this.queueId = `sockethub:data-layer:queue:${instanceId}:${sessionId}`;
-        this.debug = debug(this.uid);
+        // Create logger with full namespace (context will be prepended automatically)
+        this.log = createLogger(`data-layer:worker:${parentId}:${instanceId}`);
+
+        // Use logger's full namespace (includes context) for Redis connection name
+        this.connectionName = getLoggerNamespace(this.log);
+        redisConfig.connectionName = this.connectionName;
+
+        // Queue ID must match JobQueue's namespace (context-free) for cross-process connection
+        this.queueId = buildQueueId(parentId, instanceId);
         this.redisConfig = redisConfig;
     }
 
     protected init() {
         if (this.initialized) {
-            throw new Error(`JobWorker already initialized for ${this.uid}`);
+            throw new Error(
+                `JobWorker already initialized for ${this.queueId}`,
+            );
         }
         this.initialized = true;
-        // BullMQ v5+ prohibits colons in queue names, so replace with dashes
-        // while keeping queueId with colons for debug namespace convention
+        // BullMQ v5+ prohibits colons in queue names; derive the queue name
+        // from the canonical queue id by replacing ':' with '-'.
         const queueName = this.queueId.replace(/:/g, "-");
-        // Let BullMQ create its own connection for better lifecycle management
+        // Let BullMQ create its own connection (it duplicates them internally anyway)
         this.worker = new Worker(queueName, this.jobHandler.bind(this), {
             connection: this.redisConfig,
             // Prevent infinite retry loops when platform child process crashes mid-job.
@@ -66,7 +79,13 @@ export class JobWorker extends JobBase {
             // up to maxStalledCount times (with default 30s interval) before failing permanently.
             maxStalledCount: 3,
         });
-        this.debug("initialized");
+
+        // Handle Redis contention errors (e.g., BUSY from Lua scripts)
+        this.worker.on("error", (err) => {
+            this.log.warn(`worker error: ${err.message}`);
+        });
+
+        this.log.info("initialized");
     }
 
     /**
@@ -91,7 +110,7 @@ export class JobWorker extends JobBase {
 
     protected async jobHandler(encryptedJob: JobEncrypted) {
         const job = this.decryptJobData(encryptedJob);
-        this.debug(`handling ${job.title} ${job.msg.type}`);
+        this.log.debug(`handling ${job.title} ${job.msg.type}`);
         return await this.handler(job);
     }
 }

package/src/queue-id.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from "bun:test";
+
+import { buildCredentialsStoreId, buildQueueId } from "./queue-id";
+
+describe("queue-id helpers", () => {
+    it("buildQueueId uses canonical namespace", () => {
+        expect(buildQueueId("parent", "platform")).toBe(
+            "sockethub:parent:data-layer:queue:platform",
+        );
+    });
+
+    it("buildCredentialsStoreId uses canonical namespace", () => {
+        expect(buildCredentialsStoreId("parent", "session")).toBe(
+            "sockethub:parent:data-layer:credentials-store:session",
+        );
+    });
+});

package/src/queue-id.ts

@@ -0,0 +1,14 @@
+const REDIS_QUEUE_PREFIX = "sockethub";
+
+// Canonical ids use ':' separators; BullMQ queue names are derived by replacing
+// ':' with '-' when constructing the queue name.
+export function buildQueueId(parentId: string, instanceId: string): string {
+    return `${REDIS_QUEUE_PREFIX}:${parentId}:data-layer:queue:${instanceId}`;
+}
+
+export function buildCredentialsStoreId(
+    parentId: string,
+    sessionId: string,
+): string {
+    return `${REDIS_QUEUE_PREFIX}:${parentId}:data-layer:credentials-store:${sessionId}`;
+}

package/src/types.ts

@@ -5,6 +5,10 @@ import type {
 
 export type RedisConfig = {
     url: string;
+    connectTimeout?: number;
+    disconnectTimeout?: number;
+    maxRetriesPerRequest?: number | null;
+    connectionName?: string;
 };
 
 export interface JobDataEncrypted {