@sockethub/data-layer 1.0.0-alpha.10 → 1.0.0-alpha.12

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@sockethub/data-layer",
   "description": "Storing and RPC of data for Sockethub",
-  "version": "1.0.0-alpha.10",
+  "version": "1.0.0-alpha.12",
   "type": "module",
   "private": false,
   "author": "Nick Jennings <nick@silverbucket.net>",
@@ -41,25 +41,27 @@
   "scripts": {
     "build": "bun build src/index.ts --outdir dist --target node --format esm --sourcemap=external",
     "clean": "rm -rf dist",
+    "clean:deps": "rm -rf node_modules",
     "doc": "typedoc --options typedoc.json",
     "test": "bun test src/",
-    "test:integration": "DEBUG=ioredis*,bullmq*,sockethub* bun test ./integration/redis.integration.ts"
+    "test:integration": "bun test ./integration/redis.integration.ts"
   },
   "dependencies": {
-    "@sockethub/crypto": "^1.0.0-alpha.10",
-    "@sockethub/schemas": "^3.0.0-alpha.10",
+    "@sockethub/crypto": "^1.0.0-alpha.12",
+    "@sockethub/logger": "^1.0.0-alpha.12",
+    "@sockethub/schemas": "^3.0.0-alpha.12",
     "bullmq": "^5.66.5",
-    "debug": "^4.4.3",
     "ioredis": "^5.9.2",
-    "secure-store-redis": "^3.0.7"
+    "secure-store-redis": "^4.1.0"
   },
   "devDependencies": {
     "@types/bun": "latest",
-    "@types/debug": "^4.1.12",
     "@types/sinon": "^17.0.4",
+    "@types/winston": "^2.4.4",
     "sinon": "^17.0.2",
     "typedoc": "^0.28.16",
-    "typedoc-plugin-markdown": "^4.9.0"
+    "typedoc-plugin-markdown": "^4.9.0",
+    "winston": "^3.19.0"
   },
-  "gitHead": "8e1abf116b2a6b57d33c6e1a4af9143870517bae"
+  "gitHead": "f039dab3c3f67cbbf204476fc397532e973f82a8"
 }

package/src/credentials-store.test.ts

@@ -1,8 +1,17 @@
 import { beforeEach, describe, expect, it } from "bun:test";
 import * as sinon from "sinon";
 
+import type { Logger } from "@sockethub/schemas";
+
 import { CredentialsStore } from "./credentials-store";
 
+const mockLogger: Logger = {
+    error: () => {},
+    warn: () => {},
+    info: () => {},
+    debug: () => {},
+};
+
 describe("CredentialsStore", () => {
     let credentialsStore,
         MockSecureStore,
@@ -16,6 +25,8 @@ describe("CredentialsStore", () => {
         MockSecureStore = sinon.stub().returns({
             get: MockStoreGet,
             save: MockStoreSave,
+            isConnected: true,
+            connect: sinon.stub().resolves(),
         });
         class TestCredentialsStore extends CredentialsStore {
             initCrypto() {
@@ -34,6 +45,7 @@ describe("CredentialsStore", () => {
             "a session id",
             "a secret must be 32 chars and th",
             { url: "redis config" },
+            mockLogger,
         );
     });
 
@@ -42,11 +54,15 @@ describe("CredentialsStore", () => {
         sinon.assert.calledWith(MockSecureStore, {
             namespace: "foo",
             secret: "a secret must be 32 chars and th",
-            redis: { url: "redis config" },
+            redis: {
+                url: "redis config",
+                connectionName:
+                    "data-layer:credentials-store:a parent id:a session id",
+            },
         });
         expect(typeof credentialsStore).toEqual("object");
         expect(credentialsStore.uid).toEqual(
-            `sockethub:data-layer:credentials-store:a parent id:a session id`,
+            `sockethub:a parent id:data-layer:credentials-store:a session id`,
         );
         expect(typeof credentialsStore.get).toEqual("function");
         expect(typeof credentialsStore.save).toEqual("function");

package/src/credentials-store.ts

@@ -1,10 +1,58 @@
 import { crypto } from "@sockethub/crypto";
+import {
+    createLogger,
+    getLoggerNamespace,
+    type Logger,
+} from "@sockethub/logger";
 import type { CredentialsObject } from "@sockethub/schemas";
-import debug, { type Debugger } from "debug";
+import IORedis, { type Redis } from "ioredis";
 import SecureStore from "secure-store-redis";
 
+import { buildCredentialsStoreId } from "./queue-id.js";
 import type { RedisConfig } from "./types.js";
 
+let sharedCredentialsRedisConnection: Redis | null = null;
+
+/**
+ * Creates or returns a shared Redis connection for CredentialsStore instances.
+ * This prevents connection exhaustion by reusing a single connection across
+ * all credential storage operations.
+ *
+ * @param config - Redis configuration
+ * @returns Shared Redis connection instance
+ */
+export function createCredentialsRedisConnection(config: RedisConfig): Redis {
+    if (!sharedCredentialsRedisConnection) {
+        sharedCredentialsRedisConnection = new IORedis(config.url, {
+            connectionName: config.connectionName,
+            enableOfflineQueue: false,
+            maxRetriesPerRequest: config.maxRetriesPerRequest ?? null,
+            connectTimeout: config.connectTimeout ?? 10000,
+            disconnectTimeout: config.disconnectTimeout ?? 5000,
+            lazyConnect: false,
+            retryStrategy: (times: number) => {
+                if (times > 3) return null;
+                return Math.min(2 ** (times - 1) * 200, 2000);
+            },
+        });
+    }
+    return sharedCredentialsRedisConnection;
+}
+
+/**
+ * Resets the shared credentials Redis connection. Used primarily for testing.
+ */
+export async function resetSharedCredentialsRedisConnection(): Promise<void> {
+    if (sharedCredentialsRedisConnection) {
+        try {
+            sharedCredentialsRedisConnection.disconnect(false);
+        } catch (_err) {
+            // Ignore disconnect errors during cleanup
+        }
+        sharedCredentialsRedisConnection = null;
+    }
+}
+
 export interface CredentialsStoreInterface {
     get(
         actor: string,
@@ -14,13 +62,16 @@ export interface CredentialsStoreInterface {
 }
 
 export async function verifySecureStore(config: RedisConfig): Promise<void> {
-    const log = debug("sockethub:data-layer:credentials-store");
+    const log = createLogger("data-layer:verify-secure-store");
+    const sharedClient = createCredentialsRedisConnection(config);
     const ss = new SecureStore({
-        redis: config,
+        uid: "data-layer:verify",
+        secret: "aB3#xK9mP2qR7wZ4cT8nY6vH1jL5fD0s",
+        redis: { client: sharedClient },
     });
-    await ss.init();
+    await ss.connect();
     await ss.disconnect();
-    log("redis connection verified");
+    log.info("secure store connection verified");
 }
 
 /**
@@ -49,7 +100,7 @@ export class CredentialsStore implements CredentialsStoreInterface {
     readonly uid: string;
     store: SecureStore;
     objectHash: (o: unknown) => string;
-    private readonly log: Debugger;
+    private readonly log: Logger;
 
     /**
      * Creates a new CredentialsStore instance.
@@ -71,11 +122,19 @@ export class CredentialsStore implements CredentialsStoreInterface {
                 "CredentialsStore secret must be 32 chars in length",
             );
         }
-        this.uid = `sockethub:data-layer:credentials-store:${parentId}:${sessionId}`;
-        this.log = debug(this.uid);
+        // Create logger with full namespace (context will be prepended automatically)
+        this.log = createLogger(
+            `data-layer:credentials-store:${parentId}:${sessionId}`,
+        );
+
         this.initCrypto();
+
+        // Use the canonical, context-free namespace for credentials storage keys
+        this.uid = buildCredentialsStoreId(parentId, sessionId);
+        // Keep full logger namespace for Redis connection naming
+        redisConfig.connectionName = getLoggerNamespace(this.log);
         this.initSecureStore(secret, redisConfig);
-        this.log("initialized");
+        this.log.debug("initialized");
     }
 
     initCrypto() {
@@ -83,23 +142,29 @@ export class CredentialsStore implements CredentialsStoreInterface {
     }
 
     initSecureStore(secret: string, redisConfig: RedisConfig) {
+        // Use shared Redis connection for connection pooling
+        const sharedClient = createCredentialsRedisConnection(redisConfig);
         this.store = new SecureStore({
             uid: this.uid,
             secret: secret,
-            redis: redisConfig,
+            redis: { client: sharedClient },
         });
     }
 
     /**
-     * Gets the credentials for a given actor ID
+     * Gets the credentials for a given actor ID.
      * @param actor
-     * @param credentialsHash - Optional hash to validate credentials. If undefined, validation is skipped.
+     * @param credentialsHash - Optional hash to validate credentials.
+     *   If undefined, validation is skipped.
      */
     async get(
         actor: string,
         credentialsHash: string | undefined,
     ): Promise<CredentialsObject> {
-        this.log(`get credentials for ${actor}`);
+        this.log.debug(`get credentials for ${actor}`);
+        if (!this.store.isConnected) {
+            await this.store.connect();
+        }
         const credentials: CredentialsObject = await this.store.get(actor);
         if (!credentials) {
             throw new Error(`credentials not found for ${actor}`);
@@ -119,6 +184,9 @@ export class CredentialsStore implements CredentialsStoreInterface {
      * @param creds
      */
     async save(actor: string, creds: CredentialsObject): Promise<number> {
-        return this.store.save(actor, creds);
+        if (!this.store.isConnected) {
+            await this.store.connect();
+        }
+        return await this.store.save(actor, creds);
     }
 }

package/src/index.ts

@@ -1,19 +1,25 @@
-import debug from "debug";
+import { createLogger } from "@sockethub/logger";
 
 import {
     CredentialsStore,
     type CredentialsStoreInterface,
+    resetSharedCredentialsRedisConnection,
     verifySecureStore,
 } from "./credentials-store.js";
+import {
+    getRedisConnectionCount,
+    resetSharedRedisConnection,
+} from "./job-base.js";
 import { JobQueue, verifyJobQueue } from "./job-queue.js";
 import { JobWorker } from "./job-worker.js";
+
 export * from "./types.js";
-import type { RedisConfig } from "./types.js";
 
-const log = debug("sockethub:data-layer");
+import type { RedisConfig } from "./types.js";
 
 async function redisCheck(config: RedisConfig): Promise<void> {
-    log(`checking redis connection ${config.url}`);
+    const log = createLogger("data-layer:redis-check");
+    log.debug(`checking redis connection ${config.url}`);
     await verifySecureStore(config);
     await verifyJobQueue(config);
 }
@@ -23,5 +29,8 @@ export {
     JobQueue,
     JobWorker,
     CredentialsStore,
+    getRedisConnectionCount,
+    resetSharedRedisConnection,
+    resetSharedCredentialsRedisConnection,
     type CredentialsStoreInterface,
 };

package/src/job-base.ts

@@ -1,16 +1,83 @@
 import EventEmitter from "node:events";
-import IORedis, { type Redis } from "ioredis";
-
-import { crypto, type Crypto } from "@sockethub/crypto";
+import { type Crypto, crypto } from "@sockethub/crypto";
 import type { ActivityStream } from "@sockethub/schemas";
+import IORedis, { type Redis } from "ioredis";
 
 import type { JobDataDecrypted, JobEncrypted, RedisConfig } from "./types.js";
 
+let sharedRedisConnection: Redis | null = null;
+
+/**
+ * Creates or returns a shared Redis connection to enable connection pooling.
+ * This prevents connection exhaustion under high load by reusing a single
+ * connection across all JobQueue and JobWorker instances.
+ *
+ * @param config - Redis configuration with optional timeout and retry settings
+ * @returns Shared Redis connection instance
+ */
 export function createIORedisConnection(config: RedisConfig): Redis {
-    return new IORedis(config.url, {
-        enableOfflineQueue: false,
-        maxRetriesPerRequest: null,
-    });
+    if (!sharedRedisConnection) {
+        sharedRedisConnection = new IORedis(config.url, {
+            connectionName: config.connectionName,
+            enableOfflineQueue: false,
+            maxRetriesPerRequest: config.maxRetriesPerRequest ?? null,
+            connectTimeout: config.connectTimeout ?? 10000,
+            disconnectTimeout: config.disconnectTimeout ?? 5000,
+            lazyConnect: false,
+            retryStrategy: (times: number) => {
+                // Stop retrying after 3 attempts to fail fast
+                if (times > 3) return null;
+                // Exponential backoff: 200ms, 400ms, 800ms
+                return Math.min(2 ** (times - 1) * 200, 2000);
+            },
+        });
+    }
+    return sharedRedisConnection;
+}
+
+/**
+ * Resets the shared Redis connection. Used primarily for testing.
+ * Disconnects the current connection before resetting.
+ */
+export async function resetSharedRedisConnection(): Promise<void> {
+    if (sharedRedisConnection) {
+        try {
+            sharedRedisConnection.disconnect(false);
+        } catch (_err) {
+            // Ignore disconnect errors during cleanup
+        }
+        sharedRedisConnection = null;
+    }
+}
+
+/**
+ * Gets the total number of active Redis client connections.
+ *
+ * Note: This queries the Redis server directly using CLIENT LIST and reports
+ * ALL active connections to the Redis instance. This includes connections from
+ * Sockethub (BullMQ queues, workers, and the shared connection) as well as any
+ * other applications or services connected to the same Redis server.
+ *
+ * @returns Number of active Redis connections, or 0 if no connection exists
+ */
+export async function getRedisConnectionCount(): Promise<number> {
+    if (!sharedRedisConnection) {
+        return 0;
+    }
+
+    try {
+        const clientList = (await sharedRedisConnection.client(
+            "LIST",
+        )) as string;
+        // CLIENT LIST returns one line per connection, filter out empty lines
+        const connections = clientList
+            .split("\n")
+            .filter((line: string) => line.trim());
+        return connections.length;
+    } catch (_err) {
+        // Return 0 if Redis query fails (connection issues, etc.)
+        return 0;
+    }
 }
 
 export class JobBase extends EventEmitter {

package/src/job-queue.test.ts

@@ -1,8 +1,17 @@
 import { expect } from "chai";
 import * as sinon from "sinon";
 
+import type { Logger } from "@sockethub/schemas";
+
 import { JobQueue } from "./index";
 
+const mockLogger: Logger = {
+    error: () => {},
+    warn: () => {},
+    info: () => {},
+    debug: () => {},
+};
+
 describe("JobQueue", () => {
     let MockBull, jobQueue, cryptoMocks, sandbox;
 
@@ -31,17 +40,22 @@ describe("JobQueue", () => {
 
         class TestJobQueue extends JobQueue {
             init() {
+                // BullMQ v5+ prohibits colons in queue names, so replace with dashes
+                const queueName = this.queueId.replace(/:/g, "-");
                 this.redisConnection = MockBull();
-                this.queue = MockBull(this.uid, {
+                this.queue = MockBull(queueName, {
                     connection: this.redisConnection,
                 });
-                this.events = MockBull(this.uid, {
+                this.events = MockBull(queueName, {
                     connection: this.redisConnection,
                 });
             }
             initCrypto() {
                 this.crypto = cryptoMocks;
             }
+            getQueueId() {
+                return this.queueId;
+            }
         }
         jobQueue = new TestJobQueue(
             "a parent id",
@@ -50,6 +64,7 @@ describe("JobQueue", () => {
             {
                 url: "redis config",
             },
+            mockLogger,
         );
         jobQueue.emit = sandbox.stub();
     });
@@ -63,14 +78,14 @@ describe("JobQueue", () => {
         sinon.assert.calledThrice(MockBull);
         sinon.assert.calledWith(
             MockBull,
-            "sockethub:data-layer:queue:a parent id:a session id",
+            "sockethub-a parent id-data-layer-queue-a session id",
             {
                 connection: MockBull(),
             },
         );
         expect(typeof jobQueue).to.equal("object");
-        expect(jobQueue.uid).to.equal(
-            `sockethub:data-layer:queue:a parent id:a session id`,
+        expect(jobQueue.getQueueId()).to.equal(
+            `sockethub:a parent id:data-layer:queue:a session id`,
         );
         expect(typeof jobQueue.add).to.equal("function");
         expect(typeof jobQueue.getJob).to.equal("function");
@@ -93,7 +108,7 @@ describe("JobQueue", () => {
         it("returns expected job format", () => {
             cryptoMocks.encrypt.returns("an encrypted message");
             const job = jobQueue.createJob("a socket id", {
-                context: "some context",
+                platform: "some context",
                 id: "an identifier",
             });
             expect(job).to.eql({
@@ -106,7 +121,7 @@ describe("JobQueue", () => {
         it("uses counter when no id provided", () => {
             cryptoMocks.encrypt.returns("an encrypted message");
             let job = jobQueue.createJob("a socket id", {
-                context: "some context",
+                platform: "some context",
             });
             expect(job).to.eql({
                 title: "some context-0",
@@ -114,7 +129,7 @@ describe("JobQueue", () => {
                 sessionId: "a socket id",
             });
             job = jobQueue.createJob("a socket id", {
-                context: "some context",
+                platform: "some context",
             });
             expect(job).to.eql({
                 title: "some context-1",
@@ -186,7 +201,7 @@ describe("JobQueue", () => {
                 msg: "encrypted foo",
             };
             const res = await jobQueue.add("a socket id", {
-                context: "a platform",
+                platform: "a platform",
                 id: "an identifier",
             });
             sinon.assert.calledOnce(jobQueue.queue.isPaused);
@@ -207,7 +222,7 @@ describe("JobQueue", () => {
             jobQueue.queue.isPaused.returns(true);
             try {
                 await jobQueue.add("a socket id", {
-                    context: "a platform",
+                    platform: "a platform",
                     id: "an identifier",
                 });
             } catch (err) {