@sockethub/data-layer 1.0.0-alpha.11 → 1.0.0-alpha.13

package/dist/job-base.d.ts

@@ -0,0 +1,44 @@
+import EventEmitter from "node:events";
+import type { ActivityStream } from "@sockethub/schemas";
+import { type Crypto } from "@sockethub/util/crypto";
+import { type Redis } from "ioredis";
+import type { JobDataDecrypted, JobEncrypted, RedisConfig } from "./types.js";
+/**
+ * Creates or returns a shared Redis connection to enable connection pooling.
+ * This prevents connection exhaustion under high load by reusing a single
+ * connection across all JobQueue and JobWorker instances.
+ *
+ * @param config - Redis configuration with optional timeout and retry settings
+ * @returns Shared Redis connection instance
+ */
+export declare function createIORedisConnection(config: RedisConfig): Redis;
+/**
+ * Resets the shared Redis connection. Used primarily for testing.
+ * Disconnects the current connection before resetting.
+ */
+export declare function resetSharedRedisConnection(): Promise<void>;
+/**
+ * Gets the total number of active Redis client connections.
+ *
+ * Note: This queries the Redis server directly using CLIENT LIST and reports
+ * ALL active connections to the Redis instance. This includes connections from
+ * Sockethub (BullMQ queues, workers, and the shared connection) as well as any
+ * other applications or services connected to the same Redis server.
+ *
+ * @returns Number of active Redis connections, or 0 if no connection exists
+ */
+export declare function getRedisConnectionCount(): Promise<number>;
+export declare class JobBase extends EventEmitter {
+    protected crypto: Crypto;
+    private readonly secret;
+    constructor(secret: string);
+    initCrypto(): void;
+    disconnectBase(): void;
+    /**
+     * @param job
+     * @private
+     */
+    protected decryptJobData(job: JobEncrypted): JobDataDecrypted;
+    protected decryptActivityStream(msg: string): ActivityStream;
+    protected encryptActivityStream(msg: ActivityStream): string;
+}

package/dist/job-queue.d.ts

@@ -0,0 +1,60 @@
+import { type ActivityStream } from "@sockethub/schemas";
+import { Queue, QueueEvents } from "bullmq";
+import { JobBase } from "./job-base.js";
+import type { JobDataEncrypted, RedisConfig } from "./types.js";
+export declare function verifyJobQueue(config: RedisConfig): Promise<void>;
+/**
+ * Redis-backed job queue for managing ActivityStreams message processing.
+ *
+ * Creates isolated queues per platform instance and session, providing reliable
+ * message delivery and processing coordination between Sockethub server and
+ * platform workers.
+ *
+ * @example
+ * ```typescript
+ * const queue = new JobQueue('irc-platform', 'session123', secret, redisConfig);
+ * await queue.add('socket-id', activityStreamMessage);
+ * ```
+ */
+export declare class JobQueue extends JobBase {
+    private readonly connectionName;
+    protected readonly queueId: string;
+    protected queue: Queue;
+    protected events: QueueEvents;
+    private readonly log;
+    private counter;
+    private initialized;
+    /**
+     * Creates a new JobQueue instance.
+     *
+     * @param parentId - Sockethub instance identifier for queue isolation
+     * @param instanceId - Unique identifier for the platform instance
+     * @param secret - 32-character encryption secret for message security
+     * @param redisConfig - Redis connection configuration
+     */
+    constructor(parentId: string, instanceId: string, secret: string, redisConfig: RedisConfig);
+    protected init(redisConfig: RedisConfig): void;
+    /**
+     * Adds an ActivityStreams message to the job queue for processing.
+     *
+     * @param socketId - Socket.IO connection identifier for response routing
+     * @param msg - ActivityStreams message to be processed by platform worker
+     * @returns Promise resolving to encrypted job data
+     * @throws Error if queue is closed or Redis connection fails
+     */
+    add(socketId: string, msg: ActivityStream): Promise<JobDataEncrypted>;
+    /**
+     * Pauses job processing. New jobs can still be added but won't be processed.
+     */
+    pause(): Promise<void>;
+    /**
+     * Resumes job processing after being paused.
+     */
+    resume(): Promise<void>;
+    /**
+     * Gracefully shuts down the queue, cleaning up all resources and connections.
+     */
+    shutdown(): Promise<void>;
+    private getJob;
+    private createJob;
+}

package/dist/job-worker.d.ts

@@ -0,0 +1,58 @@
+import { Worker } from "bullmq";
+import { JobBase } from "./job-base.js";
+import type { JobEncrypted, JobHandler, RedisConfig } from "./types.js";
+/**
+ * Worker for processing jobs from a Redis queue within platform child processes.
+ *
+ * Connects to the same queue as its corresponding JobQueue instance and processes
+ * jobs using a platform-specific handler function. Provides automatic decryption
+ * of job data and error handling.
+ *
+ * @example
+ * ```typescript
+ * const worker = new JobWorker('irc-platform', 'session123', secret, redisConfig);
+ * worker.onJob(async (job) => {
+ *   // Process the decrypted ActivityStreams message
+ *   return await processMessage(job.msg);
+ * });
+ * ```
+ */
+export interface JobWorkerOptions {
+    /**
+     * Number of jobs this worker will process in parallel. Defaults to 1
+     * (serial processing). Only safe to raise for stateless platforms whose
+     * job handlers are independent of one another (e.g. feeds, metadata).
+     */
+    concurrency?: number;
+}
+export declare class JobWorker extends JobBase {
+    private readonly connectionName;
+    protected worker: Worker;
+    protected handler: JobHandler;
+    private readonly log;
+    private readonly redisConfig;
+    protected readonly queueId: string;
+    private readonly concurrency;
+    private initialized;
+    /**
+     * Creates a new JobWorker instance.
+     *
+     * @param parentId - Must match the parentId of the corresponding JobQueue
+     * @param instanceId - Must match the instanceId of the corresponding JobQueue
+     * @param secret - 32-character encryption secret, must match JobQueue secret
+     * @param redisConfig - Redis connection configuration
+     */
+    constructor(parentId: string, instanceId: string, secret: string, redisConfig: RedisConfig, options?: JobWorkerOptions);
+    protected init(): void;
+    /**
+     * Registers a job handler function and starts processing jobs from the queue.
+     *
+     * @param handler - Function that processes decrypted job data and returns results
+     */
+    onJob(handler: JobHandler): void;
+    /**
+     * Gracefully shuts down the worker, stopping job processing and cleaning up connections.
+     */
+    shutdown(): Promise<void>;
+    protected jobHandler(encryptedJob: JobEncrypted): Promise<any>;
+}

package/dist/queue-id.d.ts

@@ -0,0 +1,2 @@
+export declare function buildQueueId(parentId: string, instanceId: string): string;
+export declare function buildCredentialsStoreId(parentId: string, sessionId: string): string;

package/{src/types.ts → dist/types.d.ts}

@@ -1,8 +1,4 @@
-import type {
-    ActivityStream,
-    InternalActivityStream,
-} from "@sockethub/schemas";
-
+import type { ActivityStream, InternalActivityStream } from "@sockethub/schemas";
 export type RedisConfig = {
     url: string;
     connectTimeout?: number;
@@ -10,30 +6,23 @@ export type RedisConfig = {
     maxRetriesPerRequest?: number | null;
     connectionName?: string;
 };
-
 export interface JobDataEncrypted {
     title?: string;
     msg: string;
     sessionId: string;
 }
-
 export interface JobDataDecrypted {
     title?: string;
     msg: InternalActivityStream;
     sessionId: string;
 }
-
 export interface JobEncrypted {
     data: JobDataEncrypted;
     remove?: () => void;
 }
-
 export interface JobDecrypted {
     data: JobDataDecrypted;
     remove?: () => void;
     returnvalue: unknown;
 }
-
-export type JobHandler = (
-    job: JobDataDecrypted,
-) => Promise<string | undefined | ActivityStream>;
+export type JobHandler = (job: JobDataDecrypted) => Promise<string | undefined | ActivityStream>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@sockethub/data-layer",
   "description": "Storing and RPC of data for Sockethub",
-  "version": "1.0.0-alpha.11",
+  "version": "1.0.0-alpha.13",
   "type": "module",
   "private": false,
   "author": "Nick Jennings <nick@silverbucket.net>",
@@ -12,17 +12,16 @@
   "main": "dist/index.js",
   "exports": {
     ".": {
-      "bun": "./src/index.ts",
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     }
   },
   "files": [
-    "src/",
     "dist/"
   ],
   "engines": {
-    "bun": ">=1.2"
+    "node": ">=20"
   },
   "keywords": [
     "sockethub",
@@ -47,9 +46,9 @@
     "test:integration": "bun test ./integration/redis.integration.ts"
   },
   "dependencies": {
-    "@sockethub/crypto": "^1.0.0-alpha.11",
-    "@sockethub/logger": "^1.0.0-alpha.11",
-    "@sockethub/schemas": "^3.0.0-alpha.11",
+    "@sockethub/logger": "^1.0.0-alpha.13",
+    "@sockethub/schemas": "^3.0.0-alpha.13",
+    "@sockethub/util": "^1.0.0-alpha.1",
     "bullmq": "^5.66.5",
     "ioredis": "^5.9.2",
     "secure-store-redis": "^4.1.0"
@@ -63,5 +62,6 @@
     "typedoc-plugin-markdown": "^4.9.0",
     "winston": "^3.19.0"
   },
-  "gitHead": "c243fa9e76c688ce5ffcf524400b1dd27dcce615"
+  "gitHead": "864733e5b34449ef39542eceb4ff11aa308081bc",
+  "types": "./dist/index.d.ts"
 }

package/src/credentials-store.test.ts

@@ -1,171 +0,0 @@
-import { beforeEach, describe, expect, it } from "bun:test";
-import * as sinon from "sinon";
-
-import type { Logger } from "@sockethub/schemas";
-
-import { CredentialsStore } from "./credentials-store";
-
-const mockLogger: Logger = {
-    error: () => {},
-    warn: () => {},
-    info: () => {},
-    debug: () => {},
-};
-
-describe("CredentialsStore", () => {
-    let credentialsStore,
-        MockSecureStore,
-        MockStoreGet,
-        MockStoreSave,
-        MockObjectHash;
-    beforeEach(() => {
-        MockStoreGet = sinon.stub().returns("credential foo");
-        MockStoreSave = sinon.stub();
-        MockObjectHash = sinon.stub();
-        MockSecureStore = sinon.stub().returns({
-            get: MockStoreGet,
-            save: MockStoreSave,
-            isConnected: true,
-            connect: sinon.stub().resolves(),
-        });
-        class TestCredentialsStore extends CredentialsStore {
-            initCrypto() {
-                this.objectHash = MockObjectHash;
-            }
-            initSecureStore(secret, redisConfig) {
-                this.store = MockSecureStore({
-                    namespace: "foo",
-                    secret: secret,
-                    redis: redisConfig,
-                });
-            }
-        }
-        credentialsStore = new TestCredentialsStore(
-            "a parent id",
-            "a session id",
-            "a secret must be 32 chars and th",
-            { url: "redis config" },
-            mockLogger,
-        );
-    });
-
-    it("returns a valid CredentialsStore object", () => {
-        sinon.assert.calledOnce(MockSecureStore);
-        sinon.assert.calledWith(MockSecureStore, {
-            namespace: "foo",
-            secret: "a secret must be 32 chars and th",
-            redis: {
-                url: "redis config",
-                connectionName:
-                    "data-layer:credentials-store:a parent id:a session id",
-            },
-        });
-        expect(typeof credentialsStore).toEqual("object");
-        expect(credentialsStore.uid).toEqual(
-            `sockethub:a parent id:data-layer:credentials-store:a session id`,
-        );
-        expect(typeof credentialsStore.get).toEqual("function");
-        expect(typeof credentialsStore.save).toEqual("function");
-    });
-
-    describe("get", () => {
-        it("handles correct params", async () => {
-            const res = await credentialsStore.get("an actor");
-            sinon.assert.calledOnce(MockStoreGet);
-            sinon.assert.calledWith(MockStoreGet, "an actor");
-            sinon.assert.notCalled(MockObjectHash);
-            sinon.assert.notCalled(MockStoreSave);
-            expect(res).toEqual("credential foo");
-        });
-
-        it("handles no credentials found", async () => {
-            MockStoreGet.returns(undefined);
-            expect(async () => {
-                await credentialsStore.get("a non-existent actor");
-            }).toThrow("credentials not found for a non-existent actor");
-            sinon.assert.calledOnce(MockStoreGet);
-            sinon.assert.calledWith(MockStoreGet, "a non-existent actor");
-            sinon.assert.notCalled(MockObjectHash);
-            sinon.assert.notCalled(MockStoreSave);
-        });
-
-        it("handles an unexpected error", async () => {
-            MockStoreGet.returns(undefined);
-            try {
-                await credentialsStore.get("a problem actor");
-                expect(false).toEqual(true);
-            } catch (err) {
-                expect(err.toString()).toEqual(
-                    "Error: credentials not found for a problem actor",
-                );
-            }
-            sinon.assert.calledOnce(MockStoreGet);
-            sinon.assert.calledWith(MockStoreGet, "a problem actor");
-            sinon.assert.notCalled(MockObjectHash);
-            sinon.assert.notCalled(MockStoreSave);
-        });
-
-        it("validates credentialsHash when provided", async () => {
-            MockObjectHash.returns("a credentialsHash string");
-            MockStoreGet.returns({
-                object: "a credential",
-            });
-            const res = await credentialsStore.get(
-                "an actor",
-                "a credentialsHash string",
-            );
-            sinon.assert.calledOnce(MockStoreGet);
-            sinon.assert.calledWith(MockStoreGet, "an actor");
-            sinon.assert.calledOnce(MockObjectHash);
-            sinon.assert.calledWith(MockObjectHash, "a credential");
-            sinon.assert.notCalled(MockStoreSave);
-            expect(res).toEqual({ object: "a credential" });
-        });
-
-        it("invalidates credentialsHash when provided", async () => {
-            MockObjectHash.returns("the original credentialsHash string");
-            MockStoreGet.returns({
-                object: "a credential",
-            });
-            try {
-                expect(
-                    await credentialsStore.get(
-                        "an actor",
-                        "a different credentialsHash string",
-                    ),
-                ).toBeUndefined();
-                expect(false).toEqual(true);
-            } catch (err) {
-                expect(err.toString()).toEqual(
-                    "Error: invalid credentials for an actor",
-                );
-            }
-            sinon.assert.calledOnce(MockStoreGet);
-            sinon.assert.calledWith(MockStoreGet, "an actor");
-            sinon.assert.calledOnce(MockObjectHash);
-            sinon.assert.calledWith(MockObjectHash, "a credential");
-            sinon.assert.notCalled(MockStoreSave);
-        });
-    });
-
-    describe("save", () => {
-        it("handles success", async () => {
-            const creds = { foo: "bar" };
-            await credentialsStore.save("an actor", creds);
-            sinon.assert.calledOnce(MockStoreSave);
-            sinon.assert.calledWith(MockStoreSave, "an actor", creds);
-            sinon.assert.notCalled(MockObjectHash);
-            sinon.assert.notCalled(MockStoreGet);
-        });
-
-        it("handles failure", async () => {
-            const creds = { foo: "bar" };
-            MockStoreSave.returns(undefined);
-            await credentialsStore.save("an actor", creds);
-            sinon.assert.calledOnce(MockStoreSave);
-            sinon.assert.calledWith(MockStoreSave, "an actor", creds);
-            sinon.assert.notCalled(MockObjectHash);
-            sinon.assert.notCalled(MockStoreGet);
-        });
-    });
-});

package/src/credentials-store.ts

@@ -1,192 +0,0 @@
-import { crypto } from "@sockethub/crypto";
-import {
-    type Logger,
-    createLogger,
-    getLoggerNamespace,
-} from "@sockethub/logger";
-import type { CredentialsObject } from "@sockethub/schemas";
-import IORedis, { type Redis } from "ioredis";
-import SecureStore from "secure-store-redis";
-
-import { buildCredentialsStoreId } from "./queue-id.js";
-import type { RedisConfig } from "./types.js";
-
-let sharedCredentialsRedisConnection: Redis | null = null;
-
-/**
- * Creates or returns a shared Redis connection for CredentialsStore instances.
- * This prevents connection exhaustion by reusing a single connection across
- * all credential storage operations.
- *
- * @param config - Redis configuration
- * @returns Shared Redis connection instance
- */
-export function createCredentialsRedisConnection(config: RedisConfig): Redis {
-    if (!sharedCredentialsRedisConnection) {
-        sharedCredentialsRedisConnection = new IORedis(config.url, {
-            connectionName: config.connectionName,
-            enableOfflineQueue: false,
-            maxRetriesPerRequest: config.maxRetriesPerRequest ?? null,
-            connectTimeout: config.connectTimeout ?? 10000,
-            disconnectTimeout: config.disconnectTimeout ?? 5000,
-            lazyConnect: false,
-            retryStrategy: (times: number) => {
-                if (times > 3) return null;
-                return Math.min(2 ** (times - 1) * 200, 2000);
-            },
-        });
-    }
-    return sharedCredentialsRedisConnection;
-}
-
-/**
- * Resets the shared credentials Redis connection. Used primarily for testing.
- */
-export async function resetSharedCredentialsRedisConnection(): Promise<void> {
-    if (sharedCredentialsRedisConnection) {
-        try {
-            sharedCredentialsRedisConnection.disconnect(false);
-        } catch (err) {
-            // Ignore disconnect errors during cleanup
-        }
-        sharedCredentialsRedisConnection = null;
-    }
-}
-
-export interface CredentialsStoreInterface {
-    get(
-        actor: string,
-        credentialsHash: string | undefined,
-    ): Promise<CredentialsObject | undefined>;
-    save(actor: string, creds: CredentialsObject): Promise<number>;
-}
-
-export async function verifySecureStore(config: RedisConfig): Promise<void> {
-    const log = createLogger("data-layer:verify-secure-store");
-    const sharedClient = createCredentialsRedisConnection(config);
-    const ss = new SecureStore({
-        uid: "data-layer:verify",
-        secret: "aB3#xK9mP2qR7wZ4cT8nY6vH1jL5fD0s",
-        redis: { client: sharedClient },
-    });
-    await ss.connect();
-    await ss.disconnect();
-    log.info("secure store connection verified");
-}
-
-/**
- * Secure, encrypted storage for user credentials with session-based isolation.
- *
- * Provides automatic encryption/decryption of credential objects stored in Redis,
- * ensuring that sensitive authentication data is never stored in plaintext.
- * Each session gets its own isolated credential store.
- *
- * @example
- * ```typescript
- * const store = new CredentialsStore('session123', secret, redisConfig);
- *
- * // Store credentials
- * await store.save('user@example.com', {
- *   username: 'user',
- *   password: 'secret',
- *   server: 'irc.freenode.net'
- * });
- *
- * // Retrieve credentials
- * const creds = await store.get('user@example.com', credentialsHash);
- * ```
- */
-export class CredentialsStore implements CredentialsStoreInterface {
-    readonly uid: string;
-    store: SecureStore;
-    objectHash: (o: unknown) => string;
-    private readonly log: Logger;
-
-    /**
-     * Creates a new CredentialsStore instance.
-     *
-     * @param parentId - Unique identifier for the parent instance (e.g. server ID)
-     * @param sessionId - Client session identifier for credential isolation
-     * @param secret - 32-character encryption secret for credential security
-     * @param redisConfig - Redis connection configuration
-     * @throws Error if secret is not exactly 32 characters
-     */
-    constructor(
-        parentId: string,
-        sessionId: string,
-        secret: string,
-        redisConfig: RedisConfig,
-    ) {
-        if (secret.length !== 32) {
-            throw new Error(
-                "CredentialsStore secret must be 32 chars in length",
-            );
-        }
-        // Create logger with full namespace (context will be prepended automatically)
-        this.log = createLogger(
-            `data-layer:credentials-store:${parentId}:${sessionId}`,
-        );
-
-        this.initCrypto();
-
-        // Use the canonical, context-free namespace for credentials storage keys
-        this.uid = buildCredentialsStoreId(parentId, sessionId);
-        // Keep full logger namespace for Redis connection naming
-        redisConfig.connectionName = getLoggerNamespace(this.log);
-        this.initSecureStore(secret, redisConfig);
-        this.log.debug("initialized");
-    }
-
-    initCrypto() {
-        this.objectHash = crypto.objectHash;
-    }
-
-    initSecureStore(secret: string, redisConfig: RedisConfig) {
-        // Use shared Redis connection for connection pooling
-        const sharedClient = createCredentialsRedisConnection(redisConfig);
-        this.store = new SecureStore({
-            uid: this.uid,
-            secret: secret,
-            redis: { client: sharedClient },
-        });
-    }
-
-    /**
-     * Gets the credentials for a given actor ID.
-     * @param actor
-     * @param credentialsHash - Optional hash to validate credentials.
-     *   If undefined, validation is skipped.
-     */
-    async get(
-        actor: string,
-        credentialsHash: string | undefined,
-    ): Promise<CredentialsObject> {
-        this.log.debug(`get credentials for ${actor}`);
-        if (!this.store.isConnected) {
-            await this.store.connect();
-        }
-        const credentials: CredentialsObject = await this.store.get(actor);
-        if (!credentials) {
-            throw new Error(`credentials not found for ${actor}`);
-        }
-
-        if (credentialsHash) {
-            if (credentialsHash !== this.objectHash(credentials.object)) {
-                throw new Error(`invalid credentials for ${actor}`);
-            }
-        }
-        return credentials;
-    }
-
-    /**
-     * Saves the credentials for a given actor ID
-     * @param actor
-     * @param creds
-     */
-    async save(actor: string, creds: CredentialsObject): Promise<number> {
-        if (!this.store.isConnected) {
-            await this.store.connect();
-        }
-        return this.store.save(actor, creds);
-    }
-}

package/src/index.ts

@@ -1,34 +0,0 @@
-import { createLogger } from "@sockethub/logger";
-
-import {
-    CredentialsStore,
-    type CredentialsStoreInterface,
-    resetSharedCredentialsRedisConnection,
-    verifySecureStore,
-} from "./credentials-store.js";
-import {
-    getRedisConnectionCount,
-    resetSharedRedisConnection,
-} from "./job-base.js";
-import { JobQueue, verifyJobQueue } from "./job-queue.js";
-import { JobWorker } from "./job-worker.js";
-export * from "./types.js";
-import type { RedisConfig } from "./types.js";
-
-async function redisCheck(config: RedisConfig): Promise<void> {
-    const log = createLogger("data-layer:redis-check");
-    log.debug(`checking redis connection ${config.url}`);
-    await verifySecureStore(config);
-    await verifyJobQueue(config);
-}
-
-export {
-    redisCheck,
-    JobQueue,
-    JobWorker,
-    CredentialsStore,
-    getRedisConnectionCount,
-    resetSharedRedisConnection,
-    resetSharedCredentialsRedisConnection,
-    type CredentialsStoreInterface,
-};