@sockethub/data-layer 1.0.0-alpha.3

package/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

package/dist/credentials-store.d.ts

@@ -0,0 +1,31 @@
+import { IActivityStream } from "@sockethub/schemas";
+import { RedisConfigProps, RedisConfigUrl } from "./types";
+/**
+ * Encapsulates the storing and fetching of credential objects.
+ */
+export default class CredentialsStore {
+    readonly uid: string;
+    private readonly store;
+    private readonly log;
+    protected initialized: boolean;
+    /**
+     * @param parentId - The ID of the parent instance (eg. sockethub itself)
+     * @param sessionId - The ID of the session (socket.io connection)
+     * @param secret - The encryption secret (parent + session secrets)
+     * @param redisConfig - Connect info for redis
+     */
+    constructor(parentId: string, sessionId: string, secret: string, redisConfig: RedisConfigProps | RedisConfigUrl);
+    init(): Promise<void>;
+    /**
+     * Gets the credentials for a given actor ID
+     * @param actor
+     * @param credentialHash
+     */
+    get(actor: string, credentialHash?: string): Promise<IActivityStream>;
+    /**
+     * Saves the credentials for a given actor ID
+     * @param actor
+     * @param creds
+     */
+    save(actor: string, creds: IActivityStream): Promise<void>;
+}

package/dist/credentials-store.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const secure_store_redis_1 = __importDefault(require("secure-store-redis"));
+const debug_1 = __importDefault(require("debug"));
+const crypto_1 = __importDefault(require("@sockethub/crypto"));
+/**
+ * Encapsulates the storing and fetching of credential objects.
+ */
+class CredentialsStore {
+    /**
+     * @param parentId - The ID of the parent instance (eg. sockethub itself)
+     * @param sessionId - The ID of the session (socket.io connection)
+     * @param secret - The encryption secret (parent + session secrets)
+     * @param redisConfig - Connect info for redis
+     */
+    constructor(parentId, sessionId, secret, redisConfig) {
+        this.initialized = false;
+        this.uid = `sockethub:data-layer:credentials-store:${parentId}:${sessionId}`;
+        this.store = new secure_store_redis_1.default(this.uid, secret, { redis: redisConfig });
+        this.log = (0, debug_1.default)(this.uid);
+    }
+    async init() {
+        this.initialized = true;
+        await this.store.init();
+    }
+    /**
+     * Gets the credentials for a given actor ID
+     * @param actor
+     * @param credentialHash
+     */
+    async get(actor, credentialHash = undefined) {
+        this.log(`get credentials for ${actor}`);
+        if (!this.initialized) {
+            throw new Error('CredentialsStore not initialized');
+        }
+        const credentials = await this.store.get(actor);
+        if (!credentials) {
+            return undefined;
+        }
+        if (credentialHash) {
+            if (credentialHash !== crypto_1.default.objectHash(credentials.object)) {
+                throw new Error(`provided credentials do not match existing platform instance for actor ${actor}`);
+            }
+        }
+        return credentials;
+    }
+    /**
+     * Saves the credentials for a given actor ID
+     * @param actor
+     * @param creds
+     */
+    async save(actor, creds) {
+        if (!this.initialized) {
+            throw new Error('CredentialsStore not initialized');
+        }
+        await this.store.save(actor, creds);
+        this.log(`credentials encrypted and saved`);
+    }
+}
+exports.default = CredentialsStore;
+//# sourceMappingURL=/credentials-store.js.map

package/dist/credentials-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"credentials-store.js","sourceRoot":"/","sources":["credentials-store.ts"],"names":[],"mappings":";;;;;AAAA,4EAA6C;AAC7C,kDAAsC;AAEtC,+DAAuC;AAGvC;;GAEG;AACH,MAAqB,gBAAgB;IAMnC;;;;;OAKG;IACH,YACE,QAAgB,EAChB,SAAiB,EACjB,MAAc,EACd,WAA8C;QAZtC,gBAAW,GAAG,KAAK,CAAC;QAc5B,IAAI,CAAC,GAAG,GAAG,0CAA0C,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC7E,IAAI,CAAC,KAAK,GAAG,IAAI,4BAAW,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,GAAG,GAAG,IAAA,eAAK,EAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG,CAAC,KAAa,EAAE,iBAAyB,SAAS;QACzD,IAAI,CAAC,GAAG,CAAC,uBAAuB,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;SAAE;QAC/E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,WAAW,EAAE;YAAE,OAAO,SAAS,CAAC;SAAE;QAEvC,IAAI,cAAc,EAAE;YAClB,IAAI,cAAc,KAAK,gBAAM,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;gBAC5D,MAAM,IAAI,KAAK,CACb,0EAA0E,KAAK,EAAE,CAClF,CAAC;aACH;SACF;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CAAC,KAAa,EAAE,KAAsB;QAC9C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;SAAE;QAC/E,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC9C,CAAC;CACF;AA3DD,mCA2DC"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { default as CredentialsStore } from "./credentials-store";
+export { default as JobQueue } from "./job-queue";
+export * from "./types";

package/dist/index.js

@@ -0,0 +1,41 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JobQueue = exports.CredentialsStore = void 0;
+var credentials_store_1 = require("./credentials-store");
+Object.defineProperty(exports, "CredentialsStore", { enumerable: true, get: function () { return __importDefault(credentials_store_1).default; } });
+var job_queue_1 = require("./job-queue");
+Object.defineProperty(exports, "JobQueue", { enumerable: true, get: function () { return __importDefault(job_queue_1).default; } });
+__exportStar(require("./types"), exports);
+// export default class DataLayer {
+//   readonly uid: string;
+//   readonly jobQueue: JobQueue;
+//   readonly credentialStore: CredentialsStore;
+//
+//   constructor(instanceId: string, sessionId: string, secret: string, redisConfig: RedisConfig) {
+//     this.uid = `sockethub:data-layer:job-queue:${instanceId}:${sessionId}`;
+//     this.jobQueue = new JobQueue(
+//       instanceId, sessionId, secret, redisConfig
+//     ),
+//     this.credentialStore = new CredentialsStore(
+//       instanceId, sessionId, secret, redisConfig
+//     );
+//   }
+// }
+//# sourceMappingURL=/index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"/","sources":["index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,yDAAgE;AAAxD,sIAAA,OAAO,OAAoB;AACnC,yCAAgD;AAAxC,sHAAA,OAAO,OAAY;AAC3B,0CAAwB;AAExB,mCAAmC;AACnC,0BAA0B;AAC1B,iCAAiC;AACjC,gDAAgD;AAChD,EAAE;AACF,mGAAmG;AACnG,8EAA8E;AAC9E,oCAAoC;AACpC,mDAAmD;AACnD,SAAS;AACT,mDAAmD;AACnD,mDAAmD;AACnD,SAAS;AACT,MAAM;AACN,IAAI"}

package/dist/job-queue.d.ts

@@ -0,0 +1,32 @@
+/// <reference types="node" />
+import { JobDataDecrypted, JobDataEncrypted, JobDecrypted, RedisConfig } from "./types";
+import EventEmitter from "events";
+import { IActivityStream } from "@sockethub/schemas";
+interface JobHandler {
+    (job: JobDataDecrypted, done: CallableFunction): any;
+}
+export default class JobQueue extends EventEmitter {
+    readonly uid: string;
+    private readonly bull;
+    private readonly debug;
+    private readonly secret;
+    private handler;
+    private counter;
+    constructor(instanceId: string, sessionId: string, secret: string, redisConfig: RedisConfig);
+    add(socketId: string, msg: IActivityStream): Promise<JobDataEncrypted>;
+    initResultEvents(): void;
+    getJob(jobId: string): Promise<JobDecrypted>;
+    onJob(handler: JobHandler): void;
+    pause(): Promise<void>;
+    resume(): Promise<void>;
+    shutdown(): Promise<void>;
+    private createJob;
+    private jobHandler;
+    /**
+     * @param job
+     * @private
+     */
+    private decryptJobData;
+    private decryptActivityStream;
+}
+export {};

package/dist/job-queue.js

@@ -0,0 +1,129 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const bull_1 = __importDefault(require("bull"));
+const crypto_1 = __importDefault(require("@sockethub/crypto"));
+const debug_1 = __importDefault(require("debug"));
+const events_1 = __importDefault(require("events"));
+class JobQueue extends events_1.default {
+    constructor(instanceId, sessionId, secret, redisConfig) {
+        super();
+        this.counter = 0;
+        this.bull = new bull_1.default(instanceId + sessionId, { redis: redisConfig });
+        this.uid = `sockethub:data-layer:job-queue:${instanceId}:${sessionId}`;
+        this.secret = secret;
+        this.debug = (0, debug_1.default)(this.uid);
+        this.debug('initialized');
+    }
+    async add(socketId, msg) {
+        const job = this.createJob(socketId, msg);
+        const isPaused = await this.bull.isPaused();
+        if (isPaused) {
+            this.bull.emit('failed', job, 'queue closed');
+            return undefined;
+        }
+        this.debug(`adding ${job.title} ${msg.type}`);
+        this.bull.add(job);
+        return job;
+    }
+    initResultEvents() {
+        this.bull.on('global:completed', async (jobId, result) => {
+            const r = result ? JSON.parse(result) : "";
+            const job = await this.getJob(jobId);
+            if (job) {
+                this.debug(`completed ${job.data.title} ${job.data.msg.type}`);
+                this.emit('global:completed', job.data, r);
+                await job.remove();
+            }
+        });
+        this.bull.on('global:error', async (jobId, result) => {
+            this.debug("unknown queue error", jobId, result);
+        });
+        this.bull.on('global:failed', async (jobId, result) => {
+            const job = await this.getJob(jobId);
+            if (job) {
+                this.debug(`failed ${job.data.title} ${job.data.msg.type}`);
+                this.emit('global:failed', job.data, result);
+                await job.remove();
+            }
+        });
+        this.bull.on('failed', (job, result) => {
+            // locally failed jobs (eg. due to paused queue)
+            const unencryptedJobData = {
+                title: job.title,
+                msg: this.decryptActivityStream(job.msg),
+                sessionId: job.sessionId
+            };
+            this.debug(`failed ${unencryptedJobData.title} ${unencryptedJobData.msg.type}`);
+            this.emit('global:failed', unencryptedJobData, result);
+        });
+        this.bull.on('completed', (job) => {
+            console.log("completed ", job);
+        });
+    }
+    async getJob(jobId) {
+        const job = await this.bull.getJob(jobId);
+        if (job) {
+            job.data = this.decryptJobData(job);
+            try {
+                delete job.data.msg.sessionSecret;
+            }
+            catch (e) {
+                // this property should never be exposed externally
+            }
+        }
+        return job;
+    }
+    onJob(handler) {
+        this.handler = handler;
+        this.bull.process(this.jobHandler.bind(this));
+    }
+    async pause() {
+        await this.bull.pause();
+        this.debug('paused');
+    }
+    async resume() {
+        await this.bull.resume();
+        this.debug('resumed');
+    }
+    async shutdown() {
+        this.debug('shutdown');
+        const isPaused = await this.bull.isPaused(true);
+        if (!isPaused) {
+            await this.bull.pause();
+        }
+        await this.bull.obliterate({ force: true });
+        await this.bull.removeAllListeners();
+    }
+    createJob(socketId, msg) {
+        const title = `${msg.context}-${(msg.id) ? msg.id : this.counter++}`;
+        return {
+            title: title,
+            sessionId: socketId,
+            msg: crypto_1.default.encrypt(msg, this.secret)
+        };
+    }
+    jobHandler(encryptedJob, done) {
+        const job = this.decryptJobData(encryptedJob);
+        this.debug(`handling ${job.title} ${job.msg.type}`);
+        this.handler(job, done);
+    }
+    /**
+     * @param job
+     * @private
+     */
+    decryptJobData(job) {
+        return {
+            title: job.data.title,
+            msg: this.decryptActivityStream(job.data.msg),
+            sessionId: job.data.sessionId
+        };
+    }
+    decryptActivityStream(msg) {
+        return crypto_1.default.decrypt(msg, this.secret);
+    }
+}
+exports.default = JobQueue;
+//# sourceMappingURL=/job-queue.js.map

package/dist/job-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-queue.js","sourceRoot":"/","sources":["job-queue.ts"],"names":[],"mappings":";;;;;AAAA,gDAAyB;AACzB,+DAAuC;AAOvC,kDAAsC;AACtC,oDAAkC;AAOlC,MAAqB,QAAS,SAAQ,gBAAY;IAQhD,YAAY,UAAkB,EAAE,SAAiB,EAAE,MAAc,EAAE,WAAwB;QACzF,KAAK,EAAE,CAAC;QAHF,YAAO,GAAG,CAAC,CAAC;QAIlB,IAAI,CAAC,IAAI,GAAG,IAAI,cAAK,CAAC,UAAU,GAAG,SAAS,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,GAAG,GAAG,kCAAkC,UAAU,IAAI,SAAS,EAAE,CAAC;QACvE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAA,eAAK,EAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,GAAoB;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,QAAQ,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;YAC9C,OAAO,SAAS,CAAC;SAClB;QACD,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,gBAAgB;QACd,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,kBAAkB,EAAE,KAAK,EAAE,KAAa,EAAE,MAAc,EAAE,EAAE;YACvE,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,GAAG,EAAE;gBACP,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC3C,MAAM,GAAG,CAAC,MAAM,EAAE,CAAC;aACpB;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,EAAE,KAAa,EAAE,MAAc,EAAE,EAAE;YACnE,IAAI,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,MAAc,EAAE,EAAE;YAC5D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,GAAG,EAAE;gBACP,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC5D,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC7C,MAAM,GAAG,CAAC,MAAM,EAAE,CAAC;aACpB;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAqB,EAAE,MAAc,EAAE,EAAE;YAC/D,gDAAgD;YAChD,MAAM,kBAAkB,GAAqB;gBAC3C,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,GAAG,EAAE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC;gBACxC,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,UAAU,kBAAkB,CAAC,KAAK,IAAI,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAChF,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,kBAAkB,EAAE,MAAM,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE;YAChC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,GAAG,EAAE;YACP,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI;gBACF,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC;aACnC;YAAC,OAAO,CAAC,EAAE;gBACV,mDAAmD;aACpD;SACF;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,OAAmB;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACvB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;SACzB;QACD,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;IACvC,CAAC;IAEO,SAAS,CAAC,QAAgB,EAAE,GAAG;QACrC,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACrE,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,QAAQ;YACnB,GAAG,EAAE,gBAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;SACtC,CAAC;IACJ,CAAC;IAEO,UAAU,CAAC,YAA0B,EAAE,IAAsB;QACnE,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACK,cAAc,CAAC,GAAiB;QACtC,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;YACrB,GAAG,EAAE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YAC7C,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS;SAC9B,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAAC,GAAW;QACvC,OAAO,gBAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;CACF;AAtID,2BAsIC"}

package/dist/store.d.ts

@@ -0,0 +1,3 @@
+import SecureStore from 'secure-store-redis';
+export declare type SessionStore = SecureStore;
+export declare function getSessionStore(parentId: string, parentSecret: string, sessionId: string, sessionSecret: string): SessionStore;

package/dist/store.js

@@ -0,0 +1,17 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSessionStore = void 0;
+const secure_store_redis_1 = __importDefault(require("secure-store-redis"));
+const config_1 = __importDefault(require("../../server/src/config"));
+function getSessionStore(parentId, parentSecret, sessionId, sessionSecret) {
+    return new secure_store_redis_1.default({
+        namespace: 'sockethub:' + parentId + ':session:' + sessionId + ':store',
+        secret: parentSecret + sessionSecret,
+        redis: config_1.default.get('redis')
+    });
+}
+exports.getSessionStore = getSessionStore;
+//# sourceMappingURL=/store.js.map

package/dist/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"/","sources":["store.ts"],"names":[],"mappings":";;;;;;AAAA,4EAA6C;AAC7C,qEAA6C;AAI7C,SAAgB,eAAe,CAAC,QAAgB,EAAE,YAAoB,EACtC,SAAiB,EAAE,aAAqB;IACtE,OAAO,IAAI,4BAAW,CAAC;QACrB,SAAS,EAAE,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ;QACvE,MAAM,EAAE,YAAY,GAAG,aAAa;QACpC,KAAK,EAAE,gBAAM,CAAC,GAAG,CAAC,OAAO,CAAC;KAC3B,CAAC,CAAC;AACL,CAAC;AAPD,0CAOC"}

package/dist/types.d.ts

@@ -0,0 +1,29 @@
+import { IActivityStream } from "@sockethub/schemas";
+export declare type RedisConfigUrl = string;
+export interface RedisConfigProps {
+    host: string;
+    port: string;
+}
+export declare type RedisConfig = RedisConfigProps | RedisConfigUrl;
+export interface JobDataEncrypted {
+    title?: string;
+    msg: string;
+    sessionId: string;
+}
+export interface JobDataDecrypted {
+    title?: string;
+    msg: IActivityStream;
+    sessionId: string;
+}
+export interface JobEncrypted {
+    data: JobDataEncrypted;
+    remove?: {
+        (): void;
+    };
+}
+export interface JobDecrypted {
+    data: JobDataDecrypted;
+    remove?: {
+        (): void;
+    };
+}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=/types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"/","sources":["types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@sockethub/data-layer",
+  "description": "Storing and RPC of data for Sockethub",
+  "version": "1.0.0-alpha.3",
+  "private": false,
+  "author": "Nick Jennings <nick@silverbucket.net>",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist/",
+    "src/"
+  ],
+  "keywords": [
+    "sockethub",
+    "messaging",
+    "redis",
+    "data lyer",
+    "rpc",
+    "data store"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sockethub/sockethub.git",
+    "directory": "packages/data-layer"
+  },
+  "homepage": "https://github.com/sockethub/sockethub/tree/master/packages/data-layer",
+  "dependencies": {
+    "@sockethub/crypto": "^1.0.0-alpha.3",
+    "@sockethub/schemas": "^3.0.0-alpha.3",
+    "bull": "^4.0.0",
+    "debug": "^4.3.1",
+    "secure-store-redis": "^1.4.7"
+  },
+  "scripts": {
+    "clean": "npx rimraf dist coverage",
+    "clean:deps": "npx rimraf node_modules",
+    "compliance": "yarn run lint && yarn run test && yarn run coverage",
+    "test": "c8 -x \"src/**/*.test.*\" mocha -r ts-node/register src/*.test.ts",
+    "coverage": "c8 check-coverage --statements 85 --branches 90 --functions 100 --lines 85",
+    "lint": "eslint \"**/*.ts\"",
+    "lint:fix": "eslint --fix \"**/*.ts\"",
+    "build": "tsc"
+  },
+  "engines": {
+    "node": ">= 14"
+  },
+  "devDependencies": {
+    "@types/chai": "4.3.3",
+    "@types/debug": "4.1.7",
+    "@types/eslint": "8.4.6",
+    "@types/mocha": "9.1.1",
+    "@types/node": "17.0.13",
+    "@types/proxyquire": "1.3.28",
+    "@types/sinon": "10.0.13",
+    "@typescript-eslint/parser": "5.37.0",
+    "c8": "7.12.0",
+    "chai": "4.3.6",
+    "eslint": "8.23.1",
+    "eslint-cli": "1.1.1",
+    "mocha": "10.0.0",
+    "proxyquire": "2.1.3",
+    "sinon": "14.0.0",
+    "ts-node": "10.9.1",
+    "typescript": "4.8.3"
+  },
+  "gitHead": "1fa39894aafb49dfea7a7425bdaf196325e81e5d"
+}

package/src/credentials-store.test.ts

@@ -0,0 +1,138 @@
+import proxyquire from 'proxyquire';
+import { expect } from 'chai';
+import * as sinon from 'sinon';
+
+proxyquire.noPreserveCache();
+proxyquire.noCallThru();
+
+describe('CredentialsStore', () => {
+  let credentialsStore, MockSecureStore, MockStoreGet, MockStoreSave, MockObjectHash;
+  beforeEach(() => {
+    MockStoreGet = sinon.stub().callsArgWith(1, undefined, 'credential foo');
+    MockStoreSave = sinon.stub().callsArgWith(2, undefined);
+    MockObjectHash = sinon.stub();
+    MockSecureStore = sinon.stub().returns({
+      get: MockStoreGet,
+      save: MockStoreSave,
+    });
+    const StoreMod = proxyquire('./credentials-store', {
+      'secure-store-redis': MockSecureStore,
+      '@sockethub/crypto': {
+        objectHash: MockObjectHash
+      }
+    });
+    const CredentialsStore = StoreMod.default;
+    credentialsStore = new CredentialsStore('a parent id', 'a session id', 'a secret', 'redis config');
+  });
+
+  it('returns a valid CredentialsStore object', () => {
+    sinon.assert.calledOnce(MockSecureStore);
+    sinon.assert.calledWith(MockSecureStore, {
+      namespace: 'sockethub:data-layer:credentials-store:a parent id:a session id',
+      secret: 'a secret',
+      redis: 'redis config'
+    });
+    expect(typeof credentialsStore).to.equal('object');
+    expect(credentialsStore.uid).to.equal(`sockethub:data-layer:credentials-store:a parent id:a session id`);
+    expect(typeof credentialsStore.get).to.equal('function');
+    expect(typeof credentialsStore.save).to.equal('function');
+  });
+
+  describe('get', () => {
+    it('handles correct params', async () => {
+      let res = await credentialsStore.get('an actor');
+      sinon.assert.calledOnce(MockStoreGet);
+      sinon.assert.calledWith(MockStoreGet, 'an actor');
+      sinon.assert.notCalled(MockObjectHash);
+      sinon.assert.notCalled(MockStoreSave);
+      expect(res).to.equal('credential foo');
+    });
+
+    it('handles no credentials found', async () => {
+      MockStoreGet.callsArgWith(1, undefined, undefined);
+      let res = await credentialsStore.get('an non-existent actor');
+      sinon.assert.calledOnce(MockStoreGet);
+      sinon.assert.calledWith(MockStoreGet, 'an non-existent actor');
+      sinon.assert.notCalled(MockObjectHash);
+      sinon.assert.notCalled(MockStoreSave);
+      expect(res).to.be.undefined;
+    });
+
+    it('handles an unexpected error', async () => {
+      MockStoreGet.callsArgWith(1, 'sumting bad happen', undefined);
+      let res;
+      try {
+        res = await credentialsStore.get('a problem actor');
+        throw new Error('should not reach this spot');
+      } catch (err) {
+        expect(err).to.equal('sumting bad happen');
+      }
+      sinon.assert.calledOnce(MockStoreGet);
+      sinon.assert.calledWith(MockStoreGet, 'a problem actor');
+      sinon.assert.notCalled(MockObjectHash);
+      sinon.assert.notCalled(MockStoreSave);
+      expect(res).to.be.undefined;
+    });
+
+    it('validates credentialsHash when provided', async () => {
+      MockObjectHash.returns('a credentialHash string');
+      MockStoreGet.callsArgWith(1, undefined, {
+        object: 'a credential'
+      });
+      let res = await credentialsStore.get('an actor', 'a credentialHash string');
+      sinon.assert.calledOnce(MockStoreGet);
+      sinon.assert.calledWith(MockStoreGet, 'an actor');
+      sinon.assert.calledOnce(MockObjectHash);
+      sinon.assert.calledWith(MockObjectHash, 'a credential');
+      sinon.assert.notCalled(MockStoreSave);
+      expect(res).to.eql({object: 'a credential'});
+    });
+
+    it('invalidates credentialsHash when provided', async () => {
+      MockObjectHash.returns('the original credentialHash string');
+      MockStoreGet.callsArgWith(1, undefined, {
+        object: 'a credential'
+      });
+      let res;
+      try {
+        res = await credentialsStore.get('an actor', 'a different credentialHash string');
+        throw new Error('should not reach this spot');
+      } catch (err) {
+        expect(err).to.equal('provided credentials do not match existing platform instance for actor an actor');
+      }
+      sinon.assert.calledOnce(MockStoreGet);
+      sinon.assert.calledWith(MockStoreGet, 'an actor');
+      sinon.assert.calledOnce(MockObjectHash);
+      sinon.assert.calledWith(MockObjectHash, 'a credential');
+      sinon.assert.notCalled(MockStoreSave);
+      expect(res).to.be.undefined;
+    });
+  });
+
+  describe('save', () => {
+    it('handles success', (done) => {
+      const creds = {foo:'bar'};
+      credentialsStore.save('an actor', creds, (err) => {
+        sinon.assert.calledOnce(MockStoreSave);
+        sinon.assert.calledWith(MockStoreSave, 'an actor', creds);
+        sinon.assert.notCalled(MockObjectHash);
+        sinon.assert.notCalled(MockStoreGet);
+        expect(err).to.be.undefined;
+        done();
+      });
+    });
+
+    it('handles failure', (done) => {
+      const creds = {foo:'bar'};
+      MockStoreSave.callsArgWith(2, 'an error');
+      credentialsStore.save('an actor', creds, (err) => {
+        sinon.assert.calledOnce(MockStoreSave);
+        sinon.assert.calledWith(MockStoreSave, 'an actor', creds);
+        sinon.assert.notCalled(MockObjectHash);
+        sinon.assert.notCalled(MockStoreGet);
+        expect(err).to.equal('an error');
+        done();
+      });
+    });
+  });
+});

package/src/credentials-store.ts

@@ -0,0 +1,76 @@
+import SecureStore from 'secure-store-redis';
+import debug, {Debugger} from 'debug';
+import {IActivityStream, CallbackInterface} from "@sockethub/schemas";
+import crypto from "@sockethub/crypto";
+import {RedisConfigProps, RedisConfigUrl} from "./types";
+
+/**
+ * Encapsulates the storing and fetching of credential objects.
+ */
+export default class CredentialsStore {
+  readonly uid: string;
+  private readonly store: SecureStore;
+  private readonly log: Debugger;
+
+  /**
+   * @param parentId - The ID of the parent instance (eg. sockethub itself)
+   * @param sessionId - The ID of the session (socket.io connection)
+   * @param secret - The encryption secret (parent + session secrets)
+   * @param redisConfig - Connect info for redis
+   */
+  constructor(
+    parentId: string,
+    sessionId: string,
+    secret: string,
+    redisConfig: RedisConfigProps | RedisConfigUrl
+  ) {
+    this.uid = `sockethub:data-layer:credentials-store:${parentId}:${sessionId}`;
+    this.store = new SecureStore({
+      namespace: this.uid,
+      secret: secret,
+      redis: redisConfig
+    });
+    this.log = debug(this.uid);
+  }
+
+  /**
+   * Gets the credentials for a given actor ID
+   * @param actor
+   * @param credentialHash
+   */
+  async get(actor: string, credentialHash: string): Promise<IActivityStream> {
+    this.log(`get credentials for ${actor}`);
+    return new Promise((resolve, reject) => {
+      this.store.get(actor, (err, credentials) => {
+        if (err) { return reject(err.toString()); }
+        if (!credentials) { return resolve(undefined); }
+
+        if (credentialHash) {
+          if (credentialHash !== crypto.objectHash(credentials.object)) {
+            return reject(
+              `provided credentials do not match existing platform instance for actor ${actor}`);
+          }
+        }
+        return resolve(credentials);
+      });
+    });
+  }
+
+  /**
+   * Saves the credentials for a given actor ID
+   * @param actor
+   * @param creds
+   * @param done
+   */
+  save(actor: string, creds: IActivityStream, done: CallbackInterface): void {
+    this.store.save(actor, creds, (err) => {
+      if (err) {
+        this.log('error saving credentials to store ' + err);
+        return done(err);
+      } else {
+        this.log(`credentials encrypted and saved`);
+        return done();
+      }
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export {default as CredentialsStore} from "./credentials-store";
+export {default as JobQueue} from "./job-queue";
+export * from "./types";

package/src/job-queue.test.ts

@@ -0,0 +1,252 @@
+import proxyquire from 'proxyquire';
+import { expect } from 'chai';
+import * as sinon from 'sinon';
+
+proxyquire.noPreserveCache();
+proxyquire.noCallThru();
+
+describe('JobQueue', () => {
+  let jobQueue, MockBull, bullMocks, cryptoMocks, sandbox;
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox();
+    cryptoMocks = {
+      objectHash: sandbox.stub(),
+      decrypt: sandbox.stub(),
+      encrypt: sandbox.stub(),
+      hash: sandbox.stub(),
+    };
+    bullMocks = {
+      add: sandbox.stub(),
+      getJob: sandbox.stub(),
+      process: sandbox.stub(),
+      removeAllListeners: sandbox.stub(),
+      pause: sandbox.stub(),
+      resume: sandbox.stub(),
+      isPaused: sandbox.stub(),
+      obliterate: sandbox.stub(),
+      emit: sandbox.stub(),
+      on: sandbox.stub().callsArgWith(1, 'a job id', 'a result string')
+    };
+    MockBull = sandbox.stub().returns(bullMocks);
+    const JobQueueMod = proxyquire('./job-queue', {
+      'bull': MockBull,
+      '@sockethub/crypto': cryptoMocks
+    });
+    const JobQueue = JobQueueMod.default;
+    jobQueue = new JobQueue('a parent id', 'a session id', 'a secret', 'redis config');
+    jobQueue.emit = sandbox.stub();
+  });
+
+  afterEach(() => {
+    sinon.restore();
+    sandbox.reset();
+  });
+
+  it('returns a valid JobQueue object', () => {
+    sinon.assert.calledOnce(MockBull);
+    sinon.assert.calledWith(MockBull,
+      'a parent ida session id', { redis: 'redis config' }
+    );
+    expect(typeof jobQueue).to.equal('object');
+    expect(jobQueue.uid).to.equal(`sockethub:data-layer:job-queue:a parent id:a session id`);
+    expect(typeof jobQueue.add).to.equal('function');
+    expect(typeof jobQueue.getJob).to.equal('function');
+    expect(typeof jobQueue.onJob).to.equal('function');
+    expect(typeof jobQueue.shutdown).to.equal('function');
+  });
+
+  describe('initResultEvents', () => {
+    it('registers handlers when called', () => {
+      bullMocks.on.reset();
+      jobQueue.initResultEvents();
+      expect(bullMocks.on.callCount).to.eql(4);
+      sinon.assert.calledWith(bullMocks.on, 'global:completed');
+      sinon.assert.calledWith(bullMocks.on, 'global:error');
+      sinon.assert.calledWith(bullMocks.on, 'global:failed');
+      sinon.assert.calledWith(bullMocks.on, 'failed');
+    });
+  });
+
+  describe('createJob', () => {
+    it('returns expected job format', () => {
+      cryptoMocks.encrypt.returns('an encrypted message');
+      const job = jobQueue.createJob('a socket id', {
+        context: 'some context',
+        id: 'an identifier'
+      });
+      expect(job).to.eql({
+        title: 'some context-an identifier',
+        msg: 'an encrypted message',
+        sessionId: 'a socket id'
+      })
+    });
+
+    it('uses counter when no id provided', () => {
+      cryptoMocks.encrypt.returns('an encrypted message');
+      let job = jobQueue.createJob('a socket id', {
+        context: 'some context'
+      });
+      expect(job).to.eql({
+        title: 'some context-0',
+        msg: 'an encrypted message',
+        sessionId: 'a socket id'
+      });
+      job = jobQueue.createJob('a socket id', {
+        context: 'some context'
+      });
+      expect(job).to.eql({
+        title: 'some context-1',
+        msg: 'an encrypted message',
+        sessionId: 'a socket id'
+      });
+    })
+  });
+
+  describe('add', () => {
+    it('stores encrypted job', async () => {
+      cryptoMocks.encrypt.returns('encrypted foo')
+      bullMocks.isPaused.returns(false);
+      const resultJob = {title: 'a platform-an identifier', sessionId: 'a socket id', msg: 'encrypted foo'}
+      const res = await jobQueue.add(
+        'a socket id', {context: 'a platform', id: 'an identifier'}
+      );
+      sinon.assert.calledOnce(bullMocks.isPaused);
+      sinon.assert.notCalled(bullMocks.emit);
+      sinon.assert.calledOnceWithExactly(bullMocks.add,
+        resultJob);
+      expect(res).to.eql(resultJob);
+    });
+    it('fails job if queue paused', async () => {
+      cryptoMocks.encrypt.returns('encrypted foo')
+      bullMocks.isPaused.returns(true);
+      const resultJob = {title: 'a platform-an identifier', sessionId: 'a socket id', msg: 'encrypted foo'}
+      const res = await jobQueue.add(
+        'a socket id', {context: 'a platform', id: 'an identifier'}
+      );
+      sinon.assert.calledOnce(bullMocks.isPaused);
+      sinon.assert.calledOnceWithExactly(bullMocks.emit, 'failed', resultJob, 'queue closed');
+      sinon.assert.notCalled(bullMocks.add);
+      expect(res).to.be.undefined;
+    });
+  });
+
+  describe('getJob', () => {
+    const encryptedJob = {
+      data: {
+        title: 'a title',
+        msg: 'an encrypted msg',
+        sessionId: 'a socket id'
+      }
+    };
+
+    it('handles fetching a valid job', async () => {
+      bullMocks.getJob.returns(encryptedJob);
+      cryptoMocks.decrypt.returns('an unencrypted message')
+      const job = await jobQueue.getJob('a valid job');
+      sinon.assert.calledOnceWithExactly(bullMocks.getJob, 'a valid job');
+      encryptedJob.data.msg = 'an unencrypted message';
+      expect(job).to.eql(encryptedJob);
+    });
+
+    it('handles fetching an invalid job', async () => {
+      bullMocks.getJob.returns(undefined);
+      const job = await jobQueue.getJob('an invalid job');
+      expect(job).to.eql(undefined);
+      sinon.assert.calledOnceWithExactly(bullMocks.getJob, 'an invalid job');
+      sinon.assert.notCalled(cryptoMocks.decrypt)
+    });
+
+    it('removes sessionSecret', async () => {
+      bullMocks.getJob.returns(encryptedJob);
+      cryptoMocks.decrypt.returns({
+        foo:'bar',
+        sessionSecret: 'yarg'
+      })
+      const job = await jobQueue.getJob('a valid job');
+      sinon.assert.calledOnceWithExactly(bullMocks.getJob, 'a valid job');
+      // @ts-ignore
+      encryptedJob.data.msg = {
+        foo:'bar'
+      }
+      expect(job).to.eql(encryptedJob);
+    });
+  });
+
+  describe('onJob', () => {
+    it('queues the handler', () => {
+      jobQueue.onJob((job, done) => {
+        throw new Error('This handler should never be called');
+      });
+      sinon.assert.calledOnce(bullMocks.process);
+    });
+  });
+
+  it('pause', async () => {
+    await jobQueue.pause();
+    sinon.assert.calledOnce(bullMocks.pause)
+  });
+
+  it('resume', async () => {
+    await jobQueue.resume();
+    sinon.assert.calledOnce(bullMocks.resume)
+  });
+
+  describe('shutdown', () => {
+    it('is sure to pause when not already paused', async () => {
+      bullMocks.isPaused.returns(false)
+      await jobQueue.shutdown()
+      sinon.assert.calledOnce(bullMocks.isPaused);
+      sinon.assert.calledOnce(bullMocks.pause);
+      sinon.assert.calledOnce(bullMocks.removeAllListeners);
+      sinon.assert.calledOnce(bullMocks.obliterate);
+    });
+    it('skips pausing when already paused', async () => {
+      bullMocks.isPaused.returns(true)
+      await jobQueue.shutdown()
+      sinon.assert.calledOnce(bullMocks.isPaused);
+      sinon.assert.notCalled(bullMocks.pause);
+      sinon.assert.calledOnce(bullMocks.removeAllListeners);
+      sinon.assert.calledOnce(bullMocks.obliterate);
+    });
+  });
+
+  describe('jobHandler', () => {
+    it('calls handler as expected', (done) => {
+      cryptoMocks.decrypt.returns('an unencrypted message');
+      const encryptedJob = {
+        data: {
+          title: 'a title',
+          msg: 'an encrypted message',
+          sessionId: 'a socket id'
+        }
+      };
+      jobQueue.onJob((job, cb) => {
+        const decryptedData = encryptedJob.data;
+        decryptedData.msg = 'an unencrypted message'
+        expect(job).to.eql(decryptedData);
+        cb();
+      });
+      sinon.assert.calledOnce(bullMocks.process);
+      jobQueue.jobHandler(encryptedJob, done);
+    });
+  });
+
+  describe("decryptJobData", () => {
+    it("decrypts and returns expected object", () => {
+      cryptoMocks.decrypt.returnsArg(0);
+      const jobData = {data:{title:"foo", msg:'encryptedjobdata', sessionId:'foobar'}};
+      const secret = 'secretstring';
+      expect(jobQueue.decryptJobData(jobData, secret)).to.be.eql(jobData.data);
+    });
+  });
+
+  describe("decryptActivityStream", () => {
+    it("decrypts and returns expected object", () => {
+      cryptoMocks.decrypt.returnsArg(0);
+      const jobData = 'encryptedjobdata';
+      const secret = 'secretstring';
+      expect(jobQueue.decryptActivityStream(jobData, secret)).to.be.eql(jobData);
+    });
+  });
+});

package/src/job-queue.ts

@@ -0,0 +1,148 @@
+import Queue from 'bull';
+import crypto from '@sockethub/crypto';
+import {
+  JobDataDecrypted,
+  JobDataEncrypted, JobDecrypted,
+  JobEncrypted,
+  RedisConfig
+} from "./types";
+import debug, {Debugger} from 'debug';
+import EventEmitter from "events";
+import {IActivityStream} from "@sockethub/schemas";
+
+interface JobHandler {
+  (job: JobDataDecrypted, done: CallableFunction)
+}
+
+export default class JobQueue extends EventEmitter {
+  readonly uid: string;
+  private readonly bull: Queue;
+  private readonly debug: Debugger;
+  private readonly secret: string;
+  private handler: JobHandler;
+  private counter = 0;
+
+  constructor(instanceId: string, sessionId: string, secret: string, redisConfig: RedisConfig) {
+    super();
+    this.bull = new Queue(instanceId + sessionId, { redis: redisConfig });
+    this.uid = `sockethub:data-layer:job-queue:${instanceId}:${sessionId}`;
+    this.secret = secret;
+    this.debug = debug(this.uid);
+
+    this.debug('initialized');
+  }
+
+  async add(socketId: string, msg: IActivityStream): Promise<JobDataEncrypted> {
+    const job = this.createJob(socketId, msg);
+    const isPaused = await this.bull.isPaused();
+    if (isPaused) {
+      this.bull.emit('failed', job, 'queue closed');
+      return undefined;
+    }
+    this.debug(`adding ${job.title} ${msg.type}`);
+    this.bull.add(job);
+    return job;
+  }
+
+  initResultEvents() {
+    this.bull.on('global:completed', async (jobId: string, result: string) => {
+      const r = result ? JSON.parse(result) : "";
+      const job = await this.getJob(jobId);
+      if (job) {
+        this.debug(`completed ${job.data.title} ${job.data.msg.type}`);
+        this.emit('global:completed', job.data, r);
+        await job.remove();
+      }
+    });
+    this.bull.on('global:error', async (jobId: string, result: string) => {
+      this.debug("unknown queue error", jobId, result);
+    });
+    this.bull.on('global:failed', async (jobId, result: string) => {
+      const job = await this.getJob(jobId);
+      if (job) {
+        this.debug(`failed ${job.data.title} ${job.data.msg.type}`);
+        this.emit('global:failed', job.data, result);
+        await job.remove();
+      }
+    });
+    this.bull.on('failed', (job: JobDataEncrypted, result: string) => {
+      // locally failed jobs (eg. due to paused queue)
+      const unencryptedJobData: JobDataDecrypted = {
+        title: job.title,
+        msg: this.decryptActivityStream(job.msg),
+        sessionId: job.sessionId
+      };
+      this.debug(`failed ${unencryptedJobData.title} ${unencryptedJobData.msg.type}`);
+      this.emit('global:failed', unencryptedJobData, result);
+    });
+  }
+
+  async getJob(jobId: string): Promise<JobDecrypted> {
+    const job = await this.bull.getJob(jobId);
+    if (job) {
+      job.data = this.decryptJobData(job);
+      try {
+        delete job.data.msg.sessionSecret;
+      } catch (e) {
+        // this property should never be exposed externally
+      }
+    }
+    return job;
+  }
+
+  onJob(handler: JobHandler): void {
+    this.handler = handler;
+    this.bull.process(this.jobHandler.bind(this));
+  }
+
+  async pause() {
+    await this.bull.pause();
+    this.debug('paused');
+  }
+
+  async resume() {
+    await this.bull.resume();
+    this.debug('resumed');
+  }
+
+  async shutdown() {
+    this.debug('shutdown');
+    const isPaused = await this.bull.isPaused(true);
+    if (!isPaused) {
+      await this.bull.pause();
+    }
+    await this.bull.obliterate({ force: true });
+    await this.bull.removeAllListeners();
+  }
+
+  private createJob(socketId: string, msg): JobDataEncrypted {
+    const title = `${msg.context}-${(msg.id) ? msg.id : this.counter++}`;
+    return {
+      title: title,
+      sessionId: socketId,
+      msg: crypto.encrypt(msg, this.secret)
+    };
+  }
+
+  private jobHandler(encryptedJob: JobEncrypted, done: CallableFunction): void {
+    const job = this.decryptJobData(encryptedJob);
+    this.debug(`handling ${job.title} ${job.msg.type}`);
+    this.handler(job, done);
+  }
+
+  /**
+   * @param job
+   * @private
+   */
+  private decryptJobData(job: JobEncrypted): JobDataDecrypted {
+    return {
+      title: job.data.title,
+      msg: this.decryptActivityStream(job.data.msg),
+      sessionId: job.data.sessionId
+    };
+  }
+
+  private decryptActivityStream(msg: string): IActivityStream {
+    return crypto.decrypt(msg, this.secret);
+  }
+}

package/src/types.ts

@@ -0,0 +1,37 @@
+import {IActivityStream} from "@sockethub/schemas";
+
+export type RedisConfigUrl = string;
+
+export interface RedisConfigProps {
+  host: string,
+  port: string
+}
+
+export type RedisConfig = RedisConfigProps | RedisConfigUrl;
+
+export interface JobDataEncrypted {
+  title?: string;
+  msg: string;
+  sessionId: string;
+}
+
+export interface JobDataDecrypted {
+  title?: string;
+  msg: IActivityStream;
+  sessionId: string;
+}
+
+export interface JobEncrypted {
+  data: JobDataEncrypted,
+  remove?: {
+    (): void;
+  };
+}
+
+export interface JobDecrypted {
+  data: JobDataDecrypted,
+  remove?: {
+    (): void;
+  };
+}
+