@sockethub/server 5.0.0-alpha.3 → 5.0.0-alpha.4

package/src/middleware/store-credentials.ts

@@ -1,16 +1,10 @@
-import { IActivityStream } from "@sockethub/schemas";
-import { ISecureStoreInstance } from "../store";
+import {IActivityStream, CallbackInterface} from "@sockethub/schemas";
+import { CredentialsStore } from "@sockethub/data-layer";
 
-export default function storeCredentials(store: ISecureStoreInstance, sessionLog: Function) {
-  return (creds: IActivityStream, done: Function) => {
-    store.save(creds.actor.id, creds, (err) => {
-      if (err) {
-        sessionLog('error saving credentials to store ' + err);
-        done(err);
-      } else {
-        sessionLog('credentials encrypted and saved');
-        done();
-      }
-    });
+export default function storeCredentials(
+  credentialsStore: CredentialsStore
+) {
+  return (creds: IActivityStream, done: CallbackInterface) => {
+    credentialsStore.save(creds.actor.id, creds, done);
   };
-};
+}

package/src/middleware/validate.test.data.ts

@@ -130,10 +130,10 @@ export default [
       "image":{
         "height":250,
         "mediaType":"image/jpeg",
-        "url":"http://example.org/image.jpg",
+        "url":"https://example.org/image.jpg",
         "width":250
       },
-      "url":"http://sockethub.org"
+      "url":"https://sockethub.org"
     },
     "output":"same"
   },
@@ -145,7 +145,7 @@ export default [
       "id":"irc://sh-9K3Vk@irc.freenode.net",
       "type":"person",
       "name":"sh-9K3Vk",
-      "url":"http://sockethub.org"
+      "url":"https://sockethub.org"
     },
     "output":"same"
   },
@@ -211,7 +211,7 @@ export default [
       "context":"fake",
       "target": { "id": "irc://irc.dooder.net/a-room", "type": "room" }
     },
-    "error": "Error: activity stream: must have required property \'type\'"
+    "error": "Error: activity stream: must have required property 'type'"
   },
   {
     "name":"invalid context property",
@@ -234,7 +234,7 @@ export default [
       "context":"fake",
       "target": { "id": "irc://irc.dooder.net/a-room", "type": "room" }
     },
-    "error": "Error: activity stream: must have required property \'actor\'"
+    "error": "Error: activity stream: must have required property 'actor'"
   },
   {
     "name":"traditional message",

package/src/middleware/validate.ts

@@ -2,10 +2,8 @@
  * responsible for handling the validation and expansion (when applicable) of all incoming objects
  */
 import debug from 'debug';
-import schemas from '@sockethub/schemas';
-import { IActivityStream } from "@sockethub/schemas";
+import schemas, {IActivityStream, CallbackActivityStreamInterface} from '@sockethub/schemas';
 
-// @ts-ignore
 import init from "../bootstrap/init";
 
 init.platforms.forEach((platform) => {
@@ -18,8 +16,8 @@ init.platforms.forEach((platform) => {
 // called when registered with the middleware function, define the type of validation
 // that will be called when the middleware eventually does.
 export default function validate(type: string, sockethubId: string) {
-  const sessionLog = debug(`sockethub:validate:${sockethubId}`);
-  return (msg: IActivityStream, done: Function) => {
+  const sessionLog = debug(`sockethub:server:validate:${sockethubId}`);
+  return (msg: IActivityStream, done: CallbackActivityStreamInterface) => {
     sessionLog('applying schema validation for ' + type);
     if (type === 'activity-object') {
       const err = schemas.validateActivityObject(msg);
@@ -46,4 +44,4 @@ export default function validate(type: string, sockethubId: string) {
       }
     }
   };
-};
+}

package/src/middleware.ts

@@ -1,28 +1,43 @@
 import { debug } from 'debug';
+import {IActivityStream} from "@sockethub/schemas";
 
 export default function middleware(name: string): MiddlewareChain {
   return new MiddlewareChain(name);
 }
 
+export interface MiddlewareChainInterface {
+  (error: IActivityStream | Error,
+    data?: IActivityStream | MiddlewareChainInterface,
+    next?: MiddlewareChainInterface): void;
+}
+
+interface ErrorHandlerInterface {
+  (err: Error, data?: unknown, cb?: unknown): void;
+}
+
+export interface LogErrorInterface {
+  (msg: Error): void;
+}
+
 export class MiddlewareChain {
   public name: string;
-  private chain: Array<Function> = [];
-  private errHandler: Function = (err: Error) => { throw err; };
-  private logger: Function;
+  private chain: Array<MiddlewareChainInterface> = [];
+  private errHandler: ErrorHandlerInterface = (err: Error) => { throw err; };
+  private readonly logger: LogErrorInterface;
 
   constructor(name: string) {
     this.name = name;
     this.logger = debug(`sockethub:middleware:${name}`);
   }
 
-  use(func: Function): this {
+  use(func: ErrorHandlerInterface | MiddlewareChainInterface): this {
     if (typeof func !== 'function') {
       throw new Error('middleware use() can only take a function as an argument');
     }
     if (func.length === 3) {
-      this.errHandler = func;
+      this.errHandler = func as ErrorHandlerInterface;
     } else if (func.length === 2) {
-      this.chain.push(func);
+      this.chain.push(func as MiddlewareChainInterface);
     } else {
       throw new Error(
         'middleware function provided with incorrect number of params: ' + func.length);
@@ -31,19 +46,21 @@ export class MiddlewareChain {
   }
 
   done() {
-    return (data: any, callback: Function) => {
+    return (data: unknown, callback: MiddlewareChainInterface) => {
       let position = 0;
       if (typeof callback !== 'function') {
-        callback = () => {};
+        callback = () => {
+          // ensure we have a callback function
+        };
       }
-      const next = (_data: any) => {
+      const next = (_data: unknown) => {
         if (_data instanceof Error) {
           this.logger(_data);
           this.errHandler(_data, data, callback);
         } else if (typeof this.chain[position] === 'function') {
-          this.chain[position++](_data, next);
+          this.chain[position++](_data as IActivityStream, next);
         } else {
-          callback(_data);
+          callback(_data as IActivityStream);
         }
       };
       next(data);

package/src/platform-instance.test.ts

@@ -19,13 +19,13 @@ describe("PlatformInstance", () => {
     getSocketFake = sinon.fake.resolves(socketMock);
 
     const PlatformInstanceMod = proxyquire('./platform-instance', {
-      'bull': sandbox.stub().returns({
-        on: sandbox.stub()
-      }),
-      './store': {
-        redisConfig: {
-          createClient: () => {}
-        }
+      '@sockethub/data-layer': {
+        JobQueue: sandbox.stub().returns({
+          shutdown: sandbox.stub(),
+          on: sandbox.stub(),
+          getJob: sandbox.stub(),
+          initResultEvents: sandbox.stub()
+        })
       },
       'child_process': {
         fork: forkFake,
@@ -51,7 +51,7 @@ describe("PlatformInstance", () => {
   });
 
   describe('private instance per-actor', () => {
-    it("is set as non-global when an actor is provided", () => {
+    it("is set as non-global when an actor is provided", async () => {
       const pi = new PlatformInstance({
         identifier: 'id',
         platform: 'name',
@@ -60,7 +60,7 @@ describe("PlatformInstance", () => {
       });
       expect(pi.global).to.be.equal(false);
       sandbox.assert.calledWith(forkFake, FORK_PATH, ['parentId', 'name', 'id']);
-      pi.destroy();
+      await pi.shutdown();
     });
   });
 
@@ -82,8 +82,8 @@ describe("PlatformInstance", () => {
       };
     });
 
-    afterEach(() => {
-      pi.destroy();
+    afterEach(async () => {
+      await pi.shutdown();
     });
 
     it('has expected properties', () => {
@@ -123,23 +123,23 @@ describe("PlatformInstance", () => {
         expect(pi.sessions.has('my session id')).to.be.equal(true);
         pi.reportError('my session id', 'an error message');
         pi.sendToClient = sandbox.stub();
-        pi.destroy = sandbox.stub();
+        pi.shutdown = sandbox.stub();
         expect(pi.sessions.size).to.be.equal(0);
       });
     });
 
     it('initializes the job queue', () => {
-      expect(pi.queue).to.be.undefined;
+      expect(pi.jobQueue).to.be.undefined;
       pi.initQueue('a secret');
-      expect(pi.queue).to.be.ok;
+      expect(pi.jobQueue).to.be.ok;
     });
 
-    it("cleans up its references when destroyed", async () => {
+    it("cleans up its references when shutdown", async () => {
       pi.initQueue('a secret');
-      expect(pi.queue).to.be.ok;
+      expect(pi.jobQueue).to.be.ok;
       expect(platformInstances.has('platform identifier')).to.be.true;
-      await pi.destroy();
-      expect(pi.queue).not.to.be.ok;
+      await pi.shutdown();
+      expect(pi.jobQueue).not.to.be.ok;
       expect(platformInstances.has('platform identifier')).to.be.false;
     });
 

package/src/platform-instance.ts

@@ -1,13 +1,12 @@
-import { ChildProcess, fork } from 'child_process';
+import {ChildProcess, fork } from 'child_process';
 import { join } from 'path';
 import { debug, Debugger } from 'debug';
-import Queue from 'bull';
-import { IActivityStream } from '@sockethub/schemas';
+import {IActivityStream, CompletedJobHandler} from "@sockethub/schemas";
+import {JobQueue, JobDataDecrypted} from "@sockethub/data-layer";
 
 import config from "./config";
-import { JobDataDecrypted, JobEncrypted } from "./sockethub";
 import { getSocket } from "./listener";
-import { decryptJobData } from "./common";
+import nconf from "nconf";
 
 // collection of platform instances, stored by `id`
 export const platformInstances = new Map<string, PlatformInstance>();
@@ -19,9 +18,16 @@ export interface PlatformInstanceParams {
   actor?: string;
 }
 
+type EnvFormat = {
+  DEBUG?: string,
+  REDIS_URL?: string,
+  REDIS_HOST?: string,
+  REDIS_PORT?: string
+}
+
 interface MessageFromPlatform extends Array<string | IActivityStream>{
   0: string, 1: IActivityStream, 2: string}
-export interface MessageFromParent extends Array<string|any>{0: string, 1: any}
+export interface MessageFromParent extends Array<string|unknown>{0: string, 1: unknown}
 
 interface PlatformConfig {
   persist?: boolean;
@@ -29,11 +35,13 @@ interface PlatformConfig {
 }
 
 export default class PlatformInstance {
-  flaggedForTermination: boolean = false;
   id: string;
-  queue: Queue;
-  completedJobHandlers: Map<string, Function> = new Map();
-  config: PlatformConfig = {};
+  flaggedForTermination = false;
+  initialized = false;
+  jobQueue: JobQueue;
+  readonly global: boolean = false;
+  readonly completedJobHandlers: Map<string, CompletedJobHandler> = new Map();
+  readonly config: PlatformConfig = {};
   readonly name: string;
   readonly process: ChildProcess;
   readonly debug: Debugger;
@@ -43,7 +51,6 @@ export default class PlatformInstance {
     'close': (() => new Map())(),
     'message': (() => new Map())()
   };
-  public readonly global?: boolean = false;
   private readonly actor?: string;
 
   constructor(params: PlatformInstanceParams) {
@@ -56,60 +63,68 @@ export default class PlatformInstance {
       this.global = true;
     }
 
-    this.debug = debug(`sockethub:platform-instance:${this.id}`);
+    this.debug = debug(`sockethub:server:platform-instance:${this.id}`);
+    const env: EnvFormat = {};
+    if (process.env.DEBUG) {
+      env.DEBUG = process.env.DEBUG;
+    }
+    if (config.get('redis:url')) {
+      env.REDIS_URL = config.get('redis:url') as string;
+    } else {
+      env.REDIS_HOST = config.get('redis:host') as string;
+      env.REDIS_PORT = config.get('redis:port') as string;
+    }
+
     // spin off a process
-    const env = config.get('redis:url') ? { REDIS_URL: config.get('redis:url') }
-      : { REDIS_HOST: config.get('redis:host'), REDIS_PORT: config.get('redis:port') };
     this.process = fork(
       join(__dirname, 'platform.js'),
       [this.parentId, this.name, this.id],
-      { env: env });
+      { env: env }
+    );
   }
 
   /**
    * Destroys all references to this platform instance, internal listeners and controlled processes
    */
-  public async destroy() {
-    this.debug(`cleaning up`);
+  public async shutdown() {
+    this.debug('shutdown');
     this.flaggedForTermination = true;
+
     try {
-      await this.queue.removeAllListeners();
-    } catch (e) { }
-    try {
-      await this.queue.obliterate({ force: true });
-    } catch (e) { }
-    try {
-      delete this.queue;
       await this.process.removeAllListeners('close');
       await this.process.unref();
-      await this.process.kill();
-    } catch (e) { }
-    platformInstances.delete(this.id);
+      this.process.kill();
+    } catch (e) {
+      // needs to happen
+    }
+
+    try {
+      await this.jobQueue.shutdown();
+      delete this.jobQueue;
+    } catch (e) {
+      // this needs to happen
+    }
+
+    try {
+      platformInstances.delete(this.id);
+    } catch (e) {
+      // this needs to happen
+    }
   }
 
   /**
    * When jobs are completed or failed, we prepare the results and send them to the client socket
    */
   public initQueue(secret: string) {
-    this.queue = new Queue(this.parentId + this.id, { redis: config.get('redis') });
-
-    this.queue.on('global:completed', (jobId, resultString) => {
-      const result = resultString ? JSON.parse(resultString) : "";
-      this.queue.getJob(jobId).then(async (job: JobEncrypted) => {
-        await this.handleJobResult('completed', decryptJobData(job, secret), result);
-        await job.remove();
-      });
-    });
+    this.jobQueue = new JobQueue(this.parentId, this.id, secret, nconf.get('redis'));
+    this.jobQueue.initResultEvents();
 
-    this.queue.on('global:error', (jobId, result) => {
-      this.debug("unknown queue error", jobId, result);
+    this.jobQueue.on('global:completed', async (job: JobDataDecrypted, result: string) => {
+      await this.handleJobResult('completed', job, result);
     });
 
-    this.queue.on('global:failed', (jobId, result) => {
-      this.queue.getJob(jobId).then(async (job: JobEncrypted) => {
-        await this.handleJobResult('failed', decryptJobData(job, secret), result);
-        await job.remove();
-      });
+    this.jobQueue.on('global:failed', async (job: JobDataDecrypted, result: string) => {
+      await this.handleJobResult('failed', job, result);
     });
   }
 
@@ -120,7 +135,7 @@ export default class PlatformInstance {
   public registerSession(sessionId: string) {
     if (! this.sessions.has(sessionId)) {
       this.sessions.add(sessionId);
-      for (let type of Object.keys(this.sessionCallbacks)) {
+      for (const type of Object.keys(this.sessionCallbacks)) {
         const cb = this.callbackFunction(type, sessionId);
         this.process.on(type, cb);
         this.sessionCallbacks[type].set(sessionId, cb);
@@ -139,19 +154,20 @@ export default class PlatformInstance {
       try {
         // this property should never be exposed externally
         delete msg.sessionSecret;
-      } catch (e) {}
-      msg.context = this.name;
-      if ((msg.type === 'error') && (typeof msg.actor === 'undefined') && (this.actor)) {
-        // ensure an actor is present if not otherwise defined
-        msg.actor = { id: this.actor, type: 'unknown' };
+      } finally {
+        msg.context = this.name;
+        if ((msg.type === 'error') && (typeof msg.actor === 'undefined') && (this.actor)) {
+          // ensure an actor is present if not otherwise defined
+          msg.actor = { id: this.actor, type: 'unknown' };
+        }
+        socket.emit('message', msg);
       }
-      socket.emit('message', msg);
     }, (err) => this.debug(`sendToClient ${err}`));
   }
 
   // send message to every connected socket associated with this platform instance.
   private async broadcastToSharedPeers(sessionId: string, msg: IActivityStream) {
-    for (let sid of this.sessions.values()) {
+    for (const sid of this.sessions.values()) {
       if (sid !== sessionId) {
         this.debug(`broadcasting message to ${sid}`);
         await this.sendToClient(sid, msg);
@@ -160,30 +176,39 @@ export default class PlatformInstance {
   }
 
   // handle job results coming in on the queue from platform instances
-  private async handleJobResult(type: string, jobData: JobDataDecrypted, result) {
-    this.debug(`${type} job ${jobData.title}`);
-    delete jobData.msg.sessionSecret;
-    let msg = jobData.msg;
+  private async handleJobResult(type: string, job: JobDataDecrypted, result) {
+    const msg = job.msg;
     if (type === 'failed') {
       msg.error = result ? result : "job failed for unknown reason";
-      if ((this.config.persist) && (this.config.requireCredentials.includes(jobData.msg.type))) {
-        this.debug(`critical job type ${jobData.msg.type} failed, terminating platform instance`);
-        await this.destroy();
-      }
+      this.debug(`${job.title} ${type}: ${msg.error}`);
+
     }
 
     // send result to client
-    const callback = this.completedJobHandlers.get(jobData.title);
+    const callback = this.completedJobHandlers.get(job.title);
     if (callback) {
       callback(msg);
-      this.completedJobHandlers.delete(jobData.title);
+      this.completedJobHandlers.delete(job.title);
     } else {
-      await this.sendToClient(jobData.sessionId, msg);
+      await this.sendToClient(job.sessionId, msg);
     }
 
     // let all related peers know of result as an independent message
     // (not as part of a job completion, or failure)
-    await this.broadcastToSharedPeers(jobData.sessionId, msg);
+    await this.broadcastToSharedPeers(job.sessionId, msg);
+
+    if ((this.config.persist) && (this.config.requireCredentials.includes(job.msg.type))) {
+      if (type === 'failed') {
+        this.debug(`critical job type ${job.msg.type} failed, flagging for termination`);
+        await this.jobQueue.pause();
+        this.initialized = false;
+        this.flaggedForTermination = true;
+      } else {
+        await this.jobQueue.resume();
+        this.initialized = true;
+        this.flaggedForTermination = false;
+      }
+    }
   }
 
   /**
@@ -191,7 +216,7 @@ export default class PlatformInstance {
    * @param sessionId
    * @param errorMessage
    */
-  private async reportError(sessionId: string, errorMessage: any) {
+  private async reportError(sessionId: string, errorMessage: string) {
     const errorObject: IActivityStream = {
       context: this.name,
       type: 'error',
@@ -200,7 +225,7 @@ export default class PlatformInstance {
     };
     this.sendToClient(sessionId, errorObject);
     this.sessions.clear();
-    await this.destroy();
+    await this.shutdown();
   }
 
   /**
@@ -221,19 +246,19 @@ export default class PlatformInstance {
    */
   private callbackFunction(listener: string, sessionId: string) {
     const funcs = {
-      'close': (e: object) => {
+      'close': async (e: object) => {
         this.debug(`close even triggered ${this.id}: ${e}`);
-        this.reportError(sessionId, `Error: session thread closed unexpectedly: ${e}`);
+        await this.reportError(sessionId, `Error: session thread closed unexpectedly: ${e}`);
       },
-      'message': (data: MessageFromPlatform) => {
-        if (data[0] === 'updateActor') {
+      'message': async ([first, second, third]: MessageFromPlatform) => {
+        if (first === 'updateActor') {
           // We need to update the key to the store in order to find it in the future.
-          this.updateIdentifier(data[2]);
-        } else if (data[0] === 'error') {
-          this.reportError(sessionId, data[1]);
+          this.updateIdentifier(third);
+        } else if ((first === 'error') && (typeof second === "string")) {
+          await this.reportError(sessionId, second);
         } else {
           // treat like a message to clients
-          this.sendToClient(sessionId, data[1]);
+          this.sendToClient(sessionId, second);
         }
       }
     };