@sockethub/server 5.0.0-alpha.3

package/src/platform.ts

@@ -0,0 +1,191 @@
+import debug from 'debug';
+import hash from "object-hash";
+import Queue from 'bull';
+import { IActivityStream } from "@sockethub/schemas";
+import { getPlatformId, decryptJobData } from "./common";
+import { JobDataDecrypted, JobEncrypted } from "./sockethub";
+import { MessageFromParent } from './platform-instance';
+import { getSessionStore } from "./store";
+
+// command-line params
+const parentId = process.argv[2];
+const platformName = process.argv[3];
+let identifier = process.argv[4];
+const loggerPrefix = `sockethub:platform:${platformName}:${identifier}`;
+let logger = debug(loggerPrefix);
+
+const redisConfig = process.env.REDIS_URL ? process.env.REDIS_URL
+  : { host: process.env.REDIS_HOST, port: process.env.REDIS_PORT };
+const PlatformModule = require(`@sockethub/platform-${platformName}`);
+
+let queueStarted = false;
+let parentSecret1: string, parentSecret2: string;
+
+logger(`platform handler initialized for ${platformName} ${identifier}`);
+
+export interface PlatformSession {
+  debug(msg: string): void;
+  sendToClient(msg: IActivityStream, special?: string): void;
+  updateActor(credentials: object): void;
+}
+
+/**
+ * Handle any uncaught errors from the platform by alerting the worker and shutting down.
+ */
+process.on('uncaughtException', (err) => {
+  console.log('EXCEPTION IN PLATFORM');
+  // eslint-disable-next-line security-node/detect-crlf
+  console.log(err.stack);
+  process.send(['error', err.toString()]);
+  process.exit(1);
+});
+
+/**
+ * Incoming messages from the worker to this platform. Data is an array, the first property is the
+ * method to call, the rest are params.
+ */
+process.on('message', (data: MessageFromParent) => {
+  if (data[0] === 'secrets') {
+    parentSecret1 = data[1].parentSecret1;
+    parentSecret2 = data[1].parentSecret2;
+    startQueueListener();
+  }
+});
+
+
+/**
+ * Initialize platform module
+ */
+const platformSession: PlatformSession = {
+  debug: debug(`sockethub:platform:${platformName}:${identifier}`),
+  sendToClient: getSendFunction('message'),
+  updateActor: updateActor
+};
+const platform = new PlatformModule(platformSession);
+
+/**
+ * Get the credentials stored for this user in this sessions store, if given the correct
+ * sessionSecret.
+ * @param actorId
+ * @param sessionId
+ * @param sessionSecret
+ * @param cb
+ */
+function getCredentials(actorId: string, sessionId: string, sessionSecret: string, cb: Function) {
+  if (platform.config.noCredentials) { return cb(); }
+  const store = getSessionStore(parentId, parentSecret1, sessionId, sessionSecret);
+  store.get(actorId, (err, credentials) => {
+    if (platform.config.persist) {
+      // don't continue if we don't get credentials
+      if (err) { return cb(err); }
+    } else if (! credentials) {
+      // also skip if this is a non-persist platform with no credentials
+      return cb();
+    }
+
+    if (platform.credentialsHash) {
+      if (platform.credentialsHash !== hash(credentials.object)) {
+        return cb('provided credentials do not match existing platform instance for actor '
+            + platform.actor.id);
+      }
+    } else {
+      platform.credentialsHash = hash(credentials.object);
+    }
+    cb(undefined, credentials);
+  });
+}
+
+/**
+ * Returns a function used to handle completed jobs from the platform code (the `done` callback).
+ * @param secret the secret used to decrypt credentials
+ */
+function getJobHandler(secret: string) {
+  return (job: JobEncrypted, done: Function) => {
+    const jobData: JobDataDecrypted = decryptJobData(job, secret);
+    const jobLog = debug(`${loggerPrefix}:${jobData.sessionId}`);
+    jobLog(`received ${jobData.title} ${jobData.msg.type}`);
+    const sessionSecret = jobData.msg.sessionSecret;
+    delete jobData.msg.sessionSecret;
+
+    return getCredentials(jobData.msg.actor.id, jobData.sessionId, sessionSecret,
+      (err, credentials) => {
+        if (err) { return done(new Error(err)); }
+        let jobCallbackCalled = false;
+        const doneCallback = (err, result) => {
+          if (jobCallbackCalled) { return; }
+          jobCallbackCalled = true;
+          if (err) {
+            jobLog(`errored ${jobData.title} ${jobData.msg.type}`);
+            let errMsg;
+            // some error objects (eg. TimeoutError) don't interoplate correctly to human-readable
+            // so we have to do this little dance
+            try {
+              errMsg = err.toString();
+            } catch (e) {
+              errMsg = err;
+            }
+            done(new Error(errMsg));
+          } else {
+            jobLog(`completed ${jobData.title} ${jobData.msg.type}`);
+            done(null, result);
+          }
+        };
+        if ((Array.isArray(platform.config.requireCredentials)) &&
+          (platform.config.requireCredentials.includes(jobData.msg.type))) {
+          // add the credentials object if this method requires it
+          platform[jobData.msg.type](jobData.msg, credentials, doneCallback);
+        } else if (platform.config.persist) {
+          if (platform.initialized) {
+            platform[jobData.msg.type](jobData.msg, doneCallback);
+          } else {
+            done(new Error(`${jobData.msg.type} called on uninitialized platform`));
+          }
+        } else {
+          platform[jobData.msg.type](jobData.msg, doneCallback);
+        }
+      });
+  };
+}
+
+/**
+ * Get an function which sends a message to the parent thread (PlatformInstance). The platform
+ * can call that function to send messages back to the client.
+ * @param command string containing the type of command to be sent. 'message' or 'close'
+ */
+function getSendFunction(command: string) {
+  return function (msg: IActivityStream, special?: string) {
+    process.send([command, msg, special]);
+  };
+}
+
+/**
+ * When a user changes it's actor name, the channel identifier changes, we need to ensure that
+ * both the queue thread (listening on the channel for jobs) and the logging object are updated.
+ * @param credentials
+ */
+function updateActor(credentials) {
+  identifier = getPlatformId(platformName, credentials.actor.id);
+  logger(`platform actor updated to ${credentials.actor.id} identifier ${identifier}`);
+  logger = debug(`sockethub:platform:${identifier}`);
+  platform.credentialsHash = hash(credentials.object);
+  platform.debug = debug(`sockethub:platform:${platformName}:${identifier}`);
+  process.send(['updateActor', undefined, identifier]);
+  startQueueListener(true);
+}
+
+/**
+ * starts listening on the queue for incoming jobs
+ * @param refresh boolean if the param is true, we re-init the queue.process
+ * (used when identifier changes)
+ */
+function startQueueListener(refresh: boolean = false) {
+  const secret = parentSecret1 + parentSecret2;
+  if ((queueStarted) && (!refresh)) {
+    logger('start queue called multiple times, skipping');
+    return;
+  }
+  const queue = new Queue(parentId + identifier, { redis: redisConfig });
+  queueStarted = true;
+  logger('listening on the queue for incoming jobs');
+  queue.process(getJobHandler(secret));
+}

package/src/process-manager.ts

@@ -0,0 +1,64 @@
+import init from './bootstrap/init';
+import PlatformInstance, {
+  platformInstances, PlatformInstanceParams, MessageFromParent } from "./platform-instance";
+import { getPlatformId } from "./common";
+
+class ProcessManager {
+  private readonly parentId: string;
+  private readonly parentSecret1: string;
+  private readonly parentSecret2: string;
+
+  constructor(parentId: string, parentSecret1: string, parentSecret2: string) {
+    this.parentId = parentId;
+    this.parentSecret1 = parentSecret1;
+    this.parentSecret2 = parentSecret2;
+  }
+
+  get(platform: string, actorId: string, sessionId?: string): PlatformInstance {
+    const platformDetails = init.platforms.get(platform);
+    let pi;
+
+    if (platformDetails.config.persist) {
+      // ensure process is started - one for each actor
+      pi = this.ensureProcess(platform, sessionId, actorId);
+    } else {
+      // ensure process is started - one for all jobs
+      pi = this.ensureProcess(platform);
+    }
+    pi.config = platformDetails.config;
+    return pi;
+  }
+
+  private createPlatformInstance(identifier: string, platform: string,
+                                 actor?: string): PlatformInstance {
+    const secrets: MessageFromParent = [
+      'secrets', {
+        parentSecret1: this.parentSecret1,
+        parentSecret2: this.parentSecret2
+      }
+    ];
+    const platformInstanceConfig: PlatformInstanceParams = {
+      identifier: identifier,
+      platform: platform,
+      parentId: this.parentId,
+      actor: actor
+    };
+    const platformInstance = new PlatformInstance(platformInstanceConfig);
+    platformInstance.initQueue(this.parentSecret1 + this.parentSecret2);
+    platformInstance.process.send(secrets);
+    return platformInstance;
+  }
+
+  private ensureProcess(platform: string, sessionId?: string, actor?: string): PlatformInstance {
+    const identifier = getPlatformId(platform, actor);
+    const platformInstance = platformInstances.get(identifier) ||
+              this.createPlatformInstance(identifier, platform, actor);
+    if (sessionId) {
+      platformInstance.registerSession(sessionId);
+    }
+    platformInstances.set(identifier, platformInstance);
+    return platformInstance;
+  }
+}
+
+export default ProcessManager;

package/src/routes.test.ts

@@ -0,0 +1,100 @@
+import { expect } from 'chai';
+import * as sinon from 'sinon';
+import { existsSync } from "fs";
+
+import routes, { basePaths, examplePaths, examplePages, IRoutePaths } from "./routes";
+
+describe('routes/base', () => {
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('can find each of the base files it serves', () => {
+    Object.values(basePaths).forEach((fwd: string) => {
+      try {
+        expect(existsSync(fwd)).to.be.true;
+      } catch (e) {
+        throw new Error(`Unable to resolve path ${fwd}`);
+      }
+    });
+  });
+
+  it('can find each of the example files it serves', () => {
+    Object.values(examplePaths).forEach((fwd: string) => {
+      try {
+        expect(existsSync(fwd)).to.be.true;
+      } catch (e) {
+        throw new Error(`Unable to resolve path ${fwd}`);
+      }
+    });
+  });
+
+  it('can find each of the example page files it serves', () => {
+    Object.values(examplePages).forEach((fwd: string) => {
+      try {
+        expect(existsSync(fwd)).to.be.true;
+      } catch (e) {
+        throw new Error(`Unable to resolve path ${fwd}`);
+      }
+    });
+  });
+
+  it('adds base routes', () => {
+    const app = {
+      get: sinon.spy()
+    };
+    routes.setup(app, false);
+    sinon.assert.callCount(
+      app.get,
+      Object.keys(basePaths).length
+    );
+  });
+
+  it('adds base and example routes', () => {
+    const app = {
+      get: sinon.spy()
+    };
+    routes.setup(app, true);
+    sinon.assert.callCount(
+      app.get,
+      Object.keys(basePaths).length
+      + Object.keys(examplePaths).length
+      + Object.keys(examplePages).length
+    );
+  });
+
+  it('handles calls to base routes as expected', () => {
+    let routeHandlers: any = {};
+    let app = {
+      get: (path: string | number, route: any) => {
+        routeHandlers[path] = route;
+      }
+    };
+    routes.setup(app, true);
+
+    function verifyPathRoutes(pathMap: IRoutePaths) {
+      Object.keys(pathMap).forEach((path) => {
+        const res = {
+          setHeader: sinon.spy(),
+          sendFile: sinon.spy()
+        };
+        expect(pathMap[path].endsWith('.ejs')).to.be.false;
+        routeHandlers[path]({url: path}, res);
+        sinon.assert.called(res.setHeader);
+        sinon.assert.calledWith(res.sendFile, pathMap[path]);
+      });
+    }
+    verifyPathRoutes(basePaths);
+    verifyPathRoutes(examplePaths);
+
+    Object.keys(examplePages).forEach((path) => {
+      const res = {
+        render: sinon.spy()
+      };
+      expect(examplePages[path].endsWith('.ejs')).to.be.true;
+      routeHandlers[path]({url: path}, res);
+      sinon.assert.called(res.render);
+    });
+  });
+});

package/src/routes.ts

@@ -0,0 +1,98 @@
+import path from 'path';
+import config from "./config";
+import debug from 'debug';
+
+const debug_scope = process.env.DEBUG || '',
+      logger      = debug('sockethub:server:routes'),
+      address     = config.get('public:protocol') + '://' +
+                    config.get('public:host') + ':' +
+                    config.get('public:port') +
+                    config.get('public:path');
+
+export interface IRoutePaths {
+  [key: string]: string;
+}
+
+export const basePaths: IRoutePaths = {
+  '/sockethub-client.js':
+    path.resolve(`${__dirname}/../node_modules/@sockethub/client/dist/sockethub-client.js`),
+  '/sockethub-client.min.js':
+    path.resolve(`${__dirname}/../node_modules/@sockethub/client/dist/sockethub-client.min.js`),
+  '/sockethub-client.js.map':
+    path.resolve(`${__dirname}/../node_modules/@sockethub/client/dist/sockethub-client.js.map`),
+  '/socket.io.js': path.resolve(`${__dirname}/../node_modules/socket.io/client-dist/socket.io.js`)
+};
+
+export const examplePaths: IRoutePaths = {
+  '/jquery.js': path.resolve(`${__dirname}/../node_modules/jquery/dist/jquery.min.js`),
+  '/jquery.min.map': path.resolve(`${__dirname}/../node_modules/jquery/dist/jquery.min.map`),
+  '/examples/shared.js': path.resolve(`${__dirname}/../views/examples/shared.js`)
+};
+
+export const examplePages: IRoutePaths = {
+  '/': path.resolve(`${__dirname}/../views/index.ejs`),
+  '/examples/dummy': path.resolve(`${__dirname}/../views/examples/dummy.ejs`),
+  '/examples/feeds': path.resolve(`${__dirname}/../views/examples/feeds.ejs`),
+  '/examples/irc': path.resolve(`${__dirname}/../views/examples/irc.ejs`),
+  '/examples/xmpp': path.resolve(`${__dirname}/../views/examples/xmpp.ejs`)
+};
+
+
+function prepFileRoutes(pathMap) {
+  let _routes = [];
+  Object.keys(pathMap).forEach((key) => {
+    _routes.push({
+      meta: {
+        method: 'GET',
+        path: key
+      },
+      route: (req, res) => {
+        logger(`serving resource ${req.url}`);
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.sendFile(pathMap[req.url]);
+      }
+    });
+  });
+  return _routes;
+}
+const baseRoutes = prepFileRoutes(basePaths);
+const exampleRoutes = prepFileRoutes(examplePaths);
+
+
+Object.keys(examplePages).forEach((key) => {
+  exampleRoutes.push({
+    meta: {
+      method: 'GET',
+      path: key
+    },
+    route: (req, res) => {
+      logger(`serving page ${req.url}`);
+      res.render(examplePages[req.url], {
+        debug_scope: debug_scope,
+        address: address,
+      });
+    }
+  });
+});
+
+function addRoute(app) {
+  return (route) => {
+    app[route.meta.method.toLowerCase()](
+      route.meta.path,
+      route.route
+    );
+  };
+}
+
+/**
+ * Setup
+ */
+const routes = {
+  setup: function (app: any, examplesEnabled: boolean = config.get('examples:enabled')) {
+    baseRoutes.forEach(addRoute(app));
+    if (examplesEnabled) {
+      exampleRoutes.forEach(addRoute(app));
+    }
+  }
+};
+export default routes;

package/src/sockethub.d.ts

@@ -0,0 +1 @@
+export {};

package/src/sockethub.ts

@@ -0,0 +1,165 @@
+import debug from 'debug';
+import { Socket } from "socket.io";
+import { IActivityStream } from "@sockethub/schemas";
+
+import crypto from './crypto';
+import init from './bootstrap/init';
+import middleware from './middleware';
+import createActivityObject from "./middleware/create-activity-object";
+import expandActivityStream from "./middleware/expand-activity-stream";
+import storeCredentials from "./middleware/store-credentials";
+import validate from "./middleware/validate";
+import janitor from './janitor';
+import listener from './listener';
+import ProcessManager from "./process-manager";
+import PlatformInstance, { platformInstances } from "./platform-instance";
+import { getSessionStore } from "./store";
+
+const log = debug('sockethub:server:core');
+
+
+export interface JobDataDecrypted {
+  title?: string;
+  msg: IActivityStream;
+  sessionId: string;
+}
+
+export interface JobDataEncrypted {
+  title?: string;
+  msg: string;
+  sessionId: string;
+}
+
+export interface JobDecrypted {
+  data: JobDataDecrypted,
+  remove?: Function;
+}
+
+export interface JobEncrypted {
+  data: JobDataEncrypted,
+  remove?: Function;
+}
+
+function attachError(err, msg) {
+  if (typeof msg !== 'object') {
+    msg = { context: 'error' };
+  }
+  msg.error = err.toString();
+  delete msg.sessionSecret;
+  return msg;
+}
+
+class Sockethub {
+  private readonly parentId: string;
+  private readonly parentSecret1: string;
+  private readonly parentSecret2: string;
+  counter: number;
+  platforms: Map<string, object>;
+  status: boolean;
+  queue: any;
+  processManager: ProcessManager;
+
+  constructor() {
+    this.counter = 0;
+    this.platforms = init.platforms;
+    this.status = false;
+    this.parentId = crypto.randToken(16);
+    this.parentSecret1 = crypto.randToken(16);
+    this.parentSecret2 = crypto.randToken(16);
+    this.processManager = new ProcessManager(
+      this.parentId, this.parentSecret1, this.parentSecret2);
+    log('session id: ' + this.parentId);
+  }
+
+  /**
+   * initialization of Sockethub starts here
+   */
+  boot() {
+    if (this.status) {
+      return log('Sockethub.boot() called more than once');
+    } else {
+      this.status = true;
+    }
+
+    log('active platforms: ', [...init.platforms.keys()]);
+    janitor.clean(); // start cleanup cycle
+    listener.start();   // start external services
+    log('registering handlers');
+    listener.io.on('connection', this.handleIncomingConnection.bind(this));
+  }
+
+  async removeAllPlatformInstances() {
+    for (let platform of platformInstances) {
+      await platform[1].destroy();
+    }
+  }
+
+  private createJob(socketId: string, platformInstance: PlatformInstance, msg): JobDataEncrypted {
+    const title = `${msg.context}-${(msg.id) ? msg.id : this.counter++}`;
+    const job: JobDataEncrypted = {
+      title: title,
+      sessionId: socketId,
+      msg: crypto.encrypt(msg, this.parentSecret1 + this.parentSecret2)
+    };
+    return job;
+  };
+
+  private handleIncomingConnection(socket: Socket) {
+    // session-specific debug messages
+    const sessionLog = debug('sockethub:server:core:' + socket.id),
+          sessionSecret = crypto.randToken(16),
+          // store instance is session-specific
+          store = getSessionStore(this.parentId, this.parentSecret1, socket.id, sessionSecret);
+
+    sessionLog(`socket.io connection`);
+
+    socket.on('disconnect', () => {
+      sessionLog('disconnect received from client');
+    });
+
+    socket.on('credentials',
+      middleware('credentials')
+        .use(expandActivityStream)
+        .use(validate('credentials', socket.id))
+        .use(storeCredentials(store, sessionLog))
+        .use((err, data, next) => {
+          // error handler
+          next(attachError(err, data));
+        }).use((data, next) => { next(); })
+        .done());
+
+    // when new activity objects are created on the client side, an event is
+    // fired and we receive a copy on the server side.
+    socket.on('activity-object',
+      middleware('activity-object')
+        .use(validate('activity-object', socket.id))
+        .use(createActivityObject)
+        .use((err, data, next) => {
+          next(attachError(err, data));
+        }).use((data, next) => { next(); })
+        .done());
+
+    socket.on('message',
+      middleware('message')
+        .use(expandActivityStream)
+        .use(validate('message', socket.id))
+        .use((msg, next) => {
+          // The platform thread must find the credentials on their own using the given
+          // sessionSecret, which indicates that this specific session (socket
+          // connection) has provided credentials.
+          msg.sessionSecret = sessionSecret;
+          next(msg);
+        }).use((err, data, next) => {
+          next(attachError(err, data));
+        }).use((msg, next) => {
+          const platformInstance = this.processManager.get(msg.context, msg.actor.id, socket.id);
+          sessionLog(`queued to channel ${platformInstance.id}`);
+          const job = this.createJob(socket.id, platformInstance, msg);
+          // job validated and queued, store socket.io callback for when job is completed
+          platformInstance.completedJobHandlers.set(job.title, next);
+          platformInstance.queue.add(job);
+        }).done());
+  }
+}
+
+export default Sockethub;

package/src/store.test.ts

@@ -0,0 +1,28 @@
+import proxyquire from 'proxyquire';
+import { expect } from 'chai';
+import * as sinon from 'sinon';
+
+proxyquire.noPreserveCache();
+proxyquire.noCallThru();
+
+const MockSecureStore = sinon.fake();
+
+const StoreMod = proxyquire('./store', {
+  'secure-store-redis': MockSecureStore
+});
+
+const getSessionStore = StoreMod.getSessionStore;
+
+describe('getSessionStore', () => {
+  it('returns a valid Store object', () => {
+    const store = getSessionStore('a parent id', 'a parent secret',
+      'a session id', 'a session secret');
+    sinon.assert.calledOnce(MockSecureStore);
+    sinon.assert.calledWith(MockSecureStore, {
+      namespace: 'sockethub:a parent id:session:a session id:store',
+      secret: 'a parent secreta session secret',
+      redis: { host: '127.0.0.1', port: 6379 }
+    });
+    expect(typeof store).to.equal('object');
+  });
+});

package/src/store.ts

@@ -0,0 +1,17 @@
+import SecureStore from 'secure-store-redis';
+
+import config from "./config";
+
+export interface ISecureStoreInstance {
+  save(id: string, obj: any, cb: Function);
+  get(id: string, cb: Function);
+}
+
+export function getSessionStore(parentId: string, parentSecret: string,
+                                sessionId: string, sessionSecret: string): ISecureStoreInstance {
+  return new SecureStore({
+    namespace: 'sockethub:' + parentId + ':session:' + sessionId + ':store',
+    secret: parentSecret + sessionSecret,
+    redis: config.get('redis')
+  });
+}