@sockethub/server 5.0.0-alpha.3 → 5.0.0-alpha.4

package/src/platform.ts

@@ -1,11 +1,8 @@
 import debug from 'debug';
-import hash from "object-hash";
-import Queue from 'bull';
-import { IActivityStream } from "@sockethub/schemas";
-import { getPlatformId, decryptJobData } from "./common";
-import { JobDataDecrypted, JobEncrypted } from "./sockethub";
-import { MessageFromParent } from './platform-instance';
-import { getSessionStore } from "./store";
+import {IActivityStream, CallbackInterface} from "@sockethub/schemas";
+import crypto, {getPlatformId} from "@sockethub/crypto";
+import {CredentialsStore, JobQueue, RedisConfig} from "@sockethub/data-layer";
+import {JobDataDecrypted} from "@sockethub/data-layer/dist";
 
 // command-line params
 const parentId = process.argv[2];
@@ -16,9 +13,11 @@ let logger = debug(loggerPrefix);
 
 const redisConfig = process.env.REDIS_URL ? process.env.REDIS_URL
   : { host: process.env.REDIS_HOST, port: process.env.REDIS_PORT };
+// eslint-disable-next-line @typescript-eslint/no-var-requires
 const PlatformModule = require(`@sockethub/platform-${platformName}`);
 
-let queueStarted = false;
+let jobQueue: JobQueue;
+let jobQueueStarted = false;
 let parentSecret1: string, parentSecret2: string;
 
 logger(`platform handler initialized for ${platformName} ${identifier}`);
@@ -28,6 +27,11 @@ export interface PlatformSession {
   sendToClient(msg: IActivityStream, special?: string): void;
   updateActor(credentials: object): void;
 }
+interface SecretInterface {
+  parentSecret1: string,
+  parentSecret2: string
+}
+interface SecretFromParent extends Array<string|SecretInterface>{0: string, 1: SecretInterface}
 
 /**
  * Handle any uncaught errors from the platform by alerting the worker and shutting down.
@@ -44,15 +48,17 @@ process.on('uncaughtException', (err) => {
  * Incoming messages from the worker to this platform. Data is an array, the first property is the
  * method to call, the rest are params.
  */
-process.on('message', (data: MessageFromParent) => {
+process.on('message', async (data: SecretFromParent) => {
   if (data[0] === 'secrets') {
-    parentSecret1 = data[1].parentSecret1;
-    parentSecret2 = data[1].parentSecret2;
-    startQueueListener();
+    const {parentSecret2: parentSecret3, parentSecret1: parentSecret} = data[1];
+    parentSecret1 = parentSecret;
+    parentSecret2 = parentSecret3;
+    await startQueueListener();
+  } else {
+    throw new Error('received unknown command from parent thread');
   }
 });
 
-
 /**
  * Initialize platform module
  */
@@ -63,87 +69,54 @@ const platformSession: PlatformSession = {
 };
 const platform = new PlatformModule(platformSession);
 
-/**
- * Get the credentials stored for this user in this sessions store, if given the correct
- * sessionSecret.
- * @param actorId
- * @param sessionId
- * @param sessionSecret
- * @param cb
- */
-function getCredentials(actorId: string, sessionId: string, sessionSecret: string, cb: Function) {
-  if (platform.config.noCredentials) { return cb(); }
-  const store = getSessionStore(parentId, parentSecret1, sessionId, sessionSecret);
-  store.get(actorId, (err, credentials) => {
-    if (platform.config.persist) {
-      // don't continue if we don't get credentials
-      if (err) { return cb(err); }
-    } else if (! credentials) {
-      // also skip if this is a non-persist platform with no credentials
-      return cb();
-    }
-
-    if (platform.credentialsHash) {
-      if (platform.credentialsHash !== hash(credentials.object)) {
-        return cb('provided credentials do not match existing platform instance for actor '
-            + platform.actor.id);
-      }
-    } else {
-      platform.credentialsHash = hash(credentials.object);
-    }
-    cb(undefined, credentials);
-  });
-}
-
 /**
  * Returns a function used to handle completed jobs from the platform code (the `done` callback).
- * @param secret the secret used to decrypt credentials
  */
-function getJobHandler(secret: string) {
-  return (job: JobEncrypted, done: Function) => {
-    const jobData: JobDataDecrypted = decryptJobData(job, secret);
-    const jobLog = debug(`${loggerPrefix}:${jobData.sessionId}`);
-    jobLog(`received ${jobData.title} ${jobData.msg.type}`);
-    const sessionSecret = jobData.msg.sessionSecret;
-    delete jobData.msg.sessionSecret;
-
-    return getCredentials(jobData.msg.actor.id, jobData.sessionId, sessionSecret,
-      (err, credentials) => {
-        if (err) { return done(new Error(err)); }
-        let jobCallbackCalled = false;
-        const doneCallback = (err, result) => {
-          if (jobCallbackCalled) { return; }
-          jobCallbackCalled = true;
-          if (err) {
-            jobLog(`errored ${jobData.title} ${jobData.msg.type}`);
-            let errMsg;
-            // some error objects (eg. TimeoutError) don't interoplate correctly to human-readable
-            // so we have to do this little dance
-            try {
-              errMsg = err.toString();
-            } catch (e) {
-              errMsg = err;
-            }
-            done(new Error(errMsg));
-          } else {
-            jobLog(`completed ${jobData.title} ${jobData.msg.type}`);
-            done(null, result);
-          }
-        };
-        if ((Array.isArray(platform.config.requireCredentials)) &&
-          (platform.config.requireCredentials.includes(jobData.msg.type))) {
-          // add the credentials object if this method requires it
-          platform[jobData.msg.type](jobData.msg, credentials, doneCallback);
-        } else if (platform.config.persist) {
-          if (platform.initialized) {
-            platform[jobData.msg.type](jobData.msg, doneCallback);
-          } else {
-            done(new Error(`${jobData.msg.type} called on uninitialized platform`));
-          }
-        } else {
-          platform[jobData.msg.type](jobData.msg, doneCallback);
+function getJobHandler() {
+  return (job: JobDataDecrypted, done: CallbackInterface) => {
+    const jobLog = debug(`${loggerPrefix}:${job.sessionId}`);
+    jobLog(`received ${job.title} ${job.msg.type}`);
+    const credentialStore = new CredentialsStore(
+      parentId, job.sessionId, parentSecret1 + job.msg.sessionSecret, redisConfig as RedisConfig
+    );
+    delete job.msg.sessionSecret;
+
+    let jobCallbackCalled = false;
+    const doneCallback = (err, result) => {
+      if (jobCallbackCalled) { return; }
+      jobCallbackCalled = true;
+      if (err) {
+        jobLog(`failed ${job.title} ${job.msg.type}`);
+        let errMsg;
+        // some error objects (eg. TimeoutError) don't interpolate correctly to human-readable
+        // so we have to do this little dance
+        try {
+          errMsg = err.toString();
+        } catch (e) {
+          errMsg = err;
         }
+        done(new Error(errMsg));
+      } else {
+        jobLog(`completed ${job.title} ${job.msg.type}`);
+        done(null, result);
+      }
+    };
+
+    platform.config.requireCredentials ? platform.config.requireCredentials : [];
+    if (platform.config.requireCredentials.includes(job.msg.type)) {
+      // this method requires credentials and should be called even if the platform is not
+      // yet initialized, because they need to authenticate before they are initialized.
+      credentialStore.get(job.msg.actor.id, platform.credentialsHash).then((credentials) => {
+        platform[job.msg.type](job.msg, credentials, doneCallback);
+      }).catch((err) => {
+        jobLog('error ' + err.toString());
+        return done(new Error(err.toString()));
       });
+    } else if ((platform.config.persist) && (!platform.initialized)) {
+      done(new Error(`${job.msg.type} called on uninitialized platform`));
+    } else {
+      platform[job.msg.type](job.msg, doneCallback);
+    }
   };
 }
 
@@ -163,29 +136,34 @@ function getSendFunction(command: string) {
  * both the queue thread (listening on the channel for jobs) and the logging object are updated.
  * @param credentials
  */
-function updateActor(credentials) {
+async function updateActor(credentials) {
   identifier = getPlatformId(platformName, credentials.actor.id);
   logger(`platform actor updated to ${credentials.actor.id} identifier ${identifier}`);
   logger = debug(`sockethub:platform:${identifier}`);
-  platform.credentialsHash = hash(credentials.object);
+  platform.credentialsHash = crypto.objectHash(credentials.object);
   platform.debug = debug(`sockethub:platform:${platformName}:${identifier}`);
   process.send(['updateActor', undefined, identifier]);
-  startQueueListener(true);
+  await startQueueListener(true);
 }
 
 /**
- * starts listening on the queue for incoming jobs
+ * Starts listening on the queue for incoming jobs
  * @param refresh boolean if the param is true, we re-init the queue.process
  * (used when identifier changes)
  */
-function startQueueListener(refresh: boolean = false) {
-  const secret = parentSecret1 + parentSecret2;
-  if ((queueStarted) && (!refresh)) {
-    logger('start queue called multiple times, skipping');
-    return;
+async function startQueueListener(refresh = false) {
+  if (jobQueueStarted) {
+    if (refresh) {
+      await jobQueue.shutdown();
+    } else {
+      logger('start queue called multiple times, skipping');
+      return;
+    }
   }
-  const queue = new Queue(parentId + identifier, { redis: redisConfig });
-  queueStarted = true;
+  jobQueue = new JobQueue(
+    parentId, identifier, parentSecret1 + parentSecret2, redisConfig as RedisConfig
+  );
   logger('listening on the queue for incoming jobs');
-  queue.process(getJobHandler(secret));
+  jobQueue.onJob(getJobHandler());
+  jobQueueStarted = true;
 }

package/src/process-manager.ts

@@ -1,7 +1,7 @@
 import init from './bootstrap/init';
 import PlatformInstance, {
   platformInstances, PlatformInstanceParams, MessageFromParent } from "./platform-instance";
-import { getPlatformId } from "./common";
+import {getPlatformId} from "@sockethub/crypto";
 
 class ProcessManager {
   private readonly parentId: string;

package/src/routes.ts

@@ -39,7 +39,7 @@ export const examplePages: IRoutePaths = {
 
 
 function prepFileRoutes(pathMap) {
-  let _routes = [];
+  const _routes = [];
   Object.keys(pathMap).forEach((key) => {
     _routes.push({
       meta: {
@@ -88,7 +88,8 @@ function addRoute(app) {
  * Setup
  */
 const routes = {
-  setup: function (app: any, examplesEnabled: boolean = config.get('examples:enabled')) {
+  setup: function (app: unknown,
+                   examplesEnabled: boolean = config.get('examples:enabled') as boolean) {
     baseRoutes.forEach(addRoute(app));
     if (examplesEnabled) {
       exampleRoutes.forEach(addRoute(app));

package/src/sockethub.ts

@@ -1,10 +1,10 @@
 import debug from 'debug';
-import { Socket } from "socket.io";
-import { IActivityStream } from "@sockethub/schemas";
+import {Socket} from "socket.io";
+import crypto from "@sockethub/crypto";
+import {CredentialsStore} from "@sockethub/data-layer";
 
-import crypto from './crypto';
 import init from './bootstrap/init';
-import middleware from './middleware';
+import middleware, {MiddlewareChainInterface} from './middleware';
 import createActivityObject from "./middleware/create-activity-object";
 import expandActivityStream from "./middleware/expand-activity-stream";
 import storeCredentials from "./middleware/store-credentials";
@@ -12,34 +12,11 @@ import validate from "./middleware/validate";
 import janitor from './janitor';
 import listener from './listener';
 import ProcessManager from "./process-manager";
-import PlatformInstance, { platformInstances } from "./platform-instance";
-import { getSessionStore } from "./store";
+import nconf from "nconf";
+import {IActivityStream} from "@sockethub/schemas";
 
 const log = debug('sockethub:server:core');
 
-
-export interface JobDataDecrypted {
-  title?: string;
-  msg: IActivityStream;
-  sessionId: string;
-}
-
-export interface JobDataEncrypted {
-  title?: string;
-  msg: string;
-  sessionId: string;
-}
-
-export interface JobDecrypted {
-  data: JobDataDecrypted,
-  remove?: Function;
-}
-
-export interface JobEncrypted {
-  data: JobDataEncrypted,
-  remove?: Function;
-}
-
 function attachError(err, msg) {
   if (typeof msg !== 'object') {
     msg = { context: 'error' };
@@ -56,11 +33,9 @@ class Sockethub {
   counter: number;
   platforms: Map<string, object>;
   status: boolean;
-  queue: any;
   processManager: ProcessManager;
 
   constructor() {
-    this.counter = 0;
     this.platforms = init.platforms;
     this.status = false;
     this.parentId = crypto.randToken(16);
@@ -82,34 +57,27 @@ class Sockethub {
     }
 
     log('active platforms: ', [...init.platforms.keys()]);
-    janitor.clean(); // start cleanup cycle
     listener.start();   // start external services
+    janitor.start(); // start cleanup cycle
     log('registering handlers');
     listener.io.on('connection', this.handleIncomingConnection.bind(this));
   }
 
-  async removeAllPlatformInstances() {
-    for (let platform of platformInstances) {
-      await platform[1].destroy();
-    }
+  async shutdown() {
+    await janitor.stop();
   }
 
-  private createJob(socketId: string, platformInstance: PlatformInstance, msg): JobDataEncrypted {
-    const title = `${msg.context}-${(msg.id) ? msg.id : this.counter++}`;
-    const job: JobDataEncrypted = {
-      title: title,
-      sessionId: socketId,
-      msg: crypto.encrypt(msg, this.parentSecret1 + this.parentSecret2)
-    };
-    return job;
-  };
-
   private handleIncomingConnection(socket: Socket) {
     // session-specific debug messages
     const sessionLog = debug('sockethub:server:core:' + socket.id),
           sessionSecret = crypto.randToken(16),
           // store instance is session-specific
-          store = getSessionStore(this.parentId, this.parentSecret1, socket.id, sessionSecret);
+          // store = getSessionStore(this.parentId, this.parentSecret1, socket.id, sessionSecret);
+          credentialsStore = new CredentialsStore(
+            this.parentId, socket.id,
+            this.parentSecret1 + sessionSecret,
+            nconf.get('redis')
+          );
 
     sessionLog(`socket.io connection`);
 
@@ -121,9 +89,9 @@ class Sockethub {
       middleware('credentials')
         .use(expandActivityStream)
         .use(validate('credentials', socket.id))
-        .use(storeCredentials(store, sessionLog))
+        .use(storeCredentials(credentialsStore) as MiddlewareChainInterface)
         .use((err, data, next) => {
-          // error handler
+        // error handler
           next(attachError(err, data));
         }).use((data, next) => { next(); })
         .done());
@@ -144,20 +112,24 @@ class Sockethub {
         .use(expandActivityStream)
         .use(validate('message', socket.id))
         .use((msg, next) => {
-          // The platform thread must find the credentials on their own using the given
-          // sessionSecret, which indicates that this specific session (socket
-          // connection) has provided credentials.
+        // The platform thread must find the credentials on their own using the given
+        // sessionSecret, which indicates that this specific session (socket
+        // connection) has provided credentials.
           msg.sessionSecret = sessionSecret;
           next(msg);
         }).use((err, data, next) => {
           next(attachError(err, data));
-        }).use((msg, next) => {
+        }).use(async (msg: IActivityStream, next) => {
           const platformInstance = this.processManager.get(msg.context, msg.actor.id, socket.id);
-          sessionLog(`queued to channel ${platformInstance.id}`);
-          const job = this.createJob(socket.id, platformInstance, msg);
           // job validated and queued, store socket.io callback for when job is completed
-          platformInstance.completedJobHandlers.set(job.title, next);
-          platformInstance.queue.add(job);
+          const job = await platformInstance.jobQueue.add(socket.id, msg);
+          if (job) {
+            platformInstance.completedJobHandlers.set(job.title, next);
+          } else {
+            // failed to add job to queue, reject handler immediately
+            msg.error = 'failed to add job to queue';
+            next(msg);
+          }
         }).done());
   }
 }

package/views/examples/dummy.ejs

@@ -83,7 +83,9 @@
           }
         };
         bdebug('sending message: ', msg);
-        sc.socket.emit('message', msg);
+        sc.socket.emit('message', msg, (d) => {
+          bdebug('completed', d)
+        });
 
         $('#m').val('');
         return false;

package/views/examples/shared.js

@@ -29,7 +29,7 @@ ExamplesShared.prototype.__displayUnknownContent = function (msg) {
 };
 
 ExamplesShared.prototype.processIncomingMessage = function (msg) {
-  console.log('incoming message: ', msg);
+  console.log(msg);
   if (msg.type === 'query') {
     if (msg.object.members) {
       $('#messages').append($('<li>').text(` users in ${msg.actor.name}`))

package/views/examples/xmpp.ejs

@@ -58,6 +58,7 @@
 
     <script>
       debug = function (msg, obj) { console.log(msg, obj); };
+      let counter = 0;
       const examplesShared = new ExamplesShared();
       localStorage.debug = 'sockethub:*';
       let config = {
@@ -92,21 +93,24 @@
         sc.ActivityStreams.Object.create(asObject);
       }
 
-      function handleMessage(msg) {
-        if (msg.error) {
-          debug('received error: ', msg);
-          console.error(msg.error);
-        } else if (msg['type'] === 'connect') {
-          $('.modal').css('display', 'none');
-        } else if (msg['type'] === 'join') {
-          $('input#m').prop('disabled', false);
-          $('button#send').prop('disabled', false);
-        } else {
-          examplesShared.processIncomingMessage(msg);
-        }
+      function handleMessage(text) {
+        return (msg) => {
+          console.log(text);
+          if (msg.error) {
+            debug('error: ', msg);
+            console.error(msg.error);
+          } else if (msg['type'] === 'connect') {
+            $('.modal').css('display', 'none');
+          } else if (msg['type'] === 'join') {
+            $('input#m').prop('disabled', false);
+            $('button#send').prop('disabled', false);
+          } else {
+            examplesShared.processIncomingMessage(msg);
+          }
+        };
       }
       // handle incoming messages from the Sockethub server
-      sc.socket.on('message', handleMessage);
+      sc.socket.on('message', handleMessage("incoming message"));
 
       $('.actor').click(function (e) {
         e.preventDefault();
@@ -117,15 +121,24 @@
       $('#chat').submit(function (e) {
         e.preventDefault();
         const input = $('#m').val();
-        const [target, text_message] = input.split(':');
-
-        if ((!target || !text_message)) {
-          console.log(
-                  'unable to send message without target specified ' +
-                  '[room@server.org: hello world]: ',
-                  target, text_message, input);
-          return false;
+        let [targetId, text_message] = input.split(':');
+
+        if (!text_message) {
+          text_message = targetId;
+          targetId = undefined;
+        }
+
+        let target = {
+          id: 'kosmos-random@kosmos.chat',
+          type: 'room'
+        }
+        if (targetId) {
+          target = {
+            id: targetId,
+            type: 'room'
+          }
         }
+
         const msg = {
           type: 'send',
           context: 'xmpp',
@@ -137,9 +150,8 @@
           target: target
         };
 
-        console.log('[normalize?] sending ', msg);
         debug('sending message: ', msg);
-        sc.socket.emit('message', msg, handleMessage);
+        sc.socket.emit('message', msg, handleMessage('callback from sent message'));
         $('#m').val('');
         return false;
       });
@@ -165,27 +177,41 @@
           sc.socket.emit('credentials', credentials, (err) => {
             if (err) {
               console.error('failed to create credentials: ', err);
+            } else {
+              console.log('completed credentials');
             }
           });
 
-          const msg = {
+          const connectMsg = {
             type: 'connect',
             context: 'xmpp',
-            actor: actorString
+            actor: actorString,
+            object: {
+              type: 'message',
+              content: `debug:${++counter}`
+            }
           };
+          debug('sending connect message: ', connectMsg);
+          sc.socket.emit('message', connectMsg, handleMessage('callback from connect'));
 
-          debug('sending message: ', msg);
-          sc.socket.emit('message', msg, handleMessage);
-          msg.type = 'join';
-          msg.target = {
-            type: 'room',
-            id: 'kosmos-random@kosmos.chat'
+          const joinMsg = {
+            type: 'join',
+            context: 'xmpp',
+            actor: actorString,
+            target: {
+              type: 'room',
+              id: 'kosmos-random@kosmos.chat'
+            },
+            object: {
+              type: 'message',
+              content: `debug:${counter}`
+            }
           };
-          debug('sending message: ', msg);
-          sc.socket.emit('message', msg, handleMessage);
+          debug('sending join message: ', joinMsg);
+          sc.socket.emit('message', joinMsg, handleMessage('callback from join'));
         }, 100);
         return false;
       });
     </script>
   </body>
-</html>
+</html>