ldn-inbox-server 1.2.2 → 1.3.0

package/.env-example

@@ -9,4 +9,7 @@ LDN_SERVER_INBOX_PATH=./inbox
 LDN_SERVER_ERROR_PATH=./error
 LDN_SERVER_OUTBOX_PATH=./outbox
 LDN_SERVER_PUBLIC_PATH=./public
-LDN_SERVER_JSON_SCHEMA=./config/notification_schema.json
+LDN_SERVER_JSON_SCHEMA=./config/notification_schema.json
+LDN_SERVER_INBOX_CONFIG=./config/inbox_config.json
+LDN_SERVER_OUTBOX_CONFIG=./config/outbox_config.json
+LDN_SERVER_HAS_PUBLIC_INBOX=0

package/README.md

@@ -39,7 +39,7 @@ npx ldn-inbox-server handle @inbox -hn ./handler/demo_notification_handler.js
 Send the notifications in the outbox:
 
 ```
-npx ldn-inbox-server handle @outbox
+npx ldn-inbox-server handle @outbox -hn ./handler/send_notification_handler.js
 ```
 
 ## Environment
@@ -74,7 +74,14 @@ main();
 
 async function main() {
     await handle_inbox('./inbox', {
-        'notification_handler': 'handler/worker.js' 
+        'inbox': './inbox',
+        'outbox': './outbox',
+        'public': './public',
+        'error': './error',
+        'batch_size': 5,
+        'glob': '^.*\\.jsonld$',
+        'config': './config/inbox_config.json',
+        'notification_handler': 'handler/demo_notification_handler.js' 
     });
 }
 ```
@@ -102,6 +109,10 @@ A handler can be started on any directory. E.g. a workflow might be:
 - processed LDN messages will end up in the "outbox" box
 - invalid processing will be saved into the "error" box  
 
+## Multi handler
+
+A `handler/multi_notification_handler.js` is available to start multiple handler for each notification messages. The handlers to start are specified in a configuraton file that can be passed via the `config` parameter of an `handle_inbox`. In the commmand line tool `bin/ldn-inbox-server` the default location of such config file is `config/inbox_config.json` when processing an `@inbox`, and `config/outbox_config.json` when processing an `@outbox`.
+
 ## See also
 
 - [mellon-server](https://www.npmjs.com/package/mellon-server)

package/bin/ldn-inbox-server.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 
-const path = require('path');
+const fs = require('fs');
 const { program } = require('commander');
 const { inbox_server } = require('../lib/index');
-const { handle_inbox , defaultSendNotificationHandler } = require('../lib/handler');
+const { handle_inbox } = require('../lib/handler');
 require('dotenv').config();
 
 const HOST = process.env.LDN_SERVER_HOST ?? 'localhost';
@@ -17,6 +17,8 @@ const INBOX_PATH = process.env.LDN_SERVER_INBOX_PATH ?? './inbox';
 const ERROR_PATH = process.env.LDN_SERVER_ERROR_PATH ?? './error';
 const OUTBOX_PATH = process.env.LDN_SERVER_OUTBOX_PATH ?? './outbox';
 const JSON_SCHEMA_PATH = process.env.LDN_SERVER_JSON_SCHEMA ?? './config/notification_schema.json';
+const INBOX_CONFIG = process.env.LDN_SERVER_INBOX_CONFIG;
+const OUTBOX_CONFIG = process.env.LDN_SERVER_OUTBOX_CONFIG;
 const HAS_PUBLIC = process.env.LDN_SERVER_HAS_PUBLIC_INBOX ?? 0;
 
 program
@@ -48,6 +50,7 @@ program
   .option('--loop <seconds>', 'run in a loop',0)
   .option('--batch_size <num>','batch size to process',INBOX_BATCH_SIZE)
   .option('--glob <glob>','files to process in inbox',INBOX_GLOB)
+  .option('--config <path>','config file for handlers')
   .option('-hi,--inbox_handler <handler>','inbox handler')
   .option('-hn,--notification_handler <handler>','notification handler')
   .argument('<box>','box to process')
@@ -55,11 +58,15 @@ program
     switch (box) {
       case '@inbox':
         box = INBOX_PATH;
+        if (!options['config'] && fs.existsSync(INBOX_CONFIG)) {
+            options['config'] = INBOX_CONFIG
+        }
         break;
       case '@outbox':
         box = OUTBOX_PATH;
-        options['notification_handler'] =
-           options['notification_handler'] ?? path.join(__dirname,'..','lib','sendNotificationHandler.js');
+        if (!options['config'] && fs.existsSync(OUTBOX_CONFIG)) {
+          options['config'] = OUTBOX_CONFIG
+      }
         break;
     }
     if (options['loop']) {

package/config/inbox_config.json

@@ -0,0 +1,10 @@
+{
+    "notification_handler": {
+        "multi": {
+            "handlers": [
+                "handler/demo_notification_handler.js" ,
+                "handler/eventlog_notification_handler.js"
+            ]
+        }
+    }
+}

package/config/outbox_config.json

@@ -0,0 +1,10 @@
+{
+    "notification_handler": {
+        "multi": {
+            "handlers": [
+                "handler/send_notification_handler.js",
+                "handler/eventlog_notification_handler.js"
+            ]
+        }
+    }
+}

package/handler/demo_inbox_handler.js

@@ -1,7 +1,11 @@
 const fs = require('fs');
 
-// Demonstration of an inbox handler (don't use it the one in lib/handler.js 
-// is much more mature)
+/** 
+ * Demonstration of an alternatve inbox handler
+ *   (don't use it the one in lib/handler.js is much more mature)
+ * To use it set it via the `--hi` option of `bin/ldn-inbox-server.js`, or
+ * the `inbox_handler` option of the `handle_inbox` function.
+ */
 async function handle({path,options}) {
     console.log(`handleInbox(${path},..)`);
     fs.readdir(path, (err,files) => {

package/handler/demo_notification_handler.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const md5 = require('md5');
 const logger = require('../lib/util.js').getLogger();
 
 async function handle({path,options}) {
@@ -7,17 +8,13 @@ async function handle({path,options}) {
     try {
         const json = JSON.parse(fs.readFileSync(path, { encoding: 'utf-8'}));
         
-        const outboxFile = options['outbox'] + '/' + path.split('/').pop();
-
         const id = json['id'];
         const object = json['object'];
         const actor_id = json['actor']['id'];
         const actor_type = json['actor']['type'];
         const actor_inbox = json['actor']['inbox'];
 
-        logger.info(`storing Accept to ${outboxFile}`);
-
-        fs.writeFileSync(outboxFile, JSON.stringify({
+        const data = JSON.stringify({
             type: 'Accept',
             actor: {
                 id: 'http://my.server' ,
@@ -32,13 +29,20 @@ async function handle({path,options}) {
                 type: actor_type ,
                 inbox: actor_inbox
             }
-        },null,2));
-        return { path,options, success: true };
+        },null,4);
+
+        const outboxFile = options['outbox'] + '/' + md5(data) + '.jsonld';
+
+        logger.info(`storing Accept to ${outboxFile}`);
+
+        fs.writeFileSync(outboxFile,data);
+
+        return { path, options, success: true };
     }
     catch(e) {
         logger.error(`failed to process ${path}`);
         logger.debug(e);
-        return { path,options, success: false };
+        return { path, options, success: false };
     }
 }
 

package/handler/eventlog_notification_handler.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const md5 = require('md5');
 const fsPath = require('path');
 const lockfile = require('proper-lockfile');
 const logger = require('../lib/util.js').getLogger();
@@ -10,7 +11,7 @@ async function handle({path,options}) {
     logger.info(`parsing notification ${path}`);
     
     try {
-        const json = JSON.parse(fs.readFileSync(path, { encoding: 'utf-8'}));
+        const json = fs.readFileSync(path, { encoding: 'utf-8'});
       
         const fileName = path.split('/').pop();
         const logDir = fsPath.join(options['public'],EVENT_DIR,'log');
@@ -22,7 +23,7 @@ async function handle({path,options}) {
   
         const outboxFile = fsPath.join(logDir,fileName);
 
-        fs.writeFileSync(outboxFile, JSON.stringify(json));
+        fs.writeFileSync(outboxFile, json);
 
         // Updating metadata file
         const metaFile = outboxFile + '.meta';
@@ -34,12 +35,12 @@ async function handle({path,options}) {
 
         await updateEventLog({path,options});
 
-        return { path,options, success: true };
+        return { path, options, success: true };
     }
     catch(e) {
         logger.error(`failed to process ${path}`);
         logger.error(e);
-        return { path,options, success: false };
+        return { path, options, success: false };
     }
 }
 
@@ -47,6 +48,9 @@ async function updateEventLog({path,options}) {
     logger.info(`updating eventlog for ${path}`);
 
     try {
+        const notification = fs.readFileSync(path, { encoding: 'utf-8'});
+        const notification_checksum = md5(notification);
+
         const baseUrl = process.env.LDN_SERVER_BASEURL ?? "";
         const fileName = path.split('/').pop();
         const entry = `${baseUrl}/${EVENT_DIR}/log/${fileName}`;
@@ -72,7 +76,14 @@ async function updateEventLog({path,options}) {
         else {
             logger.info(`updating ${eventLog}`);
 
-            json['member'].push(entry);
+            json['member'].push({
+                "id": entry ,
+                "checksum": {
+                    "type": "Checksum",
+                    "algorithm": "spdx:checksumAlgorithm_md5",
+                    "checksumValue": notification_checksum
+                }
+            });
 
             if (fs.existsSync(eventLog)) {
                 try {

package/handler/multi_notification_handler.js

@@ -0,0 +1,57 @@
+const { dynamic_handler , parseAsJSON } = require('../lib/util');
+const logger = require('../lib/util.js').getLogger();
+
+async function handle({path,options}) {
+    let success = false;
+
+    const config = parseAsJSON(options['config']);
+
+    if (! config) {
+        logger.error('no configuration found for multi_notification_handler');
+        return { path, options, success: false };
+    }
+
+    const handlers = config['notification_handler']?.['multi']?.['handlers'];
+
+    if (! handlers) {
+        logger.error('no notification_handler.multi.handlers key in configuration file');
+        return { path, options, success: false };
+    }
+
+    try {
+        logger.info(`starting multi handler`);
+        
+        for (let i = 0 ; i < handlers.length ; i++) {
+            logger.info(`starting ${handlers[i]}`);
+
+            const handler = dynamic_handler(handlers[i],null);
+
+            if (! handler) {
+                throw new Error(`failed to load ${handlers[i]}`);
+            }
+
+            const result = await handler({path,options});
+           
+            if (result['success']) {
+                logger.info(`finished ${handlers[i]}`);
+            }
+            else {
+                throw new Error(`Eek! ${handlers[i]} failed`);
+            }
+        }
+
+        success = true;
+    }
+    catch (e) {
+        logger.error(`failed to process ${path}`);
+        logger.error(e);
+
+        success = false;
+    }
+
+    logger.info(`finished multi handler`);
+
+    return { path, options, success: success };
+}   
+
+module.exports = { handle };

package/lib/handler.js

@@ -17,6 +17,7 @@ async function defaultInboxHandler({path,options}) {
 
     const worker = options['notification_handler'] ?? fsPath.resolve(__dirname,'..','lib','notification.js');
 
+    // Run the notifications using a node.js worker pool
     const pool = new piscina({
         filename: worker,
         maxQueue: queue_size

package/lib/index.js

@@ -42,16 +42,17 @@ function inbox_server(options) {
     start_server({
         host: options['host'],
         port: options['port'],
+        base: options['base'],
         public: options['public'],
         registry: registry
     });
 }
 
 function doInbox(req,res) {
-    if (req.method === 'GET' && INBOX_PUBLIC_READABLE) {
+    if (req.method === 'GET' && INBOX_PUBLIC_READABLE == 1) {
         doInboxGET(req,res);
     }
-    if (req.method == 'HEAD' && INBOX_PUBLIC_READABLE) {
+    else if (req.method == 'HEAD' && INBOX_PUBLIC_READABLE == 1) {
         doInboxHEAD(req,res);
     }
     else if (req.method === 'POST') {
@@ -61,7 +62,6 @@ function doInbox(req,res) {
         logger.error(`tried method ${req.method} on inbox : forbidden`);
         res.writeHead(403);
         res.end('Forbidden');
-        return;
     }
 }
 

package/lib/util.js

@@ -125,11 +125,22 @@ function dynamic_handler(handler,fallback) {
     }
 }
 
+function parseAsJSON(path) {
+    try {
+        return JSON.parse(fs.readFileSync(path, { encoding: 'utf-8'}));
+    }
+    catch (e) {
+        logger.error(`failed to parse ${path}`);
+        return null;
+    }
+}
+
 module.exports = { 
     getLogger , 
     backOff_fetch , 
     fetchOriginal ,
     moveTo ,
     sendNotification ,
-    dynamic_handler
+    dynamic_handler ,
+    parseAsJSON 
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ldn-inbox-server",
-  "version": "1.2.2",
+  "version": "1.3.0",
   "description": "A demonstration Event Notifications Inbox server",
   "main": "lib/index.js",
   "author": "Patrick Hochstenbach <Patrick.Hochstenbach@UGent.be>",
@@ -9,9 +9,11 @@
     "server": "npx ldn-inbox-server start-server",
     "demo-post": "curl -X POST -H 'Content-Type: application/ld+json' --data-binary '@examples/offer.jsonld' http://localhost:8000/inbox/",
     "handle-inbox": "npx ldn-inbox-server handler @inbox -hn ./handler/demo_notification_handler.js",
-    "handle-outbox": "npx ldn-inbox-server handler @outbox",
+    "handle-outbox": "npx ldn-inbox-server handler @outbox -hn ./handler/send_notification_handler.js",
     "handle-eventlog": "npx ldn-inbox-server handler @inbox -hn handler/eventlog_notification_handler.js",
-    "clean": "rm error/* inbox/* outbox/*"
+    "handle-inbox-multi": "npx ldn-inbox-server handler @inbox -hn ./handler/multi_notification_handler.js",
+    "handle-outbox-multi": "npx ldn-inbox-server handler @outbox -hn ./handler/multi_notification_handler.js",
+    "clean": "rm error/* inbox/* outbox/* public/events/* public/events/log/*"
   },
   "bin": "./bin/ldn-inbox-server.js",
   "keywords": [
@@ -25,7 +27,7 @@
     "exponential-backoff": "^3.1.1",
     "jsonschema": "^1.4.1",
     "md5": "^2.3.0",
-    "mellon-server": "^1.0.4",
+    "mellon-server": "^1.0.8",
     "node-fetch": "1.7.3",
     "piscina": "^4.4.0",
     "proper-lockfile": "^4.1.2",