ldn-inbox-server 1.2.1 → 1.3.0

package/{.env → .env-example}

@@ -2,10 +2,14 @@ LOG4JS=INFO
 LDN_SERVER_HOST=localhost
 LDN_SERVER_PORT=8000
 LDN_SERVER_INBOX_GLOB="^.*\.jsonld$"
+LDN_SERVER_BASEURL=http://localhost:8000
 LDN_SERVER_INBOX_BATH_SIZE=5
 LDN_SERVER_INBOX_URL=inbox/
 LDN_SERVER_INBOX_PATH=./inbox
 LDN_SERVER_ERROR_PATH=./error
 LDN_SERVER_OUTBOX_PATH=./outbox
 LDN_SERVER_PUBLIC_PATH=./public
-LDN_SERVER_JSON_SCHEMA=./config/notification_schema.json
+LDN_SERVER_JSON_SCHEMA=./config/notification_schema.json
+LDN_SERVER_INBOX_CONFIG=./config/inbox_config.json
+LDN_SERVER_OUTBOX_CONFIG=./config/outbox_config.json
+LDN_SERVER_HAS_PUBLIC_INBOX=0

package/README.md

@@ -39,20 +39,23 @@ npx ldn-inbox-server handle @inbox -hn ./handler/demo_notification_handler.js
 Send the notifications in the outbox:
 
 ```
-npx ldn-inbox-server handle @outbox
+npx ldn-inbox-server handle @outbox -hn ./handler/send_notification_handler.js
 ```
 
 ## Environment
 
 - `LOG4JS` : log4js logging level
-- `LDN_SERVER_HOST` : default LDN inbox host
-- `LDN_SERVER_PORT` : default LDN inbox port
-- `LDN_SERVER_INBOX_URL` : default LDN inbox url (path)
-- `LDN_SERVER_INBOX_PATH` : default LDN inbox path
-- `LDN_SERVER_ERROR_PATH` : default LDN error path
-- `LDN_SERVER_OUTBOX_PATH` : default LDN outbox path
-- `LDN_SERVER_PUBLIC_PATH` : default public (HTML) path
-- `LDN_SERVER_JSON_SCHEMA` : default notification JSON validation schema
+- `LDN_SERVER_HOST` : LDN inbox host
+- `LDN_SERVER_PORT` : LDN inbox port
+- `LDN_SERVER_INBOX_URL` : LDN inbox url (path)
+- `LDN_SERVER_INBOX_PATH` : LDN inbox path
+- `LDN_SERVER_ERROR_PATH` : LDN error path
+- `LDN_SERVER_OUTBOX_PATH` : LDN outbox path
+- `LDN_SERVER_PUBLIC_PATH` : public (HTML) path
+- `LDN_SERVER_JSON_SCHEMA` : notification JSON validation schema
+- `LDN_SERVER_BASEURL` : baseurl of the LDN inbox server
+- `LDN_SERVER_INBOX_GLOB` : glob of files to process in inbox directory
+- `LDN_SERVER_HAS_PUBLIC_INBOX` : if true, then public read access is allowed on inbox
 
 ## Extend
 
@@ -71,7 +74,14 @@ main();
 
 async function main() {
     await handle_inbox('./inbox', {
-        'notification_handler': 'handler/worker.js' 
+        'inbox': './inbox',
+        'outbox': './outbox',
+        'public': './public',
+        'error': './error',
+        'batch_size': 5,
+        'glob': '^.*\\.jsonld$',
+        'config': './config/inbox_config.json',
+        'notification_handler': 'handler/demo_notification_handler.js' 
     });
 }
 ```
@@ -99,6 +109,10 @@ A handler can be started on any directory. E.g. a workflow might be:
 - processed LDN messages will end up in the "outbox" box
 - invalid processing will be saved into the "error" box  
 
+## Multi handler
+
+A `handler/multi_notification_handler.js` is available to start multiple handler for each notification messages. The handlers to start are specified in a configuraton file that can be passed via the `config` parameter of an `handle_inbox`. In the commmand line tool `bin/ldn-inbox-server` the default location of such config file is `config/inbox_config.json` when processing an `@inbox`, and `config/outbox_config.json` when processing an `@outbox`.
+
 ## See also
 
 - [mellon-server](https://www.npmjs.com/package/mellon-server)

package/bin/ldn-inbox-server.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 
-const path = require('path');
+const fs = require('fs');
 const { program } = require('commander');
 const { inbox_server } = require('../lib/index');
-const { handle_inbox , defaultSendNotificationHandler } = require('../lib/handler');
+const { handle_inbox } = require('../lib/handler');
 require('dotenv').config();
 
 const HOST = process.env.LDN_SERVER_HOST ?? 'localhost';
@@ -11,26 +11,32 @@ const PORT = process.env.LDN_SERVER_PORT ?? 8000;
 const INBOX_GLOB = process.env.LDN_SERVER_INBOX_GLOB ?? "^.*\\.jsonld$";
 const INBOX_BATCH_SIZE = process.env.LDN_SERVER_INBOX_BATH_SIZE ?? 5;
 const INBOX_URL = process.env.LDN_SERVER_INBOX_URL ?? 'inbox/';
+const INBOX_BASE_URL = process.env.LDN_SERVER_BASEURL ?? 'http://localhost:8000';
 const PUBLIC_PATH = process.env.LDN_SERVER_PUBLIC_PATH ?? './public';
 const INBOX_PATH = process.env.LDN_SERVER_INBOX_PATH ?? './inbox';
 const ERROR_PATH = process.env.LDN_SERVER_ERROR_PATH ?? './error';
 const OUTBOX_PATH = process.env.LDN_SERVER_OUTBOX_PATH ?? './outbox';
 const JSON_SCHEMA_PATH = process.env.LDN_SERVER_JSON_SCHEMA ?? './config/notification_schema.json';
+const INBOX_CONFIG = process.env.LDN_SERVER_INBOX_CONFIG;
+const OUTBOX_CONFIG = process.env.LDN_SERVER_OUTBOX_CONFIG;
+const HAS_PUBLIC = process.env.LDN_SERVER_HAS_PUBLIC_INBOX ?? 0;
 
 program
   .name('lnd-inbox-server')
-  .version('1.2.1')
+  .version('1.2.2')
   .description('A demonstration Event Notifications Inbox server');
 
 program
   .command('start-server')
   .option('--host <host>','host',HOST)
   .option('--port <port>','port',PORT)
-  .option('--url <url>','url',INBOX_URL)
+  .option('--url <path>','path',INBOX_URL)
+  .option('--base <url>','base url',INBOX_BASE_URL)
   .option('--inbox <inbox>','inbox',INBOX_PATH)
   .option('--public <public>','public',PUBLIC_PATH)
   .option('--schema <schema>','json schema',JSON_SCHEMA_PATH)
   .option('--registry <registry>','registry',null)
+  .option('--inbox-public','public readable inbox',HAS_PUBLIC)
   .action( (options) => {
     inbox_server(options);
   });
@@ -39,10 +45,12 @@ program
   .command('handler')
   .option('--inbox <inbox>','inbox',INBOX_PATH)
   .option('--outbox <outbox>','outbox',OUTBOX_PATH)
+  .option('--public <public>','public',PUBLIC_PATH)
   .option('--error <errbox>','errbox',ERROR_PATH)
   .option('--loop <seconds>', 'run in a loop',0)
   .option('--batch_size <num>','batch size to process',INBOX_BATCH_SIZE)
   .option('--glob <glob>','files to process in inbox',INBOX_GLOB)
+  .option('--config <path>','config file for handlers')
   .option('-hi,--inbox_handler <handler>','inbox handler')
   .option('-hn,--notification_handler <handler>','notification handler')
   .argument('<box>','box to process')
@@ -50,11 +58,15 @@ program
     switch (box) {
       case '@inbox':
         box = INBOX_PATH;
+        if (!options['config'] && fs.existsSync(INBOX_CONFIG)) {
+            options['config'] = INBOX_CONFIG
+        }
         break;
       case '@outbox':
         box = OUTBOX_PATH;
-        options['notification_handler'] =
-           options['notification_handler'] ?? path.join(__dirname,'..','lib','sendNotificationHandler.js');
+        if (!options['config'] && fs.existsSync(OUTBOX_CONFIG)) {
+          options['config'] = OUTBOX_CONFIG
+      }
         break;
     }
     if (options['loop']) {

package/config/inbox_config.json

@@ -0,0 +1,10 @@
+{
+    "notification_handler": {
+        "multi": {
+            "handlers": [
+                "handler/demo_notification_handler.js" ,
+                "handler/eventlog_notification_handler.js"
+            ]
+        }
+    }
+}

package/config/outbox_config.json

@@ -0,0 +1,10 @@
+{
+    "notification_handler": {
+        "multi": {
+            "handlers": [
+                "handler/send_notification_handler.js",
+                "handler/eventlog_notification_handler.js"
+            ]
+        }
+    }
+}

package/handler/demo_inbox_handler.js

@@ -1,5 +1,11 @@
 const fs = require('fs');
 
+/** 
+ * Demonstration of an alternatve inbox handler
+ *   (don't use it the one in lib/handler.js is much more mature)
+ * To use it set it via the `--hi` option of `bin/ldn-inbox-server.js`, or
+ * the `inbox_handler` option of the `handle_inbox` function.
+ */
 async function handle({path,options}) {
     console.log(`handleInbox(${path},..)`);
     fs.readdir(path, (err,files) => {

package/handler/demo_notification_handler.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const md5 = require('md5');
 const logger = require('../lib/util.js').getLogger();
 
 async function handle({path,options}) {
@@ -7,17 +8,13 @@ async function handle({path,options}) {
     try {
         const json = JSON.parse(fs.readFileSync(path, { encoding: 'utf-8'}));
         
-        const outboxFile = options['outbox'] + '/' + path.split('/').pop();
-
         const id = json['id'];
         const object = json['object'];
         const actor_id = json['actor']['id'];
         const actor_type = json['actor']['type'];
         const actor_inbox = json['actor']['inbox'];
 
-        logger.info(`storing Accept to ${outboxFile}`);
-
-        fs.writeFileSync(outboxFile, JSON.stringify({
+        const data = JSON.stringify({
             type: 'Accept',
             actor: {
                 id: 'http://my.server' ,
@@ -32,13 +29,20 @@ async function handle({path,options}) {
                 type: actor_type ,
                 inbox: actor_inbox
             }
-        },null,2));
-        return { path,options, success: true };
+        },null,4);
+
+        const outboxFile = options['outbox'] + '/' + md5(data) + '.jsonld';
+
+        logger.info(`storing Accept to ${outboxFile}`);
+
+        fs.writeFileSync(outboxFile,data);
+
+        return { path, options, success: true };
     }
     catch(e) {
         logger.error(`failed to process ${path}`);
         logger.debug(e);
-        return { path,options, success: false };
+        return { path, options, success: false };
     }
 }
 

package/handler/eventlog_notification_handler.js

@@ -0,0 +1,114 @@
+const fs = require('fs');
+const md5 = require('md5');
+const fsPath = require('path');
+const lockfile = require('proper-lockfile');
+const logger = require('../lib/util.js').getLogger();
+
+const EVENT_DIR = 'events';
+const EVENT_LOG = 'events.jsonld';
+
+async function handle({path,options}) {
+    logger.info(`parsing notification ${path}`);
+    
+    try {
+        const json = fs.readFileSync(path, { encoding: 'utf-8'});
+      
+        const fileName = path.split('/').pop();
+        const logDir = fsPath.join(options['public'],EVENT_DIR,'log');
+
+        if (! fs.existsSync(logDir)) {
+            logger.info(`creating ${logDir}`);
+            fs.mkdirSync(logDir, { recursive : true });
+        }
+  
+        const outboxFile = fsPath.join(logDir,fileName);
+
+        fs.writeFileSync(outboxFile, json);
+
+        // Updating metadata file
+        const metaFile = outboxFile + '.meta';
+
+        fs.writeFileSync(metaFile, JSON.stringify({
+            'Content-Type': 'application/ld+json',
+            'Last-Modified': nowISO()
+        },null,4));
+
+        await updateEventLog({path,options});
+
+        return { path, options, success: true };
+    }
+    catch(e) {
+        logger.error(`failed to process ${path}`);
+        logger.error(e);
+        return { path, options, success: false };
+    }
+}
+
+async function updateEventLog({path,options}) {
+    logger.info(`updating eventlog for ${path}`);
+
+    try {
+        const notification = fs.readFileSync(path, { encoding: 'utf-8'});
+        const notification_checksum = md5(notification);
+
+        const baseUrl = process.env.LDN_SERVER_BASEURL ?? "";
+        const fileName = path.split('/').pop();
+        const entry = `${baseUrl}/${EVENT_DIR}/log/${fileName}`;
+
+        const eventLog = fsPath.join(options['public'],EVENT_DIR,EVENT_LOG);
+
+        let json;
+        
+        if (fs.existsSync(eventLog)) {
+            json = JSON.parse(fs.readFileSync(eventLog, { encoding: 'utf-8'}));
+        }
+        else {
+            json = {
+                "@context": "https://labs.eventnotifications.net/contexts/eventlog.jsonld",
+                "type": "EventLog",
+                "member": []
+            };
+        }
+     
+        if (json['member'].findIndex( (e) => e === entry) >= 0) {
+            logger.info(`${entry} already in ${eventLog}`);
+        }
+        else {
+            logger.info(`updating ${eventLog}`);
+
+            json['member'].push({
+                "id": entry ,
+                "checksum": {
+                    "type": "Checksum",
+                    "algorithm": "spdx:checksumAlgorithm_md5",
+                    "checksumValue": notification_checksum
+                }
+            });
+
+            if (fs.existsSync(eventLog)) {
+                try {
+                    const lock = await lockfile(eventLog, { retries: 10 });
+                    fs.writeFileSync(eventLog,JSON.stringify(json,null,4));
+                    lock(); // release
+                }
+                catch (e) {
+                    logger.error(`failed to update ${eventLog}`);
+                }
+            }
+            else {
+                fs.writeFileSync(eventLog,JSON.stringify(json,null,4));
+            }
+        }
+
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+}
+
+function nowISO() {
+    return (new Date()).toUTCString();
+}
+
+module.exports = { handle };

package/handler/multi_notification_handler.js

@@ -0,0 +1,57 @@
+const { dynamic_handler , parseAsJSON } = require('../lib/util');
+const logger = require('../lib/util.js').getLogger();
+
+async function handle({path,options}) {
+    let success = false;
+
+    const config = parseAsJSON(options['config']);
+
+    if (! config) {
+        logger.error('no configuration found for multi_notification_handler');
+        return { path, options, success: false };
+    }
+
+    const handlers = config['notification_handler']?.['multi']?.['handlers'];
+
+    if (! handlers) {
+        logger.error('no notification_handler.multi.handlers key in configuration file');
+        return { path, options, success: false };
+    }
+
+    try {
+        logger.info(`starting multi handler`);
+        
+        for (let i = 0 ; i < handlers.length ; i++) {
+            logger.info(`starting ${handlers[i]}`);
+
+            const handler = dynamic_handler(handlers[i],null);
+
+            if (! handler) {
+                throw new Error(`failed to load ${handlers[i]}`);
+            }
+
+            const result = await handler({path,options});
+           
+            if (result['success']) {
+                logger.info(`finished ${handlers[i]}`);
+            }
+            else {
+                throw new Error(`Eek! ${handlers[i]} failed`);
+            }
+        }
+
+        success = true;
+    }
+    catch (e) {
+        logger.error(`failed to process ${path}`);
+        logger.error(e);
+
+        success = false;
+    }
+
+    logger.info(`finished multi handler`);
+
+    return { path, options, success: success };
+}   
+
+module.exports = { handle };

package/lib/handler.js

@@ -17,6 +17,7 @@ async function defaultInboxHandler({path,options}) {
 
     const worker = options['notification_handler'] ?? fsPath.resolve(__dirname,'..','lib','notification.js');
 
+    // Run the notifications using a node.js worker pool
     const pool = new piscina({
         filename: worker,
         maxQueue: queue_size

package/lib/index.js

@@ -14,11 +14,15 @@ const logger = getLogger();
 
 let INBOX_URL = 'inbox/';
 let INBOX_PATH = './inbox';
+let INBOX_BASE_URL = 'http://localhost:8000';
+let INBOX_PUBLIC_READABLE = 0;
 let JSON_SCHEMA = '';
 
 function inbox_server(options) {
     INBOX_URL = options['url'];
     INBOX_PATH = options['inbox'];
+    INBOX_BASE_URL = options['base'];
+    INBOX_PUBLIC_READABLE = options['inboxPublic'];
     JSON_SCHEMA = JSON.parse(fs.readFileSync(options['schema'], { encoding: 'utf-8'}));
     let registry = [{ path : `${INBOX_URL}.*` , do: doInbox }];
 
@@ -38,16 +42,117 @@ function inbox_server(options) {
     start_server({
         host: options['host'],
         port: options['port'],
+        base: options['base'],
         public: options['public'],
         registry: registry
     });
 }
 
 function doInbox(req,res) {
-    if (req.method !== 'POST') {
+    if (req.method === 'GET' && INBOX_PUBLIC_READABLE == 1) {
+        doInboxGET(req,res);
+    }
+    else if (req.method == 'HEAD' && INBOX_PUBLIC_READABLE == 1) {
+        doInboxHEAD(req,res);
+    }
+    else if (req.method === 'POST') {
+        doInboxPOST(req,res);
+    }
+    else { 
         logger.error(`tried method ${req.method} on inbox : forbidden`);
         res.writeHead(403);
         res.end('Forbidden');
+    }
+}
+
+function doInboxGET(req,res) {
+    const pathItem = req.url.substring(INBOX_URL.length);
+
+    logger.debug(`doInboxGET (for ${pathItem})`);
+
+    if (pathItem === '/')  {
+        doInboxGET_Index(req,res);
+    }
+    else if (pathItem.match(/^\/[a-z0-9]+\.jsonld$/)) {
+        doInboxGET_Read(req,res);
+    }
+    else {
+        res.writeHead(403);
+        res.end('Forbidden');
+    }
+}
+
+function doInboxHEAD(req,res) {
+    const pathItem = req.url.substring(INBOX_URL.length);
+
+    logger.debug(`doInboxHEAD (for ${pathItem})`);
+
+    if (pathItem === '/')  {
+        res.setHeader('Content-Type','application/ld+json');
+        res.writeHead(200);
+        res.end();
+        return;
+    }
+
+    if (pathItem.match(/^\/[a-z0-9]+\.jsonld$/)) {
+        const id = pathItem.substring(1);
+        const result = getBody(id);
+
+        if (result) {
+            res.setHeader('Content-Type','application/ld+json');
+            res.writeHead(200);
+            res.end(); 
+            return;
+        }
+        else {
+            res.writeHead(403);
+            res.end('Forbidden');
+        }
+    }
+    else {
+        res.writeHead(403);
+        res.end('Forbidden');
+        return;
+    }
+}
+
+function doInboxGET_Index(req,res) {
+    const notifications = listInbox().map( (e) => {
+        return INBOX_BASE_URL + '/' + INBOX_URL + e;
+    });
+
+    const result = {
+        "@context": "http://www.w3.org/ns/ldp",
+        "@id": INBOX_BASE_URL + '/' + INBOX_URL,
+        "contains": notifications
+    };
+
+    res.writeHead(200);
+    res.end(JSON.stringify(result,null,2));
+}
+
+function doInboxGET_Read(req,res) {
+    const pathItem = req.url.substring(INBOX_URL.length);
+
+    const result = getBody(pathItem.substring(1));
+
+    if (result) {
+        res.setHeader('Content-Type','application/ld+json');
+        res.writeHead(200);
+        res.end(result);
+    }
+    else {
+        res.writeHead(403);
+        res.end('Forbidden'); 
+    }
+}
+
+function doInboxPOST(req,res) {
+    const pathItem = req.url.substring(INBOX_URL.length);
+
+    if (pathItem !== '/') {
+        req.writeHead(403);
+        res.end('Forbidden');
         return;
     }
 
@@ -88,6 +193,35 @@ function doInbox(req,res) {
     });
 }
 
+function listInbox() {
+    const glob = new RegExp("^.*\\.jsonld$");
+
+    logger.debug(`listInbox()`);
+
+    try {
+        const entries = fs.readdirSync(INBOX_PATH).filter( (file) => {
+            return file.match(glob);
+        });
+        return entries;
+    }
+    catch(e) {
+        logger.error(e);
+        return [];
+    }
+}
+
+function getBody(id) {
+    logger.debug(`getBody(${id})`);
+
+    try {
+        return fs.readFileSync(INBOX_PATH + '/' + id, {encoding : 'utf-8'});
+    }
+    catch(e) {
+        logger.error(e);
+        return null;
+    }
+}
+
 function storeBody(data) {
     try {
         const id = md5(data);

package/lib/util.js

@@ -125,11 +125,22 @@ function dynamic_handler(handler,fallback) {
     }
 }
 
+function parseAsJSON(path) {
+    try {
+        return JSON.parse(fs.readFileSync(path, { encoding: 'utf-8'}));
+    }
+    catch (e) {
+        logger.error(`failed to parse ${path}`);
+        return null;
+    }
+}
+
 module.exports = { 
     getLogger , 
     backOff_fetch , 
     fetchOriginal ,
     moveTo ,
     sendNotification ,
-    dynamic_handler
+    dynamic_handler ,
+    parseAsJSON 
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ldn-inbox-server",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "A demonstration Event Notifications Inbox server",
   "main": "lib/index.js",
   "author": "Patrick Hochstenbach <Patrick.Hochstenbach@UGent.be>",
@@ -9,8 +9,11 @@
     "server": "npx ldn-inbox-server start-server",
     "demo-post": "curl -X POST -H 'Content-Type: application/ld+json' --data-binary '@examples/offer.jsonld' http://localhost:8000/inbox/",
     "handle-inbox": "npx ldn-inbox-server handler @inbox -hn ./handler/demo_notification_handler.js",
-    "handle-outbox": "npx ldn-inbox-server handler @outbox",
-    "clean": "rm error/* inbox/* outbox/*"
+    "handle-outbox": "npx ldn-inbox-server handler @outbox -hn ./handler/send_notification_handler.js",
+    "handle-eventlog": "npx ldn-inbox-server handler @inbox -hn handler/eventlog_notification_handler.js",
+    "handle-inbox-multi": "npx ldn-inbox-server handler @inbox -hn ./handler/multi_notification_handler.js",
+    "handle-outbox-multi": "npx ldn-inbox-server handler @outbox -hn ./handler/multi_notification_handler.js",
+    "clean": "rm error/* inbox/* outbox/* public/events/* public/events/log/*"
   },
   "bin": "./bin/ldn-inbox-server.js",
   "keywords": [
@@ -24,7 +27,7 @@
     "exponential-backoff": "^3.1.1",
     "jsonschema": "^1.4.1",
     "md5": "^2.3.0",
-    "mellon-server": "^1.0.4",
+    "mellon-server": "^1.0.8",
     "node-fetch": "1.7.3",
     "piscina": "^4.4.0",
     "proper-lockfile": "^4.1.2",