ldn-inbox-server 1.5.5 → 1.6.1

package/README.md

@@ -176,7 +176,17 @@ The json path matches when one of:
 
 ### Multi handler
 
-A `handler/multi_notification_handler.js` is available to start multiple handler for each notification messages. The handlers to start are specified in a configuration file that can be passed via the `config` parameter of an `handle_inbox`. In the commmand line tool `bin/ldn-inbox-server` the default location of such config file is `config/inbox_config.json` when processing an `@inbox`, and `config/outbox_config.json` when processing an `@outbox`.
+A `handler/multi_notification_handler.js` is available to start multiple handler for each notification messages. The handlers to start are specified in a configuration file that can be passed via the `config` parameter of an `handle_inbox`. For the commmand line tool `bin/ldn-inbox-server` the default location of such config file is `config/inbox_config.json` when processing an `@inbox`, and `config/outbox_config.json` when processing an `@outbox`.
+
+The multi handler requires a configuration file with properties `notification_handler.multi.handlers` which is an array of array. Each outer array defines a list of handlers that should be executed sequentially on a notification until one handler returns a `false` response or the last handler returns a `true` response.
+
+Each handler is defined by a hash containing as `id` property the path to the handler and optionally other property keys that will be passed to the handlers in the `config` section.
+
+Optionally when a `fallback` handler is defined as option it will be attempted when a handler in a sequence returns a `false` response.
+
+### On Error handler
+
+A handler that sets the `fallback` for a workflow sequence.
 
 ### Offer memento handler
 

package/bin/ldn-inbox-server.js

@@ -89,12 +89,13 @@ program
   .option('--outbox <outbox>','outbox',OUTBOX_PATH)
   .option('--public <public>','public',PUBLIC_PATH)
   .option('--error <errbox>','errbox',ERROR_PATH)
-  .option('--loop <seconds>', 'run in a loop',0)
+  .option('--loop', 'run in a loop')
   .option('--batch_size <num>','batch size to process',INBOX_BATCH_SIZE)
   .option('--glob <glob>','files to process in inbox',INBOX_GLOB)
   .option('--config <path>','config file for handlers')
   .option('-hi,--inbox_handler <handler>','inbox handler')
   .option('-hn,--notification_handler <handler>','notification handler')
+  .option('-s,--single','handle this one specific notification')
   .argument('<box>','box to process')
   .action( async(box,options) => {
     switch (box) {

package/config/inbox_config.json

@@ -40,6 +40,10 @@
                  }
                 ] ,
                 [
+                  {
+                    "id": "@handler/notification_handler/on_error.js",
+                    "handler": "@handler/notification_handler/nothing.js"
+                  },
                   { 
                     "id": "@handler/notification_handler/jsonpath_filter.js",
                     "anyOf": [ 

package/handler/notification_handler/accept.js

@@ -9,8 +9,6 @@ const logger = require('../../lib/util.js').getLogger();
  * it in the outbox container.
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-    
     try {
         const json = parseAsJSON(path);
         

package/handler/notification_handler/add_timemap_link.js

@@ -7,8 +7,6 @@ const logger = require('../../lib/util.js').getLogger();
  * Demonstration event log handler
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-
     const eventlog = options['eventlog'];
 
     if (!eventlog) {

package/handler/notification_handler/eventlog.js

@@ -7,8 +7,6 @@ const logger = require('../../lib/util.js').getLogger();
  * Demonstration event log handler
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-
     if (! config) {
         logger.error('no configuration found for eventlog_notification_handler');
         return { path, options, success: false };

package/handler/notification_handler/jsonpath_filter.js

@@ -7,8 +7,6 @@ const logger = require('../../lib/util.js').getLogger();
  * matches a configurable list
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-    
     if (! config) {
         logger.error('no configuration found for eventlog_notification_handler');
         return { path, options, success: false };

package/handler/notification_handler/multi.js

@@ -65,6 +65,29 @@ async function handle({path,options}) {
                 else {
                     logger.error(`workflow[${i}] : failed ${step}`);
                     thisWorkflow = false;
+
+                    if (options['fallback']) {
+                        logger.debug(`loading fallback ${options['fallback']}`);
+                        const fallback = dynamic_handler(options['fallback'],null);
+
+                        if (!fallback) {
+                            throw new Error(`failed to load ${options['fallback']}`);
+                        }
+
+                        logger.info(`workflow[${i}] : starting ${options['fallback']}`);
+                        const fallback_result = await fallback({path,options,config,notification});
+
+                        if (fallback_result['success']) {
+                            logger.info(`workflow[${i}] : finished ${options['fallback']}`);
+                        }
+                        else {
+                            logger.error(`workflow[${i}] : failed ${options['fallback']}`);
+                        }
+                    }
+                    else {
+                        logger.debug(`no fallBack defined`);                       
+                    }
+
                     break;
                 }
             }
@@ -81,6 +104,9 @@ async function handle({path,options}) {
         else {
             workflow_errors++;
         }
+
+        // Each workflow needs to install its own fallBack
+        delete options['fallback'];
     }
 
     if (handlers.length > 0 && workflow_success > 0) {

package/handler/notification_handler/nothing.js

@@ -0,0 +1,17 @@
+const logger = require('../../lib/util.js').getLogger();
+
+/**
+ * Handler that does nothing (debug purposes)
+ */
+async function handle({path,options,config}) {
+    try {
+        return { path, options, success: true };
+    }
+    catch(e) {
+        logger.error(`failed to process ${path}`);
+        logger.error(e);
+        return { path, options, success: false };
+    }
+}
+
+module.exports = { handle };

package/handler/notification_handler/offer_memento.js

@@ -8,8 +8,6 @@ const logger = require('../../lib/util.js').getLogger();
  * archivation of the event log.
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-
     if (! config) {
         logger.error('no configuration found for offer_memento');
         return { path, options, success: false };

package/handler/notification_handler/on_error.js

@@ -0,0 +1,28 @@
+const logger = require('../../lib/util.js').getLogger();
+
+/**
+ * Handler that sets the 'fallback' handler option
+ */
+async function handle({path,options,config}) {
+    try {
+        const fallback = config['handler'];
+
+        if (!fallback) {
+            logger.error('no handler configuration found');
+            return { path, options, success: true };
+        }
+
+        logger.info(`setting fallback to ${fallback}`);
+
+        options['fallback'] = fallback;
+
+        return { path, options, success: true };
+    }
+    catch(e) {
+        logger.error(`failed to process ${path}`);
+        logger.error(e);
+        return { path, options, success: false };
+    }
+}
+
+module.exports = { handle };

package/handler/notification_handler/type_filter.js

@@ -6,8 +6,6 @@ const logger = require('../../lib/util.js').getLogger();
  * matches a configurable list
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-    
     if (! config) {
         logger.error('no configuration found for eventlog_notification_handler');
         return { path, options, success: false };

package/handler/notification_handler/valid_artifact.js

@@ -6,8 +6,6 @@ const logger = require('../../lib/util.js').getLogger();
  * message contains an artifact that is known to the data node
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-    
     try {
         const json = parseAsJSON(path);
         

package/handler/notification_handler/valid_eventlog.js

@@ -6,8 +6,6 @@ const logger = require('../../lib/util.js').getLogger();
  * message contains an artifact that is known to the data node
  */
 async function handle({path,options,config}) {
-    logger.info(`parsing notification ${path}`);
-    
     try {
         const json = parseAsJSON(path);
         

package/lib/handler.js

@@ -3,11 +3,42 @@ const fsPath = require('path');
 const lockfile = require('proper-lockfile');
 const { dynamic_handler, moveTo } = require('../lib/util');
 const piscina = require('piscina');
+const chokidar = require('chokidar');
 const logger = require('../lib/util.js').getLogger();
 
 async function handle_inbox(path,options) {
-    const handler = dynamic_handler(options['inbox_handler'],defaultInboxHandler);
-    return await handler({path,options});
+    if (options.single) {
+        logger.debug(`*single file execution*`);
+        handle_single(path,options);
+    }
+    else {
+        logger.debug(`*directory execution*`);
+        const handler = dynamic_handler(options['inbox_handler'],defaultInboxHandler);
+        return await handler({path,options});
+    }
+}
+
+async function handle_single(path,options) {
+    const handler = dynamic_handler(
+                    options['notification_handler'],
+                    fsPath.resolve(__dirname,'..','lib','demoNotificationHandler.js')
+                    );
+    const result = await handler({path,options});    
+    const success = result['success'];
+    const notification = result['path'];
+
+    if (success) {
+        logger.info(`processing ${notification} is a success`);
+        if (fs.existsSync(notification)) {
+            logger.debug(`removing ${notification}`);
+            fs.unlinkSync(result['path']);
+        }
+    }
+    else {
+        logger.warn(`processing ${notification} is a failure`);
+        logger.debug(`moving ${notification} to ${options['error']}`);
+        moveTo(notification,options['error']);
+    }
 }
 
 async function defaultInboxHandler({path,options}) {
@@ -31,45 +62,61 @@ async function defaultInboxHandler({path,options}) {
         maxQueue: queue_size
     });
 
-    const loopWait = options['loop'] || 0;
+    await poolRun(pool,path,options);
+    
+    if (! options.loop) return;
 
-    do {
-        try {
-            const [prms,locks] = await inboxProcessor(pool,path,options);
-            const results = await Promise.all(prms);
+    // Loop
 
-            for (let i = 0 ; i < results.length ; i++) {
-                const result = results[i];
-                const success = result['success'];
-                const notification = result['path'];
+    const watcher = chokidar.watch(path, {
+        ignored: /(^|[\/\\])\../, // ignore dotfiles
+        persistent: true
+    });
 
-                if (success) {
-                    logger.info(`processing ${notification} is a success`);
-                    if (fs.existsSync(notification)) {
-                        logger.debug(`removing ${notification}`);
-                        fs.unlinkSync(result['path']);
-                    }
-                }
-                else {
-                    logger.warn(`processing ${notification} is a failure`);
-                    logger.debug(`moving ${notification} to ${options['error']}`);
-                    moveTo(notification,options['error']);
-                }
+    watcher.on('ready', () => {
+        logger.info(`start watching ${path}...`);
+    });
+
+    watcher.on('error', error => {
+        logger.error(error);
+    });
+
+    watcher.on('add', async notification => {
+        logger.info(`new notifiction ${notification} detected`);
+        await poolRun(pool,path,options);
+    });
+}
 
-                // release lock
-                if (locks[i]) locks[i]();
+async function poolRun(pool,path,options) {
+    try {
+        const [prms,locks] = await inboxProcessor(pool,path,options);
+        const results = await Promise.all(prms);
+
+        for (let i = 0 ; i < results.length ; i++) {
+            const result = results[i];
+            const success = result['success'];
+            const notification = result['path'];
+
+            if (success) {
+                logger.info(`processing ${notification} is a success`);
+                if (fs.existsSync(notification)) {
+                    logger.debug(`removing ${notification}`);
+                    fs.unlinkSync(result['path']);
+                }
+            }
+            else {
+                logger.warn(`processing ${notification} is a failure`);
+                logger.debug(`moving ${notification} to ${options['error']}`);
+                moveTo(notification,options['error']);
             }
-        }
-        catch (e) {
-            logger.error(e);
-        }
 
-        if (loopWait) {
-            logger.debug(`sleep ${loopWait} secs...`);
-            await new Promise(resolve => setTimeout(resolve, loopWait*1000));
+            // release lock
+            if (locks[i]) locks[i]();
         }
     }
-    while (loopWait > 0);
+    catch (e) {
+        logger.error(e);
+    }
 }
 
 async function inboxProcessor(pool,path,options) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ldn-inbox-server",
-  "version": "1.5.5",
+  "version": "1.6.1",
   "description": "A demonstration Event Notifications Inbox server",
   "main": "lib/index.js",
   "author": "Patrick Hochstenbach <Patrick.Hochstenbach@UGent.be>",
@@ -22,6 +22,7 @@
   ],
   "license": "MIT",
   "dependencies": {
+    "chokidar": "^3.6.0",
     "commander": "^12.0.0",
     "dotenv": "^16.4.5",
     "exponential-backoff": "^3.1.1",