j1-template 2021.0.13 → 2021.1.1

data/lib/starter_web/pages/public/previewer/mdi_icons_preview.adoc

@@ -1,7 +1,7 @@
 ---
 title:                                  MDI Icon Previewer
 tagline:                                search and select mdi icons
-date:                                   2020-11-08 00:00:00
+date:                                   2020-11-08 00:00:00 +100
 description: >
                                         Previewer for Material Design Icons (MDI) for J1 Template.
                                         Material Design Icons is a growing icon collection allows
@@ -89,21 +89,19 @@ include equivalent text either inside the `<i>` tag or right next to it
 click to reply, the link is usable if CSS doesn't load, and human readers
 with good eyesight and a decent browser see a pretty icon.
 
-/////
 == Asciidoc Markup
 
+For pages using Asciidoc for the source (default), a easier solution is to
+use  Markups from Asciidoc.
+
 [source, html]
 ----
-pass:[<i class="mdi mdi-3x mdi-star"></i>] <1>
-mdi:star[3x] <2>
+pass:[<i class="mdi mdi-<name> mdi-<size> mdi-<modifier>"></i>] <1>
+mdi:<name>[<size>, <modifier>] <2>
 ----
 <1> Markup using a pass block
-<2> Markup using the Asciidoc extension
+<2> Markup using the Asciidoc extension (mdi:)
 
-NOTE: The `<i>` tag provides more meaning to machines because of the HTML spec,
-it provides more meaning to *humans* because the `i` can easily associated with
-`icon`.
-/////
 
 == Sizes
 

data/lib/starter_web/pages/public/previewer/rouge.adoc

@@ -1,7 +1,7 @@
 ---
 title:                                  Rouge
 tagline:                                preview rouge themes
-date:                                   2020-11-08 00:00:00
+date:                                   2020-11-08 00:00:00 +100
 description: >
                                         Previewer page for J1 module Rouge. Rouge is a pure Ruby
                                         syntax highlighter. It can highlight over 100 different

data/lib/starter_web/pages/public/previewer/twitter_emoji_preview.adoc

@@ -1,7 +1,7 @@
 ---
 title:                                  Emoji Previewer
 tagline:                                search and select twitter emojis
-date:                                   2020-11-08 00:00:00
+date:                                   2020-11-08 00:00:00 +100
 description: >
                                         Twitter Emoji Icons V1 Previewer.Twitter Emoji is a colored
                                         icon font developed by Twitter. The full icon set  is fully

data/lib/starter_web/utilsrv/_defaults/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "utls",
-  "version": "2021.0.13",
+  "version": "2021.1.1",
   "description": "J1 Template Utility Server",
   "homepage": "https://jekyll.one",
   "author": {
@@ -19,15 +19,17 @@
     "yarn": ">=1.6"
   },
   "utilsrv": {
-    "enabled": "true",
+    "enabled": "true"
   },
   "scripts": {
     "clean": "shx rm -f *.lock && shx rm -f package-lock.json",
-    "start": "run-s utilsrv-run",
-    "develop": "run-s utilsrv",
-    "utilsrv": "cross-env UTILSRV_ENABLED=$npm_package_utilsrv_enabled if-env UTILSRV_ENABLED=true && run-s -s utilsrv-run || run-s -s utilsrv-exit",
-    "utilsrv-run": "echo Startup utility server .. && node server.js",
-    "utilsrv-exit": "echo Startup utility server globally disabled, not started."
+    "start": "run-p -s utilsrv-run",
+    "app": "run-p -s utilsrv",
+    "develop": "run-p -s utilsrv",
+    "site": "run-p -s utilsrv",
+    "utilsrv": "cross-env UTILSRV_ENABLED=$npm_package_utilsrv_enabled if-env UTILSRV_ENABLED=true && run-p -s utilsrv-run || run-p -s utilsrv-exit",
+    "utilsrv-run": "echo Startup UTILSRV .. && node server.js",
+    "utilsrv-exit": "echo UTILSRV disabled. Not started."
   },
   "dependencies": {
     "body-parser": "^1.19.0",
@@ -38,12 +40,14 @@
     "express": "^4.15.2",
     "fs": "0.0.1-security",
     "if-env": "^1.0.4",
-    "js-yaml": "^3.13.1",
+    "js-yaml": "^4.0",
     "lite-url": "^1.0.5",
+    "log4js": "^6.3.0",
     "moment": "^2.24.0",
+    "node-cron": "^3.0.0",
     "npm-run-all": "^4.1.5",
     "randomstring": "^1.1.5",
-    "shx": "^0.2.2",
+    "shx": "^0.3.3",
     "simple-git": "^1.113.0",
     "simple-oauth2": "^1.6.0",
     "sprintf-js": "^1.1.2",

data/lib/starter_web/utilsrv/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "utls",
-  "version": "2021.0.13",
+  "version": "2021.1.1",
   "description": "J1 Template Utility Server",
   "homepage": "https://jekyll.one",
   "author": {
@@ -19,12 +19,14 @@
     "yarn": ">=1.6"
   },
   "utilsrv": {
-    "enabled": "false"
+    "enabled": "true"
   },
   "scripts": {
     "clean": "shx rm -f *.lock && shx rm -f package-lock.json",
     "start": "run-p -s utilsrv-run",
+    "app": "run-p -s utilsrv",
     "develop": "run-p -s utilsrv",
+    "site": "run-p -s utilsrv",
     "utilsrv": "cross-env UTILSRV_ENABLED=$npm_package_utilsrv_enabled if-env UTILSRV_ENABLED=true && run-p -s utilsrv-run || run-p -s utilsrv-exit",
     "utilsrv-run": "echo Startup UTILSRV .. && node server.js",
     "utilsrv-exit": "echo UTILSRV disabled. Not started."
@@ -40,7 +42,9 @@
     "if-env": "^1.0.4",
     "js-yaml": "^4.0",
     "lite-url": "^1.0.5",
+    "log4js": "^6.3.0",
     "moment": "^2.24.0",
+    "node-cron": "^3.0.0",
     "npm-run-all": "^4.1.5",
     "randomstring": "^1.1.5",
     "shx": "^0.3.3",

data/lib/starter_web/utilsrv/server.js

@@ -17,7 +17,7 @@
  # Netlify-cms-github-oauth-provider is licensed under UNKNOWN License.
  # See: https://github.com/vencax/netlify-cms-github-oauth-provider/blob/master/README.md
  # -----------------------------------------------------------------------------
- # NOTE:  
+ # NOTE:
  #  To fix Webstorm NodeJS API issue see:
  #  https://stackoverflow.com/questions/19532660/webstorm-7-cannot-recognize-node-api-methods
  # -----------------------------------------------------------------------------
@@ -61,6 +61,9 @@ const exec              = util.promisify(require('child_process').exec);
 const moment            = require('moment');
 const sprintf           = require('sprintf-js').sprintf;
 const vsprintf          = require('sprintf-js').vsprintf;
+const cron              = require('node-cron');
+const log4js            = require('log4js');
+
 
 // =============================================================================
 // base settings
@@ -68,7 +71,7 @@ const vsprintf          = require('sprintf-js').vsprintf;
 const daemon_home   = path.resolve(__dirname);
 const environment   = daemon_home.indexOf('packages') !== -1 ? 'dev' : 'prod';
 const current_date  = moment().format('YYYY-MM-DD');
-// let dotenv_home;
+
 let config_home;
 let project_home;
 let log_home;
@@ -76,6 +79,7 @@ let utilsrv_options;
 let log4javascript_options;
 let private_data;
 let logStream;
+let fsStats;
 
 // timestamp settings
 //
@@ -87,18 +91,13 @@ moment().format('YYYY-MM-DD hh:mm:ss.SSS');
 if (environment === 'dev') {
   project_home    = daemon_home + '/../400_template_site';
   config_home     = daemon_home + '/../400_template_site/_data';
-//dotenv_home     = daemon_home + '/../400_template_site';
   log_home        = daemon_home + '/../..';
 } else {
   project_home    = daemon_home + '/..';
   config_home     = daemon_home + '/../_data';
-//dotenv_home     = daemon_home + '/..';
   log_home        = daemon_home + '/..';
 }
 
-// const dotenv      = require('dotenv').config({ path: dotenv_home + '/.env', silent: true });
-// const dotenv_json = JSON.stringify( dotenv, null, 2 );                          // JSON pretty print
-
 // =============================================================================
 // load configuration data
 // -----------------------------------------------------------------------------
@@ -118,68 +117,191 @@ const utilsrv_defaults_file         = util_defaults + '/' + 'util_srv.yml';
 const utilsrv_settings_file         = util_settings + '/' + 'util_srv.yml';
 
 try {
-  const log4javascript_defaults     = yaml.safeLoad(fs.readFileSync(log4javascript_defaults_file, 'utf8'));
-  const log4javascript_settings     = yaml.safeLoad(fs.readFileSync(log4javascript_settings_file, 'utf8'));
-  const utilsrv_defaults            = yaml.safeLoad(fs.readFileSync(utilsrv_defaults_file, 'utf8'));
-  const utilsrv_settings            = yaml.safeLoad(fs.readFileSync(utilsrv_settings_file, 'utf8'));
-  const private_data_settings       = yaml.safeLoad(fs.readFileSync(private_data_file, 'utf8'));
+  const log4javascript_defaults     = yaml.load(fs.readFileSync(log4javascript_defaults_file, 'utf8'));
+  const log4javascript_settings     = yaml.load(fs.readFileSync(log4javascript_settings_file, 'utf8'));
+  const utilsrv_defaults            = yaml.load(fs.readFileSync(utilsrv_defaults_file, 'utf8'));
+  const utilsrv_settings            = yaml.load(fs.readFileSync(utilsrv_settings_file, 'utf8'));
+  const private_data_settings       = yaml.load(fs.readFileSync(private_data_file, 'utf8'));
 
+  // noinspection JSUnresolvedVariable
   private_data                      = private_data_settings.util_srv;
-  log4javascript_options            = mergeData(log4javascript_defaults.defaults, log4javascript_settings_file.settings);
   utilsrv_options                   = mergeData(utilsrv_defaults.defaults, utilsrv_settings.settings);
-
+  log4javascript_options            = mergeData(log4javascript_defaults.defaults, utilsrv_settings.settings);
 } catch (e) {
   console.log(e);
 }
 const ajaxAppenderOptions           = log4javascript_options.appenders[1].appender;
 
 // -----------------------------------------------------------------------------
-// utility server (daemon) settings
+// utility server settings
 //
-const ssl             = utilsrv_options.ssl || false;
-const port            = utilsrv_options.port || 44444;
-const origin          = utilsrv_options.origin || 'localhost';
-const hostName        = utilsrv_options.host_name || '0.0.0.0';
-const verbose         = utilsrv_options.verbose || false;
-const logFileName     = ajaxAppenderOptions.log_file_name  + '_' + current_date || 'messages' + '_' + current_date;
-const logFileExt      = ajaxAppenderOptions.log_file_ext || 'log';
-const logFolder       = ajaxAppenderOptions.log_folder || 'log';
-const logFileNamePath = log_home + '/' + logFolder + '/' + logFileName + '.' +  logFileExt;
-const util_srv_url    = ssl ? 'https://' +  origin + ':' +  port : 'http://' +  origin + ':' +  port;
+const enabled             = utilsrv_options.enabled || false;
+const ssl                 = utilsrv_options.ssl || false;
+const port                = utilsrv_options.port || 40020;
+const origin              = utilsrv_options.origin || 'localhost';
+const hostName            = utilsrv_options.host_name || '0.0.0.0';
+const verbose             = utilsrv_options.verbose || false;
+const logFileName         = ajaxAppenderOptions.log_file_name  + '_' + current_date || 'messages' + '_' + current_date;
+const logFileExt          = ajaxAppenderOptions.log_file_ext || 'log';
+const logFolder           = ajaxAppenderOptions.log_folder || 'log';
+const logFileNamePath     = log_home + '/' + logFolder + '/' + logFileName + '.' +  logFileExt;
+const current_logFile     = log_home + '/' + logFolder + '/' + 'messages.current';
+const util_srv_url        = ssl ? 'https://' +  origin + ':' +  port : 'http://' +  origin + ':' +  port;
+const thread_id           = generateId (11);
+const page                = '/util_srv';
+const isWin               = process.platform === "win32";
 
 // -----------------------------------------------------------------------------
-// print utility server issue
+// logger settings
+// See: https://github.com/log4js-node/log4js-node/blob/master/docs/layouts.md
 //
-if (environment === 'dev') {
-  console.log('Environment detected as: ' + environment);
-  console.log('Daemon path set to:      ' + daemon_home);
-  console.log('Daemon verbosity set to: ' + verbose);
-  console.log('Project path set to:     ' + project_home);
-  console.log('Data path set to:        ' + config_home);
-  console.log('Log file set to:         ' + logFileNamePath);
-}
+log4js.configure({
+  appenders: {
+    stdout: {
+      type: 'stdout',
+      layout: {
+        type: 'pattern',
+        pattern: '[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] [%-40c] %m%n'
+      }
+    },
+    file: {
+      type: 'file',
+      filename: logFileNamePath,
+      layout: {
+        type: 'pattern',
+        pattern: '[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%-11x{thread}] [%-5p] [%-60x{page}] [%-40c] %m',
+        tokens: {
+          thread: thread_id,
+          page: page
+        }
+      }
+    }
+  },
+  categories: {
+    default: {
+      appenders: ['stdout'],
+      level: 'info'
+    },
+    'j1.util_srv': {
+      appenders: ['file'],
+      level: 'info'
+    },
+    'j1.util_srv.preflight': {
+      appenders: ['file'],
+      level: 'info'
+    },
+    'j1.util_srv.task': {
+      appenders: ['file'],
+      level: 'info'
+    }
+  }
+});
 
 // -----------------------------------------------------------------------------
-// initialize log client
-// fs.writeFile(logFileNamePath, '',  function(){console.log('Reset file: ' +logFileNamePath)});
+// create loggers
 //
+let stdout    = log4js.getLogger('stdout');
+let preflight = log4js.getLogger('j1.util_srv.preflight');
+let logger    = log4js.getLogger('j1.util_srv.core');
 
-if ( ajaxAppenderOptions.create_on_start === 'true') {
-  touch(logFileNamePath);
-} else {
+// -----------------------------------------------------------------------------
+// scheduler task settings
+// See: https://github.com/node-cron/node-cron
+//
+let test_per_minute = cron.schedule('* * * * *', () =>  {
+  let timestamp = moment().format('YYYY-MM-DD HH:mm');
+  console.log(timestamp + ': scheduled test task running every minute');
+}, {
+  scheduled: false
+});
+
+let rolling_logs = cron.schedule('* * * * *', () =>  {
+  let logger = log4js.getLogger( 'j1.util_srv.task');
+//  logger_stdout.info('rolling log task running every minute');
+  logger.info('rolling log task running every minute');
+  let timestamp = moment().format('YYYY-MM-DD HH:mm');
+  console.log(timestamp + ': rolling log task running every minute');
+}, {
+  scheduled: false
+});
+
+// -----------------------------------------------------------------------------
+// initialize the logfile
+
+// check if the logfile exists
+try {
+  fsStats = fs.statSync(logFileNamePath);
+  console.log('Log file exists :        ' + logFileName);
+  preflight.info('log file exists: ' + logFileName)
+
+  if ( ajaxAppenderOptions.reset_on_start === true) {
+    fs.truncate(logFileNamePath, 0, function(){console.log('Reset file: ' + logFileName)});
+  }
+}
+catch (e) {
+  console.log('Create Log file :' + logFileName);
+  preflight.info('create log file  :' + logFileName);
+  // create empty logfile
   touch(logFileNamePath);
 }
 
-if ( ajaxAppenderOptions.reset_on_start === 'true') {
-  fs.truncate(logFileNamePath, 0, function(){console.log('Reset file: ' + logFileNamePath)});
+// symlinks on Windows are only supported by elevated user rights
+//
+if (isWin === false) {
+  // (Re-)Create symlink to current logfile
+  //
+  fs.unlink(current_logFile, (err => {
+    if (err) {
+      fs.symlink (
+          logFileNamePath,
+          current_logFile,
+          function (err) { console.log(err || 'Symlink to current log created.'); }
+      );
+    } else {
+      // See: https://stackoverflow.com/questions/29777506/create-relative-symlinks-using-absolute-paths-in-node-js
+      //
+      fs.symlink (
+          logFileNamePath,
+          current_logFile,
+          function (err) { console.log(err || 'Symlink to current log re-created.'); }
+      );
+    }
+  }));
 }
 
-if ( ajaxAppenderOptions.mode === 'append') {
+
+// check if logs should be appended
+//
+preflight.info('appender options, mode: ' + ajaxAppenderOptions.mode);
+if (ajaxAppenderOptions.mode === 'append') {
   logStream = fs.createWriteStream(logFileNamePath, {'flags': 'a'});
 } else {
+  fs.truncate(logFileNamePath, 0, function(){console.log('Reset file: ' + logFileName)});
   logStream = fs.createWriteStream(logFileNamePath, {'flags': 'a'});
 }
 
+// check if logs should be rolled (e.g. daily)
+//
+preflight.info('appender options, rolling files: ' + ajaxAppenderOptions.rolling_files);
+if (ajaxAppenderOptions.rolling_files === true) {
+  // start the scheduled task for rolling (log) files
+  //
+  rolling_logs.start();
+}
+
+// -----------------------------------------------------------------------------
+// print utility server issue
+//
+if (environment === 'dev') {
+  console.log('Server enabled:          ' + enabled);
+  console.log('Environment detected as: ' + environment);
+  console.log('Daemon path set to:      ' + daemon_home);
+  console.log('Daemon verbosity set to: ' + verbose);
+  console.log('Project path set to:     ' + project_home);
+  console.log('Data path set to:        ' + config_home);
+  console.log('Log file set to:         ' + logFileNamePath);
+}
+
 // -----------------------------------------------------------------------------
 // Github OAuth client settings (used for CC)
 //
@@ -193,11 +315,10 @@ const oauthProviderClientScope    = private_data.oauth.client_scope;
 const oauthProviderClientId       = private_data.oauth.client_id;
 const oauthProviderClientSecret   = private_data.oauth.client_secret;
 
-
 // -----------------------------------------------------------------------------
 // cors settings
 //
-var corsSettings = {
+let corsSettings = {
   origin:               '*',
   optionsSuccessStatus: 200                                                     // Some legacy browsers (IE11, various SmartTVs) choke on 204
 }
@@ -211,7 +332,7 @@ var corsSettings = {
 //
 
 // =============================================================================
-// initialize libraries
+// initialize runtime libraries
 // -----------------------------------------------------------------------------
 const app    = express();
 const oauth2 = simpleOauthModule.create({
@@ -322,6 +443,7 @@ app.get('/auth/github/callback', (req, res) => {
     }
 
     // see: http://usefulangle.com/post/4/javascript-communication-parent-child-window
+    //
     const script = `
     <script>
     (function() {
@@ -524,6 +646,10 @@ app.post('/log2disk', (req, res) => {
   // ---------------------------------------------------------------------------
   // globals
   let pageID   = req.headers['x-page-id'];
+  let tzOffset = req.headers['x-tz-offset'];
+  const tz_offset = tzOffset.replace(/GMT/g, '');
+  let tz_factor;
+  let tz_offset_milli;
   let logLine;
   let msgDate2Int;
   let timestamp;
@@ -532,6 +658,7 @@ app.post('/log2disk', (req, res) => {
 
   // ---------------------------------------------------------------------------
   // process the POST response body
+  //
   if (req.body.layout === 'XmlLayout') {
     logLine = req.body.data;
   } else if (req.body.layout === 'JsonLayout' || req.body.layout === 'PatternLayout' || req.body.layout === 'SimpleLayout' || req.body.layout === 'NullLayout') {
@@ -539,18 +666,35 @@ app.post('/log2disk', (req, res) => {
   } else if (req.body.layout === 'HttpPostDataLayout') {
     url = new parseURL(req.body.url);
     path = url.pathname;
-    msgDate2Int = parseInt(req.body.timestamp, 10);                              // ISOString: ±YYYY-MM-DDTHH:mm:ss.sssZ
+
+    msgDate2Int = parseInt(req.body.timestamp, 10);
+
+    // calculate TZ offset
+    //
+    let tz_split = tz_offset.split(':');
+    let tz_offset_hours = eval(tz_split[0]*1);
+    let tz_offset_minutes = eval(tz_split[1]*1);
+
+    tz_factor = tz_offset_hours < 0 ? -1 : 1;
+
+    tz_offset_hours = tz_factor*tz_offset_hours;
+    tz_offset_milli = tz_factor*((3600*1000*tz_offset_hours) + (tz_offset_minutes*60*1000));
+    msgDate2Int += tz_offset_milli;
+
+    // ISOString: yyyy-MM-ddThh:mm:ss.sssZ
     timestamp   = new Date(msgDate2Int).toISOString().slice(0, 23).replace('T', ' ');
+
     // [10:05:12.666] [INFO ] [j1.logger.writer                   ] [logger.js:154] [state: finished]
     // [http://localhost:41000/assets/themes/j1/adapter/js/logger.js:154]
-    logLine     = sprintf('[%s] [%s] [%-5s] [%-25s] [%-35s] %s\n', timestamp, pageID, req.body.level, path, req.body.logger, req.body.message);
+    logLine = sprintf('[%s] [%s] [%-5s] [%-60s] [%-40s] %s\n', timestamp, pageID, req.body.level, path, req.body.logger, req.body.message);
   } else {
     logLine = req.body + '\n';
   }
 
-  // if (verbose) console.log('Utility Server: endpoint /log2disk entered');
-  // if (verbose) console.log('Utility Server: processing request: ' + req.query.request);
-  if (verbose) console.log('Utility Server: write message: ' + logLine);
+//  if (verbose) console.log('Utility Server: endpoint /log2disk entered');
+//  if (verbose) console.log('Utility Server: processing request: ' + req.query.request);
+//  if (verbose) console.log('Utility Server: write message: ' + logLine);
+
   logStream.write(logLine);
   res.send('');
 
@@ -577,6 +721,20 @@ function mergeData () {
   return o;
 }
 
+// -------------------------------------------------------------------------
+// generateId()
+// Generate a unique (thread) id used by the logger
+// -------------------------------------------------------------------------
+function generateId (length) {
+  let result           = '';
+  let characters       = 'abcdefghijklmnopqrstuvwxyz0123456789';
+  let charactersLength = characters.length;
+  for ( let i = 0; i < length; i++ ) {
+    result += characters.charAt(Math.floor(Math.random() * charactersLength));
+  }
+  return result;
+} // END generateId
+
 
 // =============================================================================
 // main
@@ -594,8 +752,23 @@ process.on('uncaughtException', function(err) {
   process.exit;
 });
 
-// run the daemon (use IPV4, all interfaces)
-// see https://github.com/expressjs/express/issues/3528
-app.listen(port, hostName, () => {
-  console.log("Utility Server is listening on port: " + port);
-});
+if (utilsrv_options.enabled) {
+
+  // test_per_minute.start();
+
+  logger.info('utility server is starting');
+
+  // run the daemon (use IPV4, all interfaces)
+  // see https://github.com/expressjs/express/issues/3528
+  app.listen(port, hostName, () => {
+    console.log("Utility Server is listening on port: " + port);
+    logger.info('utility server is listening on port: ' + port);
+  });
+} else {
+  logger.info('found utility server: disabled');
+  logger.info('stop the server');
+  console.log('Stop the server. Exiting ...');
+}
+
+// END main
+// -----------------------------------------------------------------------------