@tangelo/tangelo-configuration-toolkit 1.20.2 → 1.21.0

package/index.js

@@ -72,6 +72,7 @@ global._git = {
     if (!_appdata.gitUser) _appdata._update({gitUser: execGitCommand(`config --get user.email`, _paths.repo)});
     return _appdata.gitUser;
   },
+  status: memoize(() => execGitCommand(`status -s`, _paths.repo)),
   commitLocal: memoize(() => execGitCommand(`log -1 --format=%D;%H;%cd --date=iso-strict`, _paths.repo, ['branch', 'hash', 'date'])),
   commitRemote: memoize(() => execGitCommand('log -1 --format=%cd --date=iso-strict origin/' + _git.commitLocal().branch, _paths.repo, ['date'])),
   commitTdi: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangelo/tangelo-configuration-toolkit",
-  "version": "1.20.2",
+  "version": "1.21.0",
   "engines": {
     "node": ">=14.0.0"
   },

package/src/lib/gulp-sftp.js

@@ -21,13 +21,15 @@ module.exports = function(ftpConfig, remotedir) {
       return cb();
     },
 
-    function flush(cb) {
+    async function flush(cb) {
       if (!files[0]) cb();
 
       const paths = [...new Set(files.map(({destination}) => path.dirname(destination)))] // collect unique paths
         .filter((p1, i, a) => !a.find(p2 => p1 != p2 && p2.includes(p1+'/'))) // remove paths being part of others
       ;
 
+      await ftpConfig.eventBeforeAll(files);
+
       sftp.connect(ftpConfig)
         .then(() => Promise.all(paths.map(p => parallel( // check if all directories exist
           () => sftp.exists(p).then(r => {
@@ -48,8 +50,8 @@ module.exports = function(ftpConfig, remotedir) {
             ).catch(err => _warn(`File transfer failed${err ? ': '+err : ''}`));
           }
         ))))
-        .then(() => {
-          _info(`${files.length} file(s) transferred`);
+        .then(async () => {
+          await ftpConfig.eventAfterAll(files);
           cb();
           return sftp.end();
         })

package/src/lib/remote-exec.js

@@ -0,0 +1,44 @@
+const {NodeSSH} = require('node-ssh');
+const pLimit    = require('p-limit');
+
+
+module.exports = class RemoteExec {
+
+  #ftpConfig;
+  #queue = [];
+
+  constructor(ftpConfig) {
+    this.#ftpConfig = ftpConfig;
+  }
+
+  add(cmd, msg) {
+    if (!this.#queue.find(e => e[0]==cmd)) this.#queue.push([cmd, msg]);
+    return this;
+  }
+
+  async process() {
+    if (!this.#queue[0]) return;
+
+    const sshI = new NodeSSH();
+    const limit = pLimit(this.#ftpConfig.parallel);
+
+    await sshI.connect(this.#ftpConfig) // set up connection once
+      .then(() => Promise.all(this.#queue.map(([cmd, msg]) => limit(
+        () => sshI.execCommand(cmd).then(result => {
+          if (result.stderr) _warn(result.stderr);
+          else if (msg === 'STDOUT') _write(result.stdout);
+          else if (msg) _info(msg === 'CMD' ? cmd : msg);
+        })
+      ))))
+      .catch(err => {
+        if (err.code=='ECONNRESET' || err.code=='ECONNABORTED') _warn(`Server aborted connection. Retrying.`);
+        else _warn(err);
+        this.process(); // retry set up connection
+      })
+      .finally(() => {
+        sshI.dispose();
+        this.#queue = [];
+      });
+  }
+
+};

package/src/modules/deploy/execute.js

@@ -3,7 +3,6 @@ const execGitCommand  = require('../../lib/exec-git-command');
 const fs              = require('fs-extra');
 const globby          = require('globby');
 const gulp            = require('gulp');
-const {NodeSSH}       = require('node-ssh');
 const g_tcl           = require('../../lib/gulp-tcl-config');
 const through2        = require('through2');
 const {spawnSync}     = require('child_process');
@@ -24,59 +23,6 @@ const c = require('./config');
 const s = require('./srcset');
 
 
-const remote = {
-  queue: [],
-  add(cmd, msg) {
-    if (!this.queue.find(e => e[0]==cmd)) this.queue.push([cmd, msg]);
-    return this;
-  },
-  do(sshI) {
-    let executing = Math.min(this.queue.length, c.server.ftpConfig.parallel);
-
-    if (executing===0) { // close connection and return callback after last exec command
-      sshI.dispose();
-      _write();
-    }
-
-    // execute each command from the queue in batches
-    this.queue.filter((e, i) => i < executing).forEach(()=>{
-      const command = this.queue.shift();
-
-      sshI.execCommand(command[0])
-        .then(result => {
-          if (result.stderr) _warn(result.stderr);
-          else if (command[1] === '') _write(result.stdout);
-          else _info(command[1] || command[0], true);
-
-          executing--;
-          if (executing===0) this.do(sshI);
-        })
-        .catch(() => {
-          _warn(`Server aborted connection. Retrying.`);
-          this.queue.unshift(command);
-
-          executing--;
-          if (executing===0) this.do(sshI);
-        });
-    });
-  },
-  process() {
-    if (!this.queue[0]) return;
-
-    const sshI = new NodeSSH();
-    sshI.connect(c.server.ftpConfig) // set up connection once
-      .then(() => {
-        this.do(sshI);
-      })
-      .catch(err => {
-        if (err.code=='ECONNRESET' || err.code=='ECONNABORTED') _warn(`Server aborted connection. Retrying.`);
-        else _warn(err);
-        this.process(); // retry set up connection
-      });
-  }
-};
-
-
 const createDeliveryPack = () => { // create install scripts if necessary, zip output and remove temp folder
   const dbiSql = [];
   const dbiPs1 = [];
@@ -120,7 +66,7 @@ const createDeliveryPack = () => { // create install scripts if necessary, zip o
 };
 
 
-const transfer = (paths, {watch, lrServer} = {}) => {
+module.exports = function transfer (paths, {watch, lrServer} = {}) {
   const jsFGlob = _settingsTdi().transpileToES5 ? ['**/*.js', '!**/hce/**/template_*', '!**/fonto/**', '!**/vendor/**', '!**/*.min.js'] : [];
   const jsF     = g_filter(jsFGlob, {restore: true});
   const xpsF    = g_filter(['**/xopus/**/*.x*'], {restore: true});
@@ -189,8 +135,6 @@ const transfer = (paths, {watch, lrServer} = {}) => {
       _info('Finished transferring\n', true);
       if (fontoPaths.outdated[0]) _warn(`Fonto build files in the following folders were outdated and therefore skipped:\n  ${fontoPaths.outdated.map(v => v.slice(0, -1)).join('\n  ')}`);
 
-      remote.process();
-
       if (lrServer) { // reload specific resources if files are all image or css, else reload page
         const reloadPage = !files.every(f => /.(?:css|jpe?g|png|gif)$/i.test(f));
         lrServer.changed({params: {files: reloadPage ? ['page'] : files}});
@@ -199,7 +143,4 @@ const transfer = (paths, {watch, lrServer} = {}) => {
 
       if (c.deliveryPack) createDeliveryPack();
     });
-};
-
-
-module.exports = {remote, transfer};
+};

package/src/modules/deploy/index.js

@@ -1,10 +1,11 @@
 const del         = require('del');
 const gulp        = require('gulp');
 const path        = require('path');
+const RemoteExec  = require('../../lib/remote-exec');
 const sftpClient  = require('ssh2-sftp-client');
 const tinylr      = require('tiny-lr');
 
-const {remote, transfer} = require('./execute');
+const transfer = require('./execute');
 const c = require('./config');
 const s = require('./srcset');
 
@@ -16,17 +17,34 @@ module.exports = function deploy (argv) {
   c.setServer(argv.server);
   c.prepareForCopy(argv.filter);
 
-  if (c.server.ftpConfig) {
-    c.server.ftpConfig.eventPut = file => {
-      _write(file.destination.replace(c.server.remotedir, ''));
+  const {ftpConfig, remotedir} = c.server;
+
+  if (ftpConfig) {
+    const logPath = path.join(remotedir, 'log/deployments.log').toFws;
+    const re = new RemoteExec(ftpConfig);
+
+    ftpConfig.eventPut = file => {
+      _write(file.destination.replace(remotedir, ''));
       if      (path.extname(file.destination)=='.sh')
-        remote.add('chmod 755 '+file.destination, 'Permissions set: '+file.destination);
+        re.add('chmod 755 '+file.destination, 'Permissions set: '+file.destination);
       else if (file.destination.match(/cmscustom.*hce.*config.xml/))
-        remote.add('touch '+c.getRemotePath('hce/hce-config.xml'), 'Touched: '+c.getRemotePath('hce/hce-config.xml'));
+        re.add('touch '+c.getRemotePath('hce/hce-config.xml'), 'Touched: '+c.getRemotePath('hce/hce-config.xml'));
       else if (file.destination.match(/cmscustom.*od[fts].*config.xml/))
-        remote.add('touch '+c.getRemotePath('odf/odf-config.xml'), 'Touched: '+c.getRemotePath('odf/odf-config.xml'));
+        re.add('touch '+c.getRemotePath('odf/odf-config.xml'), 'Touched: '+c.getRemotePath('odf/odf-config.xml'));
       else if (file.destination.match(/txp\/site-configs/))
-        remote.add('touch '+c.getRemotePath('txp/xmlpages-config.xml'), 'Touched: '+c.getRemotePath('txp/xmlpages-config.xml'));
+        re.add('touch '+c.getRemotePath('txp/xmlpages-config.xml'), 'Touched: '+c.getRemotePath('txp/xmlpages-config.xml'));
+    };
+
+    ftpConfig.eventBeforeAll = async files => {
+      const dli = deployLogInfo(!argv.copy, c.transferPatterns[0], files, 'START');
+      await re.add(`echo -e "${dli}" >> ${logPath}`).process();
+    };
+
+    ftpConfig.eventAfterAll = async files => {
+      _info(`${files.length} file(s) transferred`);
+      const dli = deployLogInfo(!argv.copy, c.transferPatterns[0], files, 'DONE');
+      re.add(`echo -e "${dli}" >> ${logPath}`).add(`echo "$(tail -10000 ${logPath})" > ${logPath}`);
+      await re.process();
     };
   }
 
@@ -64,9 +82,9 @@ module.exports = function deploy (argv) {
     };
 
     // check connection
-    if (c.server.ftpConfig) {
+    if (ftpConfig) {
       const sftp = new sftpClient();
-      sftp.connect(c.server.ftpConfig).then(() => sftp.end()).catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
+      sftp.connect(ftpConfig).then(() => sftp.end()).catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
     }
 
     gulp.watch(c.transferPatterns)
@@ -89,8 +107,8 @@ module.exports = function deploy (argv) {
 
           if (!path.parse(filepath).base.match(/\.scss/)) {
             const rp  = c.getRemotePath(filepath);
-            const msg = 'Removed: ' + rp.replace(c.server.remotedir, '').white;
-            if (c.server.ftpConfig) remote.add(`rm -rf "${rp}"`, msg).process();
+            const msg = 'Removed: ' + rp.replace(remotedir, '').white;
+            if (ftpConfig) new RemoteExec(ftpConfig).add(`rm -rf "${rp}"`, msg).process();
             else del([rp], {force: true}).then(() => _info(msg, true));
           }
         }
@@ -98,4 +116,46 @@ module.exports = function deploy (argv) {
 
   }
 
-};
+};
+
+
+function deployLogInfo (watch, filter, files, action) {
+  const timestamp = new Date().toISOString();
+  const user = _git.user().split('@')[0];
+  const {branch, hash} = _git.commitLocal();
+  const uncommittedChanges = _git.status().trim() ? ':~' : '';
+
+  let logline = `${timestamp} ${action.padEnd(5)} [${user}] [${branch}:${hash.substring(0, 7)}${uncommittedChanges}] [${filter}]`;
+
+  if (action == 'START') {
+    if (uncommittedChanges) {
+      const uncommittedChanges = _git.status().replace(/\?\?/g, ' U').split('\n');
+      logline += '\n  Uncommitted changes:\n    ' + uncommittedChanges.join('\n   ');
+    }
+    const filepaths = files.map(({destination}) => destination);
+    logline += `\n  Transferring ${filepaths.length} file${filepaths.length > 1 ? 's' : ''}`;
+    if (watch || filepaths.length == 1) logline += ':\n    ' + filepaths.join('\n    ');
+    else logline += ' in dir:\n    ' + getCommonPath(filepaths);
+  }
+  return logline.replace(/"/g, '\\"');
+}
+
+
+function getCommonPath(paths) {
+  const splitPaths = paths.map(p => p.split('/'));
+  let commonPath = splitPaths[0];
+
+  for (let i = 1; i < splitPaths.length; i++) {
+    const pathParts = splitPaths[i];
+    commonPath = commonPath.slice(0, Math.min(commonPath.length, pathParts.length));
+
+    for (let j = 0; j < commonPath.length; j++) {
+      if (commonPath[j] !== pathParts[j]) {
+        commonPath = commonPath.slice(0, j);
+        break;
+      }
+    }
+  }
+
+  return commonPath.length === 0 ? '' : commonPath.join('/');
+}

package/src/modules/info/index.js

@@ -6,7 +6,7 @@ const {Table} = require('console-table-printer');
 const execGitCommand  = require('../../lib/exec-git-command');
 const getTdiBranch    = require('../../lib/get-tdi-branch');
 const c               = require('../deploy/config');
-const {remote}        = require('../deploy/execute');
+const RemoteExec      = require('../../lib/remote-exec');
 
 
 const getGitInfo = () => {
@@ -142,7 +142,7 @@ const getServerInfo = (server) => {
 
   if (!c.envDev) {
     _info(`Remote version info for '${c.server.ftpConfig.host}':\n`);
-    remote.add('sudo ~root/scripts/version.sh', '').process();
+    new RemoteExec(c.server.ftpConfig).add('sudo ~root/scripts/version.sh', 'STDOUT').process();
   }
   else {
     _info('For development environments no server version information is available. Check rancher / database for this information.\nAdd the --server option with a non-dev environment to see version information for that server.');