@tangelo/tangelo-configuration-toolkit 1.20.2 → 1.22.1

package/README.md

@@ -83,7 +83,7 @@ ENDLOCAL
 EXIT /b %errorlevel%
 :find_dp0
 SET dp0=%~dp0
-EXIT
+EXIT /b
 ```
 
 Then:

package/index.js

@@ -72,6 +72,7 @@ global._git = {
     if (!_appdata.gitUser) _appdata._update({gitUser: execGitCommand(`config --get user.email`, _paths.repo)});
     return _appdata.gitUser;
   },
+  status: memoize(() => execGitCommand(`status -s`, _paths.repo)),
   commitLocal: memoize(() => execGitCommand(`log -1 --format=%D;%H;%cd --date=iso-strict`, _paths.repo, ['branch', 'hash', 'date'])),
   commitRemote: memoize(() => execGitCommand('log -1 --format=%cd --date=iso-strict origin/' + _git.commitLocal().branch, _paths.repo, ['date'])),
   commitTdi: {
@@ -82,8 +83,10 @@ global._git = {
       {commitHash: '77b8ea9', regex: /^7\.14\./}, // 7.14.x
       {commitHash: '8066c44', regex: /^7\.13\./}, // 7.13.x
       {commitHash: 'a2b2d4e', regex: /^7\.12\./}  // 7.12.x
-    ]
-  }
+    ],
+    hasUncommittedFiles: memoize(() => execGitCommand(`status --short`, path.join(_paths.repo, _paths.tdi)).length > 0),
+  },
+  hasUncommittedFiles: memoize(() => execGitCommand(`status --short`, _paths.repo).length > 0),
 };
 
 global._tdiSubmoduleExists = () => fs.existsSync(path.join(_paths.repo, _paths.tdi));
@@ -121,6 +124,9 @@ global._modulesTdi = {
 
 global._settingsTdi = memoize(() => Object.assign(_package.settingsTdiDefault, _modulesTdi.require('settings.json', {throws: false, message: false}) ?? {}));
 
+global._filters = {
+  fontoSources: ['!**/node_modules/**', '!**/fonto/packages/**', '!**/fonto/packages_shared/**', '!**/fonto/platform/**']
+};
 
 process.on('beforeExit', () => {
   _write(); // print empty line before and after update check

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangelo/tangelo-configuration-toolkit",
-  "version": "1.20.2",
+  "version": "1.22.1",
   "engines": {
     "node": ">=14.0.0"
   },

package/src/lib/gulp-sftp.js

@@ -21,13 +21,15 @@ module.exports = function(ftpConfig, remotedir) {
       return cb();
     },
 
-    function flush(cb) {
+    async function flush(cb) {
       if (!files[0]) cb();
 
       const paths = [...new Set(files.map(({destination}) => path.dirname(destination)))] // collect unique paths
         .filter((p1, i, a) => !a.find(p2 => p1 != p2 && p2.includes(p1+'/'))) // remove paths being part of others
       ;
 
+      await ftpConfig.eventBeforeAll(files);
+
       sftp.connect(ftpConfig)
         .then(() => Promise.all(paths.map(p => parallel( // check if all directories exist
           () => sftp.exists(p).then(r => {
@@ -48,8 +50,8 @@ module.exports = function(ftpConfig, remotedir) {
             ).catch(err => _warn(`File transfer failed${err ? ': '+err : ''}`));
           }
         ))))
-        .then(() => {
-          _info(`${files.length} file(s) transferred`);
+        .then(async () => {
+          await ftpConfig.eventAfterAll(files);
           cb();
           return sftp.end();
         })

package/src/lib/remote-exec.js

@@ -0,0 +1,44 @@
+const {NodeSSH} = require('node-ssh');
+const pLimit    = require('p-limit');
+
+
+module.exports = class RemoteExec {
+
+  #ftpConfig;
+  #queue = [];
+
+  constructor(ftpConfig) {
+    this.#ftpConfig = ftpConfig;
+  }
+
+  add(cmd, msg) {
+    if (!this.#queue.find(e => e[0]==cmd)) this.#queue.push([cmd, msg]);
+    return this;
+  }
+
+  async process() {
+    if (!this.#queue[0]) return;
+
+    const sshI = new NodeSSH();
+    const limit = pLimit(this.#ftpConfig.parallel);
+
+    await sshI.connect(this.#ftpConfig) // set up connection once
+      .then(() => Promise.all(this.#queue.map(([cmd, msg]) => limit(
+        () => sshI.execCommand(cmd).then(result => {
+          if (result.stderr) _warn(result.stderr);
+          else if (msg === 'STDOUT') _write(result.stdout);
+          else if (msg) _info(msg === 'CMD' ? cmd : msg);
+        })
+      ))))
+      .catch(err => {
+        if (err.code=='ECONNRESET' || err.code=='ECONNABORTED') _warn(`Server aborted connection. Retrying.`);
+        else _warn(err);
+        this.process(); // retry set up connection
+      })
+      .finally(() => {
+        sshI.dispose();
+        this.#queue = [];
+      });
+  }
+
+};

package/src/lib/tcl-config.js

@@ -1,4 +1,5 @@
 const fs = require('fs-extra');
+const globby = require('globby');
 const path = require('path');
 const sass = require('sass');
 const SaxonJS = require('saxon-js');
@@ -53,6 +54,10 @@ module.exports = class TclConfig {
     }
   }
 
+  static findProjects() {
+    return globby.sync('config/cmscustom/!(tdi)/!(fonts)').map(p => ({path: p, tcl: fs.existsSync(path.join(p, this.#TCL_FILE_NAME))}));
+  }
+
   #outputFn;
   #doModifyFontoHtml;
   #tclFilePathRel;

package/src/modules/build/index.js

@@ -4,11 +4,12 @@ const inquirer  = require('inquirer');
 const oxygen    = require('./oxygen');
 const path      = require('path');
 
+
 const createSymlink = (t, p) => {
   const up = p.replace(/[^/]+/g, '..').replace(/^..\//, '');
   fs.symlink(path.join(up, _paths.tdi, t), p, 'dir', err => {
     if (err) {
-      if (err.code=='EEXIST') _write(`Symlink already exists: ${p}`);
+      if (err.code==='EEXIST') _write(`Symlink already exists: ${p}`);
       else _error((err.code!=='EPERM' ? err : `Can't create symlinks, restart your console as admin, or fix it permanently:\n  1. Open 'Local Security Policy' (secpol.msc)\n  2. Go to 'Security Settings > Local Policies > User Rights Assignment > Create symbolic links'\n  3. Add your own account\n  4. Restart Windows`));
     }
     else _write(`Symlink created: ${p}`);
@@ -22,7 +23,12 @@ const createSymlinks = () => {
   globby
     .sync('config/cmscustom/!(tdi)/**/fonto')
     .forEach(p => {
-      createSymlink(src + '/config/cmscustom/tdi/fonto/packages-shared', p + '/packages-shared');
+      const tdiCodebasePath =
+      _settingsTdi().codebase === 'dual' && fs.existsSync(path.join(p, '../project.tcl')) ? '/tcl' :
+        _settingsTdi().codebase === 'dual' ? '/traditional' :
+          ''
+      ;
+      createSymlink(src + `/config/cmscustom/tdi${tdiCodebasePath}/fonto/packages-shared`, p + '/packages-shared');
     });
 };
 

package/src/modules/deploy/config.js

@@ -129,9 +129,6 @@ function prepareForCopy (filter) {
   _info(`Target: ${this.getRemotePath(transferPattern)}`);
   _write();
 
-  if (_git.commitLocal().date < _git.commitRemote().date) {
-    _warn(`You're not deploying from the most recent git commit!\n`);
-  }
 }
 
 

package/src/modules/deploy/execute.js

@@ -3,7 +3,6 @@ const execGitCommand  = require('../../lib/exec-git-command');
 const fs              = require('fs-extra');
 const globby          = require('globby');
 const gulp            = require('gulp');
-const {NodeSSH}       = require('node-ssh');
 const g_tcl           = require('../../lib/gulp-tcl-config');
 const through2        = require('through2');
 const {spawnSync}     = require('child_process');
@@ -24,59 +23,6 @@ const c = require('./config');
 const s = require('./srcset');
 
 
-const remote = {
-  queue: [],
-  add(cmd, msg) {
-    if (!this.queue.find(e => e[0]==cmd)) this.queue.push([cmd, msg]);
-    return this;
-  },
-  do(sshI) {
-    let executing = Math.min(this.queue.length, c.server.ftpConfig.parallel);
-
-    if (executing===0) { // close connection and return callback after last exec command
-      sshI.dispose();
-      _write();
-    }
-
-    // execute each command from the queue in batches
-    this.queue.filter((e, i) => i < executing).forEach(()=>{
-      const command = this.queue.shift();
-
-      sshI.execCommand(command[0])
-        .then(result => {
-          if (result.stderr) _warn(result.stderr);
-          else if (command[1] === '') _write(result.stdout);
-          else _info(command[1] || command[0], true);
-
-          executing--;
-          if (executing===0) this.do(sshI);
-        })
-        .catch(() => {
-          _warn(`Server aborted connection. Retrying.`);
-          this.queue.unshift(command);
-
-          executing--;
-          if (executing===0) this.do(sshI);
-        });
-    });
-  },
-  process() {
-    if (!this.queue[0]) return;
-
-    const sshI = new NodeSSH();
-    sshI.connect(c.server.ftpConfig) // set up connection once
-      .then(() => {
-        this.do(sshI);
-      })
-      .catch(err => {
-        if (err.code=='ECONNRESET' || err.code=='ECONNABORTED') _warn(`Server aborted connection. Retrying.`);
-        else _warn(err);
-        this.process(); // retry set up connection
-      });
-  }
-};
-
-
 const createDeliveryPack = () => { // create install scripts if necessary, zip output and remove temp folder
   const dbiSql = [];
   const dbiPs1 = [];
@@ -120,7 +66,7 @@ const createDeliveryPack = () => { // create install scripts if necessary, zip o
 };
 
 
-const transfer = (paths, {watch, lrServer} = {}) => {
+module.exports = function transfer (paths, {watch, lrServer} = {}) {
   const jsFGlob = _settingsTdi().transpileToES5 ? ['**/*.js', '!**/hce/**/template_*', '!**/fonto/**', '!**/vendor/**', '!**/*.min.js'] : [];
   const jsF     = g_filter(jsFGlob, {restore: true});
   const xpsF    = g_filter(['**/xopus/**/*.x*'], {restore: true});
@@ -189,8 +135,6 @@ const transfer = (paths, {watch, lrServer} = {}) => {
       _info('Finished transferring\n', true);
       if (fontoPaths.outdated[0]) _warn(`Fonto build files in the following folders were outdated and therefore skipped:\n  ${fontoPaths.outdated.map(v => v.slice(0, -1)).join('\n  ')}`);
 
-      remote.process();
-
       if (lrServer) { // reload specific resources if files are all image or css, else reload page
         const reloadPage = !files.every(f => /.(?:css|jpe?g|png|gif)$/i.test(f));
         lrServer.changed({params: {files: reloadPage ? ['page'] : files}});
@@ -199,7 +143,4 @@ const transfer = (paths, {watch, lrServer} = {}) => {
 
       if (c.deliveryPack) createDeliveryPack();
     });
-};
-
-
-module.exports = {remote, transfer};
+};

package/src/modules/deploy/index.js

@@ -1,32 +1,93 @@
 const del         = require('del');
 const gulp        = require('gulp');
 const path        = require('path');
+const RemoteExec  = require('../../lib/remote-exec');
 const sftpClient  = require('ssh2-sftp-client');
 const tinylr      = require('tiny-lr');
 
-const {remote, transfer} = require('./execute');
+const transfer = require('./execute');
 const c = require('./config');
 const s = require('./srcset');
+const inquirer = require('inquirer');
+
+const checkDeployment = async (ftpConfig) => {
+  const deployToDevServer = /\.dev\./.test( ftpConfig?.host );
+  const deployToTestServer = /\.t(e)?st\./.test( ftpConfig?.host );
+  const deployToProdServer = c?.deliveryPack || !( deployToDevServer || deployToTestServer );
+  const isBehind = _git.commitLocal().date < _git.commitRemote().date;
+  // In most cases productionBranches are called 'master' or 'main'; Some customers use a development, test and production branches structure; hence we allow 'producti(on|e)'
+  const isPrdBranch = ( /master|main|producti(on|e)/.test( _git.commitLocal().branch ) );
+
+  if (!deployToTestServer && !deployToProdServer) {
+    // dev-environements
+    if (isBehind) {
+      /* Warn for dev deployements that are behind */
+      _warn(`You're not deploying from the most recent commit in branch "${_git.commitLocal().branch}"! Update your repository: ${'tct git --update-repo'.cyan}`);
+    }
+    return;
+  }
 
+  // Checks for test / prd environments:
+  if ( !isPrdBranch ) {
+    _warn(`You want to deploy from branch "${_git.commitLocal().branch}" to a production server. It is strongly advised to deploy from the master branch.`);
+  }
+  if (_git.hasUncommittedFiles()) {
+    _warn(`You have uncommitted changes!`);
+    if (_git.commitTdi.hasUncommittedFiles()) _warn(`You have uncommitted changes in the TDI submodule!`);
+  }
+  if ( isBehind ) {
+    _error(`You're not deploying from the most recent commit in branch "${_git.commitLocal().branch}"! Update your repository: ${'tct git --update-repo'.cyan}`);
+  }
+
+  // Ask for continuation of deployment for production environements only; and only if not deploying from a prd branch or when there exist uncommitted changes
+  if ( (!isPrdBranch || _git.hasUncommittedFiles()) && deployToProdServer &&
+        !( await inquirer.prompt(
+          {type: 'confirm', message: 'Are you sure you want to continue?', name: 'confirm', default: false}
+        ).then( res => res.confirm )
+        )
+  ) {
+    return {cancel: true};
+  }
+};
 
-module.exports = function deploy (argv) {
+module.exports = async function deploy (argv) {
 
   // common setup
 
   c.setServer(argv.server);
   c.prepareForCopy(argv.filter);
+  const {ftpConfig, remotedir} = c.server;
+
+  // Deployment checks; cancel deployment when checks require
+  const deploy = await checkDeployment(ftpConfig);
+  if (deploy?.cancel) { return; }
 
-  if (c.server.ftpConfig) {
-    c.server.ftpConfig.eventPut = file => {
-      _write(file.destination.replace(c.server.remotedir, ''));
+  if (ftpConfig) {
+    const logPath = path.join(remotedir, 'log/deployments.log').toFws;
+    const re = new RemoteExec(ftpConfig);
+
+    ftpConfig.eventPut = file => {
+      _write(file.destination.replace(remotedir, ''));
       if      (path.extname(file.destination)=='.sh')
-        remote.add('chmod 755 '+file.destination, 'Permissions set: '+file.destination);
+        re.add('chmod 755 '+file.destination, 'Permissions set: '+file.destination);
       else if (file.destination.match(/cmscustom.*hce.*config.xml/))
-        remote.add('touch '+c.getRemotePath('hce/hce-config.xml'), 'Touched: '+c.getRemotePath('hce/hce-config.xml'));
+        re.add('touch '+c.getRemotePath('hce/hce-config.xml'), 'Touched: '+c.getRemotePath('hce/hce-config.xml'));
       else if (file.destination.match(/cmscustom.*od[fts].*config.xml/))
-        remote.add('touch '+c.getRemotePath('odf/odf-config.xml'), 'Touched: '+c.getRemotePath('odf/odf-config.xml'));
+        re.add('touch '+c.getRemotePath('odf/odf-config.xml'), 'Touched: '+c.getRemotePath('odf/odf-config.xml'));
       else if (file.destination.match(/txp\/site-configs/))
-        remote.add('touch '+c.getRemotePath('txp/xmlpages-config.xml'), 'Touched: '+c.getRemotePath('txp/xmlpages-config.xml'));
+        re.add('touch '+c.getRemotePath('txp/xmlpages-config.xml'), 'Touched: '+c.getRemotePath('txp/xmlpages-config.xml'));
+    };
+
+    ftpConfig.eventBeforeAll = async files => {
+      const dli = deployLogInfo(!argv.copy, c.transferPatterns[0], files, 'START');
+      await re.add(`echo -e "${dli}" >> ${logPath}`).process();
+    };
+
+    ftpConfig.eventAfterAll = async files => {
+      _info(`${files.length} file(s) transferred`);
+      const dli = deployLogInfo(!argv.copy, c.transferPatterns[0], files, 'DONE');
+      re.add(`echo -e "${dli}" >> ${logPath}`).add(`echo "$(tail -10000 ${logPath})" > ${logPath}`);
+      await re.process();
     };
   }
 
@@ -41,6 +102,7 @@ module.exports = function deploy (argv) {
   }
 
   if (argv.watch || argv.live) {
+    if (c?.deliveryPack) _error('You cannot use watch with a delivery-pack.');
     _write(`Watching... (press ctrl+c to quit)\n`);
 
     const lrServer = argv.live && tinylr();
@@ -64,9 +126,9 @@ module.exports = function deploy (argv) {
     };
 
     // check connection
-    if (c.server.ftpConfig) {
+    if (ftpConfig) {
       const sftp = new sftpClient();
-      sftp.connect(c.server.ftpConfig).then(() => sftp.end()).catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
+      sftp.connect(ftpConfig).then(() => sftp.end()).catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
     }
 
     gulp.watch(c.transferPatterns)
@@ -89,8 +151,8 @@ module.exports = function deploy (argv) {
 
           if (!path.parse(filepath).base.match(/\.scss/)) {
             const rp  = c.getRemotePath(filepath);
-            const msg = 'Removed: ' + rp.replace(c.server.remotedir, '').white;
-            if (c.server.ftpConfig) remote.add(`rm -rf "${rp}"`, msg).process();
+            const msg = 'Removed: ' + rp.replace(remotedir, '').white;
+            if (ftpConfig) new RemoteExec(ftpConfig).add(`rm -rf "${rp}"`, msg).process();
             else del([rp], {force: true}).then(() => _info(msg, true));
           }
         }
@@ -98,4 +160,46 @@ module.exports = function deploy (argv) {
 
   }
 
-};
+};
+
+
+function deployLogInfo (watch, filter, files, action) {
+  const timestamp = new Date().toISOString();
+  const user = _git.user().split('@')[0];
+  const {branch, hash} = _git.commitLocal();
+  const uncommittedChanges = _git.status().trim() ? ':~' : '';
+
+  let logline = `${timestamp} ${action.padEnd(5)} [${user}] [${branch}:${hash.substring(0, 7)}${uncommittedChanges}] [${filter}]`;
+
+  if (action == 'START') {
+    if (uncommittedChanges) {
+      const uncommittedChanges = _git.status().replace(/\?\?/g, ' U').split('\n');
+      logline += '\n  Uncommitted changes:\n    ' + uncommittedChanges.join('\n   ');
+    }
+    const filepaths = files.map(({destination}) => destination);
+    logline += `\n  Transferring ${filepaths.length} file${filepaths.length > 1 ? 's' : ''}`;
+    if (watch || filepaths.length == 1) logline += ':\n    ' + filepaths.join('\n    ');
+    else logline += ' in dir:\n    ' + getCommonPath(filepaths);
+  }
+  return logline.replace(/"/g, '\\"');
+}
+
+
+function getCommonPath(paths) {
+  const splitPaths = paths.map(p => p.split('/'));
+  let commonPath = splitPaths[0];
+
+  for (let i = 1; i < splitPaths.length; i++) {
+    const pathParts = splitPaths[i];
+    commonPath = commonPath.slice(0, Math.min(commonPath.length, pathParts.length));
+
+    for (let j = 0; j < commonPath.length; j++) {
+      if (commonPath[j] !== pathParts[j]) {
+        commonPath = commonPath.slice(0, j);
+        break;
+      }
+    }
+  }
+
+  return commonPath.length === 0 ? '' : commonPath.join('/');
+}

package/src/modules/deploy/srcset.js

@@ -12,8 +12,10 @@ module.exports = {
   create (paths, {test, watch} = {}) {
     const startTime = new Date();
 
-    this.filesToTransfer            = watch ? paths : globby.sync(paths, {nodir: true});  // get list of files to transfer
-    this.potentialBaseFilesCache  ??= globby.sync('**/*.{xml,xsd,xsl,scss}');             // get list of potential base files, if not previously done
+    this.filesToTransfer            = watch ? paths : globby.sync([...paths, ..._filters.fontoSources], {nodir: true});  // get list of files to transfer
+
+    // get list of potential base files, if not previously done. Globby 6.x does not expand ** to symlinks, so use **/** to include those too
+    this.potentialBaseFilesCache  ??= globby.sync(['**/**/*.{xml,xsd,xsl,scss}', ..._filters.fontoSources]);
     this.potentialBaseFiles         = this.potentialBaseFilesCache
       .filter(p => !this.filesToTransfer.includes(p)) // subtract files to transfer from list of potential base files
       .map(path => [path]);                           // make array so file contents can be added later

package/src/modules/fonto/commands.js

@@ -10,7 +10,7 @@ const TclConfig = require('../../lib/tcl-config');
 const wws       = require('../../lib/worker-with-spinner');
 
 
-g_print.setLogFunction((filepath) => _write(filepath.nostyle));
+g_print.setLogFunction(filepath => _write(filepath.nostyle));
 
 
 const compileSass = () => new Promise((resolve, reject) => {
@@ -26,14 +26,13 @@ const compileSass = () => new Promise((resolve, reject) => {
   ;
 });
 
-const cmdExec = command => new Promise((resolve) => {
+const cmdExec = command => new Promise(resolve => {
   _info(`Executing command (${command}):`);
   const cp  = exec(command);
   const log = msg => {
     const line = msg.replace(/\s+$/, ''); // remove excessive whitespace
     if (line) console.log(line);
   };
-
   cp.stdout.setEncoding('utf8');
   cp.stdout.on('data', data => {
     if (/Error|Input error/.test(data)) _warn(data);
@@ -48,13 +47,19 @@ const cmdExec = command => new Promise((resolve) => {
 });
 
 
+
 module.exports = {
 
   init ([fdt, fontoVersion]) {
     return new Promise((resolve, reject) => {
       _info('Ensure symlink exists:');
-      fs.symlink(path.join('../../../tdi/fonto/packages-shared'), 'packages-shared', 'dir', err => {
-        if (err && err.code!='EEXIST') reject(err.code=='EPERM' ? 'Start your console as admin for symlink creation!' : err);
+      const tdiCodebasePath =
+      _settingsTdi().codebase === 'dual' && fs.existsSync('../project.tcl') ? '/tcl' :
+        _settingsTdi().codebase === 'dual' ? '/traditional' :
+          ''
+      ;
+      fs.symlink(path.join(`../../../tdi${tdiCodebasePath}/fonto/packages-shared`), 'packages-shared', 'dir', err => {
+        if (err && err.code!=='EEXIST') reject(err.code==='EPERM' ? 'Start your console as admin for symlink creation!' : err);
         else _write('Done.\n');
         resolve();
       });
@@ -180,7 +185,7 @@ module.exports = {
       Object.entries(rootSchemas).forEach(([, obj]) => {
         const data              = execSync(`${fdt} attributes --schema packages/${obj.packageName}/src/assets/schemas/${obj.packageName}.json`, {encoding: 'UTF-8'});
         const attributes        = data.replace(/(^.*?Default value\s+)|(\s+Printed name\*.*$)/gs, '').split(/\n\s+/).map(a => a.split(/\s+/)).map(a =>
-          a[0] + (a[2]=='required' ? ' (required)' : '') + (a[3]=='-' ? ' (no default)' : '')
+          a[0] + (a[2]==='required' ? ' (required)' : '') + (a[3]==='-' ? ' (no default)' : '')
         );
         const customAttributes  = [...new Set(attributes)].filter(e => e && !ignoreAttributes.includes(e.replace(/ .*/, '')));
 

package/src/modules/fonto/index.js

@@ -29,7 +29,7 @@ module.exports = function fonto (argv) {
   process.chdir(_paths.apply || '.');
 
   // find fonto instances by searching for fonto/manifest.json files
-  const fontoPaths = globby.sync(['**/fonto/manifest.json', 'manifest.json', `!${_paths.tdi}/**`])
+  const fontoPaths = globby.sync(['**/fonto/manifest.json', 'manifest.json', `!${_paths.tdi}/**`, ..._filters.fontoSources])
     .map(p => ([p.replace('manifest.json', ''), fs.readJsonSync(p).sdkVersion.replace(/Nightlies.*/, 'nightly')]))
   ;
 

package/src/modules/git/index.js

@@ -169,9 +169,15 @@ function updateRepo () {
  * @param {string[]} options.dates - An array of two strings representing the start and end dates for the custom date range.
  * @param {string} options.updateSubmodule - The target branch name for the submodule update.
  */
-function updateSubmodule({dates = [], updateSubmodule: toBranchName}) {
+function updateSubmodule({dates = [], updateSubmodule: releaseBranchName}) {
   const [dateCustomFrom, dateCustomTo] = dates;
-  const branch = getTdiBranch(typeof toBranchName !== 'boolean' && toBranchName);
+  const toBranchName = typeof releaseBranchName !== 'boolean' ? releaseBranchName : null;
+
+  if (dateCustomFrom) _warn(`While testing the "TDI commits requiring migration" note that the submodule is not changed.`);
+  if (dateCustomFrom && !toBranchName) _error(`Specify a release branch for testing the migration, e.g. "--us 5.6".`);
+
+  // For testing the migrations (dates argument is specified); submodule pull, fetch, updates fail or result in switching to the release-branch; while we want to test the migration in the current submodule branch / symlink to TDI-project. Therefore all submodule actions should not be applied when the dates argument is specified. As the release of the current submodule cannot be determined you also need to specify the toBranchName so you can test the migrations you want.
+  const branch = !dateCustomFrom ? getTdiBranch(toBranchName) : {'name': `release/${toBranchName}`};
   const branchUpgrade = branch.from ? branch.from.name < branch.name : false;
 
   if (branchUpgrade) _info(`Current branch '${branch.from.name}' will be updated to '${branch.name}'\nCommon ancestor will be used in selecting TDI commits requiring migration: ${_formatDate(branch.commonAncestor.date)}`);
@@ -184,10 +190,12 @@ function updateSubmodule({dates = [], updateSubmodule: toBranchName}) {
   if (branch.commitsBehind > 0 || dateCustomFrom) {
     const dateBeforeUpdate = _git.commitTdi.local().date;
 
-    // update submodule
-    const updateSubmoduleMsg = execGitCommand(`submodule update --remote`, _paths.repo);
-    if (updateSubmoduleMsg.error) _error(`Update submodule failed\n${updateSubmoduleMsg.error}`);
-    if (updateSubmoduleMsg) _info(`TDI submodule updated:\n${updateSubmoduleMsg}\n`);
+    if (!dateCustomFrom) {
+      // update submodule
+      const updateSubmoduleMsg = execGitCommand(`submodule update --remote`, _paths.repo);
+      if (updateSubmoduleMsg.error) _error(`Update submodule failed\n${updateSubmoduleMsg.error}`);
+      _info(`TDI submodule updated:\n${updateSubmoduleMsg}\n`);
+    }
 
     // tdiMigrationFilePath should exist in latest commits of releases 5.3+; As we updated to the latest version this should work
     const migrations = _modulesTdi.require('git/tdiCommitsRequiringMigration.js');

package/src/modules/info/index.js

@@ -6,7 +6,7 @@ const {Table} = require('console-table-printer');
 const execGitCommand  = require('../../lib/exec-git-command');
 const getTdiBranch    = require('../../lib/get-tdi-branch');
 const c               = require('../deploy/config');
-const {remote}        = require('../deploy/execute');
+const RemoteExec      = require('../../lib/remote-exec');
 
 
 const getGitInfo = () => {
@@ -142,7 +142,7 @@ const getServerInfo = (server) => {
 
   if (!c.envDev) {
     _info(`Remote version info for '${c.server.ftpConfig.host}':\n`);
-    remote.add('sudo ~root/scripts/version.sh', '').process();
+    new RemoteExec(c.server.ftpConfig).add('sudo ~root/scripts/version.sh', 'STDOUT').process();
   }
   else {
     _info('For development environments no server version information is available. Check rancher / database for this information.\nAdd the --server option with a non-dev environment to see version information for that server.');

package/src/modules/migrate/index.js

@@ -1,6 +1,7 @@
 const globby    = require('globby');
 const inquirer  = require('inquirer');
 const path      = require('path');
+const TclConfig = require('../../lib/tcl-config');
 
 
 module.exports = function migrate (argv) {
@@ -76,8 +77,9 @@ module.exports = function migrate (argv) {
           _write();
           const startTime = new Date();
 
+          const projectsWithType = TclConfig.findProjects();
           const scriptPath = path.join('migrate', a.script);
-          _modulesTdi.require(scriptPath).forEach(step => require('./steps')(step, argv.dry, filter));
+          _modulesTdi.require(scriptPath).forEach(step => require('./steps')(step, argv.dry, filter, projectsWithType));
 
           _perf(startTime);
         });

package/src/modules/migrate/steps.js

@@ -5,26 +5,37 @@ const path      = require('path');
 const rif       = require('replace-in-file');
 
 
-const getPaths = (search, filter) =>
-  globby
-    .sync(search, {dot: true, ignore: [_paths.tdi + '/**', '**/cmscustom/tdi/**']})
-    .filter(p => !filter || minimatch(p, filter)
-    );
-
+class PathSearcher {
+  #filter;
+  #ignorePaths = [_paths.tdi + '/**', 'config/cmscustom/tdi/**'];
+  constructor (filter, ignoreProjectPaths) {
+    this.#filter = filter;
+    this.#ignorePaths.push(...ignoreProjectPaths);
+  }
+  #doFilter (paths) {
+    return this.#filter ? paths.filter(p => minimatch(p, this.#filter)) : paths;
+  }
+  get(globPath) {
+    return this.#doFilter(globby.sync(globPath, {dot: true, ignore: this.#ignorePaths}));
+  }
+}
 
-module.exports = function steps (step, dry, filter) {
 
-  const {deletePaths, renamePaths, replaceInFiles, customAction, applyFilter = true} = step;
-  const setFilter = applyFilter && filter;
+module.exports = function steps (step, dry, passedFilter, projectsWithType) {
 
   if (Array.isArray(step)) {
-    step.forEach(substep => require('./steps')(substep, dry, filter));
+    step.forEach(substep => steps(substep, dry, passedFilter, projectsWithType));
+    return;
   }
 
+  const {projectType = 'all', applyFilter = true, deletePaths, renamePaths, replaceInFiles, customAction} = step;
+  const projectsToIgnore = projectType === 'all' ? [] : projectsWithType.filter(p => p.tcl !== (projectType === 'tcl'));
+  const pathSearcher = new PathSearcher(applyFilter && passedFilter, projectsToIgnore.map(p => p.path + '/**'));
+
   if (deletePaths) {
     deletePaths.forEach(curpath => {
       _info(`Deleting paths: ${curpath}`);
-      const paths = getPaths(curpath, setFilter);
+      const paths = pathSearcher.get(curpath);
       if (!dry) paths.forEach(p => fs.removeSync(p));
       _write('Paths deleted:', paths.length);
     });
@@ -33,7 +44,7 @@ module.exports = function steps (step, dry, filter) {
   if (renamePaths) {
     renamePaths.forEach(([curpath, change]) => {
       _info(`Renaming paths: ${curpath}`);
-      const paths = getPaths(curpath, setFilter);
+      const paths = pathSearcher.get(curpath);
       if (!dry) paths.forEach(p => fs.moveSync(p, path.join(p, change), {overwrite: true}));
       _write('Paths renamed:', paths.length);
     });
@@ -43,7 +54,7 @@ module.exports = function steps (step, dry, filter) {
     replaceInFiles.forEach(r => {
       _info(`Executing replacements in files: ${r.files}`);
 
-      r.files = r.files.map(curpath => getPaths(curpath, setFilter)).flat();
+      r.files = r.files.map(curpath => pathSearcher.get(curpath)).flat();
       r.dry   = dry;
 
       if (r.fromtoPairs) {
@@ -51,12 +62,11 @@ module.exports = function steps (step, dry, filter) {
         r.to    = r.fromtoPairs.map(p => p[1]);
       }
 
-      let filesModCount = 0;
-      let maxRepeat = 15;
-      for (let i=0; i<maxRepeat && r.files[0]; i++) { // execute repeatedly for modified files only (with safety limit of 20)
+      let filesModCount;
+      for (let i=15/* safety limit*/; i>0 && r.files[0]; i--) { // execute repeatedly for modified files only
         r.files = rif.sync(r).filter(f => f.hasChanged).map(f => f.file);
-        if (i===0) filesModCount = r.files.length; // save count only after first run (after this only subsets are processed)
-        if (i===maxRepeat-1 && r.files[0]) _warn(`Repeated replacement stopped by safety limit - check file changes for too many content occurrences`);
+        filesModCount ??= r.files.length; // save count only after first run (after this only subsets are processed)
+        if (i===1 && r.files[0]) _warn(`Repeated replacement stopped by safety limit - check file changes for too many content occurrences`);
         if (dry) break;
       }
 
@@ -66,7 +76,7 @@ module.exports = function steps (step, dry, filter) {
 
   if (!dry && customAction) {
     customAction.forEach(([dirpath, actionFn]) => {
-      getPaths(dirpath, setFilter).forEach(p => actionFn(p));
+      pathSearcher.get(dirpath).forEach(p => actionFn(p));
     });
   }