@tangelo/tangelo-configuration-toolkit 1.8.3 → 1.9.2

package/index.js

@@ -106,6 +106,7 @@ else {
 }
 
 
+global._tdiSubmoduleExists = fs.existsSync(path.join(_paths.repo, _paths.tdi));
 global._isPre42 = fs.existsSync(path.join(_paths.repo, _paths.tdi, 'create_new_project')); // folder changed in 4.2
 global._isPre51 = !fs.existsSync(path.join(_paths.repo, _paths.tdi, 'src')); // folder changed in 5.1 (check new folder because old one could still exist after branch switch)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangelo/tangelo-configuration-toolkit",
-  "version": "1.8.3",
+  "version": "1.9.2",
   "description": "Tangelo Configuration Toolkit is a command-line toolkit which offers support for developing a Tangelo configuration.",
   "bin": {
     "tct": "bin/index.js",
@@ -17,8 +17,8 @@
   "author": "Iddo Fontijn <iddo.fontijn@tangelo.nl>",
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
-    "babel-core": "^6.26.3",
-    "babel-preset-es2015-without-strict": "0.0.4",
+    "@babel/core": "^7.17.4",
+    "@babel/preset-env": "^7.16.11",
     "cli-spinner": "^0.2.10",
     "compare-versions": "^4.1.1",
     "del": "^6.0.0",
@@ -28,7 +28,7 @@
     "global-modules-path": "^2.3.1",
     "globby": "^6.1.0",
     "gulp": "^4.0.2",
-    "gulp-babel": "^7.0.1",
+    "gulp-babel": "^8.0.0",
     "gulp-eol": "^0.2.0",
     "gulp-filter": "^7.0.0",
     "gulp-plumber": "^1.2.1",
@@ -37,13 +37,14 @@
     "gulp-sourcemaps": "^3.0.0",
     "inquirer": "^8.2.0",
     "istextorbinary": "^6.0.0",
-    "minimatch": "^3.0.4",
+    "minimatch": "^5.0.0",
     "node-ssh": "^12.0.2",
     "object-assign-deep": "^0.4.0",
-    "replace-in-file": "^3.4.4",
+    "p-limit": "^3.1.0",
+    "replace-in-file": "^6.3.2",
     "sass": "^1.43.5",
     "saxon-js": "^2.3.0",
-    "scp2": "^0.5.0",
+    "ssh2-sftp-client": "^7.2.2",
     "through2": "^4.0.2",
     "tiny-lr": "^2.0.0",
     "yargs": "^16.2.0"

package/src/lib/gulp-sftp.js

@@ -0,0 +1,61 @@
+const path        = require('path');
+const pLimit      = require('p-limit');
+const sftpClient  = require('ssh2-sftp-client');
+const through2    = require('through2');
+
+
+module.exports = function(ftpConfig, remotedir) {
+  const sftp      = new sftpClient();
+  const serial    = pLimit(1);
+  const parallel  = pLimit(ftpConfig.parallel);
+  const files     = [];
+
+  return through2.obj(
+
+    function transform (file, _, cb) {
+      if (file.isStream()) return cb(new Error('Streaming not supported.'));
+      if (file.stat.isDirectory()) return cb();
+
+      file.destination = path.join(remotedir, file.relative).toFws();
+      files.push(file); // collect all files in array
+      return cb();
+    },
+
+    function flush(cb) {
+      if (!files[0]) cb();
+
+      const paths = [...new Set(files.map(({destination}) => path.dirname(destination)))] // collect unique paths
+        .filter((p1, i, a) => !a.find(p2 => p1 != p2 && p2.includes(p1+'/'))) // remove paths being part of others
+      ;
+
+      sftp.connect(ftpConfig)
+      .then(() => Promise.all(paths.map(p => parallel( // check if all directories exist
+        () => sftp.exists(p).then(r => {
+          if (!!r) paths.splice(paths.indexOf(p), 1) // if exists, remove from paths
+        })
+      ))))
+      .then(() => Promise.all(paths.map(p => serial( // create directories that do not exist, which cannot be done parallel
+        () => {
+          _write(p.replace(remotedir, '').lblack);
+          return sftp.mkdir(p, true).catch(err => _warn(err));
+        }
+      ))))
+      .then(() => Promise.all(files.map(file => parallel( // upload files
+        () => {
+          ftpConfig.eventPut(file);
+          return ( // use fastPut for files larger than 3mb; since it requires a local path we assume these files are not changed in the gulp pipeline
+            file.contents.length > 3000000 ? sftp.fastPut(file.originalRelativePath, file.destination) : sftp.put(file.contents, file.destination)
+          ).catch(err => _warn(`File transfer failed${err ? ': '+err : ''}`));
+        }
+      ))))
+      .then(() => {
+        _info(`${files.length} file(s) transferred`);
+        cb();
+        return sftp.end();
+      })
+      .catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
+    }
+
+  )
+  .resume(); // required for triggering 'end' event
+};

package/src/modules/deploy/config.js

@@ -54,7 +54,9 @@ module.exports = {
         parallel: serverConfig.config.parallel,
         username: serverConfig.config.username,
         agent: process.platform=='win32' ? 'pageant' : process.env.SSH_AUTH_SOCK,
-        agentForward: process.platform!='win32'
+        agentForward: process.platform!='win32',
+        readyTimeout: 10000,
+        retries: 1
       }
     };
 
@@ -105,7 +107,7 @@ module.exports = {
 
     this.transferPatterns = [transferPattern, tdiPattern, ...this.giPatterns, '!**/*.crdownload']; // ignore patterns must come last
 
-    if (this.deliveryPack && transferPattern.includes(_paths.tdi)) this.transferPatterns.push(`${_paths.tdi}/src/database/create.sql`); // create.sql is called by tdi install
+    if (this.deliveryPack && transferPattern.includes(_paths.tdi)) this.transferPatterns.push(`${_paths.tdi}/src/database/create*.sql`); // create(_exists).sql is called by tdi install
 
     // add time parameters to all xopus requests to overcome cache
     const ts = new Date().toISOString().replace(/-|:/g, '').substring(0,15);

package/src/modules/deploy/execute.js

@@ -1,23 +1,22 @@
-const fs            = require('fs-extra');
-const globby        = require('globby');
-const gulp          = require('gulp');
-const {NodeSSH}     = require('node-ssh');
-const through2      = require('through2');
-const {spawnSync}   = require('child_process');
-
-const g_babel_env   = require('babel-preset-es2015-without-strict');
-const g_babel       = require('gulp-babel');
-const g_eol         = require('gulp-eol');
-const g_filter      = require('gulp-filter');
-const g_plumber     = require('gulp-plumber');
-const g_print       = require('gulp-print');
-const g_replace     = require('../../lib/gulp-batch-replace-with-filter');
-const g_resolveIncl = require('../../lib/gulp-resolve-includes');
-const g_sass        = require('gulp-sass')(require('sass'));
-const g_scp         = require('../../lib/gulp-scp2-async');
-const g_sourcemaps  = require('gulp-sourcemaps');
-
-const execGitCommand = require('../../lib/exec-git-command');
+const execGitCommand  = require('../../lib/exec-git-command');
+const fs              = require('fs-extra');
+const globby          = require('globby');
+const gulp            = require('gulp');
+const {NodeSSH}       = require('node-ssh');
+const through2        = require('through2');
+const {spawnSync}     = require('child_process');
+
+const g_babel         = require('gulp-babel')({presets: [[require('@babel/preset-env'), {modules: false}]], comments: false, minified: true});
+const g_eol           = require('gulp-eol');
+const g_filter        = require('gulp-filter');
+const g_plumber       = require('gulp-plumber');
+const g_print         = require('gulp-print');
+const g_replace       = require('../../lib/gulp-batch-replace-with-filter');
+const g_resolveIncl   = require('../../lib/gulp-resolve-includes');
+const g_sass          = require('gulp-sass')(require('sass'));
+const g_sftp          = require('../../lib/gulp-sftp');
+const g_sourcemaps    = require('gulp-sourcemaps');
+
 
 const c = require('./config');
 const s = require('./srcset');
@@ -123,26 +122,27 @@ const transfer = (paths, lrServer) => {
   const sassF   = g_filter(['**/*.scss'], {restore: true});
   const jsF     = g_filter(['**/*.js', '!**/hce/**/template_*', '!**/fonto/**', '!**/vendor/**', '!**/*.min.js'], {restore: true});
   const shF     = g_filter(['**/*.sh'], {restore: true});
-  const ftpC    = Object.assign({dest: c.server.remotedir}, c.server.ftpConfig); // new obj because of async transfers
   const srcset  = s.create(paths);
   const files   = [];
 
-  const skippedFontoBuildFiles = [];
+  const fontoPaths = {outdated: [], uptodate: []};
   const removeOutdatedFontoBuild = g_filter(file => { // fonto build files must be newer than the last commit date containing fonto changes
     const prPath = 'config\\'+file.relative.replace(/fonto.+$/, '');
-    if (skippedFontoBuildFiles.includes(prPath)) return false;
+    if (fontoPaths.outdated.includes(prPath)) return false;
+    if (fontoPaths.uptodate.includes(prPath)) return true;
     if (file.relative.match(/fonto[\\\/]dist/)) {
       const paths = `${prPath}\\fonto ${prPath}\\schema ${_paths.tdi}`; // only check paths containing fonto sources
       const lastFontoCommitDate = execGitCommand(`log -1 --format=%cd --date=iso-strict origin/${_commit.branch} ${paths}`, _paths.repo);
       if (fs.statSync(file.relative).mtime < lastFontoCommitDate) {
-        skippedFontoBuildFiles.push(prPath);
+        fontoPaths.outdated.push(prPath);
         return false;
       }
+      else fontoPaths.uptodate.push(prPath);
     }
     return true;
   });
 
-  // disable file logging for ftp (c.ftpConfig.watch function does it already)
+  // disable file logging for ftp (gulp-sftp does it already)
   g_print.setLogFunction((filepath) => c.localTransfer ? _write(filepath.nostyle) : null);
 
   _info('Start transferring', true);
@@ -155,7 +155,7 @@ const transfer = (paths, lrServer) => {
   .pipe(xpsF.restore)
   .pipe(jsF)
   .pipe(g_sourcemaps.init())
-  .pipe(g_babel({presets: [g_babel_env], comments: false, minified: true}))
+  .pipe(g_babel)
   .pipe(g_sourcemaps.write('.'))
   .pipe(jsF.restore)
   .pipe(sassF)
@@ -169,15 +169,16 @@ const transfer = (paths, lrServer) => {
   .pipe(g_plumber.stop())
   .pipe(g_replace(c.replaceStrings))
   .pipe(through2.obj((file, enc, cb) => {
-    file.path = file.path.replace(/(fonto)(?:[\\\/]dist|.+([\\\/]assets[\\\/]schemas))/, '$1$2'); // change destination path for fonto build and schemas
+    file.originalRelativePath = file.relative; // original path needed for sftp.fastPut
+    file.path = file.path.replace(/(fonto)[\\\/](dev|dist)/, '$1'); // change destination path for fonto build
     if (!file.relative.endsWith('.map')) files.push(file.relative); // collect all file paths in array for livereload
     cb(null, file);
   }))
   .pipe(g_print.default())
-  .pipe(c.localTransfer ? gulp.dest(c.server.remotedir) : g_scp(ftpC))
+  .pipe(c.localTransfer ? gulp.dest(c.server.remotedir) : g_sftp(c.server.ftpConfig, c.server.remotedir))
   .on('end', () => {
     _info('Finished transferring\n', true);
-    if (skippedFontoBuildFiles[0]) _warn(`Fonto build files in the following folders were outdated and therefore skipped:\n  ${skippedFontoBuildFiles.map(v => v.slice(0, -1)).join('\n  ')}`);
+    if (fontoPaths.outdated[0]) _warn(`Fonto build files in the following folders were outdated and therefore skipped:\n  ${fontoPaths.outdated.map(v => v.slice(0, -1)).join('\n  ')}`);
 
     remote.process();
 

package/src/modules/deploy/index.js

@@ -1,7 +1,8 @@
-const del     = require('del');
-const gulp    = require('gulp');
-const path    = require('path');
-const tinylr  = require('tiny-lr');
+const del         = require('del');
+const gulp        = require('gulp');
+const path        = require('path');
+const sftpClient  = require('ssh2-sftp-client');
+const tinylr      = require('tiny-lr');
 
 const {remote, transfer} = require('./execute');
 const c = require('./config');
@@ -16,7 +17,7 @@ module.exports = function deploy (argv) {
   c.prepareForCopy(argv.filter);
 
   if (!c.localTransfer) {
-    c.server.ftpConfig.eventWrite = (file) => {
+    c.server.ftpConfig.eventPut = (file) => {
       _write(file.destination.replace(c.server.remotedir, ''));
       if      (path.extname(file.destination)=='.sh')
         remote.add('chmod 755 '+file.destination, 'Permissions set: '+file.destination);
@@ -60,12 +61,18 @@ module.exports = function deploy (argv) {
       }
     };
 
-    // for fonto, only watch packages (for schemas) and dist (for the build), but not dist/assets (causes build error / does not detect all files)
-    gulp.watch([...c.transferPatterns, '!**/fonto/{*.*, !({dist,packages})/**, dist/*/**}'])
+    // check connection
+    if (!c.localTransfer) {
+      const sftp = new sftpClient();
+      sftp.connect(c.server.ftpConfig).then(() => sftp.end()).catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
+    }
+
+    // for fonto, only watch dev/dist (for the build, does not detect "assets" most of the times btw)
+    gulp.watch([...c.transferPatterns, `!**/fonto/!(${c.envDev?'dev|':''}dist)/**`])
     .on('all', (event, filepath) => {
 
       if ((event=='add' || event=='change') &&
-          (/fonto(.dist|.+schemas.+\.json)/.test(filepath) || !/fonto/.test(filepath)) // within fonto, only transfer build and schemas
+          (/fonto.(dev|dist)/.test(filepath) || !/fonto/.test(filepath)) // within fonto, only transfer build files
       ) {
         transfers.add(filepath);
       }

package/src/modules/fonto/commands.js

@@ -43,7 +43,7 @@ const cmdExec = command => new Promise((resolve, reject) => {
 
   cp.stdout.setEncoding('utf8');
   cp.stdout.on('data', data => {
-    if (data.includes('Error') || data.includes('Input error')) reject(data);
+    if (/Error|Input error/.test(data)) _warn(data);
     else log(data);
   });
   cp.stderr.setEncoding('utf8');
@@ -203,7 +203,7 @@ module.exports = {
       fs.removeSync('dist/assets');
       _write('Done.\n');
     })
-    .then(() => cmdExec('fdt editor build -M'))
+    .then(() => cmdExec('fdt editor build'))
     ;
   },
 
@@ -226,7 +226,7 @@ module.exports = {
       });
       _write('Done.\n');
     })
-    .then(() => cmdExec('fdt editor run'))
+    .then(() => cmdExec('fdt editor run --write-to-disk'))
     ;
   }
 

package/src/modules/fonto/index.js

@@ -6,18 +6,20 @@ const path      = require('path');
 
 module.exports = function fonto (argv) {
 
+  if (!_tdiSubmoduleExists) _error('TDI submodule folder is missing.');
+
   process.chdir(_paths.apply || '.');
-  
+
   // find fonto instances by searching for manifest.json files having a fonto property
   const fontoPaths = globby.sync(['**/manifest.json', `!${_paths.tdi}/**`])
     .filter(p => (fs.readJsonSync(p, {throws: false}) || {}).edition == 'fontoxml-platform-base')
     .map(p => p.replace('manifest.json', ''))
   ;
-  
+
   if (fontoPaths.length==0) _error('No Fonto instance found.');
-  
+
   let promiseChain = Promise.resolve(); // for sequentially executing commands for each fonto instance
-  
+
   fontoPaths.forEach((p, i) => {
     promiseChain = promiseChain
     .then(() => {
@@ -40,5 +42,5 @@ module.exports = function fonto (argv) {
   .catch(error => {
     if (error) _warn(error.message || error);
   });
-  
+
 };

package/src/modules/migrate/steps.js

@@ -2,10 +2,10 @@ const fs        = require('fs-extra');
 const globby    = require('globby');
 const minimatch = require('minimatch');
 const path      = require('path');
-const rif       = require('replace-in-file');
+const rif      = require('replace-in-file');
 
 
-const getPaths  = (search, filter)  =>
+const getPaths = (search, filter) =>
   globby
   .sync(search, {dot: true, ignore: [_paths.tdi + '/**','**/cmscustom/tdi/**']})
   .filter(p => !filter || minimatch(p, filter)
@@ -47,9 +47,11 @@ module.exports = function steps (step, dry, filter) {
         r.to    = r.fromtoPairs.map(p => p[1]);
       }
 
-      r.files = rif.sync(r);                                                // execute first time
-      const filesModCount = r.files.length;                                 // save file count
-      for (let i=0; i<20 && !dry && r.files[0]; i++) r.files = rif.sync(r); // execute repeatedly for modified files only (with safety limit of 20)
+      let filesModCount;
+      for (let i=0; i<20 && !dry && r.files[0]; i++) { // execute repeatedly for modified files only (with safety limit of 20)
+        r.files = rif.sync(r).filter(f => f.hasChanged).map(f => f.file);
+        if (i==0) filesModCount = r.files.length; // save count only after first run (after this only subsets are processed)
+      }
 
       _write('Files modified:', filesModCount);
     });

package/src/lib/gulp-scp2-async.js

@@ -1,95 +0,0 @@
-// based on gulp-scp2
-
-// core
-const path = require('path');
-// other
-const through2 = require('through2'), Client = require('scp2').Client;
-
-const fixWinPath = (str) => str.replace(/\\/g, '/');
-
-module.exports = function(options) {
-  const files = [];
-
-  options.watch = function(client){ // set file transfer watch function
-    client
-    .on('write', options.eventWrite)
-    .on('error', err => {})
-    ;
-  };
-
-  const stream = through2.obj(
-    function transform(file, enc, callback) {
-      if (file.isStream()) return callback(new Error('Streaming not supported.'));
-      if (file.stat.isDirectory()) return callback();
-
-      files.push(file); // collect all files in array
-
-      return callback();
-    },
-    function flush(callback) { // on callback start transfering all files in parallel threads
-      if (!files[0]) callback();
-
-      let transferring = 0, transferred = 0;
-
-      function uploadFile(client) {
-        const file = files.shift();
-
-        if (file) {
-          const filepath = fixWinPath(path.join(options.dest, file.relative));
-          transferring++;
-
-          client.mkdir(path.dirname(filepath), (err) => { // try creating dir
-            if (err) {
-              const fail = err.message=='Permission denied';
-              transferring--;
-              if (!fail) files.unshift(file); // add file back to queue
-              _warn((fail ? `Fail: ${err.message}` : `Server aborted connection. Retrying.`));
-              uploadFile(client); // retry
-            }
-            else {
-              client.write({ // upload file
-                destination: filepath,
-                content: file.contents
-              }, (err) => {
-                transferring--;
-                transferred++;
-                uploadFile(client);
-              });
-            }
-          });
-
-        }
-        else {
-          client.close();
-          if (transferring==0) {
-            transferring--;
-            _info(`${transferred} file(s) transferred`);
-            return callback();
-          }
-        }
-      }
-
-      function uploadThread() {
-        const client = new Client(options);
-        options.watch(client);
-        client.sftp(err => {
-          if (err) {
-            if      (err.toString().toLowerCase().includes('timed out')) _warn(`Connection timed out. Retrying.`);
-            else if (err.code=='ECONNRESET' || err.code=='ECONNABORTED') _warn(`Server aborted connection. Retrying.`);
-            else if (err.message=='All configured authentication methods failed') _error('Authentication failed. Check if SSH key is loaded.');
-            else _warn(err.toString());
-
-            uploadThread(); // retry
-          }
-          else uploadFile(client);
-        });
-      }
-
-      const parallel = Math.min(Math.round(files.length / 2), options.parallel);
-      for (let i = 0; i < parallel; i++) setTimeout(uploadThread, 100 * i);
-    }
-  );
-
-  stream.resume(); // required for triggering 'end' event
-  return stream;
-};