@tangelo/tangelo-configuration-toolkit 1.8.3 → 1.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangelo/tangelo-configuration-toolkit",
-  "version": "1.8.3",
+  "version": "1.9.0",
   "description": "Tangelo Configuration Toolkit is a command-line toolkit which offers support for developing a Tangelo configuration.",
   "bin": {
     "tct": "bin/index.js",
@@ -40,10 +40,11 @@
     "minimatch": "^3.0.4",
     "node-ssh": "^12.0.2",
     "object-assign-deep": "^0.4.0",
+    "p-limit": "^3.1.0",
     "replace-in-file": "^3.4.4",
     "sass": "^1.43.5",
     "saxon-js": "^2.3.0",
-    "scp2": "^0.5.0",
+    "ssh2-sftp-client": "^7.2.2",
     "through2": "^4.0.2",
     "tiny-lr": "^2.0.0",
     "yargs": "^16.2.0"

package/src/lib/gulp-sftp.js

@@ -0,0 +1,61 @@
+const path        = require('path');
+const pLimit      = require('p-limit');
+const sftpClient  = require('ssh2-sftp-client');
+const through2    = require('through2');
+
+
+module.exports = function(ftpConfig, remotedir) {
+  const sftp      = new sftpClient();
+  const serial    = pLimit(1);
+  const parallel  = pLimit(ftpConfig.parallel);
+  const files     = [];
+
+  return through2.obj(
+
+    function transform (file, _, cb) {
+      if (file.isStream()) return cb(new Error('Streaming not supported.'));
+      if (file.stat.isDirectory()) return cb();
+
+      file.destination = path.join(remotedir, file.relative).toFws();
+      files.push(file); // collect all files in array
+      return cb();
+    },
+
+    function flush(cb) {
+      if (!files[0]) cb();
+
+      const paths = [...new Set(files.map(({destination}) => path.dirname(destination)))] // collect unique paths
+        .filter((p1, i, a) => !a.find(p2 => p1 != p2 && p2.includes(p1))) // remove paths being part of others
+      ;
+
+      sftp.connect(ftpConfig)
+      .then(() => Promise.all(paths.map(p => parallel( // check if all directories exist
+        () => sftp.exists(p).then(r => {
+          if (!!r) paths.splice(paths.indexOf(p), 1) // if exists, remove from paths
+        })
+      ))))
+      .then(() => Promise.all(paths.map(p => serial( // create directories that do not exist, which cannot be done parallel
+        () => {
+          _write(p.replace(remotedir, '').lblack);
+          return sftp.mkdir(p, true).catch(err => _warn(err));
+        }
+      ))))
+      .then(() => Promise.all(files.map(file => parallel( // upload files
+        () => {
+          ftpConfig.eventPut(file);
+          return ( // use fastPut for files larger than 3mb; since it requires a local path we assume these files are not changed in the gulp pipeline
+            file.contents.length > 3000000 ? sftp.fastPut(file.originalRelativePath, file.destination) : sftp.put(file.contents, file.destination)
+          ).catch(err => _warn(`File transfer failed${err ? ': '+err : ''}`));
+        }
+      ))))
+      .then(() => {
+        _info(`${files.length} file(s) transferred`);
+        cb();
+        return sftp.end();
+      })
+      .catch(err => _error(err));
+    }
+
+  )
+  .resume(); // required for triggering 'end' event
+};

package/src/modules/deploy/execute.js

@@ -1,23 +1,23 @@
-const fs            = require('fs-extra');
-const globby        = require('globby');
-const gulp          = require('gulp');
-const {NodeSSH}     = require('node-ssh');
-const through2      = require('through2');
-const {spawnSync}   = require('child_process');
-
-const g_babel_env   = require('babel-preset-es2015-without-strict');
-const g_babel       = require('gulp-babel');
-const g_eol         = require('gulp-eol');
-const g_filter      = require('gulp-filter');
-const g_plumber     = require('gulp-plumber');
-const g_print       = require('gulp-print');
-const g_replace     = require('../../lib/gulp-batch-replace-with-filter');
-const g_resolveIncl = require('../../lib/gulp-resolve-includes');
-const g_sass        = require('gulp-sass')(require('sass'));
-const g_scp         = require('../../lib/gulp-scp2-async');
-const g_sourcemaps  = require('gulp-sourcemaps');
-
-const execGitCommand = require('../../lib/exec-git-command');
+const execGitCommand  = require('../../lib/exec-git-command');
+const fs              = require('fs-extra');
+const globby          = require('globby');
+const gulp            = require('gulp');
+const {NodeSSH}       = require('node-ssh');
+const through2        = require('through2');
+const {spawnSync}     = require('child_process');
+
+const g_babel_env     = require('babel-preset-es2015-without-strict');
+const g_babel         = require('gulp-babel');
+const g_eol           = require('gulp-eol');
+const g_filter        = require('gulp-filter');
+const g_plumber       = require('gulp-plumber');
+const g_print         = require('gulp-print');
+const g_replace       = require('../../lib/gulp-batch-replace-with-filter');
+const g_resolveIncl   = require('../../lib/gulp-resolve-includes');
+const g_sass          = require('gulp-sass')(require('sass'));
+const g_sftp          = require('../../lib/gulp-sftp');
+const g_sourcemaps    = require('gulp-sourcemaps');
+
 
 const c = require('./config');
 const s = require('./srcset');
@@ -123,26 +123,27 @@ const transfer = (paths, lrServer) => {
   const sassF   = g_filter(['**/*.scss'], {restore: true});
   const jsF     = g_filter(['**/*.js', '!**/hce/**/template_*', '!**/fonto/**', '!**/vendor/**', '!**/*.min.js'], {restore: true});
   const shF     = g_filter(['**/*.sh'], {restore: true});
-  const ftpC    = Object.assign({dest: c.server.remotedir}, c.server.ftpConfig); // new obj because of async transfers
   const srcset  = s.create(paths);
   const files   = [];
 
-  const skippedFontoBuildFiles = [];
+  const fontoPaths = {outdated: [], uptodate: []};
   const removeOutdatedFontoBuild = g_filter(file => { // fonto build files must be newer than the last commit date containing fonto changes
     const prPath = 'config\\'+file.relative.replace(/fonto.+$/, '');
-    if (skippedFontoBuildFiles.includes(prPath)) return false;
+    if (fontoPaths.outdated.includes(prPath)) return false;
+    if (fontoPaths.uptodate.includes(prPath)) return true;
     if (file.relative.match(/fonto[\\\/]dist/)) {
       const paths = `${prPath}\\fonto ${prPath}\\schema ${_paths.tdi}`; // only check paths containing fonto sources
       const lastFontoCommitDate = execGitCommand(`log -1 --format=%cd --date=iso-strict origin/${_commit.branch} ${paths}`, _paths.repo);
       if (fs.statSync(file.relative).mtime < lastFontoCommitDate) {
-        skippedFontoBuildFiles.push(prPath);
+        fontoPaths.outdated.push(prPath);
         return false;
       }
+      else fontoPaths.uptodate.push(prPath);
     }
     return true;
   });
 
-  // disable file logging for ftp (c.ftpConfig.watch function does it already)
+  // disable file logging for ftp (gulp-sftp does it already)
   g_print.setLogFunction((filepath) => c.localTransfer ? _write(filepath.nostyle) : null);
 
   _info('Start transferring', true);
@@ -169,15 +170,16 @@ const transfer = (paths, lrServer) => {
   .pipe(g_plumber.stop())
   .pipe(g_replace(c.replaceStrings))
   .pipe(through2.obj((file, enc, cb) => {
-    file.path = file.path.replace(/(fonto)(?:[\\\/]dist|.+([\\\/]assets[\\\/]schemas))/, '$1$2'); // change destination path for fonto build and schemas
+    file.originalRelativePath = file.relative; // original path needed for sftp.fastPut
+    file.path = file.path.replace(/(fonto)[\\\/](dev|dist)/, '$1'); // change destination path for fonto build
     if (!file.relative.endsWith('.map')) files.push(file.relative); // collect all file paths in array for livereload
     cb(null, file);
   }))
   .pipe(g_print.default())
-  .pipe(c.localTransfer ? gulp.dest(c.server.remotedir) : g_scp(ftpC))
+  .pipe(c.localTransfer ? gulp.dest(c.server.remotedir) : g_sftp(c.server.ftpConfig, c.server.remotedir))
   .on('end', () => {
     _info('Finished transferring\n', true);
-    if (skippedFontoBuildFiles[0]) _warn(`Fonto build files in the following folders were outdated and therefore skipped:\n  ${skippedFontoBuildFiles.map(v => v.slice(0, -1)).join('\n  ')}`);
+    if (fontoPaths.outdated[0]) _warn(`Fonto build files in the following folders were outdated and therefore skipped:\n  ${fontoPaths.outdated.map(v => v.slice(0, -1)).join('\n  ')}`);
 
     remote.process();
 

package/src/modules/deploy/index.js

@@ -16,7 +16,7 @@ module.exports = function deploy (argv) {
   c.prepareForCopy(argv.filter);
 
   if (!c.localTransfer) {
-    c.server.ftpConfig.eventWrite = (file) => {
+    c.server.ftpConfig.eventPut = (file) => {
       _write(file.destination.replace(c.server.remotedir, ''));
       if      (path.extname(file.destination)=='.sh')
         remote.add('chmod 755 '+file.destination, 'Permissions set: '+file.destination);
@@ -60,12 +60,12 @@ module.exports = function deploy (argv) {
       }
     };
 
-    // for fonto, only watch packages (for schemas) and dist (for the build), but not dist/assets (causes build error / does not detect all files)
-    gulp.watch([...c.transferPatterns, '!**/fonto/{*.*, !({dist,packages})/**, dist/*/**}'])
+    // for fonto, only watch dev/dist (for the build, does not detect "assets" most of the times btw)
+    gulp.watch([...c.transferPatterns, `!**/fonto/!(${c.envDev?'dev|':''}dist)/**`])
     .on('all', (event, filepath) => {
 
       if ((event=='add' || event=='change') &&
-          (/fonto(.dist|.+schemas.+\.json)/.test(filepath) || !/fonto/.test(filepath)) // within fonto, only transfer build and schemas
+          (/fonto.(dev|dist)/.test(filepath) || !/fonto/.test(filepath)) // within fonto, only transfer build files
       ) {
         transfers.add(filepath);
       }

package/src/modules/fonto/commands.js

@@ -43,7 +43,7 @@ const cmdExec = command => new Promise((resolve, reject) => {
 
   cp.stdout.setEncoding('utf8');
   cp.stdout.on('data', data => {
-    if (data.includes('Error') || data.includes('Input error')) reject(data);
+    if (/Error|Input error/.test(data)) _warn(data);
     else log(data);
   });
   cp.stderr.setEncoding('utf8');
@@ -203,7 +203,7 @@ module.exports = {
       fs.removeSync('dist/assets');
       _write('Done.\n');
     })
-    .then(() => cmdExec('fdt editor build -M'))
+    .then(() => cmdExec('fdt editor build'))
     ;
   },
 
@@ -226,7 +226,7 @@ module.exports = {
       });
       _write('Done.\n');
     })
-    .then(() => cmdExec('fdt editor run'))
+    .then(() => cmdExec('fdt editor run --write-to-disk'))
     ;
   }
 

package/src/lib/gulp-scp2-async.js

@@ -1,95 +0,0 @@
-// based on gulp-scp2
-
-// core
-const path = require('path');
-// other
-const through2 = require('through2'), Client = require('scp2').Client;
-
-const fixWinPath = (str) => str.replace(/\\/g, '/');
-
-module.exports = function(options) {
-  const files = [];
-
-  options.watch = function(client){ // set file transfer watch function
-    client
-    .on('write', options.eventWrite)
-    .on('error', err => {})
-    ;
-  };
-
-  const stream = through2.obj(
-    function transform(file, enc, callback) {
-      if (file.isStream()) return callback(new Error('Streaming not supported.'));
-      if (file.stat.isDirectory()) return callback();
-
-      files.push(file); // collect all files in array
-
-      return callback();
-    },
-    function flush(callback) { // on callback start transfering all files in parallel threads
-      if (!files[0]) callback();
-
-      let transferring = 0, transferred = 0;
-
-      function uploadFile(client) {
-        const file = files.shift();
-
-        if (file) {
-          const filepath = fixWinPath(path.join(options.dest, file.relative));
-          transferring++;
-
-          client.mkdir(path.dirname(filepath), (err) => { // try creating dir
-            if (err) {
-              const fail = err.message=='Permission denied';
-              transferring--;
-              if (!fail) files.unshift(file); // add file back to queue
-              _warn((fail ? `Fail: ${err.message}` : `Server aborted connection. Retrying.`));
-              uploadFile(client); // retry
-            }
-            else {
-              client.write({ // upload file
-                destination: filepath,
-                content: file.contents
-              }, (err) => {
-                transferring--;
-                transferred++;
-                uploadFile(client);
-              });
-            }
-          });
-
-        }
-        else {
-          client.close();
-          if (transferring==0) {
-            transferring--;
-            _info(`${transferred} file(s) transferred`);
-            return callback();
-          }
-        }
-      }
-
-      function uploadThread() {
-        const client = new Client(options);
-        options.watch(client);
-        client.sftp(err => {
-          if (err) {
-            if      (err.toString().toLowerCase().includes('timed out')) _warn(`Connection timed out. Retrying.`);
-            else if (err.code=='ECONNRESET' || err.code=='ECONNABORTED') _warn(`Server aborted connection. Retrying.`);
-            else if (err.message=='All configured authentication methods failed') _error('Authentication failed. Check if SSH key is loaded.');
-            else _warn(err.toString());
-
-            uploadThread(); // retry
-          }
-          else uploadFile(client);
-        });
-      }
-
-      const parallel = Math.min(Math.round(files.length / 2), options.parallel);
-      for (let i = 0; i < parallel; i++) setTimeout(uploadThread, 100 * i);
-    }
-  );
-
-  stream.resume(); // required for triggering 'end' event
-  return stream;
-};