@tangelo/tangelo-configuration-toolkit 1.9.1 → 1.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangelo/tangelo-configuration-toolkit",
-  "version": "1.9.1",
+  "version": "1.10.0",
   "description": "Tangelo Configuration Toolkit is a command-line toolkit which offers support for developing a Tangelo configuration.",
   "bin": {
     "tct": "bin/index.js",
@@ -41,7 +41,7 @@
     "node-ssh": "^12.0.2",
     "object-assign-deep": "^0.4.0",
     "p-limit": "^3.1.0",
-    "replace-in-file": "^3.4.4",
+    "replace-in-file": "^6.3.2",
     "sass": "^1.43.5",
     "saxon-js": "^2.3.0",
     "ssh2-sftp-client": "^7.2.2",

package/src/cli.js

@@ -72,6 +72,7 @@ module.exports = function cli () {
       init: {alias: 'i', desc: `Initialize editor (optionally pass version number or 'latest')`},
       schema: {alias: 's', desc: 'Compile schemas to json'},
       elements: {alias: 'e', desc: 'List schema elements not having a default configuration'},
+      attributes: {alias: 'a', desc: 'List schema attributes not having a default configuration'},
       localize: {alias: 'l', desc: 'Update localization file'},
       build: {alias: 'b', desc: 'Build the editor instance'},
       run: {alias: 'r', desc: 'Start editor instance on localhost and watch changes'}

package/src/lib/gulp-sftp.js

@@ -25,7 +25,7 @@ module.exports = function(ftpConfig, remotedir) {
       if (!files[0]) cb();
 
       const paths = [...new Set(files.map(({destination}) => path.dirname(destination)))] // collect unique paths
-        .filter((p1, i, a) => !a.find(p2 => p1 != p2 && p2.includes(p1))) // remove paths being part of others
+        .filter((p1, i, a) => !a.find(p2 => p1 != p2 && p2.includes(p1+'/'))) // remove paths being part of others
       ;
 
       sftp.connect(ftpConfig)

package/src/modules/deploy/execute.js

@@ -1,3 +1,4 @@
+const babel_pe        = require('@babel/preset-env');
 const execGitCommand  = require('../../lib/exec-git-command');
 const fs              = require('fs-extra');
 const globby          = require('globby');
@@ -6,7 +7,7 @@ const {NodeSSH}       = require('node-ssh');
 const through2        = require('through2');
 const {spawnSync}     = require('child_process');
 
-const g_babel         = require('gulp-babel')({presets: [[require('@babel/preset-env'), {modules: false}]], comments: false, minified: true});
+const g_babel         = require('gulp-babel');
 const g_eol           = require('gulp-eol');
 const g_filter        = require('gulp-filter');
 const g_plumber       = require('gulp-plumber');
@@ -155,7 +156,7 @@ const transfer = (paths, lrServer) => {
   .pipe(xpsF.restore)
   .pipe(jsF)
   .pipe(g_sourcemaps.init())
-  .pipe(g_babel)
+  .pipe(g_babel({presets: [[babel_pe, {modules: false}]], comments: false, minified: true})) // function must be executed here, otherwise second execution during watch fails
   .pipe(g_sourcemaps.write('.'))
   .pipe(jsF.restore)
   .pipe(sassF)

package/src/modules/deploy/index.js

@@ -49,15 +49,17 @@ module.exports = function deploy (argv) {
     const transfers = {
       queue: [],
       do () {
+        this.queue = this.queue.map(p => p.replace(/(fonto.)(dist|dev|packages.sx-shell-.*.src)(.assets.schemas)/, '$1dist$3')) // fonto schema changes can be detected in different paths at the same time
         this.queue = [...new Set(this.queue)]; // remove duplicates
         this.queue.forEach(v => s.addToCache(v));
         transfer(this.queue, lrServer);
         this.queue = [];
       },
       add (fp) {
+        const delay = !fp.match(/fonto/) ? 500 : 1500; // longer delay for fonto build files
         this.queue.push(fp);
         clearTimeout(this.delayedExec);
-        this.delayedExec = setTimeout(()=>transfers.do(), 500);
+        this.delayedExec = setTimeout(()=>transfers.do(), delay);
       }
     };
 
@@ -67,12 +69,17 @@ module.exports = function deploy (argv) {
       sftp.connect(c.server.ftpConfig).then(() => sftp.end()).catch(err => _error(`Could not connect to server${err ? ': '+err.message : ''}`));
     }
 
-    // for fonto, only watch dev/dist (for the build, does not detect "assets" most of the times btw)
-    gulp.watch([...c.transferPatterns, `!**/fonto/!(${c.envDev?'dev|':''}dist)/**`])
+    gulp.watch(c.transferPatterns)
     .on('all', (event, filepath) => {
 
-      if ((event=='add' || event=='change') &&
-          (/fonto.(dev|dist)/.test(filepath) || !/fonto/.test(filepath)) // within fonto, only transfer build files
+      if ((event=='add' || event=='change') && (
+            // within fonto, transfer build files only, but also schema files, because
+            // the "dist" folder isn't watched properly: it does not detect "assets" anymore after building once
+            !/fonto[\\\/]/.test(filepath) ||
+            /fonto[\\\/]dist/.test(filepath) ||
+            (/fonto[\\\/]dev/.test(filepath) && c.envDev) ||
+            /fonto[\\\/]packages[\\\/]sx-shell-.*?[\\\/]assets[\\\/]schemas[\\\/]/.test(filepath)
+          )
       ) {
         transfers.add(filepath);
       }

package/src/modules/fonto/commands.js

@@ -92,6 +92,7 @@ module.exports = {
 
       const {rootSchemas} = fs.readJsonSync('../schema/fonto.json');
       const fontoManifest = fs.readJsonSync('config/fonto-manifest.json');
+      let jsonPaths = [];
 
       // remove all schema packages (sx-shell-*) from fonto-manifest.json
       Object.keys(fontoManifest.dependencies).filter(k => k.includes('sx-shell-')).forEach(k => {
@@ -100,13 +101,14 @@ module.exports = {
 
       // add dependency for sx-module to each package fonto-manifest.json
       // add schema packages in fonto.json to config/fonto-manifest.json
-      Object.entries(rootSchemas).forEach(([path, obj]) => {
+      Object.entries(rootSchemas).forEach(([xsdPath, obj]) => {
         const pckPath = 'packages/' + obj.packageName;
         const pckFontoManifest = fs.readJsonSync(pckPath + '/fonto-manifest.json', {throws: false}) || {dependencies: {}};
         pckFontoManifest.dependencies['sx-module'] = 'packages/sx-module';
         fs.outputJsonSync(pckPath + '/fonto-manifest.json', pckFontoManifest, {spaces: 2});
         _write(`${pckPath}/fonto-manifest.json`);
         fontoManifest.dependencies[obj.packageName] = pckPath;
+        jsonPaths.push(`${pckPath}/src/assets/schemas/${obj.packageName}.json`);
       });
 
       fs.outputJsonSync('config/fonto-manifest.json', fontoManifest, {spaces: 2});
@@ -119,6 +121,13 @@ module.exports = {
       _write('config/schemaExperienceResolver.js');
 
       _write();
+      _info('Copying schema json files to build folders:');
+      // deploy-watch copies schema's to server directly, so update local builds to keep integrity
+      jsonPaths.forEach(jsonPath => {
+        fs.copySync(jsonPath, 'dev/assets/schemas/' + jsonPath.match(/[^//]*$/)[0]);
+        fs.copySync(jsonPath, 'dist/assets/schemas/' + jsonPath.match(/[^//]*$/)[0]);
+      })
+      _write('Done.\n');
     })
     ;
   },
@@ -133,7 +142,7 @@ module.exports = {
       let ignoreElements = [];
       fs.readdirSync(tdiXsdPath).forEach(filename => {
         const file = fs.readFileSync(tdiXsdPath + filename, {encoding: 'UTF-8'});
-        ignoreElements.push(...file.match(/(?<=xs:element name=")[^"]+(?=")/g));
+        ignoreElements.push(...(file.match(/(?<=xs:element name=")[^"]+(?=")/g)||[]));
       });
       ignoreElements = [...new Set(ignoreElements)]; // remove duplicates
 
@@ -169,6 +178,54 @@ module.exports = {
     );
   },
 
+  attributes () {
+    _info('Attributes requiring custom configuration:');
+
+    const findAttributes = ({rootSchemas, tdiXsdPath}) => { // cannot use variables from outside this function
+      const {execSync} = require('child_process'), fs = require('fs');
+
+      // get all attributes defined in default schema
+      let ignoreAttributes = [];
+      fs.readdirSync(tdiXsdPath).forEach(filename => {
+        const file = fs.readFileSync(tdiXsdPath + filename, {encoding: 'UTF-8'});
+        ignoreAttributes.push(...(file.match(/(?<=xs:attribute name=")[^"]+(?=")/g)||[]));
+      });
+      ignoreAttributes = [...new Set(ignoreAttributes)]; // remove duplicates
+
+      // execute "fdt attributes" for each schema package, ignore default attributes, and combine results
+      const schemasPerAttribute = {};
+      Object.entries(rootSchemas).forEach(([path, obj]) => {
+        const data              = execSync(`fdt attributes --schema ${obj.packageName}`, {encoding: 'UTF-8'});
+        const attributes        = data.replace(/^.*?Default value\s+|\s+Printed name\*.*$/gs, '').split(/\n\s+/).map(a => a.split(/\s+/)).map(a =>
+          a[0] + (a[2]=='required' ? ' (required)' : '') + (a[3]=='-' ? ' (no default)' : '')
+        );
+        const customAttributes  = [...new Set(attributes)].filter(e => e && !ignoreAttributes.includes(e.replace(/ .*/, '')));
+
+        customAttributes.forEach(e => {
+          if (schemasPerAttribute[e]) schemasPerAttribute[e] = [...schemasPerAttribute[e], obj.packageName];
+          else schemasPerAttribute[e] = [obj.packageName];
+        });
+      });
+
+      const result = [];
+      Object.entries(schemasPerAttribute).forEach(([e, s]) => result.push([e, s])); // convert object to array
+      return result[0] ? result : 'No results.';
+    };
+
+    return wws(
+      'searching',
+      findAttributes,
+      {
+        rootSchemas: fs.readJsonSync('../schema/fonto.json').rootSchemas,
+        tdiXsdPath: path.join(_paths.repo, _paths.tdi, 'src/config/cmscustom/[customer]/[project]/schema/xsd/')
+      },
+      result => {
+        if (Array.isArray(result)) result.forEach(([e, s]) => _write(e, s, '\n'));
+        else _write(result, '\n');
+      }
+    );
+  },
+
   localize () {
     const templateFile = 'messages-template-packages.json';
     const messagesFile = 'packages/localization/src/messages.nl.json';

package/src/modules/fonto/index.js

@@ -31,6 +31,7 @@ module.exports = function fonto (argv) {
       .then(argv.init && (()=>commands.init(argv.init)))
       .then(argv.schema && commands.schema)
       .then(argv.elements && commands.elements)
+      .then(argv.attributes && commands.attributes)
       .then(argv.localize && commands.localize)
       .then(argv.build && commands.build)
       .then(argv.run && commands.run)

package/src/modules/migrate/steps.js

@@ -2,10 +2,10 @@ const fs        = require('fs-extra');
 const globby    = require('globby');
 const minimatch = require('minimatch');
 const path      = require('path');
-const rif       = require('replace-in-file');
+const rif      = require('replace-in-file');
 
 
-const getPaths  = (search, filter)  =>
+const getPaths = (search, filter) =>
   globby
   .sync(search, {dot: true, ignore: [_paths.tdi + '/**','**/cmscustom/tdi/**']})
   .filter(p => !filter || minimatch(p, filter)
@@ -47,9 +47,11 @@ module.exports = function steps (step, dry, filter) {
         r.to    = r.fromtoPairs.map(p => p[1]);
       }
 
-      r.files = rif.sync(r);                                                // execute first time
-      const filesModCount = r.files.length;                                 // save file count
-      for (let i=0; i<20 && !dry && r.files[0]; i++) r.files = rif.sync(r); // execute repeatedly for modified files only (with safety limit of 20)
+      let filesModCount = 0;
+      for (let i=0; i<20 && !dry && r.files[0]; i++) { // execute repeatedly for modified files only (with safety limit of 20)
+        r.files = rif.sync(r).filter(f => f.hasChanged).map(f => f.file);
+        if (i==0) filesModCount = r.files.length; // save count only after first run (after this only subsets are processed)
+      }
 
       _write('Files modified:', filesModCount);
     });