apostrophe 3.18.0 → 3.19.0

package/.stylelintrc

@@ -1,6 +1,7 @@
 {
   plugins: [
-    'stylelint-order', 'stylelint-declaration-strict-value'
+    'stylelint-order',
+    'stylelint-declaration-strict-value'
   ],
   rules: {
     'color-hex-length': 'short',

package/CHANGELOG.md

@@ -1,6 +1,27 @@
 # Changelog
 
-# 3.18.0 (2022-05-03)
+## 3.19.0
+
+### Adds
+
+* New schema field type `dateAndTime` added. This schema field type saves in ISO8601 format, as UTC (Universal Coordinated Time), but is edited in a user-friendly way in the user's current time zone and locale.
+* Webpack disk cache for better build performance in development and, if appropriately configured, production as well.
+* In development, Webpack rebuilds the front end without the need to restart the Node.js process, yielding an additional speedup. To get this speedup for existing projects, see the `nodemonConfig` section of the latest `package.json` in [a3-boilerplate](https://github.com/apostrophecms/a3-boilerplate) for the new "ignore" rules you'll need to prevent nodemon from stopping the process and restarting.
+* Added the new command line task `apostrophecms/asset:clear-cache` for clearing the webpack disk cache. This should be necessary only in rare cases where the configuration has changed in ways Apostrophe can't automatically detect.
+
+### 3.18.1
+
+### Adds
+
+* Webpack cache for build performance in development and production mode.
+* Add a watcher in development mode to rebuild the assets on changes in the same process.
+* Add asset task `apostrophecms/asset:clear-cache` for force clearing the webpack build cache.
+
+### Fixes
+
+* The admin UI now rebuilds properly in a development environment when new npm modules are installed in a multisite project (`apos.rootDir` differs from `apos.npmRootDir`).
+
+## 3.18.0 (2022-05-03)
 
 ### Adds
 
@@ -26,6 +47,7 @@
 * The `self.email` method of modules now correctly accepts a default `from` address configured for a specific module via the `from` subproperty of the `email` option to that module. Thanks to `chmdebeer` for pointing out the issue and the fix.
 * Fixes `_urls` not added on attachment fields when pieces API index is requested (#3643)
 * Fixes float field UI bug that transforms the value to integer when there is no field error and the first number after the decimal is `0`.
+* The `nestedModuleSubdirs` feature no longer throws an error and interrupts startup if a project contains both `@apostrophecms/asset` and `asset`, which should be considered separate module names.
 
 ## 3.17.0 (2022-03-31)
 

package/lib/moog-require.js

@@ -5,6 +5,7 @@ const path = require('path');
 const glob = require('glob');
 const importFresh = require('import-fresh');
 const resolveFrom = require('resolve-from');
+const regExpQuote = require('regexp-quote');
 
 module.exports = function(options) {
   const self = require('./moog')(options);
@@ -62,7 +63,12 @@ module.exports = function(options) {
         self._indexes = glob.sync(self.options.localModules + '/**/index.js');
       }
       const matches = self._indexes.filter(function(index) {
-        return index.endsWith('/' + type + '/index.js');
+        // Double-check that we're not confusing "@apostrophecms/asset" with "asset" by
+        // making sure "type" is not preceded by an npm namespace folder. If type is itself namespaced
+        // this never comes up (because npm namespaces don't nest). The risk is that a legitimate
+        // project level module that happens to end with the same name as a namespaced module
+        // will be rejected as a duplicate when nestedModuleSubdirs is present
+        return index.endsWith('/' + type + '/index.js') && !index.match(new RegExp(`/@[^/]+/${regExpQuote(type)}/index\\.js$`));
       });
       if (matches.length > 1) {
         throw new Error('The module ' + type + ' appears in multiple locations:\n' + matches.join('\n'));

package/modules/@apostrophecms/admin-bar/ui/apos/components/TheAposAdminBarLocale.vue

@@ -309,8 +309,8 @@ export default {
 
 .apos-available-locale {
   display: inline-block;
-  font-size: 10px;
   color: var(--a-primary);
+  font-size: var(--a-type-small);
 }
 
 .apos-available-locale:not(:last-of-type) {

package/modules/@apostrophecms/asset/index.js

@@ -4,10 +4,13 @@ const Promise = require('bluebird');
 const webpackModule = require('webpack');
 const globalIcons = require('./lib/globalIcons');
 const path = require('path');
+const util = require('util');
 const express = require('express');
 const { stripIndent } = require('common-tags');
 const { merge: webpackMerge } = require('webpack-merge');
 const cuid = require('cuid');
+const chokidar = require('chokidar');
+const _ = require('lodash');
 const {
   checkModulesWebpackConfig,
   getWebpackExtensions,
@@ -28,7 +31,13 @@ module.exports = {
     // If this option is true and process.env.NODE_ENV is not `production`,
     // the browser will refresh when the Apostrophe application
     // restarts. A useful companion to `nodemon`.
-    refreshOnRestart: false
+    refreshOnRestart: false,
+    // If false no UI assets sources will be watched in development.
+    // This option has no effect in production (watch disabled).
+    watch: true,
+    // Miliseconds to wait between asset sources changes before
+    // performing a build.
+    watchDebounceMs: 1000
   },
 
   async init(self) {
@@ -47,26 +56,17 @@ module.exports = {
     self.extraBundles = fillExtraBundles(verifiedBundles);
     self.webpackExtensions = extensions;
     self.verifiedBundles = verifiedBundles;
+    self.buildWatcherEnable = process.env.APOS_ASSET_WATCH !== '0' && self.options.watch !== false;
+    self.buildWatcherDebounceMs = parseInt(self.options.watchDebounceMs || 1000, 10);
+    self.buildWatcher = null;
   },
   handlers (self) {
     return {
       'apostrophe:modulesRegistered': {
         async runUiBuildTask() {
-          if (
-            // Do not automatically build the UI if we're starting from a task
-            !self.apos.isTask() &&
-            // Or if we're in production
-            process.env.NODE_ENV !== 'production' &&
-            // Or if we've set an app option to skip the auto build
-            self.apos.options.autoBuild !== false
-          ) {
-
-            checkModulesWebpackConfig(self.apos.modules, self.apos.task.getReq().t);
-            // If starting up normally, run the build task, checking if we
-            // really need to update the apos build
-            await self.apos.task.invoke('@apostrophecms/asset:build', {
-              'check-apos-build': true
-            });
+          const ran = await self.autorunUiBuildTask();
+          if (ran) {
+            await self.watchUiAndRebuild();
           }
         },
         injectAssetsPlaceholders() {
@@ -79,6 +79,12 @@ module.exports = {
           if (self.uploadfs && (self.uploadfs !== self.apos.uploadfs)) {
             await Promise.promisify(self.uploadfs.destroy)();
           }
+        },
+        async destroyBuildWatcher() {
+          if (self.buildWatcher) {
+            await self.buildWatcher.close();
+            self.buildWatcher = null;
+          }
         }
       }
     };
@@ -111,6 +117,8 @@ module.exports = {
         usage: 'Build Apostrophe frontend CSS and JS bundles',
         afterModuleInit: true,
         async task(argv) {
+          // The lock could become huge, cache it, see computeCacheMeta()
+          let packageLockContentCached;
           const req = self.apos.task.getReq();
           const namespace = self.getNamespace();
           const buildDir = `${self.apos.rootDir}/apos-build/${namespace}`;
@@ -317,7 +325,13 @@ module.exports = {
                 ? webpackMerge(webpackInstanceConfig, ...Object.values(self.webpackExtensions))
                 : webpackInstanceConfig;
 
+              // Inject the cache location at the end - we need the merged
+              const cacheMeta = await computeCacheMeta(name, webpackInstanceConfigMerged);
+              webpackInstanceConfigMerged.cache.cacheLocation = cacheMeta.location;
+
               const result = await webpack(webpackInstanceConfigMerged);
+              await writeCacheMeta(name, cacheMeta);
+
               if (result.compilation.errors.length) {
                 // Throwing a string is appropriate in a command line task
                 throw cleanErrors(result.toString('errors'));
@@ -632,11 +646,9 @@ module.exports = {
           async function lockFileIsNewer(name) {
             const timestamp = fs.readFileSync(`${bundleDir}/${name}-build-timestamp.txt`, 'utf8');
             let pkgStats;
-
-            if (await fs.pathExists(`${self.apos.rootDir}/package-lock.json`)) {
-              pkgStats = await fs.stat(`${self.apos.rootDir}/package-lock.json`);
-            } else if (await fs.pathExists(`${self.apos.rootDir}/yarn.lock`)) {
-              pkgStats = await fs.stat(`${self.apos.rootDir}/yarn.lock`);
+            const packageLock = await findPackageLock();
+            if (packageLock) {
+              pkgStats = await fs.stat(packageLock);
             }
 
             const pkgTimestamp = pkgStats && pkgStats.mtimeMs;
@@ -644,6 +656,18 @@ module.exports = {
             return pkgTimestamp > parseInt(timestamp);
           }
 
+          async function findPackageLock() {
+            const packageLockPath = path.join(self.apos.npmRootDir, 'package-lock.json');
+            const yarnPath = path.join(self.apos.npmRootDir, 'yarn.lock');
+            if (await fs.pathExists(packageLockPath)) {
+              return packageLockPath;
+            } else if (await fs.pathExists(yarnPath)) {
+              return yarnPath;
+            } else {
+              return false;
+            }
+          }
+
           function getComponentName(component, { enumerateImports } = {}, i) {
             return path
               .basename(component)
@@ -657,6 +681,93 @@ module.exports = {
             // confused by this
             return errors.replace(/(ERROR in[\s\S]*?Module parse failed[\s\S]*)You may need an appropriate loader.*/, '$1');
           }
+
+          // A (CPU intensive) webpack cache helper to compute a hash and build paths.
+          // Cache the result when possible.
+          // The base cache path is by default `data/temp/webpack-cache`
+          // but it can be overridden by an APOS_ASSET_CACHE environment.
+          // In order to compute an accurate hash, this helper needs
+          // the final, merged webpack configuration.
+          async function computeCacheMeta(name, webpackConfig) {
+            const cacheBase = self.getCacheBasePath();
+
+            if (!packageLockContentCached) {
+              const packageLock = await findPackageLock();
+              if (packageLock === false) {
+                // this should happen only when testing and
+                // we don't want to break all non-core module tests
+                packageLockContentCached = 'none';
+              } else {
+                packageLockContentCached = await fs.readFile(packageLock, 'utf8');
+              }
+            }
+
+            // Plugins can output timestamps or other random information as
+            // configuration (e.g. StylelintWebpackPlugin). As we don't have
+            // control over plugins, we need to remove their configuration values.
+            // We keep only the plugin name and config keys sorted list.
+            // Additionally plugins are sorted by their constructor names.
+            // A shallow clone is enough to avoid modification of the original
+            // config.
+            const config = { ...webpackConfig };
+            config.plugins = (config.plugins || []).map(p => {
+              const result = [];
+              if (p.constructor && p.constructor.name) {
+                result.push(p.constructor.name);
+              }
+              const keys = Object.keys(p);
+              keys.sort();
+              result.push(...keys);
+              return result;
+            });
+            config.plugins.sort((a, b) => (a[0] || '').localeCompare(b[0]));
+            const configString = util.inspect(config, {
+              depth: 10,
+              compact: true,
+              breakLength: Infinity
+            });
+            const hash = self.apos.util.md5(
+              `${self.getNamespace()}:${name}:${packageLockContentCached}:${configString}`
+            );
+            const location = path.resolve(cacheBase, hash);
+
+            return {
+              base: cacheBase,
+              hash,
+              location
+            };
+          }
+
+          // Add .apos, useful for debugging, testing and cache invalidation.
+          // It also keeps in sync the modified time of the cache folder.
+          async function writeCacheMeta(name, cacheMeta) {
+            try {
+              const cachePath = path.join(cacheMeta.location, '.apos');
+              const lastModified = new Date();
+              await fs.writeFile(
+                cachePath,
+                `${lastModified.toISOString()} ${self.getNamespace()}:${name}`,
+                'utf8'
+              );
+              // should be the same as the meta
+              await fs.utimes(cachePath, lastModified, lastModified);
+            } catch (e) {
+              // Build probably failed, path is missing, ignore
+            }
+          }
+        }
+      },
+
+      'clear-cache': {
+        usage: 'Clear build cache',
+        afterModuleInit: true,
+        async task(argv) {
+          const cacheBaseDir = self.getCacheBasePath();
+
+          await fs.emptyDir(cacheBaseDir);
+          self.apos.util.log(
+            self.apos.task.getReq().t('apostrophe:assetWebpackCacheCleared')
+          );
         }
       }
     };
@@ -757,6 +868,144 @@ module.exports = {
           return `/apos-frontend/${namespace}`;
         }
       },
+      getCacheBasePath() {
+        return process.env.APOS_ASSET_CACHE ||
+              path.join(self.apos.rootDir, 'data/temp/webpack-cache');
+      },
+      // Run build task automatically when appropriate
+      async autorunUiBuildTask() {
+        if (
+        // Do not automatically build the UI if we're starting from a task
+          !self.apos.isTask() &&
+            // Or if we're in production
+            process.env.NODE_ENV !== 'production' &&
+            // Or if we've set an app option to skip the auto build
+            self.apos.options.autoBuild !== false
+        ) {
+
+          checkModulesWebpackConfig(self.apos.modules, self.apos.task.getReq().t);
+          // If starting up normally, run the build task, checking if we
+          // really need to update the apos build
+          await self.apos.task.invoke('@apostrophecms/asset:build', {
+            'check-apos-build': true
+          });
+          return true;
+        }
+        return false;
+      },
+      // Start watching assets from `modules/` and
+      // every symlinked package found in `node_modules/`.
+      // `rebuildCallback` is invoked with queue length argument
+      //  on actual build attempt only.
+      // It's there mainly for testing and debugging purposes.
+      async watchUiAndRebuild(rebuildCallback) {
+        if (!self.buildWatcherEnable || self.buildWatcher) {
+          return;
+        }
+        const rootDir = self.apos.rootDir;
+        // chokidar may invoke ready event multiple times,
+        // we want one "watch enabled" message.
+        let loggedOnce = false;
+        const logOnce = (...msg) => {
+          if (!loggedOnce) {
+            self.apos.util.log(...msg);
+            loggedOnce = true;
+          }
+        };
+        const error = self.apos.util.error;
+        const queue = [];
+        let queueLength = 0;
+        let queueRunning = false;
+
+        const debounceRebuild = _.debounce(chain, self.buildWatcherDebounceMs, {
+          leading: false,
+          trailing: true
+        });
+
+        const symLinkModules = await findSymlinks();
+        const watchDirs = [
+          './modules/**/ui/apos/**',
+          './modules/**/ui/src/**',
+          './modules/**/ui/public/**',
+          ...symLinkModules.reduce(
+            (prev, m) => [
+              ...prev,
+              `./node_modules/${m}/modules/**/ui/apos/**`,
+              `./node_modules/${m}/modules/**/ui/src/**`,
+              `./node_modules/${m}/modules/**/ui/public/**`
+            ],
+            []
+          )
+        ];
+        self.buildWatcher = chokidar.watch(watchDirs, {
+          cwd: rootDir,
+          ignoreInitial: true
+        });
+
+        self.buildWatcher
+          .on('add', debounceRebuild)
+          .on('change', debounceRebuild)
+          .on('unlink', debounceRebuild)
+          .on('addDir', debounceRebuild)
+          .on('unlinkDir', debounceRebuild)
+          .on('error', e => error(`Watcher error: ${e}`))
+          .on('ready', () => logOnce(
+            self.apos.task.getReq().t('apostrophe:assetBuildWatchStarted')
+          ));
+
+        async function rebuild() {
+          await self.autorunUiBuildTask();
+          self.restartId = self.apos.util.generateId();
+          if (typeof rebuildCallback === 'function') {
+            rebuildCallback(queueLength);
+          };
+        };
+
+        // Simple, capped, self-exhausting queue implementation.
+        function enqueue(fn) {
+          if (queueLength === 2) {
+            return;
+          }
+          queue.push(fn);
+          queueLength++;
+        };
+        async function dequeue() {
+          if (!queueLength) {
+            queueRunning = false;
+            return;
+          }
+          queueRunning = true;
+          await queue.pop()();
+          queueLength--;
+          await dequeue();
+        }
+        async function chain(f) {
+          enqueue(rebuild);
+          if (!queueRunning) {
+            await dequeue();
+          }
+        }
+
+        // Find all symlinks in node modules.
+        // This would find both `module-name` and `@company/module-name`
+        // package symlinks
+        async function findSymlinks(sub = '') {
+          let result = [];
+          const handle = await fs.promises.opendir(path.join(rootDir, 'node_modules', sub));
+          let mod = await handle.read();
+          while (mod) {
+            if (mod.isSymbolicLink()) {
+              result.push(sub + mod.name);
+            } else if (!sub && mod.name.startsWith('@')) {
+              const dres = await findSymlinks(`${mod.name}/`);
+              result = [ ...result, ...dres ];
+            }
+            mod = await handle.read();
+          }
+          await handle.close();
+          return result;
+        }
+      },
       // An implementation method that you should not need to call.
       // Sets a predetermined configuration for the frontend builds.
       // If you are trying to enable IE11 support for ui/src, use the
@@ -903,9 +1152,17 @@ module.exports = {
           // Long polling: keep the logs quiet by responding slowly, except the
           // first time. If we restart, the request will fail immediately,
           // and the client will know to try again with `fast`. The client also
-          // uses `fast` the first time
-          if (!req.query.fast) {
-            await Promise.delay(30000);
+          // uses `fast` the first time.
+          if (req.query.fast) {
+            return self.restartId;
+          }
+          // Long polling will be interrupted if restartId changes.
+          let delay = 30000;
+          const step = 300;
+          const oldRestartId = self.restartId;
+          while (delay > 0 && oldRestartId === self.restartId) {
+            delay -= step;
+            await Promise.delay(step);
           }
           return self.restartId;
         }

package/modules/@apostrophecms/asset/lib/webpack/apos/webpack.config.js

@@ -33,6 +33,13 @@ module.exports = ({
       path: outputPath,
       filename: outputFilename
     },
+    // cacheLocation will be added dynamically later
+    cache: {
+      type: 'filesystem',
+      buildDependencies: {
+        config: [ __filename ]
+      }
+    },
     // we could extend this with aliases for other apostrophe modules
     // at a later date if needed
     resolveLoader: {

package/modules/@apostrophecms/asset/lib/webpack/src/webpack.config.js

@@ -50,6 +50,13 @@ module.exports = ({
           : `[name]-${moduleName}-bundle.js`;
       }
     },
+    // cacheLocation will be added dynamically later
+    cache: {
+      type: 'filesystem',
+      buildDependencies: {
+        config: [ __filename ]
+      }
+    },
     resolveLoader: {
       extensions: [ '*', '.js' ],
       // Make sure css-loader and postcss-loader can always be found, even

package/modules/@apostrophecms/doc-type/ui/apos/components/AposDocEditor.vue

@@ -125,6 +125,11 @@ export default {
     AposAdvisoryLockMixin,
     AposArchiveMixin
   ],
+  provide () {
+    return {
+      originalDoc: this.originalDoc
+    };
+  },
   props: {
     moduleName: {
       type: String,
@@ -153,6 +158,9 @@ export default {
       },
       triggerValidation: false,
       original: null,
+      originalDoc: {
+        ref: null
+      },
       published: null,
       errorCount: 0,
       restoreOnly: false,
@@ -363,7 +371,8 @@ export default {
     manuallyPublished() {
       this.saveMenu = this.computeSaveMenu();
     },
-    original() {
+    original(newVal) {
+      this.originalDoc.ref = newVal;
       this.saveMenu = this.computeSaveMenu();
     },
     tabs() {

package/modules/@apostrophecms/i18n/i18n/en.json

@@ -35,6 +35,9 @@
   "assetTypeBuilding": "🧑‍💻 Building the {{ label }}...",
   "assetWebpackConfigWarning": "⚠️ In the module {{ module }}, your webpack config is incorrect. It must be an object and should contain only two properties extensions and bundles.",
   "assetWebpackBundlesWarning": "⚠️ In the module {{ module }} your webpack config is incorrect. Each bundle can only have one property 'templates' that must be an array of strings.",
+  "assetWebpackCacheCleared": "Build cache cleared.",
+  "assetBuildWatchStarted": "Watching for UI changes...",
+  "at": "at",
   "back": "Back",
   "backToHome": "Back to Home",
   "basics": "Basics",

package/modules/@apostrophecms/i18n/i18n/es.json

@@ -25,6 +25,7 @@
   "archivingWillLoseDraftChanges": "Cambios al borrador de este {{ type }} sin publicar serán eliminados permanentemente.",
   "archivingWillUnpublish": "Esto también despublicará el {{ type }}.",
   "areYouSure": "¿Está Seguro?",
+  "at": "a las",
   "aspectRatio": "Relación de aspecto",
   "aspectRatioFree": "Libre",
   "aspectRatioWarning": "La relación de aspecto no se puede cambiar para este widget",

package/modules/@apostrophecms/i18n/i18n/pt-BR.json

@@ -30,6 +30,7 @@
   "aspectRatioWarning": "A proporção não pode ser alterada para este widget",
   "assetTypeBuildComplete": "👍 {{ label }} completado!",
   "assetTypeBuilding": "🧑‍💻 Fazendo o build: {{ label }}...",
+  "at": "às",
   "back": "Voltar",
   "backToHome": "Voltar ao início",
   "basics": "Básico",

package/modules/@apostrophecms/i18n/i18n/sk.json

@@ -33,6 +33,7 @@
   "aspectRatioWarning": "Pre túto miniaplikáciu nie je možné zmeniť pomer strán",
   "assetTypeBuildComplete": "👍 {{ label }} je kompletný!",
   "assetTypeBuilding": "🧑‍💻 Budovanie {{ label }}...",
+  "at": "o",
   "back": "Naspäť",
   "backToHome": "Naspäť na domovskú stránku",
   "basics": "Základy",

package/modules/@apostrophecms/image/ui/apos/components/AposImageCropper.vue

@@ -428,15 +428,15 @@ export default {
   // flex-grow: 0;
 
   .apos-image-focal-point {
+    z-index: $z-index-default;
     position: absolute;
-    z-index: 1;
     width: 10px;
     height: 10px;
     border-radius: 50%;
     border: 1px solid var(--a-white);
     background-color: var(--a-primary);
     box-shadow: 0 0 4px var(--a-black);
-    transition: left 0.1s ease, top 0.1s ease;
+    transition: left 0.15s ease, top 0.15s ease;
     cursor: grab;
   }
 }

package/modules/@apostrophecms/image/ui/apos/components/AposImageRelationshipEditor.vue

@@ -427,7 +427,7 @@ export default {
 .apos-schema {
   margin: 30px 15px 0;
 
- .apos-field {
+  .apos-field {
     margin-bottom: 20px;
 
     &__label {
@@ -448,9 +448,9 @@ export default {
   flex-direction: row;
 
   .apos-field {
+    position: relative;
     display: flex;
     align-items: center;
-    position: relative;
     flex-grow: 1;
 
     &:first-child {
@@ -472,7 +472,7 @@ export default {
   }
 }
 
-.apos-input[type="number"] {
+.apos-input[type='number'] {
   padding-right: 5px;
 }
 
@@ -495,16 +495,16 @@ export default {
 }
 
 .apos-field__label--aligned {
-  margin: 0
+  margin: 0;
 }
 
 .apos-image-cropper__container {
   display: flex;
   justify-content: center;
   align-items: center;
-  margin: 30px 10%;
   // We remove the modal's paddings - header height - container margin
   height: calc(100vh - 40px - 75px - 60px);
+  margin: 30px 10%;
   box-sizing: border-box;
 }
 </style>

package/modules/@apostrophecms/login/ui/apos/components/TheAposLoginHeader.vue

@@ -89,15 +89,16 @@ export default {
 
   &__header--tiny {
     flex-direction: row;
+    // stylelint-disable-next-line scale-unlimited/declaration-strict-value
     color: #F8F9FA;
 
     .apos-login__project {
-      opacity: 0.7
+      opacity: 0.7;
     }
 
     .apos-login__project-name {
+      // stylelint-disable-next-line scale-unlimited/declaration-strict-value
       font-size: 21px;
-
     }
 
     .apos-login__project-env {