apostrophe 3.18.0 → 3.19.0

package/test/assets.js

@@ -1,7 +1,8 @@
 const t = require('../test-lib/test.js');
-const assert = require('assert');
+const assert = require('assert').strict;
 const fs = require('fs-extra');
 const path = require('path');
+const Promise = require('bluebird');
 
 const {
   checkModulesWebpackConfig,
@@ -13,11 +14,17 @@ let apos;
 
 const badModules = {
   badModuleConfig: {
+    options: {
+      ignoreNoCodeWarning: true
+    },
     webpack: {
       badprop: {}
     }
   },
   badModuleConfig2: {
+    options: {
+      ignoreNoCodeWarning: true
+    },
     webpack: []
   }
 };
@@ -78,19 +85,24 @@ const modules = {
 describe('Assets', function() {
   const {
     publicFolderPath,
+    cacheFolderPath,
     getScriptMarkup,
     getStylesheetMarkup,
     expectedBundlesNames,
     deleteBuiltFolders,
-    allBundlesAreIncluded
+    allBundlesAreIncluded,
+    removeCache,
+    getCacheMeta,
+    retryAssertTrue
   } = loadUtils();
 
   after(async function() {
     await deleteBuiltFolders(publicFolderPath, true);
+    await removeCache();
     return t.destroy(apos);
   });
 
-  this.timeout(90000);
+  this.timeout(5 * 60 * 1000);
 
   it('should exist on the apos object', async function() {
     apos = await t.create({
@@ -284,10 +296,407 @@ describe('Assets', function() {
     assert(bundlePage.includes(getScriptMarkup('extra2-module-bundle', 'module')));
     assert(bundlePage.includes(getScriptMarkup('extra2-nomodule-bundle', 'nomodule')));
   });
+
+  it('should build with cache and gain performance', async function() {
+    await t.destroy(apos);
+    await removeCache();
+    await removeCache(cacheFolderPath.replace('/webpack-cache', '/changed'));
+
+    const es5Modules = {
+      ...modules,
+      '@apostrophecms/asset': {
+        options: {
+          es5: true
+        }
+      }
+    };
+
+    apos = await t.create({
+      root: module,
+      modules: es5Modules
+    });
+    assert.throws(() => fs.readdirSync(cacheFolderPath), {
+      code: 'ENOENT'
+    });
+
+    let startTime;
+
+    // Cold run
+    startTime = Date.now();
+    await apos.asset.tasks.build.task({
+      'check-apos-build': false
+    });
+    const execTime = Date.now() - startTime;
+    const { meta, folders } = getCacheMeta();
+    assert.equal(folders.length, 3);
+    assert.equal(Object.keys(meta).length, 3);
+    assert(meta['default:apos']);
+    assert(meta['default:src']);
+    assert(meta['default:src-es5']);
+
+    // Cache
+    startTime = Date.now();
+    await apos.asset.tasks.build.task({
+      'check-apos-build': false
+    });
+    const execTimeCached = Date.now() - startTime;
+    const { meta: meta2, folders: folders2 } = getCacheMeta();
+    assert.equal(folders2.length, 3);
+    assert.equal(Object.keys(meta2).length, 3);
+    assert(meta2['default:apos']);
+    assert(meta2['default:src']);
+    assert(meta2['default:src-es5']);
+
+    // Expect at least 40% gain, in reallity it should be 50+
+    const gain = (execTime - execTimeCached) / execTime * 100;
+    assert(gain >= 20, `Expected gain >=20%, got ${gain}%`);
+
+    // Modification times
+    assert(meta['default:apos'].mdate);
+    assert(meta2['default:apos'].mdate);
+    assert(meta['default:src'].mdate);
+    assert(meta2['default:src'].mdate);
+    assert(meta['default:src-es5'].mdate);
+    assert(meta2['default:src-es5'].mdate);
+    assert(
+      new Date(meta['default:apos'].mdate) < new Date(meta2['default:apos'].mdate)
+    );
+    assert.equal(
+      new Date(meta2['default:apos'].mdate).toISOString(),
+      fs.statSync(meta2['default:apos'].location).mtime.toISOString()
+    );
+    assert(
+      new Date(meta['default:src'].mdate) < new Date(meta2['default:src'].mdate)
+    );
+    assert.equal(
+      new Date(meta2['default:src'].mdate).toISOString(),
+      fs.statSync(meta2['default:src'].location).mtime.toISOString()
+    );
+    assert(
+      new Date(meta['default:src-es5'].mdate) < new Date(meta2['default:src-es5'].mdate)
+    );
+    assert.equal(
+      new Date(meta2['default:src-es5'].mdate).toISOString(),
+      fs.statSync(meta2['default:src-es5'].location).mtime.toISOString()
+    );
+  });
+
+  it('should invalidate build cache when namespace changes', async function() {
+    process.env.APOS_DEBUG_NAMESPACE = 'test';
+    await apos.asset.tasks.build.task({
+      'check-apos-build': false
+    });
+    const { meta, folders } = getCacheMeta();
+    assert.equal(folders.length, 6);
+    assert.equal(Object.keys(meta).length, 6);
+    assert(meta['test:apos']);
+    assert(meta['test:src']);
+    assert(meta['test:src-es5']);
+    assert(meta['default:apos']);
+    assert(meta['default:src']);
+    assert(meta['default:src-es5']);
+    delete process.env.APOS_DEBUG_NAMESPACE;
+  });
+
+  it('should invalidate build cache when packages change', async function() {
+    await t.destroy(apos);
+    const lock = require('./package-lock.json');
+    assert.equal(lock.version, 'current');
+    lock.version = 'new';
+    fs.writeFileSync(
+      path.join(process.cwd(), 'test/package-lock.json'),
+      JSON.stringify(lock, null, '  '),
+      'utf8'
+    );
+    const es5Modules = {
+      ...modules,
+      '@apostrophecms/asset': {
+        options: {
+          es5: true
+        }
+      }
+    };
+
+    apos = await t.create({
+      root: module,
+      modules: es5Modules
+    });
+    await apos.asset.tasks.build.task({
+      'check-apos-build': false
+    });
+
+    const { meta, folders } = getCacheMeta();
+    assert.equal(folders.length, 9);
+    assert.equal(Object.keys(meta).length, 9);
+    assert(meta['default:apos_2']);
+    assert(meta['default:src_2']);
+    assert(meta['default:src-es5_2']);
+  });
+
+  it('should invalidate build cache when configuration changes', async function() {
+    await t.destroy(apos);
+    const es5Modules = {
+      ...modules,
+      'bundle-page': {
+        webpack: {
+          extensions: {
+            ext1: {
+              resolve: {
+                alias: {
+                  ext1: 'changed'
+                }
+              }
+            }
+          }
+        }
+      },
+      '@apostrophecms/asset': {
+        options: {
+          es5: true
+        }
+      }
+    };
+    apos = await t.create({
+      root: module,
+      modules: es5Modules
+    });
+    await apos.asset.tasks.build.task({
+      'check-apos-build': false
+    });
+
+    const { meta, folders } = getCacheMeta();
+    assert.equal(folders.length, 11);
+    assert.equal(Object.keys(meta).length, 11);
+    assert(!meta['default:apos_3']);
+    assert(meta['default:src_3']);
+    assert(meta['default:src-es5_3']);
+  });
+
+  it('should clear build cache', async function() {
+    const cacheFolders = fs.readdirSync(cacheFolderPath, 'utf8');
+    assert(cacheFolders.length > 0);
+    await apos.asset.tasks['clear-cache'].task();
+
+    assert.equal(fs.readdirSync(cacheFolderPath, 'utf8').length, 0);
+  });
+
+  it('should be able to override the build cache location via APOS_ASSET_CACHE', async function() {
+    await t.destroy(apos);
+    await removeCache();
+    const altCacheLoc = cacheFolderPath.replace('/webpack-cache', '/changed');
+    await removeCache(altCacheLoc);
+    process.env.APOS_ASSET_CACHE = altCacheLoc;
+
+    apos = await t.create({
+      root: module,
+      modules
+    });
+    assert.throws(() => fs.readdirSync(altCacheLoc), {
+      code: 'ENOENT'
+    });
+    await apos.asset.tasks.build.task({
+      'check-apos-build': false
+    });
+    const { meta, folders } = getCacheMeta(altCacheLoc);
+    assert.equal(folders.length, 2);
+    assert.equal(Object.keys(meta).length, 2);
+    assert(meta['default:apos']);
+    assert(meta['default:src']);
+
+    delete process.env.APOS_ASSET_CACHE;
+    await removeCache(altCacheLoc);
+  });
+
+  it('should watch and rebuild assets and reload page in development', async function() {
+    await t.destroy(apos);
+
+    apos = await t.create({
+      root: module,
+      autoBuild: true,
+      modules
+    });
+    const restartId = apos.asset.restartId;
+    assert(apos.asset.buildWatcher);
+    assert(apos.asset.restartId);
+
+    // Modify asset and rebuild
+    const assetPath = path.join(process.cwd(), 'test/modules/bundle-page/ui/src/extra.js');
+    const assetPathPublic = path.join(process.cwd(), 'test/public/apos-frontend/default/extra-module-bundle.js');
+    const assetContent = 'export default () => {};\n';
+    fs.writeFileSync(
+      assetPath,
+      'export default () => { \'bundle-page-watcher-test\'; };\n',
+      'utf8'
+    );
+
+    await retryAssertTrue(
+      async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test/),
+      'Unable to verify public asset was rebuilt by the watcher',
+      500,
+      10000
+    );
+
+    await retryAssertTrue(
+      () => apos.asset.restartId !== restartId,
+      'Unable to verify restartId has been changed',
+      500,
+      10000
+    );
+
+    await t.destroy(apos);
+    assert.equal(apos.asset.buildWatcher, null);
+    apos = null;
+    fs.writeFileSync(assetPath, assetContent, 'utf8');
+  });
+
+  it('should watch and rebuild assets in a debounced queue', async function() {
+    await t.destroy(apos);
+    let timesRebuilt = 0;
+    const inc = () => {
+      timesRebuilt += 1;
+    };
+
+    apos = await t.create({
+      root: module,
+      autoBuild: true,
+      modules: {
+        ...modules,
+        '@apostrophecms/asset': {
+          extendMethods() {
+            return {
+              async watchUiAndRebuild(_super) {
+                return _super(inc);
+              }
+            };
+          }
+        }
+      }
+    });
+    assert(apos.asset.buildWatcher);
+
+    const assetPath = path.join(process.cwd(), 'test/modules/bundle-page/ui/src/extra.js');
+    const assetPathPublic = path.join(process.cwd(), 'test/public/apos-frontend/default/extra-module-bundle.js');
+    const assetContent = 'export default () => {};\n';
+
+    // Modify below the debounce rate
+    for (const i of [ 1, 2, 3 ]) {
+      await fs.writeFile(
+        assetPath,
+        `export default () => { 'bundle-page-watcher-test-${i}'; };\n`,
+        'utf8'
+      );
+      await Promise.delay(300);
+    }
+    await retryAssertTrue(
+      async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test-3/),
+      'Unable to verify public asset rebuilding by the watcher',
+      500,
+      10000
+    );
+    await retryAssertTrue(
+      () => timesRebuilt === 1,
+      `Expected to rebuild 1 time, got ${timesRebuilt}`,
+      100,
+      5000
+    );
+
+    // Modify above the debounce rate, test the queue cap
+    timesRebuilt = 0;
+    for (const i of [ 1, 2, 3 ]) {
+      await fs.writeFile(
+        assetPath,
+        `export default () => { 'bundle-page-watcher-test-${i}0'; };\n`,
+        'utf8'
+      );
+      await Promise.delay(1050);
+    }
+    await retryAssertTrue(
+      async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test-30/),
+      'Unable to verify public asset rebuilding by the watcher',
+      500,
+      10000
+    );
+    await retryAssertTrue(
+      () => timesRebuilt === 2,
+      `Expected to rebuild 2 times, got ${timesRebuilt}`,
+      100,
+      5000
+    );
+
+    await t.destroy(apos);
+    apos = null;
+    fs.writeFileSync(assetPath, assetContent, 'utf8');
+  });
+
+  it('should be able to setup the debounce time', async function() {
+    await t.destroy(apos);
+
+    apos = await t.create({
+      root: module,
+      modules: {
+        '@apostrophecms/asset': {
+          options: {
+            watchDebounceMs: 500
+          }
+        }
+      }
+    });
+    assert.equal(apos.asset.buildWatcherDebounceMs, 500);
+  });
+
+  it('should not watch if explicitly disabled by option or env in development', async function() {
+    await t.destroy(apos);
+    process.env.APOS_ASSET_WATCH = '0';
+
+    apos = await t.create({
+      root: module,
+      autoBuild: true
+    });
+    assert(!apos.asset.buildWatcher);
+    delete process.env.APOS_ASSET_WATCH;
+    await t.destroy(apos);
+
+    apos = await t.create({
+      root: module,
+      autoBuild: true,
+      modules: {
+        '@apostrophecms/asset': {
+          options: {
+            watch: false
+          }
+        }
+      }
+    });
+    assert(!apos.asset.buildWatcher);
+  });
+
+  it('should not watch if autoBuild is disabled', async function() {
+    await t.destroy(apos);
+
+    apos = await t.create({
+      root: module
+    });
+    assert(!apos.asset.buildWatcher);
+  });
+
+  it('should not watch in production', async function() {
+    await t.destroy(apos);
+    process.env.NODE_ENV = 'production';
+
+    apos = await t.create({
+      root: module,
+      autoBuild: true,
+      modules
+    });
+    assert(!apos.asset.buildWatcher);
+    process.env.NODE_ENV = 'development';
+  });
 });
 
 function loadUtils () {
   const publicFolderPath = path.join(process.cwd(), 'test/public');
+  const cacheFolderPath = process.env.APOS_ASSET_CACHE ||
+              path.join(process.cwd(), 'test/data/temp/webpack-cache');
 
   const getScriptMarkup = (file, mod) => {
     const moduleStr = mod === 'module' ? ' type="module"' : ' nomodule';
@@ -318,12 +727,61 @@ function loadUtils () {
     assert(page.includes(getScriptMarkup('extra2-module-bundle')));
   };
 
+  const removeCache = async (loc) => {
+    await fs.remove(loc || cacheFolderPath);
+  };
+
+  const getCacheMeta = (loc) => {
+    const cacheFolders = fs.readdirSync(loc || cacheFolderPath, 'utf8');
+    const i = {};
+    const meta = cacheFolders
+      .reduce((prev, folder) => {
+        const location = `${loc || cacheFolderPath}/${folder}/.apos`;
+        const m = fs.readFileSync(location, 'utf8');
+        let [ mdate, id ] = m.split(' ');
+        // e.g. default:apos_2, default:apos_3, etc
+        if (prev[id]) {
+          i[id] = (i[id] || 1) + 1;
+          id = `${id}_${i[id]}`;
+        }
+        return {
+          ...prev,
+          [id]: {
+            mdate: new Date(mdate),
+            folder,
+            location
+          }
+        };
+      }, {});
+    return {
+      folders: cacheFolders,
+      meta
+    };
+  };
+
+  // Retry `max` ms with `delay` ms between the retries
+  // until `assertFn` returns true or fail with `failMsg`
+  const retryAssertTrue = async (assertFn, failMsg, delay, max) => {
+    let current = 0;
+    while (!(await assertFn())) {
+      await Promise.delay(delay);
+      current += delay;
+      if (current >= max) {
+        assert.fail(`${failMsg}`);
+      }
+    }
+  };
+
   return {
     publicFolderPath,
+    cacheFolderPath,
     getScriptMarkup,
     getStylesheetMarkup,
     expectedBundlesNames,
     deleteBuiltFolders,
-    allBundlesAreIncluded
+    allBundlesAreIncluded,
+    removeCache,
+    getCacheMeta,
+    retryAssertTrue
   };
 }

package/test/schemas.js

@@ -1652,6 +1652,31 @@ describe('Schemas', function() {
       age: null
     }, 'age', 'required');
   });
+
+  it('should save date and time with the right format', async function () {
+    const req = apos.task.getReq();
+    const schema = apos.schema.compose({
+      addFields: [
+        {
+          name: 'emptyValue',
+          type: 'dateAndTime'
+        },
+        {
+          name: 'goodValue',
+          type: 'dateAndTime'
+        }
+      ]
+    });
+
+    const output = {};
+    await apos.schema.convert(req, schema, {
+      emptyValue: null,
+      goodValue: '2022-05-09T22:36:00.000Z'
+    }, output);
+
+    assert(output.emptyValue === null);
+    assert(output.goodValue === '2022-05-09T22:36:00.000Z');
+  });
 });
 
 async function testSchemaError(schema, input, path, name) {

package/test-lib/test.js

@@ -34,4 +34,16 @@ for (const dir of dirs) {
 
 fs.writeFileSync(packageJson, JSON.stringify(packageJsonInfo, null, '  '));
 
+// A "project level" package-lock.json for checking webpack build cache
+
+const packageLockJson = path.join(__dirname, '/../test/package-lock.json');
+const packageLockJsonInfo = {
+  _: 'Do not change, fake lock used for testing',
+  name: 'apostrophe',
+  version: 'current',
+  packages: {}
+};
+fs.removeSync(packageLockJson);
+fs.writeFileSync(packageLockJson, JSON.stringify(packageLockJsonInfo, null, '  '));
+
 module.exports = require('./util.js');

package/test-lib/util.js

@@ -10,10 +10,11 @@ const mongodbConnect = require('../lib/mongodb-connect');
 // If `apos` is null, no work is done.
 
 async function destroy(apos) {
-  const dbName = apos.db && apos.db.databaseName;
-  if (apos) {
-    await apos.destroy();
+  if (!apos) {
+    return;
   }
+  await apos.destroy();
+  const dbName = apos.db && apos.db.databaseName;
   // TODO at some point accommodate nonsense like testing remote databases
   // that won't let us use dropDatabase, no shell available etc., but the
   // important principle here is that we should not have to have an apos