@maka/meteor-sdk 0.2.8 → 0.2.10

package/tools/isobuild/meteor-npm.js

@@ -23,6 +23,7 @@ import {
 import { wrap as wrapOptimistic } from "optimism";
 import {
   dirtyNodeModulesDirectory,
+  dirtyReplacedDirectory,
   optimisticLStat,
   optimisticStatOrNull,
   optimisticReadJsonOrNull,
@@ -791,6 +792,7 @@ async function restoreNodeModulesFromNpmDepsCache(packageName, packageNpmDir) {
     await files.cp_r(cachedNodeModules, tmpDir, { preserveSymlinks: true });
     await files.renameDirAlmostAtomically(tmpDir, nodeModulesDir);
     dirtyNodeModulesDirectory(nodeModulesDir);
+    dirtyReplacedDirectory(nodeModulesDir);
     return true;
   } catch (e) {
     try {
@@ -893,6 +895,71 @@ function pruneNpmDepsCache(packageCacheDir, keepDir) {
   } catch (e) {}
 }
 
+// Install a package's npm deps purely from its shipped
+// npm-shrinkwrap.json -- no evaluated Npm.depends needed, which is what
+// makes prewarm-time (pre-package.js) installs possible. Mirrors
+// updateExistingNpmDirectory's preserved-shrinkwrap branch exactly:
+// mapped shrinkwrap + minimal package.json into a temp dir, `npm
+// install`, then completeNpmDirectory (which also seeds the cache).
+// The top-level dependency map is derived from the shrinkwrap itself
+// (exact versions; resolved URLs for non-registry deps) -- a superset
+// of the real Npm.depends, which npm accepts since every entry is
+// pinned to what the lock installs. If the shipped shrinkwrap turns
+// out stale relative to the package's actual Npm.depends, the build's
+// own subtree verification catches it and reinstalls normally.
+// Callers must hold the npm package lock for packageNpmDir.
+async function preinstallNpmDepsFromShrinkwrap(packageName, packageNpmDir) {
+  const shrinkwrappedDependenciesTree =
+    getShrinkwrappedDependenciesTree(packageNpmDir);
+  const npmDependencies = treeToDependencies(
+    minimizeDependencyTree(shrinkwrappedDependenciesTree));
+  if (_.isEmpty(npmDependencies)) {
+    return false;
+  }
+  const cacheKey = npmDepsCacheKey(packageNpmDir);
+
+  const newPackageNpmDir =
+    convertColonsInPath(packageNpmDir) + '-prewarm-' + utils.randomToken();
+  tmpDirs.push(newPackageNpmDir);
+  try {
+    makeNewPackageNpmDir(newPackageNpmDir);
+
+    const mappedDependencies = Object.entries(
+      shrinkwrappedDependenciesTree.dependencies
+    ).reduce((acc, [name, info]) => {
+      return { ...acc, [`node_modules/${name}`]: info };
+    }, {});
+
+    files.writeFile(
+      files.pathJoin(newPackageNpmDir, 'npm-shrinkwrap.json'),
+      JSON.stringify({
+        ...shrinkwrappedDependenciesTree,
+        dependencies: mappedDependencies,
+      }, null, 2)
+    );
+    files.writeFile(
+      files.pathJoin(newPackageNpmDir, 'package.json'),
+      JSON.stringify({ dependencies: npmDependencies }, null, 2)
+    );
+
+    logUpdateDependencies(packageName, npmDependencies);
+    await installFromShrinkwrap(newPackageNpmDir);
+
+    files.unlink(files.pathJoin(newPackageNpmDir, 'npm-shrinkwrap.json'));
+    files.unlink(files.pathJoin(newPackageNpmDir, 'package.json'));
+
+    await completeNpmDirectory(
+      packageName, newPackageNpmDir, packageNpmDir, npmDependencies,
+      cacheKey);
+    return true;
+  } finally {
+    if (files.exists(newPackageNpmDir)) {
+      await files.rm_recursive_deferred(newPackageNpmDir);
+    }
+    tmpDirs = _.without(tmpDirs, newPackageNpmDir);
+  }
+}
+
 // Restore every restorable package under packagesDir from the cache, in
 // parallel, before the (serial) per-package build pipeline reaches
 // them. Purely filesystem-driven: a candidate is any package dir with a
@@ -900,69 +967,129 @@ function pruneNpmDepsCache(packageCacheDir, keepDir) {
 // package.js evaluation is needed. Safe to run concurrently with a
 // build -- each restore holds the same cross-process lock
 // updateDependencies() takes, and re-checks its precondition under it.
-// Returns the number of packages restored.
+//
+// Options:
+// - onlyPackages: array of package names to consider (REQUIRED for
+//   installOnMiss -- without a scope, cold-cache installs would cover
+//   every bundled package instead of just the app's).
+// - installOnMiss: after the restore phase, npm-install the remaining
+//   misses from their shipped shrinkwraps, in parallel.
+// - concurrency (restore pool, default 8), installConcurrency (npm
+//   pool, default 3 -- kept small because concurrent npm processes
+//   contend on npm's shared content cache).
+//
+// Returns {restored, installed}.
 meteorNpm.prewarmNpmDepsCache = async function (packagesDir, options) {
   options = options || {};
+  const none = { restored: 0, installed: 0 };
   if (npmDepsCacheDisabled()) {
-    return 0;
+    return none;
   }
   packagesDir = files.convertToStandardPath(packagesDir);
 
+  const only = options.onlyPackages &&
+    new Set(options.onlyPackages);
   const candidates = [];
+  // A dir is a candidate when it has a shipped shrinkwrap but no
+  // installed tree. `name` keys the cache entry, so it must match what
+  // the normal install path passes as packageName: the package name for
+  // .npm/package, the plugin name for .npm/plugin/<pluginName>.
+  function considerCandidate(name, packageNpmDir) {
+    if (! files.exists(
+          files.pathJoin(packageNpmDir, "npm-shrinkwrap.json"))) {
+      return;
+    }
+    if (files.exists(files.pathJoin(
+          packageNpmDir, "node_modules", ".package-lock.json"))) {
+      return;
+    }
+    candidates.push({ name, packageNpmDir });
+  }
   try {
     files.readdir(packagesDir).forEach(name => {
-      const packageNpmDir =
-        files.pathJoin(packagesDir, name, ".npm", "package");
-      if (! files.exists(
-            files.pathJoin(packageNpmDir, "npm-shrinkwrap.json"))) {
+      if (only && ! only.has(name)) {
         return;
       }
-      if (files.exists(files.pathJoin(
-            packageNpmDir, "node_modules", ".package-lock.json"))) {
+      considerCandidate(
+        name, files.pathJoin(packagesDir, name, ".npm", "package"));
+      // Build plugins keep their own npm trees at .npm/plugin/<plugin
+      // name> -- these install during the runner's FIRST build, i.e.
+      // inside the file-watch window, where the resulting .npm churn
+      // can restart the build (post-update "flapping"). Prewarming them
+      // here moves that work before the watcher arms.
+      const pluginRoot = files.pathJoin(packagesDir, name, ".npm", "plugin");
+      let pluginNames;
+      try {
+        pluginNames = files.readdir(pluginRoot);
+      } catch (e) {
         return;
       }
-      candidates.push({ name, packageNpmDir });
+      pluginNames.forEach(pluginName => {
+        if (pluginName.startsWith(".") ||
+            pluginName.endsWith(".lock") ||
+            pluginName.includes("-new-") ||
+            pluginName.includes("-prewarm-")) {
+          return;
+        }
+        considerCandidate(
+          pluginName, files.pathJoin(pluginRoot, pluginName));
+      });
     });
   } catch (e) {
-    return 0;
+    return none;
   }
   if (! candidates.length) {
-    return 0;
+    return none;
   }
 
-  let restored = 0;
-  let nextIndex = 0;
-  async function worker() {
-    while (nextIndex < candidates.length) {
-      const candidate = candidates[nextIndex++];
-      try {
-        const ok = await withNpmPackageLock(
-          candidate.packageNpmDir,
-          async () => {
-            // Re-check under the lock: a concurrent build may have
-            // installed (or restored) this package already.
-            if (files.exists(files.pathJoin(
-                  candidate.packageNpmDir,
-                  "node_modules", ".package-lock.json"))) {
-              return false;
-            }
-            return restoreNodeModulesFromNpmDepsCache(
-              candidate.name, candidate.packageNpmDir);
+  function installedAlready(candidate) {
+    return files.exists(files.pathJoin(
+      candidate.packageNpmDir, "node_modules", ".package-lock.json"));
+  }
+
+  async function runPool(queue, size, fn) {
+    let nextIndex = 0;
+    let completed = 0;
+    async function worker() {
+      while (nextIndex < queue.length) {
+        const candidate = queue[nextIndex++];
+        try {
+          // Re-check under the lock: a concurrent build (or the other
+          // prewarm phase) may have handled this package already.
+          const ok = await withNpmPackageLock(
+            candidate.packageNpmDir,
+            async () => ! installedAlready(candidate) && fn(candidate)
+          );
+          if (ok) {
+            completed++;
+          }
+        } catch (e) {
+          // Best-effort; the build's own updateDependencies() will
+          // handle this package the normal way.
+          if (options.debugThrow) {
+            console.error("prewarm error for " + candidate.name + ":",
+              (e && e.stack) || e);
           }
-        );
-        if (ok) {
-          restored++;
         }
-      } catch (e) {
-        // Best-effort; the build's own updateDependencies() will handle
-        // this package the normal way.
       }
     }
+    await Promise.all(
+      _.times(Math.min(size, queue.length || 1), worker));
+    return completed;
+  }
+
+  const restored = await runPool(
+    candidates, options.concurrency || 8,
+    c => restoreNodeModulesFromNpmDepsCache(c.name, c.packageNpmDir));
+
+  let installed = 0;
+  if (options.installOnMiss && only) {
+    const misses = candidates.filter(c => ! installedAlready(c));
+    installed = await runPool(
+      misses, options.installConcurrency || 3,
+      c => preinstallNpmDepsFromShrinkwrap(c.name, c.packageNpmDir));
   }
-  const concurrency = Math.min(
-    options.concurrency || 8, candidates.length);
-  await Promise.all(_.times(concurrency, worker));
-  return restored;
+  return { restored, installed };
 };
 
 var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
@@ -1293,6 +1420,11 @@ async function completeNpmDirectory(
   await files.renameDirAlmostAtomically(newPackageNpmDir, packageNpmDir);
 
   dirtyNodeModulesDirectory(files.pathJoin(packageNpmDir, "node_modules"));
+  // The whole .npm dir was just atomically replaced; any optimistic
+  // result cached during the swap window (e.g. a null hash for
+  // npm-shrinkwrap.json) must be forgotten deterministically -- watcher
+  // events can't be relied on for this (see dirtyReplacedDirectory).
+  dirtyReplacedDirectory(packageNpmDir);
 
   if (! _.isEmpty(npmDependencies)) {
     scheduleNpmDepsCachePopulate(

package/isopackets/combined/npm/node_modules/meteor/fetch/node_modules/node-fetch/.meteor-portable-2.json

@@ -1 +0,0 @@
-true