@maka/meteor-sdk 0.2.8 → 0.2.9

package/README.md

@@ -243,11 +243,17 @@ per-user cache of completed `node_modules` trees at `~/.meteor-npm-cache`,
 keyed by shrinkwrap content + Node compatibility version + platform/arch.
 On the first build after an SDK update, packages whose dependencies
 didn't change restore from the cache by parallel directory copy instead
-of spawning npm -- silently and near-instantly. Cache writes happen in
-the background off the build's critical path, entries are pruned to the
-3 newest per package, and everything is best-effort: any cache problem
-falls back to a normal npm install, and a stale restore is caught by the
-existing shrinkwrap-vs-installed verification.
+of spawning npm -- silently and near-instantly. Cache *misses* are
+handled in parallel too: once `loadProject()` knows the app's package
+set (from `.meteor/versions`), the remaining installs run through a
+small npm worker pool ahead of the build's serial per-package walk
+(bounded to 3 processes -- concurrent npm invocations contend on npm's
+shared content cache), which also keeps those installs' `.npm` churn
+from re-firing the app runner's file watcher mid-build. Cache writes
+happen in the background off the build's critical path, entries are
+pruned to the 3 newest per package, and everything is best-effort: any
+cache problem falls back to a normal npm install, and a stale restore is
+caught by the existing shrinkwrap-vs-installed verification.
 
 Control it with `METEOR_PACKAGE_NPM_CACHE`: unset means enabled at the
 default location, `0`/`false`/`no`/`off` disables it, and any other

package/isopackets/combined/npm/node_modules/meteor/babel-compiler/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1m6rn6thduz",
+  "name": "package-new-14mcukbut2o",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/boilerplate-generator/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-e9rbm1cl3n4",
+  "name": "package-new-1nxrqkqgiun",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/constraint-solver/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1ul11s6qh6",
+  "name": "package-new-lhgmlf5mlyk",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/ddp-client/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-ee0ne0pml8l",
+  "name": "package-new-9dm5oc7j4sc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/ecmascript/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1g24wbdxvx8",
+  "name": "package-new-1wcn7u5f6o9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/ecmascript-runtime/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-6fxj13oxx1b",
+  "name": "package-new-1kdw71by1kb",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/ecmascript-runtime-client/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-16q2eii8237",
+  "name": "package-new-11ay5a2qmvv",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/ecmascript-runtime-server/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-g8qjgrro1ac",
+  "name": "package-new-jkr5smb2te",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/fetch/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-286qo524b55",
+  "name": "package-new-x0lqsl14am",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/inter-process-messaging/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1pcp8sbti9w",
+  "name": "package-new-xth6zuvyoge",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/logging/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1c4mjv2utk7h",
+  "name": "package-new-3l78651i8vb",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/logic-solver/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-9t84b3my7i",
+  "name": "package-new-obu6cxhlj2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/meteor/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1bjt87sifq2",
+  "name": "package-new-27q6iwo9mjek",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/modules/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-6gguidiwzx",
+  "name": "package-new-16qpf002t31",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/modules-runtime/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-ovtntj9ibd",
+  "name": "package-new-1avxcuqbu7h",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/npm-mongo/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-t8jk4dq32z",
+  "name": "package-new-1om3fpd6efo",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/package-version-parser/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1t8h3r5p5mc",
+  "name": "package-new-dclewbgria6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/promise/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-19ro17vxabi",
+  "name": "package-new-mt7pq41l8d9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/react-fast-refresh/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1mbjo8u1bdw",
+  "name": "package-new-i4l2akdfgip",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/socket-stream-client/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-1bskl16cqu8",
+  "name": "package-new-4dhnqh88rfe",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@maka/meteor-sdk",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "description": "Programmatic Node API over the Meteor build tool (bundle/run/create/etc.) -- for driving Meteor's build system from plain Node code instead of shelling out to a meteor CLI (there isn't one anymore; see README.md). \"dependencies\" is generated by scripts/write-tool-package-json.js from scripts/dev-bundle-tool-package.js at the monorepo root -- do not hand-edit that field, edit that file and re-run the script instead.",
   "main": "src/index.js",
   "exports": {

package/src/project.js

@@ -65,6 +65,19 @@ async function loadProject(projectDir) {
     projectDir: standardProjectDir,
   });
 
+  // Second-stage npm-deps prewarm (bootstrap.js fired the restore-only
+  // first stage): now that an app is known, its .meteor/versions names
+  // the exact core-package set the build will need, so cache MISSES can
+  // be npm-installed too -- in parallel (small pool; concurrent npm
+  // processes contend on npm's shared content cache) and BEFORE the
+  // build's serial per-package walk, which also keeps those installs'
+  // .npm churn from re-firing the app runner's file watcher mid-build.
+  // Un-awaited: every prewarm operation holds the same cross-process
+  // per-package lock updateDependencies() takes, so the build below
+  // simply waits on (or wins) each package's lock. Best-effort by
+  // design -- on any failure the build installs serially as before.
+  startAppNpmDepsPrewarm(standardProjectDir);
+
   await runCaptured("preparing project", () =>
     projectContext.prepareProjectForBuild()
   );
@@ -72,4 +85,31 @@ async function loadProject(projectDir) {
   return new Project(projectContext);
 }
 
+function startAppNpmDepsPrewarm(standardProjectDir) {
+  try {
+    const files = require("../tools/fs/files");
+    const versionsFile = files.pathJoin(
+      standardProjectDir, ".meteor", "versions");
+    // name@version lines; author:package names never match the bundled
+    // core-package directory names, so no special-casing needed. A
+    // missing versions file (brand-new app) just skips this stage.
+    const names = files.readFile(versionsFile, "utf8")
+      .split("\n")
+      .map((line) => line.trim().split("@")[0])
+      .filter(Boolean);
+    if (!names.length) {
+      return;
+    }
+    const meteorNpm = require("../tools/isobuild/meteor-npm.js");
+    Promise.resolve(
+      meteorNpm.prewarmNpmDepsCache(
+        path.resolve(__dirname, "..", "packages"),
+        { onlyPackages: names, installOnMiss: true }
+      )
+    ).catch(() => {});
+  } catch (e) {
+    // Best-effort: the build's own per-package install path still works.
+  }
+}
+
 module.exports = { loadProject, Project };

package/tools/isobuild/meteor-npm.js

@@ -893,6 +893,71 @@ function pruneNpmDepsCache(packageCacheDir, keepDir) {
   } catch (e) {}
 }
 
+// Install a package's npm deps purely from its shipped
+// npm-shrinkwrap.json -- no evaluated Npm.depends needed, which is what
+// makes prewarm-time (pre-package.js) installs possible. Mirrors
+// updateExistingNpmDirectory's preserved-shrinkwrap branch exactly:
+// mapped shrinkwrap + minimal package.json into a temp dir, `npm
+// install`, then completeNpmDirectory (which also seeds the cache).
+// The top-level dependency map is derived from the shrinkwrap itself
+// (exact versions; resolved URLs for non-registry deps) -- a superset
+// of the real Npm.depends, which npm accepts since every entry is
+// pinned to what the lock installs. If the shipped shrinkwrap turns
+// out stale relative to the package's actual Npm.depends, the build's
+// own subtree verification catches it and reinstalls normally.
+// Callers must hold the npm package lock for packageNpmDir.
+async function preinstallNpmDepsFromShrinkwrap(packageName, packageNpmDir) {
+  const shrinkwrappedDependenciesTree =
+    getShrinkwrappedDependenciesTree(packageNpmDir);
+  const npmDependencies = treeToDependencies(
+    minimizeDependencyTree(shrinkwrappedDependenciesTree));
+  if (_.isEmpty(npmDependencies)) {
+    return false;
+  }
+  const cacheKey = npmDepsCacheKey(packageNpmDir);
+
+  const newPackageNpmDir =
+    convertColonsInPath(packageNpmDir) + '-prewarm-' + utils.randomToken();
+  tmpDirs.push(newPackageNpmDir);
+  try {
+    makeNewPackageNpmDir(newPackageNpmDir);
+
+    const mappedDependencies = Object.entries(
+      shrinkwrappedDependenciesTree.dependencies
+    ).reduce((acc, [name, info]) => {
+      return { ...acc, [`node_modules/${name}`]: info };
+    }, {});
+
+    files.writeFile(
+      files.pathJoin(newPackageNpmDir, 'npm-shrinkwrap.json'),
+      JSON.stringify({
+        ...shrinkwrappedDependenciesTree,
+        dependencies: mappedDependencies,
+      }, null, 2)
+    );
+    files.writeFile(
+      files.pathJoin(newPackageNpmDir, 'package.json'),
+      JSON.stringify({ dependencies: npmDependencies }, null, 2)
+    );
+
+    logUpdateDependencies(packageName, npmDependencies);
+    await installFromShrinkwrap(newPackageNpmDir);
+
+    files.unlink(files.pathJoin(newPackageNpmDir, 'npm-shrinkwrap.json'));
+    files.unlink(files.pathJoin(newPackageNpmDir, 'package.json'));
+
+    await completeNpmDirectory(
+      packageName, newPackageNpmDir, packageNpmDir, npmDependencies,
+      cacheKey);
+    return true;
+  } finally {
+    if (files.exists(newPackageNpmDir)) {
+      await files.rm_recursive_deferred(newPackageNpmDir);
+    }
+    tmpDirs = _.without(tmpDirs, newPackageNpmDir);
+  }
+}
+
 // Restore every restorable package under packagesDir from the cache, in
 // parallel, before the (serial) per-package build pipeline reaches
 // them. Purely filesystem-driven: a candidate is any package dir with a
@@ -900,17 +965,34 @@ function pruneNpmDepsCache(packageCacheDir, keepDir) {
 // package.js evaluation is needed. Safe to run concurrently with a
 // build -- each restore holds the same cross-process lock
 // updateDependencies() takes, and re-checks its precondition under it.
-// Returns the number of packages restored.
+//
+// Options:
+// - onlyPackages: array of package names to consider (REQUIRED for
+//   installOnMiss -- without a scope, cold-cache installs would cover
+//   every bundled package instead of just the app's).
+// - installOnMiss: after the restore phase, npm-install the remaining
+//   misses from their shipped shrinkwraps, in parallel.
+// - concurrency (restore pool, default 8), installConcurrency (npm
+//   pool, default 3 -- kept small because concurrent npm processes
+//   contend on npm's shared content cache).
+//
+// Returns {restored, installed}.
 meteorNpm.prewarmNpmDepsCache = async function (packagesDir, options) {
   options = options || {};
+  const none = { restored: 0, installed: 0 };
   if (npmDepsCacheDisabled()) {
-    return 0;
+    return none;
   }
   packagesDir = files.convertToStandardPath(packagesDir);
 
+  const only = options.onlyPackages &&
+    new Set(options.onlyPackages);
   const candidates = [];
   try {
     files.readdir(packagesDir).forEach(name => {
+      if (only && ! only.has(name)) {
+        return;
+      }
       const packageNpmDir =
         files.pathJoin(packagesDir, name, ".npm", "package");
       if (! files.exists(
@@ -924,45 +1006,60 @@ meteorNpm.prewarmNpmDepsCache = async function (packagesDir, options) {
       candidates.push({ name, packageNpmDir });
     });
   } catch (e) {
-    return 0;
+    return none;
   }
   if (! candidates.length) {
-    return 0;
+    return none;
   }
 
-  let restored = 0;
-  let nextIndex = 0;
-  async function worker() {
-    while (nextIndex < candidates.length) {
-      const candidate = candidates[nextIndex++];
-      try {
-        const ok = await withNpmPackageLock(
-          candidate.packageNpmDir,
-          async () => {
-            // Re-check under the lock: a concurrent build may have
-            // installed (or restored) this package already.
-            if (files.exists(files.pathJoin(
-                  candidate.packageNpmDir,
-                  "node_modules", ".package-lock.json"))) {
-              return false;
-            }
-            return restoreNodeModulesFromNpmDepsCache(
-              candidate.name, candidate.packageNpmDir);
+  function installedAlready(candidate) {
+    return files.exists(files.pathJoin(
+      candidate.packageNpmDir, "node_modules", ".package-lock.json"));
+  }
+
+  async function runPool(queue, size, fn) {
+    let nextIndex = 0;
+    let completed = 0;
+    async function worker() {
+      while (nextIndex < queue.length) {
+        const candidate = queue[nextIndex++];
+        try {
+          // Re-check under the lock: a concurrent build (or the other
+          // prewarm phase) may have handled this package already.
+          const ok = await withNpmPackageLock(
+            candidate.packageNpmDir,
+            async () => ! installedAlready(candidate) && fn(candidate)
+          );
+          if (ok) {
+            completed++;
+          }
+        } catch (e) {
+          // Best-effort; the build's own updateDependencies() will
+          // handle this package the normal way.
+          if (options.debugThrow) {
+            console.error("prewarm error for " + candidate.name + ":",
+              (e && e.stack) || e);
           }
-        );
-        if (ok) {
-          restored++;
         }
-      } catch (e) {
-        // Best-effort; the build's own updateDependencies() will handle
-        // this package the normal way.
       }
     }
+    await Promise.all(
+      _.times(Math.min(size, queue.length || 1), worker));
+    return completed;
+  }
+
+  const restored = await runPool(
+    candidates, options.concurrency || 8,
+    c => restoreNodeModulesFromNpmDepsCache(c.name, c.packageNpmDir));
+
+  let installed = 0;
+  if (options.installOnMiss && only) {
+    const misses = candidates.filter(c => ! installedAlready(c));
+    installed = await runPool(
+      misses, options.installConcurrency || 3,
+      c => preinstallNpmDepsFromShrinkwrap(c.name, c.packageNpmDir));
   }
-  const concurrency = Math.min(
-    options.concurrency || 8, candidates.length);
-  await Promise.all(_.times(concurrency, worker));
-  return restored;
+  return { restored, installed };
 };
 
 var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,