@maka/meteor-sdk 0.2.7 → 0.2.9

package/isopackets/combined/npm/node_modules/meteor/promise/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-d6edvk3v9nl",
+  "name": "package-new-mt7pq41l8d9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/react-fast-refresh/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-8e8v01oivjc",
+  "name": "package-new-i4l2akdfgip",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/socket-stream-client/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-3efvozmolln",
+  "name": "package-new-4dhnqh88rfe",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@maka/meteor-sdk",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "Programmatic Node API over the Meteor build tool (bundle/run/create/etc.) -- for driving Meteor's build system from plain Node code instead of shelling out to a meteor CLI (there isn't one anymore; see README.md). \"dependencies\" is generated by scripts/write-tool-package-json.js from scripts/dev-bundle-tool-package.js at the monorepo root -- do not hand-edit that field, edit that file and re-run the script instead.",
   "main": "src/index.js",
   "exports": {

package/src/bootstrap.js

@@ -120,6 +120,31 @@ function ensureMainModuleShim() {
   };
 }
 
+// Kick off the persistent npm-deps cache prewarm (see tools/isobuild/
+// meteor-npm.js): restore every bundled package's node_modules from
+// ~/.meteor-npm-cache in parallel, so the first build after a fresh SDK
+// install doesn't pay a serial `npm install` per package. Deliberately
+// NOT awaited -- each restore holds the same cross-process lock
+// updateDependencies() takes, so a build racing ahead of the prewarm
+// simply waits on (or wins) the per-package lock; firing it before
+// ensureRuntimeReady() lets even the isopacket builds overlap with it.
+let npmDepsCachePrewarm = null;
+function startNpmDepsCachePrewarm() {
+  if (npmDepsCachePrewarm) {
+    return;
+  }
+  try {
+    const meteorNpm = require("../tools/isobuild/meteor-npm.js");
+    const path = require("path");
+    npmDepsCachePrewarm = Promise.resolve(
+      meteorNpm.prewarmNpmDepsCache(path.resolve(__dirname, "..", "packages"))
+    ).catch(() => {});
+  } catch (e) {
+    // Best-effort: the build's own per-package install path still works.
+    npmDepsCachePrewarm = Promise.resolve();
+  }
+}
+
 // Idempotent: safe to call from every public SDK entry point. The actual
 // Babel require() hook installed by install-babel.js is itself naturally
 // idempotent too (Node's own require() cache means requiring the same
@@ -152,6 +177,7 @@ async function bootstrap(options) {
   require("../tools/tool-env/install-babel.js");
   ensureAsyncLocalStorage();
   consoleAdapter.patch();
+  startNpmDepsCachePrewarm();
   await ensureRuntimeReady(options && options.offline);
   bootstrapped = true;
 }

package/src/bundle.js

@@ -45,6 +45,11 @@ async function bundle(project, options) {
     throwIfHasMessages(result.errors);
   }
 
+  // Background npm-deps cache writes (see meteor-npm.js) ran concurrently
+  // with the build; wait for them here so a build-and-exit process still
+  // gets its cache populated. Usually a no-op by this point.
+  await require("../tools/isobuild/meteor-npm.js").flushNpmDepsCacheWrites();
+
   return {
     outputPath: options.outputPath,
     starManifest: result.starManifest,

package/src/project.js

@@ -65,6 +65,19 @@ async function loadProject(projectDir) {
     projectDir: standardProjectDir,
   });
 
+  // Second-stage npm-deps prewarm (bootstrap.js fired the restore-only
+  // first stage): now that an app is known, its .meteor/versions names
+  // the exact core-package set the build will need, so cache MISSES can
+  // be npm-installed too -- in parallel (small pool; concurrent npm
+  // processes contend on npm's shared content cache) and BEFORE the
+  // build's serial per-package walk, which also keeps those installs'
+  // .npm churn from re-firing the app runner's file watcher mid-build.
+  // Un-awaited: every prewarm operation holds the same cross-process
+  // per-package lock updateDependencies() takes, so the build below
+  // simply waits on (or wins) each package's lock. Best-effort by
+  // design -- on any failure the build installs serially as before.
+  startAppNpmDepsPrewarm(standardProjectDir);
+
   await runCaptured("preparing project", () =>
     projectContext.prepareProjectForBuild()
   );
@@ -72,4 +85,31 @@ async function loadProject(projectDir) {
   return new Project(projectContext);
 }
 
+function startAppNpmDepsPrewarm(standardProjectDir) {
+  try {
+    const files = require("../tools/fs/files");
+    const versionsFile = files.pathJoin(
+      standardProjectDir, ".meteor", "versions");
+    // name@version lines; author:package names never match the bundled
+    // core-package directory names, so no special-casing needed. A
+    // missing versions file (brand-new app) just skips this stage.
+    const names = files.readFile(versionsFile, "utf8")
+      .split("\n")
+      .map((line) => line.trim().split("@")[0])
+      .filter(Boolean);
+    if (!names.length) {
+      return;
+    }
+    const meteorNpm = require("../tools/isobuild/meteor-npm.js");
+    Promise.resolve(
+      meteorNpm.prewarmNpmDepsCache(
+        path.resolve(__dirname, "..", "packages"),
+        { onlyPackages: names, installOnMiss: true }
+      )
+    ).catch(() => {});
+  } catch (e) {
+    // Best-effort: the build's own per-package install path still works.
+  }
+}
+
 module.exports = { loadProject, Project };

package/tools/isobuild/meteor-npm.js

@@ -676,6 +676,392 @@ var makeNewPackageNpmDir = function (newPackageNpmDir) {
      ''/*git diff complains without trailing newline*/].join('\n'));
 };
 
+// --- Persistent npm-deps cache ---------------------------------------
+//
+// A package's .npm/package/node_modules is fully determined by three
+// things: the npm-shrinkwrap.json content, the Node compatibility
+// version, and the platform/arch. Yet every fresh install of the tool
+// (e.g. `npm i -g` of a new SDK version) starts with empty node_modules
+// for every bundled package and pays a serial `npm install` per package
+// on the first build. Since shrinkwraps rarely change between releases,
+// keep a per-user cache of completed node_modules trees keyed by a hash
+// of those three inputs, and restore by directory copy instead of
+// spawning npm.
+//
+// Everything here is best-effort: any failure falls through to the
+// normal npm install path, and a stale/corrupt restore is caught by the
+// existing shrinkwrap-vs-installed subtree comparison (which then
+// triggers a real reinstall over it).
+//
+// METEOR_PACKAGE_NPM_CACHE: unset -> enabled at ~/.meteor-npm-cache;
+// "0"/"false"/"no"/"off" -> disabled; any other value -> cache root.
+
+function npmDepsCacheDisabled() {
+  const v = process.env.METEOR_PACKAGE_NPM_CACHE;
+  return typeof v === "string" && /^\s*(0|false|no|off)\s*$/i.test(v);
+}
+
+function npmDepsCacheRoot() {
+  const v = process.env.METEOR_PACKAGE_NPM_CACHE;
+  if (v && ! npmDepsCacheDisabled()) {
+    return files.convertToStandardPath(v);
+  }
+  return files.pathJoin(
+    files.convertToStandardPath(require("os").homedir()),
+    ".meteor-npm-cache"
+  );
+}
+
+// JSON.stringify with deterministically sorted object keys, so the hash
+// is insensitive to key order, whitespace, and line endings of the
+// on-disk shrinkwrap file.
+function stableStringify(value) {
+  if (Array.isArray(value)) {
+    return "[" + value.map(stableStringify).join(",") + "]";
+  }
+  if (value && typeof value === "object") {
+    return "{" + Object.keys(value).sort().map(
+      key => JSON.stringify(key) + ":" + stableStringify(value[key])
+    ).join(",") + "}";
+  }
+  return JSON.stringify(value);
+}
+
+// Hash of the package's current npm-shrinkwrap.json plus everything
+// else that determines node_modules content. Returns null if the
+// shrinkwrap is missing or unparsable.
+function npmDepsCacheKey(packageNpmDir) {
+  let shrinkwrap;
+  try {
+    shrinkwrap = JSON.parse(files.readFile(
+      files.pathJoin(packageNpmDir, "npm-shrinkwrap.json"), "utf8"));
+  } catch (e) {
+    return null;
+  }
+  return require("crypto").createHash("sha256").update(stableStringify({
+    shrinkwrap,
+    node: currentNodeCompatibilityVersion(),
+    platform: process.platform,
+    arch: process.arch,
+  })).digest("hex").slice(0, 32);
+}
+
+function npmDepsCacheEntryDir(packageName, key) {
+  // Meteor package names may contain characters that are not valid in
+  // paths everywhere (e.g. ":" on Windows); the hash disambiguates, so
+  // a lossy sanitization is fine.
+  const safeName = packageName.replace(/[^A-Za-z0-9._-]/g, "_");
+  return files.pathJoin(npmDepsCacheRoot(), safeName, key);
+}
+
+// Copy a cached node_modules into packageNpmDir (via temp dir + rename,
+// so a crash never leaves a half-restored node_modules in place).
+// Returns true on success. Callers must hold the npm package lock for
+// packageNpmDir.
+async function restoreNodeModulesFromNpmDepsCache(packageName, packageNpmDir) {
+  if (npmDepsCacheDisabled()) {
+    return false;
+  }
+  const key = npmDepsCacheKey(packageNpmDir);
+  if (! key) {
+    return false;
+  }
+  const cachedNodeModules = files.pathJoin(
+    npmDepsCacheEntryDir(packageName, key), "node_modules");
+  if (! files.exists(
+        files.pathJoin(cachedNodeModules, ".package-lock.json"))) {
+    return false;
+  }
+
+  // Sweep temp dirs left behind by a crashed earlier restore (safe: we
+  // hold the package lock).
+  try {
+    files.readdir(packageNpmDir).forEach(entry => {
+      if (entry.startsWith(".node_modules-from-cache-")) {
+        files.rm_recursive(files.pathJoin(packageNpmDir, entry));
+      }
+    });
+  } catch (e) {}
+
+  const nodeModulesDir = files.pathJoin(packageNpmDir, "node_modules");
+  const tmpDir = files.pathJoin(
+    packageNpmDir, ".node_modules-from-cache-" + utils.randomToken());
+  tmpDirs.push(tmpDir);
+  try {
+    await files.cp_r(cachedNodeModules, tmpDir, { preserveSymlinks: true });
+    await files.renameDirAlmostAtomically(tmpDir, nodeModulesDir);
+    dirtyNodeModulesDirectory(nodeModulesDir);
+    return true;
+  } catch (e) {
+    try {
+      if (files.exists(tmpDir)) {
+        await files.rm_recursive(tmpDir);
+      }
+    } catch (cleanupError) {}
+    return false;
+  } finally {
+    tmpDirs = _.without(tmpDirs, tmpDir);
+  }
+}
+
+// Cache writes run in the background (fire-and-forget with an atomic
+// temp-dir + rename publish) so they never sit on the build's critical
+// path. flushNpmDepsCacheWrites() lets one-shot processes (e.g. a
+// build-and-exit embedder) wait for them before exiting; if a process
+// dies first, the orphaned .tmp-* dir is swept by a later prune.
+const pendingNpmDepsCacheWrites = [];
+
+meteorNpm.flushNpmDepsCacheWrites = function () {
+  return Promise.allSettled(pendingNpmDepsCacheWrites);
+};
+
+function scheduleNpmDepsCachePopulate(packageName, packageNpmDir, key) {
+  if (npmDepsCacheDisabled()) {
+    return;
+  }
+  pendingNpmDepsCacheWrites.push(
+    populateNpmDepsCache(packageName, packageNpmDir, key).catch(e => {})
+  );
+}
+
+async function populateNpmDepsCache(packageName, packageNpmDir, key) {
+  key = key || npmDepsCacheKey(packageNpmDir);
+  if (! key) {
+    return;
+  }
+  const entryDir = npmDepsCacheEntryDir(packageName, key);
+  if (files.exists(entryDir)) {
+    return;
+  }
+  const packageCacheDir = files.pathDirname(entryDir);
+  files.mkdir_p(packageCacheDir);
+  const tmpDir = files.pathJoin(
+    packageCacheDir, ".tmp-" + utils.randomToken());
+  try {
+    await files.cp_r(
+      files.pathJoin(packageNpmDir, "node_modules"),
+      files.pathJoin(tmpDir, "node_modules"),
+      { preserveSymlinks: true }
+    );
+    files.rename(tmpDir, entryDir);
+  } catch (e) {
+    // Concurrent populate may have won the rename race, or the copy
+    // failed midway; either way discard our temp dir and move on.
+    try {
+      if (files.exists(tmpDir)) {
+        await files.rm_recursive(tmpDir);
+      }
+    } catch (cleanupError) {}
+    return;
+  }
+  pruneNpmDepsCache(packageCacheDir, entryDir);
+}
+
+// Keep the 3 newest entries per package (different keys = different
+// shrinkwrap/node/platform combos) and sweep stale temp dirs, so the
+// cache stays bounded without any global bookkeeping.
+function pruneNpmDepsCache(packageCacheDir, keepDir) {
+  try {
+    const keepBase = files.pathBasename(keepDir);
+    const entries = [];
+    files.readdir(packageCacheDir).forEach(entry => {
+      const fullPath = files.pathJoin(packageCacheDir, entry);
+      if (entry.startsWith(".tmp-")) {
+        // Only sweep temp dirs that are clearly abandoned; a concurrent
+        // process may still be filling a fresh one.
+        try {
+          const ageMs = Date.now() - files.stat(fullPath).mtime.getTime();
+          if (ageMs > 60 * 60 * 1000) {
+            files.rm_recursive(fullPath);
+          }
+        } catch (e) {}
+        return;
+      }
+      if (entry === keepBase) {
+        return;
+      }
+      try {
+        entries.push({ fullPath, mtimeMs: files.stat(fullPath).mtime.getTime() });
+      } catch (e) {}
+    });
+    entries.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    entries.slice(2).forEach(entry => {
+      try {
+        files.rm_recursive(entry.fullPath);
+      } catch (e) {}
+    });
+  } catch (e) {}
+}
+
+// Install a package's npm deps purely from its shipped
+// npm-shrinkwrap.json -- no evaluated Npm.depends needed, which is what
+// makes prewarm-time (pre-package.js) installs possible. Mirrors
+// updateExistingNpmDirectory's preserved-shrinkwrap branch exactly:
+// mapped shrinkwrap + minimal package.json into a temp dir, `npm
+// install`, then completeNpmDirectory (which also seeds the cache).
+// The top-level dependency map is derived from the shrinkwrap itself
+// (exact versions; resolved URLs for non-registry deps) -- a superset
+// of the real Npm.depends, which npm accepts since every entry is
+// pinned to what the lock installs. If the shipped shrinkwrap turns
+// out stale relative to the package's actual Npm.depends, the build's
+// own subtree verification catches it and reinstalls normally.
+// Callers must hold the npm package lock for packageNpmDir.
+async function preinstallNpmDepsFromShrinkwrap(packageName, packageNpmDir) {
+  const shrinkwrappedDependenciesTree =
+    getShrinkwrappedDependenciesTree(packageNpmDir);
+  const npmDependencies = treeToDependencies(
+    minimizeDependencyTree(shrinkwrappedDependenciesTree));
+  if (_.isEmpty(npmDependencies)) {
+    return false;
+  }
+  const cacheKey = npmDepsCacheKey(packageNpmDir);
+
+  const newPackageNpmDir =
+    convertColonsInPath(packageNpmDir) + '-prewarm-' + utils.randomToken();
+  tmpDirs.push(newPackageNpmDir);
+  try {
+    makeNewPackageNpmDir(newPackageNpmDir);
+
+    const mappedDependencies = Object.entries(
+      shrinkwrappedDependenciesTree.dependencies
+    ).reduce((acc, [name, info]) => {
+      return { ...acc, [`node_modules/${name}`]: info };
+    }, {});
+
+    files.writeFile(
+      files.pathJoin(newPackageNpmDir, 'npm-shrinkwrap.json'),
+      JSON.stringify({
+        ...shrinkwrappedDependenciesTree,
+        dependencies: mappedDependencies,
+      }, null, 2)
+    );
+    files.writeFile(
+      files.pathJoin(newPackageNpmDir, 'package.json'),
+      JSON.stringify({ dependencies: npmDependencies }, null, 2)
+    );
+
+    logUpdateDependencies(packageName, npmDependencies);
+    await installFromShrinkwrap(newPackageNpmDir);
+
+    files.unlink(files.pathJoin(newPackageNpmDir, 'npm-shrinkwrap.json'));
+    files.unlink(files.pathJoin(newPackageNpmDir, 'package.json'));
+
+    await completeNpmDirectory(
+      packageName, newPackageNpmDir, packageNpmDir, npmDependencies,
+      cacheKey);
+    return true;
+  } finally {
+    if (files.exists(newPackageNpmDir)) {
+      await files.rm_recursive_deferred(newPackageNpmDir);
+    }
+    tmpDirs = _.without(tmpDirs, newPackageNpmDir);
+  }
+}
+
+// Restore every restorable package under packagesDir from the cache, in
+// parallel, before the (serial) per-package build pipeline reaches
+// them. Purely filesystem-driven: a candidate is any package dir with a
+// .npm/package/npm-shrinkwrap.json but no installed node_modules, so no
+// package.js evaluation is needed. Safe to run concurrently with a
+// build -- each restore holds the same cross-process lock
+// updateDependencies() takes, and re-checks its precondition under it.
+//
+// Options:
+// - onlyPackages: array of package names to consider (REQUIRED for
+//   installOnMiss -- without a scope, cold-cache installs would cover
+//   every bundled package instead of just the app's).
+// - installOnMiss: after the restore phase, npm-install the remaining
+//   misses from their shipped shrinkwraps, in parallel.
+// - concurrency (restore pool, default 8), installConcurrency (npm
+//   pool, default 3 -- kept small because concurrent npm processes
+//   contend on npm's shared content cache).
+//
+// Returns {restored, installed}.
+meteorNpm.prewarmNpmDepsCache = async function (packagesDir, options) {
+  options = options || {};
+  const none = { restored: 0, installed: 0 };
+  if (npmDepsCacheDisabled()) {
+    return none;
+  }
+  packagesDir = files.convertToStandardPath(packagesDir);
+
+  const only = options.onlyPackages &&
+    new Set(options.onlyPackages);
+  const candidates = [];
+  try {
+    files.readdir(packagesDir).forEach(name => {
+      if (only && ! only.has(name)) {
+        return;
+      }
+      const packageNpmDir =
+        files.pathJoin(packagesDir, name, ".npm", "package");
+      if (! files.exists(
+            files.pathJoin(packageNpmDir, "npm-shrinkwrap.json"))) {
+        return;
+      }
+      if (files.exists(files.pathJoin(
+            packageNpmDir, "node_modules", ".package-lock.json"))) {
+        return;
+      }
+      candidates.push({ name, packageNpmDir });
+    });
+  } catch (e) {
+    return none;
+  }
+  if (! candidates.length) {
+    return none;
+  }
+
+  function installedAlready(candidate) {
+    return files.exists(files.pathJoin(
+      candidate.packageNpmDir, "node_modules", ".package-lock.json"));
+  }
+
+  async function runPool(queue, size, fn) {
+    let nextIndex = 0;
+    let completed = 0;
+    async function worker() {
+      while (nextIndex < queue.length) {
+        const candidate = queue[nextIndex++];
+        try {
+          // Re-check under the lock: a concurrent build (or the other
+          // prewarm phase) may have handled this package already.
+          const ok = await withNpmPackageLock(
+            candidate.packageNpmDir,
+            async () => ! installedAlready(candidate) && fn(candidate)
+          );
+          if (ok) {
+            completed++;
+          }
+        } catch (e) {
+          // Best-effort; the build's own updateDependencies() will
+          // handle this package the normal way.
+          if (options.debugThrow) {
+            console.error("prewarm error for " + candidate.name + ":",
+              (e && e.stack) || e);
+          }
+        }
+      }
+    }
+    await Promise.all(
+      _.times(Math.min(size, queue.length || 1), worker));
+    return completed;
+  }
+
+  const restored = await runPool(
+    candidates, options.concurrency || 8,
+    c => restoreNodeModulesFromNpmDepsCache(c.name, c.packageNpmDir));
+
+  let installed = 0;
+  if (options.installOnMiss && only) {
+    const misses = candidates.filter(c => ! installedAlready(c));
+    installed = await runPool(
+      misses, options.installConcurrency || 3,
+      c => preinstallNpmDepsFromShrinkwrap(c.name, c.packageNpmDir));
+  }
+  return { restored, installed };
+};
+
 var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
                                            packageNpmDir, npmDependencies,
                                            quiet) {
@@ -713,6 +1099,14 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
     }
   }
 
+  // No installed tree yet (fresh tool install, node-version wipe above,
+  // or first build of a cloned package)? Try the persistent cache
+  // before paying for an npm install.
+  if (! _.isEmpty(npmDependencies) &&
+      ! files.exists(files.pathJoin(nodeModulesDir, ".package-lock.json"))) {
+    await restoreNodeModulesFromNpmDepsCache(packageName, packageNpmDir);
+  }
+
   // Make sure node_modules is present (fix for #1761). Prevents npm install
   // from installing to an existing node_modules dir higher up in the
   // filesystem.  node_modules may be absent due to a change in Node version or
@@ -759,6 +1153,16 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
 
   let preservedShrinkwrap;
 
+  // Key the eventual cache entry by the shrinkwrap that PRODUCED the
+  // install, captured before completeNpmDirectory() regenerates the
+  // file. When installing from the existing (shipped) shrinkwrap, that
+  // is the same content the next fresh install of this tool version
+  // will probe with -- the regenerated file can differ (e.g.
+  // platform-specific optional deps), which would make every probe
+  // miss. When NOT installing from the shipped shrinkwrap, leave this
+  // null so the populate step keys off the regenerated file instead.
+  let npmDepsCachePopulateKey = null;
+
   if (_.isEmpty(npmDependencies)) {
     // If there are no npmDependencies, make sure nothing is installed.
     preservedShrinkwrap = { dependencies: {} };
@@ -767,6 +1171,7 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
     // If the top-level npm dependencies are already encompassed by the
     // npm-shrinkwrap.json file, then reuse that file.
     preservedShrinkwrap = shrinkwrappedDependenciesTree;
+    npmDepsCachePopulateKey = npmDepsCacheKey(packageNpmDir);
 
   } else {
     // Otherwise install npmTree.dependencies as if we were creating a new
@@ -837,7 +1242,7 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
   }
 
   await completeNpmDirectory(packageName, newPackageNpmDir, packageNpmDir,
-                       npmDependencies);
+                       npmDependencies, npmDepsCachePopulateKey);
 };
 
 // Git-URL dependencies could never satisfy the strict string comparison
@@ -969,6 +1374,7 @@ async function completeNpmDirectory(
   newPackageNpmDir,
   packageNpmDir,
   npmDependencies,
+  npmDepsCachePopulateKey,
 ) {
   // Create a shrinkwrap file.
   shrinkwrap(newPackageNpmDir);
@@ -984,6 +1390,11 @@ async function completeNpmDirectory(
   await files.renameDirAlmostAtomically(newPackageNpmDir, packageNpmDir);
 
   dirtyNodeModulesDirectory(files.pathJoin(packageNpmDir, "node_modules"));
+
+  if (! _.isEmpty(npmDependencies)) {
+    scheduleNpmDepsCachePopulate(
+      packageName, packageNpmDir, npmDepsCachePopulateKey);
+  }
 }
 
 var createReadme = function (newPackageNpmDir) {