@maka/meteor-sdk 0.2.7 → 0.2.8

package/isopackets/combined/npm/node_modules/meteor/promise/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-d6edvk3v9nl",
+  "name": "package-new-19ro17vxabi",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/react-fast-refresh/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-8e8v01oivjc",
+  "name": "package-new-1mbjo8u1bdw",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/isopackets/combined/npm/node_modules/meteor/socket-stream-client/node_modules/.package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "package-new-3efvozmolln",
+  "name": "package-new-1bskl16cqu8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@maka/meteor-sdk",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "Programmatic Node API over the Meteor build tool (bundle/run/create/etc.) -- for driving Meteor's build system from plain Node code instead of shelling out to a meteor CLI (there isn't one anymore; see README.md). \"dependencies\" is generated by scripts/write-tool-package-json.js from scripts/dev-bundle-tool-package.js at the monorepo root -- do not hand-edit that field, edit that file and re-run the script instead.",
   "main": "src/index.js",
   "exports": {

package/src/bootstrap.js

@@ -120,6 +120,31 @@ function ensureMainModuleShim() {
   };
 }
 
+// Kick off the persistent npm-deps cache prewarm (see tools/isobuild/
+// meteor-npm.js): restore every bundled package's node_modules from
+// ~/.meteor-npm-cache in parallel, so the first build after a fresh SDK
+// install doesn't pay a serial `npm install` per package. Deliberately
+// NOT awaited -- each restore holds the same cross-process lock
+// updateDependencies() takes, so a build racing ahead of the prewarm
+// simply waits on (or wins) the per-package lock; firing it before
+// ensureRuntimeReady() lets even the isopacket builds overlap with it.
+let npmDepsCachePrewarm = null;
+function startNpmDepsCachePrewarm() {
+  if (npmDepsCachePrewarm) {
+    return;
+  }
+  try {
+    const meteorNpm = require("../tools/isobuild/meteor-npm.js");
+    const path = require("path");
+    npmDepsCachePrewarm = Promise.resolve(
+      meteorNpm.prewarmNpmDepsCache(path.resolve(__dirname, "..", "packages"))
+    ).catch(() => {});
+  } catch (e) {
+    // Best-effort: the build's own per-package install path still works.
+    npmDepsCachePrewarm = Promise.resolve();
+  }
+}
+
 // Idempotent: safe to call from every public SDK entry point. The actual
 // Babel require() hook installed by install-babel.js is itself naturally
 // idempotent too (Node's own require() cache means requiring the same
@@ -152,6 +177,7 @@ async function bootstrap(options) {
   require("../tools/tool-env/install-babel.js");
   ensureAsyncLocalStorage();
   consoleAdapter.patch();
+  startNpmDepsCachePrewarm();
   await ensureRuntimeReady(options && options.offline);
   bootstrapped = true;
 }

package/src/bundle.js

@@ -45,6 +45,11 @@ async function bundle(project, options) {
     throwIfHasMessages(result.errors);
   }
 
+  // Background npm-deps cache writes (see meteor-npm.js) ran concurrently
+  // with the build; wait for them here so a build-and-exit process still
+  // gets its cache populated. Usually a no-op by this point.
+  await require("../tools/isobuild/meteor-npm.js").flushNpmDepsCacheWrites();
+
   return {
     outputPath: options.outputPath,
     starManifest: result.starManifest,

package/tools/isobuild/meteor-npm.js

@@ -676,6 +676,295 @@ var makeNewPackageNpmDir = function (newPackageNpmDir) {
      ''/*git diff complains without trailing newline*/].join('\n'));
 };
 
+// --- Persistent npm-deps cache ---------------------------------------
+//
+// A package's .npm/package/node_modules is fully determined by three
+// things: the npm-shrinkwrap.json content, the Node compatibility
+// version, and the platform/arch. Yet every fresh install of the tool
+// (e.g. `npm i -g` of a new SDK version) starts with empty node_modules
+// for every bundled package and pays a serial `npm install` per package
+// on the first build. Since shrinkwraps rarely change between releases,
+// keep a per-user cache of completed node_modules trees keyed by a hash
+// of those three inputs, and restore by directory copy instead of
+// spawning npm.
+//
+// Everything here is best-effort: any failure falls through to the
+// normal npm install path, and a stale/corrupt restore is caught by the
+// existing shrinkwrap-vs-installed subtree comparison (which then
+// triggers a real reinstall over it).
+//
+// METEOR_PACKAGE_NPM_CACHE: unset -> enabled at ~/.meteor-npm-cache;
+// "0"/"false"/"no"/"off" -> disabled; any other value -> cache root.
+
+function npmDepsCacheDisabled() {
+  const v = process.env.METEOR_PACKAGE_NPM_CACHE;
+  return typeof v === "string" && /^\s*(0|false|no|off)\s*$/i.test(v);
+}
+
+function npmDepsCacheRoot() {
+  const v = process.env.METEOR_PACKAGE_NPM_CACHE;
+  if (v && ! npmDepsCacheDisabled()) {
+    return files.convertToStandardPath(v);
+  }
+  return files.pathJoin(
+    files.convertToStandardPath(require("os").homedir()),
+    ".meteor-npm-cache"
+  );
+}
+
+// JSON.stringify with deterministically sorted object keys, so the hash
+// is insensitive to key order, whitespace, and line endings of the
+// on-disk shrinkwrap file.
+function stableStringify(value) {
+  if (Array.isArray(value)) {
+    return "[" + value.map(stableStringify).join(",") + "]";
+  }
+  if (value && typeof value === "object") {
+    return "{" + Object.keys(value).sort().map(
+      key => JSON.stringify(key) + ":" + stableStringify(value[key])
+    ).join(",") + "}";
+  }
+  return JSON.stringify(value);
+}
+
+// Hash of the package's current npm-shrinkwrap.json plus everything
+// else that determines node_modules content. Returns null if the
+// shrinkwrap is missing or unparsable.
+function npmDepsCacheKey(packageNpmDir) {
+  let shrinkwrap;
+  try {
+    shrinkwrap = JSON.parse(files.readFile(
+      files.pathJoin(packageNpmDir, "npm-shrinkwrap.json"), "utf8"));
+  } catch (e) {
+    return null;
+  }
+  return require("crypto").createHash("sha256").update(stableStringify({
+    shrinkwrap,
+    node: currentNodeCompatibilityVersion(),
+    platform: process.platform,
+    arch: process.arch,
+  })).digest("hex").slice(0, 32);
+}
+
+function npmDepsCacheEntryDir(packageName, key) {
+  // Meteor package names may contain characters that are not valid in
+  // paths everywhere (e.g. ":" on Windows); the hash disambiguates, so
+  // a lossy sanitization is fine.
+  const safeName = packageName.replace(/[^A-Za-z0-9._-]/g, "_");
+  return files.pathJoin(npmDepsCacheRoot(), safeName, key);
+}
+
+// Copy a cached node_modules into packageNpmDir (via temp dir + rename,
+// so a crash never leaves a half-restored node_modules in place).
+// Returns true on success. Callers must hold the npm package lock for
+// packageNpmDir.
+async function restoreNodeModulesFromNpmDepsCache(packageName, packageNpmDir) {
+  if (npmDepsCacheDisabled()) {
+    return false;
+  }
+  const key = npmDepsCacheKey(packageNpmDir);
+  if (! key) {
+    return false;
+  }
+  const cachedNodeModules = files.pathJoin(
+    npmDepsCacheEntryDir(packageName, key), "node_modules");
+  if (! files.exists(
+        files.pathJoin(cachedNodeModules, ".package-lock.json"))) {
+    return false;
+  }
+
+  // Sweep temp dirs left behind by a crashed earlier restore (safe: we
+  // hold the package lock).
+  try {
+    files.readdir(packageNpmDir).forEach(entry => {
+      if (entry.startsWith(".node_modules-from-cache-")) {
+        files.rm_recursive(files.pathJoin(packageNpmDir, entry));
+      }
+    });
+  } catch (e) {}
+
+  const nodeModulesDir = files.pathJoin(packageNpmDir, "node_modules");
+  const tmpDir = files.pathJoin(
+    packageNpmDir, ".node_modules-from-cache-" + utils.randomToken());
+  tmpDirs.push(tmpDir);
+  try {
+    await files.cp_r(cachedNodeModules, tmpDir, { preserveSymlinks: true });
+    await files.renameDirAlmostAtomically(tmpDir, nodeModulesDir);
+    dirtyNodeModulesDirectory(nodeModulesDir);
+    return true;
+  } catch (e) {
+    try {
+      if (files.exists(tmpDir)) {
+        await files.rm_recursive(tmpDir);
+      }
+    } catch (cleanupError) {}
+    return false;
+  } finally {
+    tmpDirs = _.without(tmpDirs, tmpDir);
+  }
+}
+
+// Cache writes run in the background (fire-and-forget with an atomic
+// temp-dir + rename publish) so they never sit on the build's critical
+// path. flushNpmDepsCacheWrites() lets one-shot processes (e.g. a
+// build-and-exit embedder) wait for them before exiting; if a process
+// dies first, the orphaned .tmp-* dir is swept by a later prune.
+const pendingNpmDepsCacheWrites = [];
+
+meteorNpm.flushNpmDepsCacheWrites = function () {
+  return Promise.allSettled(pendingNpmDepsCacheWrites);
+};
+
+function scheduleNpmDepsCachePopulate(packageName, packageNpmDir, key) {
+  if (npmDepsCacheDisabled()) {
+    return;
+  }
+  pendingNpmDepsCacheWrites.push(
+    populateNpmDepsCache(packageName, packageNpmDir, key).catch(e => {})
+  );
+}
+
+async function populateNpmDepsCache(packageName, packageNpmDir, key) {
+  key = key || npmDepsCacheKey(packageNpmDir);
+  if (! key) {
+    return;
+  }
+  const entryDir = npmDepsCacheEntryDir(packageName, key);
+  if (files.exists(entryDir)) {
+    return;
+  }
+  const packageCacheDir = files.pathDirname(entryDir);
+  files.mkdir_p(packageCacheDir);
+  const tmpDir = files.pathJoin(
+    packageCacheDir, ".tmp-" + utils.randomToken());
+  try {
+    await files.cp_r(
+      files.pathJoin(packageNpmDir, "node_modules"),
+      files.pathJoin(tmpDir, "node_modules"),
+      { preserveSymlinks: true }
+    );
+    files.rename(tmpDir, entryDir);
+  } catch (e) {
+    // Concurrent populate may have won the rename race, or the copy
+    // failed midway; either way discard our temp dir and move on.
+    try {
+      if (files.exists(tmpDir)) {
+        await files.rm_recursive(tmpDir);
+      }
+    } catch (cleanupError) {}
+    return;
+  }
+  pruneNpmDepsCache(packageCacheDir, entryDir);
+}
+
+// Keep the 3 newest entries per package (different keys = different
+// shrinkwrap/node/platform combos) and sweep stale temp dirs, so the
+// cache stays bounded without any global bookkeeping.
+function pruneNpmDepsCache(packageCacheDir, keepDir) {
+  try {
+    const keepBase = files.pathBasename(keepDir);
+    const entries = [];
+    files.readdir(packageCacheDir).forEach(entry => {
+      const fullPath = files.pathJoin(packageCacheDir, entry);
+      if (entry.startsWith(".tmp-")) {
+        // Only sweep temp dirs that are clearly abandoned; a concurrent
+        // process may still be filling a fresh one.
+        try {
+          const ageMs = Date.now() - files.stat(fullPath).mtime.getTime();
+          if (ageMs > 60 * 60 * 1000) {
+            files.rm_recursive(fullPath);
+          }
+        } catch (e) {}
+        return;
+      }
+      if (entry === keepBase) {
+        return;
+      }
+      try {
+        entries.push({ fullPath, mtimeMs: files.stat(fullPath).mtime.getTime() });
+      } catch (e) {}
+    });
+    entries.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    entries.slice(2).forEach(entry => {
+      try {
+        files.rm_recursive(entry.fullPath);
+      } catch (e) {}
+    });
+  } catch (e) {}
+}
+
+// Restore every restorable package under packagesDir from the cache, in
+// parallel, before the (serial) per-package build pipeline reaches
+// them. Purely filesystem-driven: a candidate is any package dir with a
+// .npm/package/npm-shrinkwrap.json but no installed node_modules, so no
+// package.js evaluation is needed. Safe to run concurrently with a
+// build -- each restore holds the same cross-process lock
+// updateDependencies() takes, and re-checks its precondition under it.
+// Returns the number of packages restored.
+meteorNpm.prewarmNpmDepsCache = async function (packagesDir, options) {
+  options = options || {};
+  if (npmDepsCacheDisabled()) {
+    return 0;
+  }
+  packagesDir = files.convertToStandardPath(packagesDir);
+
+  const candidates = [];
+  try {
+    files.readdir(packagesDir).forEach(name => {
+      const packageNpmDir =
+        files.pathJoin(packagesDir, name, ".npm", "package");
+      if (! files.exists(
+            files.pathJoin(packageNpmDir, "npm-shrinkwrap.json"))) {
+        return;
+      }
+      if (files.exists(files.pathJoin(
+            packageNpmDir, "node_modules", ".package-lock.json"))) {
+        return;
+      }
+      candidates.push({ name, packageNpmDir });
+    });
+  } catch (e) {
+    return 0;
+  }
+  if (! candidates.length) {
+    return 0;
+  }
+
+  let restored = 0;
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < candidates.length) {
+      const candidate = candidates[nextIndex++];
+      try {
+        const ok = await withNpmPackageLock(
+          candidate.packageNpmDir,
+          async () => {
+            // Re-check under the lock: a concurrent build may have
+            // installed (or restored) this package already.
+            if (files.exists(files.pathJoin(
+                  candidate.packageNpmDir,
+                  "node_modules", ".package-lock.json"))) {
+              return false;
+            }
+            return restoreNodeModulesFromNpmDepsCache(
+              candidate.name, candidate.packageNpmDir);
+          }
+        );
+        if (ok) {
+          restored++;
+        }
+      } catch (e) {
+        // Best-effort; the build's own updateDependencies() will handle
+        // this package the normal way.
+      }
+    }
+  }
+  const concurrency = Math.min(
+    options.concurrency || 8, candidates.length);
+  await Promise.all(_.times(concurrency, worker));
+  return restored;
+};
+
 var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
                                            packageNpmDir, npmDependencies,
                                            quiet) {
@@ -713,6 +1002,14 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
     }
   }
 
+  // No installed tree yet (fresh tool install, node-version wipe above,
+  // or first build of a cloned package)? Try the persistent cache
+  // before paying for an npm install.
+  if (! _.isEmpty(npmDependencies) &&
+      ! files.exists(files.pathJoin(nodeModulesDir, ".package-lock.json"))) {
+    await restoreNodeModulesFromNpmDepsCache(packageName, packageNpmDir);
+  }
+
   // Make sure node_modules is present (fix for #1761). Prevents npm install
   // from installing to an existing node_modules dir higher up in the
   // filesystem.  node_modules may be absent due to a change in Node version or
@@ -759,6 +1056,16 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
 
   let preservedShrinkwrap;
 
+  // Key the eventual cache entry by the shrinkwrap that PRODUCED the
+  // install, captured before completeNpmDirectory() regenerates the
+  // file. When installing from the existing (shipped) shrinkwrap, that
+  // is the same content the next fresh install of this tool version
+  // will probe with -- the regenerated file can differ (e.g.
+  // platform-specific optional deps), which would make every probe
+  // miss. When NOT installing from the shipped shrinkwrap, leave this
+  // null so the populate step keys off the regenerated file instead.
+  let npmDepsCachePopulateKey = null;
+
   if (_.isEmpty(npmDependencies)) {
     // If there are no npmDependencies, make sure nothing is installed.
     preservedShrinkwrap = { dependencies: {} };
@@ -767,6 +1074,7 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
     // If the top-level npm dependencies are already encompassed by the
     // npm-shrinkwrap.json file, then reuse that file.
     preservedShrinkwrap = shrinkwrappedDependenciesTree;
+    npmDepsCachePopulateKey = npmDepsCacheKey(packageNpmDir);
 
   } else {
     // Otherwise install npmTree.dependencies as if we were creating a new
@@ -837,7 +1145,7 @@ var updateExistingNpmDirectory = async function (packageName, newPackageNpmDir,
   }
 
   await completeNpmDirectory(packageName, newPackageNpmDir, packageNpmDir,
-                       npmDependencies);
+                       npmDependencies, npmDepsCachePopulateKey);
 };
 
 // Git-URL dependencies could never satisfy the strict string comparison
@@ -969,6 +1277,7 @@ async function completeNpmDirectory(
   newPackageNpmDir,
   packageNpmDir,
   npmDependencies,
+  npmDepsCachePopulateKey,
 ) {
   // Create a shrinkwrap file.
   shrinkwrap(newPackageNpmDir);
@@ -984,6 +1293,11 @@ async function completeNpmDirectory(
   await files.renameDirAlmostAtomically(newPackageNpmDir, packageNpmDir);
 
   dirtyNodeModulesDirectory(files.pathJoin(packageNpmDir, "node_modules"));
+
+  if (! _.isEmpty(npmDependencies)) {
+    scheduleNpmDepsCachePopulate(
+      packageName, packageNpmDir, npmDepsCachePopulateKey);
+  }
 }
 
 var createReadme = function (newPackageNpmDir) {