@agoric/xsnap 0.14.3-u19.1 → 0.14.3-u20.0

package/api.js

@@ -7,7 +7,7 @@
  * Also, update golden master test/xs-perf.test.js to reflect new meter
  * version.
  */
-export const METER_TYPE = 'xs-meter-34';
+export const METER_TYPE = 'xs-meter-35';
 
 export const ExitCode = {
   E_UNKNOWN_ERROR: -1,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/xsnap",
-  "version": "0.14.3-u19.1",
+  "version": "0.14.3-u20.0",
   "description": "Snapshotting VM worker based on Moddable's XS Javascript engine",
   "author": "Agoric",
   "license": "Apache-2.0",
@@ -28,25 +28,26 @@
     "test:xs": "exit 0"
   },
   "dependencies": {
-    "@agoric/internal": "^0.4.0-u19.1",
-    "@agoric/xsnap-lockdown": "^0.14.1-u19.0",
-    "@endo/bundle-source": "^3.5.1",
-    "@endo/errors": "^1.2.9",
-    "@endo/eventual-send": "^1.3.0",
-    "@endo/init": "^1.1.8",
-    "@endo/netstring": "^1.0.14",
-    "@endo/promise-kit": "^1.1.9",
-    "@endo/stream": "^1.2.9",
-    "@endo/stream-node": "^1.1.9",
+    "@agoric/internal": "^0.4.0-u20.0",
+    "@agoric/xsnap-lockdown": "^0.14.1-u20.0",
+    "@endo/bundle-source": "^4.0.0",
+    "@endo/errors": "^1.2.10",
+    "@endo/eventual-send": "^1.3.1",
+    "@endo/init": "^1.1.9",
+    "@endo/netstring": "^1.0.15",
+    "@endo/promise-kit": "^1.1.10",
+    "@endo/stream": "^1.2.10",
+    "@endo/stream-node": "^1.1.10",
     "glob": "^7.1.6",
     "tmp": "^0.2.1"
   },
   "devDependencies": {
     "@endo/base64": "^1.0.9",
-    "@endo/nat": "^5.0.14",
+    "@endo/nat": "^5.1.0",
     "@types/glob": "^8.1.0",
     "ava": "^5.3.0",
-    "c8": "^10.1.2"
+    "c8": "^10.1.2",
+    "execa": "^7.2.0"
   },
   "files": [
     "LICENSE*",
@@ -76,7 +77,7 @@
     "workerThreads": false
   },
   "typeCoverage": {
-    "atLeast": 93.51
+    "atLeast": 93.56
   },
-  "gitHead": "a04d2bf43a9753d123954b52c6ba8d35083a3c8f"
+  "gitHead": "8e4207fa19dabf76c1f91f8779b5b5b93570ecea"
 }

package/src/build.js

@@ -10,9 +10,18 @@ const { freeze } = Object;
 /** @param {string} path */
 const asset = path => fileURLToPath(new URL(path, import.meta.url));
 
+/** @param {Promise<unknown>} p */
+const isRejected = p =>
+  p.then(
+    () => false,
+    () => true,
+  );
+
+/** @typedef {{ path: string, make?: string }} ModdablePlatform */
+
 const ModdableSDK = {
   MODDABLE: asset('../moddable'),
-  /** @type { Record<string, { path: string, make?: string }>} */
+  /** @type { Record<string, ModdablePlatform>} */
   platforms: {
     Linux: { path: 'lin' },
     Darwin: { path: 'mac' },
@@ -22,7 +31,8 @@ const ModdableSDK = {
 };
 
 /**
- * Adapt spawn to Promises style.
+ * Create promise-returning functions for asynchronous execution of no-input
+ * commands.
  *
  * @param {string} command
  * @param {{
@@ -48,6 +58,8 @@ function makeCLI(command, { spawn }) {
 
   return freeze({
     /**
+     * Run the command, writing directly to stdout and stderr.
+     *
      * @param {string[]} args
      * @param {{ cwd?: string }} [opts]
      */
@@ -55,26 +67,30 @@ function makeCLI(command, { spawn }) {
       const { cwd = '.' } = opts || {};
       const child = spawn(command, args, {
         cwd,
-        stdio: ['inherit', 'inherit', 'inherit'],
+        stdio: ['ignore', 'inherit', 'inherit'],
       });
       return wait(child);
     },
     /**
+     * Run the command, writing directly to stderr but capturing and returning
+     * stdout.
+     *
      * @param {string[]} args
-     * @param {{ cwd?: string }} [opts]
+     * @param {{ cwd?: string, fullOutput?: boolean }} [opts]
+     * @returns {Promise<string>} command output, stripped of trailing
+     *   whitespace unless option "fullOutput" is true
      */
-    pipe: (args, opts) => {
-      const { cwd = '.' } = opts || {};
+    pipe: async (args, opts) => {
+      const { cwd = '.', fullOutput = false } = opts || {};
       const child = spawn(command, args, {
         cwd,
-        stdio: ['inherit', 'pipe', 'inherit'],
-      });
-      let output = '';
-      child.stdout.setEncoding('utf8');
-      child.stdout.on('data', data => {
-        output += data.toString();
+        stdio: ['ignore', 'pipe', 'inherit'],
       });
-      return wait(child).then(() => output);
+      const chunks = [];
+      child.stdout.on('data', chunk => chunks.push(chunk));
+      await wait(child);
+      const output = Buffer.concat(chunks).toString('utf8');
+      return fullOutput ? output : output.trimEnd();
     },
   });
 }
@@ -85,28 +101,6 @@ function makeCLI(command, { spawn }) {
  * @param {{ git: ReturnType<typeof makeCLI> }} io
  */
 const makeSubmodule = (path, repoUrl, { git }) => {
-  /** @param {string} text */
-  const parseStatus = text =>
-    text
-      .split('\n')
-      // From `git submodule --help`:
-      // Show the status of the submodules. This will print the SHA-1 of the
-      // currently checked out commit for each submodule, along with the
-      // submodule path and the output of git describe for the SHA-1. Each
-      // SHA-1 will possibly be prefixed with - if the submodule is not
-      // initialized, + if the currently checked out submodule commit does
-      // not match the SHA-1 found in the index of the containing repository
-      // and U if the submodule has merge conflicts.
-      //
-      // We discovered that in other cases, the prefix is a single space.
-      .map(line => [line[0], ...line.slice(1).split(' ', 3)])
-      .map(([prefix, hash, statusPath, describe]) => ({
-        prefix,
-        hash,
-        path: statusPath,
-        describe,
-      }));
-
   return freeze({
     path,
     clone: async () => git.run(['clone', repoUrl, path]),
@@ -114,90 +108,107 @@ const makeSubmodule = (path, repoUrl, { git }) => {
     checkout: async commitHash =>
       git.run(['checkout', commitHash], { cwd: path }),
     init: async () => git.run(['submodule', 'update', '--init', '--checkout']),
-    status: async () =>
-      git.pipe(['submodule', 'status', path]).then(parseStatus),
-    /** @param {string} leaf */
+    status: async () => {
+      const line = await git.pipe(['submodule', 'status', path]);
+      // From `git submodule --help`:
+      // status [--cached] [--recursive] [--] [<path>...]
+      //     Show the status of the submodules. This will print the SHA-1 of the
+      //     currently checked out commit for each submodule, along with the
+      //     submodule path and the output of git describe for the SHA-1. Each
+      //     SHA-1 will possibly be prefixed with - if the submodule is not
+      //     initialized, + if the currently checked out submodule commit does
+      //     not match the SHA-1 found in the index of the containing repository
+      //     and U if the submodule has merge conflicts.
+      //
+      // We discovered that in other cases, the prefix is a single space.
+      const prefix = line[0];
+      const [hash, statusPath, ...describe] = line.slice(1).split(' ');
+      return {
+        prefix,
+        hash,
+        path: statusPath,
+        describe: describe.join(' '),
+      };
+    },
+    /**
+     * Read a specific configuration value for this submodule (e.g., "path" or
+     * "url") from the top-level .gitmodules.
+     *
+     * @param {string} leaf
+     */
     config: async leaf => {
       // git rev-parse --show-toplevel
-      const top = await git
-        .pipe(['rev-parse', '--show-toplevel'])
-        .then(l => l.trimEnd());
-      // assume full paths
-      const name = path.slice(top.length + 1);
-      // git config -f ../../.gitmodules --get submodule."$name".url
-      const value = await git
-        .pipe([
-          'config',
-          '-f',
-          `${top}/.gitmodules`,
-          '--get',
-          `submodule.${name}.${leaf}`,
-        ])
-        .then(l => l.trimEnd());
+      const repoRoot = await git.pipe(['rev-parse', '--show-toplevel']);
+      if (!path.startsWith(`${repoRoot}/`)) {
+        throw Error(
+          `Expected submodule path ${path} to be a subdirectory of repository ${repoRoot}`,
+        );
+      }
+      const relativePath = path.slice(repoRoot.length + 1);
+      // git config -f ../../.gitmodules --get submodule.${relativePath}.${leaf}
+      const value = await git.pipe([
+        'config',
+        '-f',
+        `${repoRoot}/.gitmodules`,
+        '--get',
+        `submodule.${relativePath}.${leaf}`,
+      ]);
       return value;
     },
   });
 };
 
 /**
- * @param {boolean} showEnv
+ * @typedef {{
+ *   url: string,
+ *   path: string,
+ *   commitHash?: string,
+ *   envPrefix: string,
+ * }} SubmoduleDescriptor
+ */
+
+/**
+ * @param {SubmoduleDescriptor[]} submodules
  * @param {{
- *   env: Record<string, string | undefined>,
+ *   git: ReturnType<typeof makeCLI>,
  *   stdout: typeof process.stdout,
- *   spawn: typeof import('child_process').spawn,
- *   fs: {
- *     existsSync: typeof import('fs').existsSync,
- *     rmdirSync: typeof import('fs').rmdirSync,
- *     readFile: typeof import('fs').promises.readFile,
- *   },
  * }} io
  */
-const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => {
-  const git = makeCLI('git', { spawn });
-
-  // When changing/adding entries here, make sure to search the whole project
-  // for `@@AGORIC_DOCKER_SUBMODULES@@`
-  const submodules = [
-    {
-      url: env.MODDABLE_URL || 'https://github.com/agoric-labs/moddable.git',
-      path: ModdableSDK.MODDABLE,
-      commitHash: env.MODDABLE_COMMIT_HASH,
-      envPrefix: 'MODDABLE_',
-    },
-    {
-      url:
-        env.XSNAP_NATIVE_URL || 'https://github.com/agoric-labs/xsnap-pub.git',
-      path: asset('../xsnap-native'),
-      commitHash: env.XSNAP_NATIVE_COMMIT_HASH,
-      envPrefix: 'XSNAP_NATIVE_',
-    },
-  ];
-
+const showEnv = async (submodules, { git, stdout }) => {
   await null;
-  if (showEnv) {
-    for (const submodule of submodules) {
-      const { path, envPrefix, commitHash } = submodule;
-      if (!commitHash) {
-        // We need to glean the commitHash and url from Git.
-        const sm = makeSubmodule(path, '?', { git });
-        const [[{ hash }], url] = await Promise.all([
-          sm.status(),
-          sm.config('url'),
-        ]);
-        submodule.commitHash = hash;
-        submodule.url = url;
-      }
-      stdout.write(`${envPrefix}URL=${submodule.url}\n`);
-      stdout.write(`${envPrefix}COMMIT_HASH=${submodule.commitHash}\n`);
+  for (const desc of submodules) {
+    const { path, envPrefix } = desc;
+    let { url, commitHash } = desc;
+    if (!commitHash) {
+      // We need to glean the commitHash and url from Git.
+      const submodule = makeSubmodule(path, '?', { git });
+      const [{ hash }, gitUrl] = await Promise.all([
+        submodule.status(),
+        submodule.config('url'),
+      ]);
+      commitHash = hash;
+      url = gitUrl;
     }
-    return;
+    stdout.write(`${envPrefix}URL=${url}\n`);
+    stdout.write(`${envPrefix}COMMIT_HASH=${commitHash}\n`);
   }
+};
 
+/**
+ * @param {SubmoduleDescriptor[]} submodules
+ * @param {{
+ *   fs: Pick<typeof import('fs'), 'existsSync' | 'rmdirSync'>,
+ *   git: ReturnType<typeof makeCLI>,
+ * }} io
+ */
+const updateSubmodules = async (submodules, { fs, git }) => {
+  await null;
   for (const { url, path, commitHash } of submodules) {
     const submodule = makeSubmodule(path, url, { git });
 
-    // Allow overriding of the checked-out version of the submodule.
-    if (commitHash) {
+    if (!commitHash) {
+      await submodule.init();
+    } else {
       // Do the moral equivalent of submodule update when explicitly overriding.
       try {
         fs.rmdirSync(submodule.path);
@@ -208,29 +219,20 @@ const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => {
         await submodule.clone();
       }
       await submodule.checkout(commitHash);
-    } else {
-      await submodule.init();
     }
   }
 };
 
 /**
+ * @param {ModdablePlatform} platform
+ * @param {boolean} force
  * @param {{
- *   spawn: typeof import('child_process').spawn,
- *   fs: {
- *     existsSync: typeof import('fs').existsSync,
- *     rmdirSync: typeof import('fs').rmdirSync,
- *     readFile: typeof import('fs').promises.readFile,
- *     writeFile: typeof import('fs').promises.writeFile,
- *   },
- *   os: {
- *     type: typeof import('os').type,
- *   }
+ *   fs: Pick<typeof import('fs'), 'existsSync'> &
+ *     Pick<typeof import('fs').promises, 'readFile' | 'writeFile'>,
+ *   make: ReturnType<typeof makeCLI>,
  * }} io
- * @param {object} [options]
- * @param {boolean} [options.forceBuild]
  */
-const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => {
+const buildXsnap = async (platform, force, { fs, make }) => {
   const pjson = await fs.readFile(asset('../package.json'), 'utf-8');
   const pkg = JSON.parse(pjson);
 
@@ -245,16 +247,10 @@ const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => {
     : '';
 
   const expectedConfigEnvs = configEnvs.concat('').join('\n');
-  if (forceBuild || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) {
+  if (force || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) {
     await fs.writeFile(configEnvFile, expectedConfigEnvs);
   }
 
-  const platform = ModdableSDK.platforms[os.type()];
-  if (!platform) {
-    throw Error(`Unsupported OS found: ${os.type()}`);
-  }
-
-  const make = makeCLI(platform.make || 'make', { spawn });
   for (const goal of ModdableSDK.buildGoals) {
     await make.run(
       [
@@ -280,15 +276,9 @@ const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => {
  *   env: Record<string, string | undefined>,
  *   stdout: typeof process.stdout,
  *   spawn: typeof import('child_process').spawn,
- *   fs: {
- *     existsSync: typeof import('fs').existsSync,
- *     rmdirSync: typeof import('fs').rmdirSync,
- *     readFile: typeof import('fs').promises.readFile,
- *     writeFile: typeof import('fs').promises.writeFile,
- *   },
- *   os: {
- *     type: typeof import('os').type,
- *   }
+ *   fs: Pick<typeof import('fs'), 'existsSync' | 'rmdirSync'> &
+ *     Pick<typeof import('fs').promises, 'readFile' | 'writeFile'>,
+ *   os: Pick<typeof import('os'), 'type'>,
  * }} io
  */
 async function main(args, { env, stdout, spawn, fs, os }) {
@@ -297,76 +287,77 @@ async function main(args, { env, stdout, spawn, fs, os }) {
   await null;
 
   const osType = os.type();
-  const platform = {
-    Linux: 'lin',
-    Darwin: 'mac',
-    // Windows_NT: 'win', // One can dream.
-  }[osType];
-  if (platform === undefined) {
-    throw Error(`xsnap does not support platform ${osType}`);
+  const platform = ModdableSDK.platforms[osType];
+  if (!platform) {
+    throw Error(`xsnap does not support OS ${osType}`);
   }
 
-  // If this is a working copy of xsnap in a checkout of agoric-sdk, we need to
-  // either clone or update submodules.
-  // Otherwise, we are running from an extracted npm tarball and we should not
-  // attempt to update Git submodules and should make the binary from the
-  // published source.
-  //
-  // These steps will avoid rebuilding native xsnap in the common case for end
-  // users.
-  //
-  //                    ||      | X    || git
-  //                    || X    | X    || make
-  //                    || ---- | ---- || ----
-  // | bin | src | .git || pack | work ||
-  // | --- | --- | ---- || ---- | ---- ||
-  // |     |     |      ||      | X    ||
-  // |     |     | X    ||      |      ||
-  // |     | X   |      || X    |      ||
-  // |     | X   | X    ||      | X    ||
-  // | X   |     |      ||      |      ||
-  // | X   |     | X    ||      |      ||
-  // | X   | X   |      || X    |      ||
-  // | X   | X   | X    ||      | X    ||
-  //
-  // We build both release and debug, so checking for one should suffice.
+  const git = makeCLI('git', { spawn });
+  const make = makeCLI(platform.make || 'make', { spawn });
+
+  // When changing/adding entries here, make sure to search the whole project
+  // for `@@AGORIC_DOCKER_SUBMODULES@@`
+  const submodules = [
+    {
+      url: env.MODDABLE_URL || 'https://github.com/agoric-labs/moddable.git',
+      path: ModdableSDK.MODDABLE,
+      commitHash: env.MODDABLE_COMMIT_HASH,
+      envPrefix: 'MODDABLE_',
+    },
+    {
+      url:
+        env.XSNAP_NATIVE_URL || 'https://github.com/agoric-labs/xsnap-pub.git',
+      path: asset('../xsnap-native'),
+      commitHash: env.XSNAP_NATIVE_COMMIT_HASH,
+      envPrefix: 'XSNAP_NATIVE_',
+    },
+  ];
+
+  // We build both release and debug executables, so checking for only the
+  // former is fine.
   // XXX This will need to account for the .exe extension if we recover support
   // for Windows.
-  const hasBin = fs.existsSync(
-    asset(`../xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker`),
+  const bin = asset(
+    `../xsnap-native/xsnap/build/bin/${platform.path}/release/xsnap-worker`,
   );
-  let hasSource = fs.existsSync(asset('../moddable/xs/includes/xs.h'));
+  const hasBin = fs.existsSync(bin);
+  const hasSource = fs.existsSync(asset('../moddable/xs/includes/xs.h'));
   const hasGit = fs.existsSync(asset('../moddable/.git'));
+
+  // If a git submodule is present or source files and prebuilt executables are
+  // both absent, consider ourselves to be in an active git checkout (as opposed
+  // to e.g. an extracted npm tarball).
   const isWorkingCopy = hasGit || (!hasSource && !hasBin);
-  const showEnv = args.includes('--show-env');
 
-  if (isWorkingCopy || showEnv) {
-    if (showEnv && !isWorkingCopy) {
+  // --show-env reports submodule status without making changes.
+  if (args.includes('--show-env')) {
+    if (!isWorkingCopy) {
       throw Error('XSnap requires a working copy and git to --show-env');
     }
-    await updateSubmodules(showEnv, { env, stdout, spawn, fs });
-    hasSource = true;
+    await showEnv(submodules, { git, stdout });
+    return;
   }
 
-  if (!showEnv) {
-    if (hasSource) {
-      // Force a rebuild if for some reason the binary is out of date
-      // Since the make checks may not always detect that situation
-      let forceBuild = !hasBin;
-      if (hasBin) {
-        const npm = makeCLI('npm', { spawn });
-        await npm
-          .run(['run', '-s', 'check-version'], { cwd: asset('..') })
-          .catch(() => {
-            forceBuild = true;
-          });
-      }
-      await makeXsnap({ spawn, fs, os }, { forceBuild });
-    } else if (!hasBin) {
-      throw Error(
-        'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?',
-      );
-    }
+  // Fetch/update source files via `git submodule` as appropriate.
+  if (isWorkingCopy) {
+    await updateSubmodules(submodules, { fs, git });
+  }
+
+  // If we now have source files, (re)build from them.
+  // Otherwise, require presence of a previously-built executable.
+  if (hasSource || isWorkingCopy) {
+    // Force a rebuild if for some reason the binary is out of date
+    // since the make checks may not always detect that situation.
+    const npm = makeCLI('npm', { spawn });
+    const force = await (!hasBin ||
+      isRejected(
+        npm.run(['run', '-s', 'check-version'], { cwd: asset('..') }),
+      ));
+    await buildXsnap(platform, force, { fs, make });
+  } else if (!hasBin) {
+    throw Error(
+      'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?',
+    );
   }
 }
 
@@ -376,10 +367,10 @@ const run = () =>
     stdout: process.stdout,
     spawn: childProcessTop.spawn,
     fs: {
-      readFile: fsTop.promises.readFile,
-      writeFile: fsTop.promises.writeFile,
       existsSync: fsTop.existsSync,
       rmdirSync: fsTop.rmdirSync,
+      readFile: fsTop.promises.readFile,
+      writeFile: fsTop.promises.writeFile,
     },
     os: {
       type: osTop.type,