@agoric/xsnap 0.14.3-other-dev-8f8782b.0 → 0.14.3-other-dev-fbe72e7.0.fbe72e7

package/src/build.js

@@ -1,18 +1,27 @@
 #!/usr/bin/env node
-/* global process */
-/* eslint-disable @jessie.js/no-nested-await -- test/build code */
+/* eslint-env node */
 import * as childProcessTop from 'child_process';
+import { fileURLToPath } from 'url';
 import fsTop from 'fs';
 import osTop from 'os';
 
 const { freeze } = Object;
 
 /** @param {string} path */
-const asset = path => new URL(path, import.meta.url).pathname;
+const asset = path => fileURLToPath(new URL(path, import.meta.url));
+
+/** @param {Promise<unknown>} p */
+const isRejected = p =>
+  p.then(
+    () => false,
+    () => true,
+  );
+
+/** @typedef {{ path: string, make?: string }} ModdablePlatform */
 
 const ModdableSDK = {
   MODDABLE: asset('../moddable'),
-  /** @type { Record<string, { path: string, make?: string }>} */
+  /** @type { Record<string, ModdablePlatform>} */
   platforms: {
     Linux: { path: 'lin' },
     Darwin: { path: 'mac' },
@@ -22,7 +31,8 @@ const ModdableSDK = {
 };
 
 /**
- * Adapt spawn to Promises style.
+ * Create promise-returning functions for asynchronous execution of no-input
+ * commands.
  *
  * @param {string} command
  * @param {{
@@ -48,6 +58,8 @@ function makeCLI(command, { spawn }) {
 
   return freeze({
     /**
+     * Run the command, writing directly to stdout and stderr.
+     *
      * @param {string[]} args
      * @param {{ cwd?: string }} [opts]
      */
@@ -55,26 +67,30 @@ function makeCLI(command, { spawn }) {
       const { cwd = '.' } = opts || {};
       const child = spawn(command, args, {
         cwd,
-        stdio: ['inherit', 'inherit', 'inherit'],
+        stdio: ['ignore', 'inherit', 'inherit'],
       });
       return wait(child);
     },
     /**
+     * Run the command, writing directly to stderr but capturing and returning
+     * stdout.
+     *
      * @param {string[]} args
-     * @param {{ cwd?: string }} [opts]
+     * @param {{ cwd?: string, fullOutput?: boolean }} [opts]
+     * @returns {Promise<string>} command output, stripped of trailing
+     *   whitespace unless option "fullOutput" is true
      */
-    pipe: (args, opts) => {
-      const { cwd = '.' } = opts || {};
+    pipe: async (args, opts) => {
+      const { cwd = '.', fullOutput = false } = opts || {};
       const child = spawn(command, args, {
         cwd,
-        stdio: ['inherit', 'pipe', 'inherit'],
+        stdio: ['ignore', 'pipe', 'inherit'],
       });
-      let output = '';
-      child.stdout.setEncoding('utf8');
-      child.stdout.on('data', data => {
-        output += data.toString();
-      });
-      return wait(child).then(() => output);
+      const chunks = [];
+      child.stdout.on('data', chunk => chunks.push(chunk));
+      await wait(child);
+      const output = Buffer.concat(chunks).toString('utf8');
+      return fullOutput ? output : output.trimEnd();
     },
   });
 }
@@ -85,28 +101,6 @@ function makeCLI(command, { spawn }) {
  * @param {{ git: ReturnType<typeof makeCLI> }} io
  */
 const makeSubmodule = (path, repoUrl, { git }) => {
-  /** @param {string} text */
-  const parseStatus = text =>
-    text
-      .split('\n')
-      // From `git submodule --help`:
-      // Show the status of the submodules. This will print the SHA-1 of the
-      // currently checked out commit for each submodule, along with the
-      // submodule path and the output of git describe for the SHA-1. Each
-      // SHA-1 will possibly be prefixed with - if the submodule is not
-      // initialized, + if the currently checked out submodule commit does
-      // not match the SHA-1 found in the index of the containing repository
-      // and U if the submodule has merge conflicts.
-      //
-      // We discovered that in other cases, the prefix is a single space.
-      .map(line => [line[0], ...line.slice(1).split(' ', 3)])
-      .map(([prefix, hash, statusPath, describe]) => ({
-        prefix,
-        hash,
-        path: statusPath,
-        describe,
-      }));
-
   return freeze({
     path,
     clone: async () => git.run(['clone', repoUrl, path]),
@@ -114,93 +108,107 @@ const makeSubmodule = (path, repoUrl, { git }) => {
     checkout: async commitHash =>
       git.run(['checkout', commitHash], { cwd: path }),
     init: async () => git.run(['submodule', 'update', '--init', '--checkout']),
-    status: async () =>
-      git.pipe(['submodule', 'status', path]).then(parseStatus),
-    /** @param {string} leaf */
+    status: async () => {
+      const line = await git.pipe(['submodule', 'status', path]);
+      // From `git submodule --help`:
+      // status [--cached] [--recursive] [--] [<path>...]
+      //     Show the status of the submodules. This will print the SHA-1 of the
+      //     currently checked out commit for each submodule, along with the
+      //     submodule path and the output of git describe for the SHA-1. Each
+      //     SHA-1 will possibly be prefixed with - if the submodule is not
+      //     initialized, + if the currently checked out submodule commit does
+      //     not match the SHA-1 found in the index of the containing repository
+      //     and U if the submodule has merge conflicts.
+      //
+      // We discovered that in other cases, the prefix is a single space.
+      const prefix = line[0];
+      const [hash, statusPath, ...describe] = line.slice(1).split(' ');
+      return {
+        prefix,
+        hash,
+        path: statusPath,
+        describe: describe.join(' '),
+      };
+    },
+    /**
+     * Read a specific configuration value for this submodule (e.g., "path" or
+     * "url") from the top-level .gitmodules.
+     *
+     * @param {string} leaf
+     */
     config: async leaf => {
       // git rev-parse --show-toplevel
-      const top = await git
-        .pipe(['rev-parse', '--show-toplevel'])
-        .then(l => l.trimEnd());
-      // assume full paths
-      const name = path.slice(top.length + 1);
-      // git config -f ../../.gitmodules --get submodule."$name".url
-      const value = await git
-        .pipe([
-          'config',
-          '-f',
-          `${top}/.gitmodules`,
-          '--get',
-          `submodule.${name}.${leaf}`,
-        ])
-        .then(l => l.trimEnd());
+      const repoRoot = await git.pipe(['rev-parse', '--show-toplevel']);
+      if (!path.startsWith(`${repoRoot}/`)) {
+        throw Error(
+          `Expected submodule path ${path} to be a subdirectory of repository ${repoRoot}`,
+        );
+      }
+      const relativePath = path.slice(repoRoot.length + 1);
+      // git config -f ../../.gitmodules --get submodule.${relativePath}.${leaf}
+      const value = await git.pipe([
+        'config',
+        '-f',
+        `${repoRoot}/.gitmodules`,
+        '--get',
+        `submodule.${relativePath}.${leaf}`,
+      ]);
       return value;
     },
   });
 };
 
 /**
- * @param {string[]} args
+ * @typedef {{
+ *   url: string,
+ *   path: string,
+ *   commitHash?: string,
+ *   envPrefix: string,
+ * }} SubmoduleDescriptor
+ */
+
+/**
+ * @param {SubmoduleDescriptor[]} submodules
  * @param {{
- *   env: Record<string, string | undefined>,
+ *   git: ReturnType<typeof makeCLI>,
  *   stdout: typeof process.stdout,
- *   spawn: typeof import('child_process').spawn,
- *   fs: {
- *     existsSync: typeof import('fs').existsSync,
- *     rmdirSync: typeof import('fs').rmdirSync,
- *     readFile: typeof import('fs').promises.readFile,
- *   },
- *   os: {
- *     type: typeof import('os').type,
- *   }
  * }} io
  */
-async function main(args, { env, stdout, spawn, fs, os }) {
-  const git = makeCLI('git', { spawn });
-
-  // When changing/adding entries here, make sure to search the whole project
-  // for `@@AGORIC_DOCKER_SUBMODULES@@`
-  const submodules = [
-    {
-      url: env.MODDABLE_URL || 'https://github.com/agoric-labs/moddable.git',
-      path: ModdableSDK.MODDABLE,
-      commitHash: env.MODDABLE_COMMIT_HASH,
-      envPrefix: 'MODDABLE_',
-    },
-    {
-      url:
-        env.XSNAP_NATIVE_URL || 'https://github.com/agoric-labs/xsnap-pub.git',
-      path: asset('../xsnap-native'),
-      commitHash: env.XSNAP_NATIVE_COMMIT_HASH,
-      envPrefix: 'XSNAP_NATIVE_',
-    },
-  ];
-
-  if (args.includes('--show-env')) {
-    for (const submodule of submodules) {
-      const { path, envPrefix, commitHash } = submodule;
-      if (!commitHash) {
-        // We need to glean the commitHash and url from Git.
-        const sm = makeSubmodule(path, '?', { git });
-        // eslint-disable-next-line no-await-in-loop
-        const [[{ hash }], url] = await Promise.all([
-          sm.status(),
-          sm.config('url'),
-        ]);
-        submodule.commitHash = hash;
-        submodule.url = url;
-      }
-      stdout.write(`${envPrefix}URL=${submodule.url}\n`);
-      stdout.write(`${envPrefix}COMMIT_HASH=${submodule.commitHash}\n`);
+const showEnv = async (submodules, { git, stdout }) => {
+  await null;
+  for (const desc of submodules) {
+    const { path, envPrefix } = desc;
+    let { url, commitHash } = desc;
+    if (!commitHash) {
+      // We need to glean the commitHash and url from Git.
+      const submodule = makeSubmodule(path, '?', { git });
+      const [{ hash }, gitUrl] = await Promise.all([
+        submodule.status(),
+        submodule.config('url'),
+      ]);
+      commitHash = hash;
+      url = gitUrl;
     }
-    return;
+    stdout.write(`${envPrefix}URL=${url}\n`);
+    stdout.write(`${envPrefix}COMMIT_HASH=${commitHash}\n`);
   }
+};
 
+/**
+ * @param {SubmoduleDescriptor[]} submodules
+ * @param {{
+ *   fs: Pick<typeof import('fs'), 'existsSync' | 'rmdirSync'>,
+ *   git: ReturnType<typeof makeCLI>,
+ * }} io
+ */
+const updateSubmodules = async (submodules, { fs, git }) => {
+  await null;
   for (const { url, path, commitHash } of submodules) {
     const submodule = makeSubmodule(path, url, { git });
 
-    // Allow overriding of the checked-out version of the submodule.
-    if (commitHash) {
+    if (!commitHash) {
+      await submodule.init();
+    } else {
       // Do the moral equivalent of submodule update when explicitly overriding.
       try {
         fs.rmdirSync(submodule.path);
@@ -208,34 +216,50 @@ async function main(args, { env, stdout, spawn, fs, os }) {
         // ignore
       }
       if (!fs.existsSync(submodule.path)) {
-        // eslint-disable-next-line no-await-in-loop
         await submodule.clone();
       }
-      // eslint-disable-next-line no-await-in-loop
       await submodule.checkout(commitHash);
-    } else {
-      // eslint-disable-next-line no-await-in-loop
-      await submodule.init();
     }
   }
+};
 
+/**
+ * @param {ModdablePlatform} platform
+ * @param {boolean} force
+ * @param {{
+ *   fs: Pick<typeof import('fs'), 'existsSync'> &
+ *     Pick<typeof import('fs').promises, 'readFile' | 'writeFile'>,
+ *   make: ReturnType<typeof makeCLI>,
+ * }} io
+ */
+const buildXsnap = async (platform, force, { fs, make }) => {
   const pjson = await fs.readFile(asset('../package.json'), 'utf-8');
   const pkg = JSON.parse(pjson);
 
-  const platform = ModdableSDK.platforms[os.type()];
-  if (!platform) {
-    throw Error(`Unsupported OS found: ${os.type()}`);
+  const configEnvs = [
+    `XSNAP_VERSION=${pkg.version}`,
+    `CC=cc "-D__has_builtin(x)=1"`,
+  ];
+
+  const configEnvFile = asset('../build.config.env');
+  const existingConfigEnvs = fs.existsSync(configEnvFile)
+    ? await fs.readFile(configEnvFile, 'utf-8')
+    : '';
+
+  const expectedConfigEnvs = configEnvs.concat('').join('\n');
+  if (force || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) {
+    await fs.writeFile(configEnvFile, expectedConfigEnvs);
   }
 
-  const make = makeCLI(platform.make || 'make', { spawn });
   for (const goal of ModdableSDK.buildGoals) {
-    // eslint-disable-next-line no-await-in-loop
     await make.run(
       [
         `MODDABLE=${ModdableSDK.MODDABLE}`,
         `GOAL=${goal}`,
-        `XSNAP_VERSION=${pkg.version}`,
-        `CC=cc "-D__has_builtin(x)=1"`,
+        // Any other configuration variables that affect the build output
+        // should be placed in `configEnvs` to force a rebuild if they change
+        ...configEnvs,
+        `EXTRA_DEPS=${configEnvFile}`,
         '-f',
         'xsnap-worker.mk',
       ],
@@ -244,21 +268,122 @@ async function main(args, { env, stdout, spawn, fs, os }) {
       },
     );
   }
+};
+
+/**
+ * @param {string[]} args
+ * @param {{
+ *   env: Record<string, string | undefined>,
+ *   stdout: typeof process.stdout,
+ *   spawn: typeof import('child_process').spawn,
+ *   fs: Pick<typeof import('fs'), 'existsSync' | 'rmdirSync'> &
+ *     Pick<typeof import('fs').promises, 'readFile' | 'writeFile'>,
+ *   os: Pick<typeof import('os'), 'type'>,
+ * }} io
+ */
+async function main(args, { env, stdout, spawn, fs, os }) {
+  // I solemnly swear I will do no synchronous work followed by a variable
+  // number turns of the event loop.
+  await null;
+
+  const osType = os.type();
+  const platform = ModdableSDK.platforms[osType];
+  if (!platform) {
+    throw Error(`xsnap does not support OS ${osType}`);
+  }
+
+  const git = makeCLI('git', { spawn });
+  const make = makeCLI(platform.make || 'make', { spawn });
+
+  // When changing/adding entries here, make sure to search the whole project
+  // for `@@AGORIC_DOCKER_SUBMODULES@@`
+  const submodules = [
+    {
+      url: env.MODDABLE_URL || 'https://github.com/agoric-labs/moddable.git',
+      path: ModdableSDK.MODDABLE,
+      commitHash: env.MODDABLE_COMMIT_HASH,
+      envPrefix: 'MODDABLE_',
+    },
+    {
+      url:
+        env.XSNAP_NATIVE_URL || 'https://github.com/agoric-labs/xsnap-pub.git',
+      path: asset('../xsnap-native'),
+      commitHash: env.XSNAP_NATIVE_COMMIT_HASH,
+      envPrefix: 'XSNAP_NATIVE_',
+    },
+  ];
+
+  // We build both release and debug executables, so checking for only the
+  // former is fine.
+  // XXX This will need to account for the .exe extension if we recover support
+  // for Windows.
+  const bin = asset(
+    `../xsnap-native/xsnap/build/bin/${platform.path}/release/xsnap-worker`,
+  );
+  const hasBin = fs.existsSync(bin);
+  const hasSource = fs.existsSync(asset('../moddable/xs/includes/xs.h'));
+  const hasGit = fs.existsSync(asset('../moddable/.git'));
+
+  // If a git submodule is present or source files and prebuilt executables are
+  // both absent, consider ourselves to be in an active git checkout (as opposed
+  // to e.g. an extracted npm tarball).
+  const isWorkingCopy = hasGit || (!hasSource && !hasBin);
+
+  // --show-env reports submodule status without making changes.
+  if (args.includes('--show-env')) {
+    if (!isWorkingCopy) {
+      throw Error('XSnap requires a working copy and git to --show-env');
+    }
+    await showEnv(submodules, { git, stdout });
+    return;
+  }
+
+  // Fetch/update source files via `git submodule` as appropriate.
+  if (isWorkingCopy) {
+    await updateSubmodules(submodules, { fs, git });
+  }
+
+  // If we now have source files, (re)build from them.
+  // Otherwise, require presence of a previously-built executable.
+  if (hasSource || isWorkingCopy) {
+    // Force a rebuild if for some reason the binary is out of date
+    // since the make checks may not always detect that situation.
+    const npm = makeCLI('npm', { spawn });
+    const force = await (!hasBin ||
+      isRejected(
+        npm.run(['run', '-s', 'check-version'], { cwd: asset('..') }),
+      ));
+    await buildXsnap(platform, force, { fs, make });
+  } else if (!hasBin) {
+    throw Error(
+      'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?',
+    );
+  }
 }
 
-main(process.argv.slice(2), {
-  env: { ...process.env },
-  stdout: process.stdout,
-  spawn: childProcessTop.spawn,
-  fs: {
-    readFile: fsTop.promises.readFile,
-    existsSync: fsTop.existsSync,
-    rmdirSync: fsTop.rmdirSync,
+const run = () =>
+  main(process.argv.slice(2), {
+    env: { ...process.env },
+    stdout: process.stdout,
+    spawn: childProcessTop.spawn,
+    fs: {
+      existsSync: fsTop.existsSync,
+      rmdirSync: fsTop.rmdirSync,
+      readFile: fsTop.promises.readFile,
+      writeFile: fsTop.promises.writeFile,
+    },
+    os: {
+      type: osTop.type,
+    },
+  });
+
+process.exitCode = 1;
+run().then(
+  () => {
+    process.exitCode = 0;
   },
-  os: {
-    type: osTop.type,
+  err => {
+    console.error('Failed with', err);
+    process.exit(process.exitCode || 1);
   },
-}).catch(e => {
-  console.error(e);
-  process.exit(1);
-});
+);

package/src/globals.d.ts

@@ -1,5 +1,5 @@
-declare var issueCommand: (msg: ArrayBuffer) => ArrayBuffer;
+declare var issueCommand: (msg: ArrayBufferLike) => ArrayBuffer;
 
 namespace global {
-  declare var issueCommand: (msg: ArrayBuffer) => ArrayBuffer;
+  declare var issueCommand: (msg: ArrayBufferLike) => ArrayBuffer;
 }

package/src/replay.js

@@ -13,6 +13,7 @@ import osPowers from 'os';
 import fsPowers from 'fs';
 import { Readable } from 'stream';
 import { tmpName as tmpNamePower } from 'tmp';
+import { fileURLToPath } from 'url';
 import { makeQueue } from '@endo/stream';
 import { xsnap, DEFAULT_CRANK_METERING_LIMIT } from './xsnap.js';
 
@@ -38,7 +39,9 @@ function makeSyncStorage(path, { writeFileSync }) {
     file: fn => {
       /** @param {Uint8Array} data */
       const put = data =>
-        writeFileSync(new URL(fn, base).pathname, data, { flag: 'wx' });
+        writeFileSync(fileURLToPath(new URL(fn, base)), data, {
+          flag: 'wx',
+        });
 
       return freeze({
         put,
@@ -60,14 +63,18 @@ function makeSyncAccess(path, { readdirSync, readFileSync }) {
   const base = new URL(path, 'file://');
   /** @param {string} fn */
   const file = fn => {
-    const fullname = new URL(fn, base).pathname;
+    const fullname = fileURLToPath(new URL(fn, base));
 
     return freeze({
       getData: () => readFileSync(fullname),
       getText: () => readFileSync(fullname, 'utf-8'),
     });
   };
-  return freeze({ path, file, readdir: () => readdirSync(base.pathname) });
+  return freeze({
+    path,
+    file,
+    readdir: () => readdirSync(fileURLToPath(base)),
+  });
 }
 
 /**
@@ -227,14 +234,12 @@ export async function replayXSnap(
       const seq = parseInt(digits, 10);
       console.log(folder, seq, kind);
       if (running && !['command', 'reply'].includes(kind)) {
-        // eslint-disable-next-line @jessie.js/no-nested-await
         await running;
         running = undefined;
       }
       const file = rd.file(step);
       switch (kind) {
         case 'isReady':
-          // eslint-disable-next-line @jessie.js/no-nested-await
           await it.isReady();
           break;
         case 'evaluate':
@@ -254,7 +259,6 @@ export async function replayXSnap(
             console.log(folder, step, 'ignoring remaining steps from', folder);
             return;
           } else {
-            // eslint-disable-next-line @jessie.js/no-nested-await
             await (async () => {
               const snapshotPath = file.getText();
               const snapFile = await opts.fs.open(snapshotPath, 'w');
@@ -322,8 +326,8 @@ export async function main(
   await replayXSnap(options, folders, { readdirSync, readFileSync });
 }
 
-/* global process */
-if (process.argv[1] === new URL(import.meta.url).pathname) {
+/* eslint-env node */
+if (process.argv[1] === fileURLToPath(new URL(import.meta.url))) {
   main([...process.argv.slice(2)], {
     spawn: childProcessPowers.spawn,
     fs: { ...fsPowers, ...fsPowers.promises },