@agoric/swingset-vat 0.32.3-dev-3dacdc3.0.3dacdc3 → 0.32.3-dev-2db363c.0.2db363c

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swingset-vat",
-  "version": "0.32.3-dev-3dacdc3.0.3dacdc3",
+  "version": "0.32.3-dev-2db363c.0.2db363c",
   "description": "Vat/Container Launcher",
   "type": "module",
   "main": "src/index.js",
@@ -22,7 +22,7 @@
     "lint:eslint": "yarn run -T eslint ."
   },
   "devDependencies": {
-    "@agoric/xsnap": "0.14.3-dev-3dacdc3.0.3dacdc3",
+    "@agoric/xsnap": "0.14.3-dev-2db363c.0.2db363c",
     "@types/better-sqlite3": "^7.6.13",
     "@types/microtime": "^2.1.0",
     "@types/tmp": "^0.2.0",
@@ -30,15 +30,15 @@
     "ava": "^6.4.1"
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-3dacdc3.0.3dacdc3",
-    "@agoric/kmarshal": "0.1.1-dev-3dacdc3.0.3dacdc3",
-    "@agoric/store": "0.9.3-dev-3dacdc3.0.3dacdc3",
-    "@agoric/swing-store": "0.9.2-dev-3dacdc3.0.3dacdc3",
-    "@agoric/swingset-liveslots": "0.10.3-dev-3dacdc3.0.3dacdc3",
-    "@agoric/swingset-xsnap-supervisor": "0.10.3-dev-3dacdc3.0.3dacdc3",
-    "@agoric/time": "0.3.3-dev-3dacdc3.0.3dacdc3",
-    "@agoric/vat-data": "0.5.3-dev-3dacdc3.0.3dacdc3",
-    "@agoric/xsnap-lockdown": "0.14.1-dev-3dacdc3.0.3dacdc3",
+    "@agoric/internal": "0.3.3-dev-2db363c.0.2db363c",
+    "@agoric/kmarshal": "0.1.1-dev-2db363c.0.2db363c",
+    "@agoric/store": "0.9.3-dev-2db363c.0.2db363c",
+    "@agoric/swing-store": "0.9.2-dev-2db363c.0.2db363c",
+    "@agoric/swingset-liveslots": "0.10.3-dev-2db363c.0.2db363c",
+    "@agoric/swingset-xsnap-supervisor": "0.10.3-dev-2db363c.0.2db363c",
+    "@agoric/time": "0.3.3-dev-2db363c.0.2db363c",
+    "@agoric/vat-data": "0.5.3-dev-2db363c.0.2db363c",
+    "@agoric/xsnap-lockdown": "0.14.1-dev-2db363c.0.2db363c",
     "@endo/base64": "^1.0.12",
     "@endo/bundle-source": "^4.1.2",
     "@endo/captp": "^4.4.8",
@@ -89,10 +89,18 @@
     "example": "examples"
   },
   "ava": {
+    "extensions": {
+      "js": true,
+      "ts": "module"
+    },
     "workerThreads": false,
     "files": [
       "test/**/*.test.*"
     ],
+    "nodeArguments": [
+      "--import=ts-blank-space/register",
+      "--no-warnings"
+    ],
     "require": [
       "@agoric/internal/src/ava-force-exit.mjs",
       "@endo/init/debug.js"
@@ -105,5 +113,5 @@
   "typeCoverage": {
     "atLeast": 76.72
   },
-  "gitHead": "3dacdc33b578ee52258e5771192bf8e57beb603b"
+  "gitHead": "2db363cfdc60b99cc644823e99bced301cf748b5"
 }

package/tools/bundleTool.js

@@ -1,15 +1,20 @@
 import { makeNodeBundleCache as wrappedMaker } from '@endo/bundle-source/cache.js';
 import styles from 'ansi-styles'; // less authority than 'chalk'
-import { mkdir, readFile, rm, stat, writeFile } from 'fs/promises';
+import * as fsPromises from 'fs/promises';
+import { createHash } from 'node:crypto';
 import { fileURLToPath } from 'url';
 import path from 'path';
 import { setTimeout as delay } from 'timers/promises';
+import { makeDirectoryLock } from '@agoric/internal/src/build-cache.js';
 
 /**
  * @import {EReturn} from '@endo/far';
+ * @import {BuildCacheEvent} from '@agoric/internal/src/build-cache-types.js';
+ * @import {BundleOptions, ModuleFormat} from '@endo/bundle-source';
  */
 
 /** @typedef {EReturn<typeof wrappedMaker>} WrappedBundleCache */
+/** @typedef {BundleOptions<ModuleFormat> & Parameters<typeof wrappedMaker>[1]} BundleCacheOptions */
 /**
  * @typedef {{
  *   sourceSpec?: string;
@@ -31,18 +36,139 @@ import { setTimeout as delay } from 'timers/promises';
  *   loadRegistry: <T extends Record<string, BundleRegistryEntry>>(registry: T) => Promise<LoadedRegistryBundles<T>>;
  * }} BundleCache
  */
+/**
+ * @typedef {{
+ *   onBundleToolEvent?: (event: BundleToolEvent) => void,
+ * }} BundleToolEventSink
+ */
+/**
+ * @typedef {{
+ *   type: 'bundle-source-log';
+ *   args: unknown[];
+ *   phase: string;
+ *   timestamp: number;
+ * }} BundleSourceLogEvent
+ */
+/**
+ * @typedef {{
+ *   type: 'registry-loaded';
+ *   args: string[];
+ *   phase: '[registry]';
+ *   elapsedMs: number;
+ *   bundleCount: number;
+ * }} RegistryLoadedEvent
+ */
+/**
+ * @typedef {BundleSourceLogEvent | RegistryLoadedEvent | BuildCacheEvent} BundleToolEvent
+ */
+/**
+ * @typedef {{
+ *   delayMs: (ms: number) => Promise<unknown>,
+ *   eventSink?: BundleToolEventSink,
+ *   isPidAlive: (pid: number) => boolean,
+ *   loadModule: Parameters<typeof wrappedMaker>[2],
+ *   monotonicNow: () => number,
+ *   now: () => number,
+ *   pid: number,
+ * }} BundleToolPowers
+ */
 
 const BUNDLE_LOCK_ACQUIRE_TIMEOUT_MS = 5 * 60_000;
 const BUNDLE_STALE_LOCK_MS = 60_000;
 
+const defaultBundleToolEventSink = {
+  onBundleToolEvent: event => {
+    if (
+      event.type !== 'bundle-source-log' &&
+      event.type !== 'registry-loaded'
+    ) {
+      return;
+    }
+    const { args = [], phase = '[check]', type } = event;
+    if (type === 'bundle-source-log') {
+      const joined = args.map(arg => String(arg)).join(' ');
+      if (joined.includes(' valid:')) {
+        return;
+      }
+    }
+    const flattened = args.map(arg =>
+      // Don't print stack traces.
+      arg instanceof Error ? arg.message : arg,
+    );
+    console.log(
+      `${styles.dim.open}[bundleTool]${phase}`,
+      ...flattened,
+      styles.dim.close,
+    );
+  },
+};
+harden(defaultBundleToolEventSink);
+
+const inferLogPhase = args => {
+  const joined = args.map(arg => String(arg)).join(' ');
+  let phase = '[check]';
+  if (joined.includes(' add:') || joined.includes(' bundled ')) {
+    phase = '[rebuilt]';
+  }
+  return phase;
+};
+harden(inferLogPhase);
+
 /**
- * @param {string} dest
- * @param {Parameters<typeof wrappedMaker>[1]} options
- * @param {Parameters<typeof wrappedMaker>[2]} loadModule
- * @param {number} [pid]
+ * @param {Partial<BundleToolPowers>} [options]
+ * @returns {BundleToolPowers}
+ * @alpha
+ */
+export const makeAmbientBundleToolPowers = (options = {}) => {
+  const {
+    delayMs = ms => delay(ms),
+    eventSink = defaultBundleToolEventSink,
+    isPidAlive = lockPid => {
+      if (!Number.isInteger(lockPid) || lockPid <= 0) {
+        return false;
+      }
+      try {
+        process.kill(lockPid, 0);
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    loadModule = specifier => import(specifier),
+    monotonicNow = () => performance.now(),
+    now = () => Date.now(),
+    pid = process.pid,
+  } = options;
+  return harden({
+    delayMs,
+    eventSink,
+    isPidAlive,
+    loadModule,
+    monotonicNow,
+    now,
+    pid,
+  });
+};
+harden(makeAmbientBundleToolPowers);
+
+/**
+ * @param {string} destPath
+ * @param {BundleToolPowers} powers
+ * @param {BundleCacheOptions} [options]
  * @returns {Promise<BundleCache>}
+ * @alpha
  */
-export const makeNodeBundleCache = async (dest, options, loadModule, pid) => {
+export const makeNodeBundleCache = async (destPath, powers, options = {}) => {
+  const {
+    delayMs,
+    eventSink = defaultBundleToolEventSink,
+    isPidAlive,
+    loadModule,
+    monotonicNow,
+    now,
+    pid,
+  } = powers;
+
   /** @param {string} sourceSpec */
   const canonicalizeSourceSpec = sourceSpec => {
     if (sourceSpec.startsWith('file:')) {
@@ -66,147 +192,83 @@ export const makeNodeBundleCache = async (dest, options, loadModule, pid) => {
     }
   };
 
-  const log = (...args) => {
-    const joined = args.map(arg => String(arg)).join(' ');
-    const isCacheHit = joined.includes(' valid:');
-    if (isCacheHit) {
-      return;
-    }
-    let phase = '[check]';
-    if (joined.includes(' add:') || joined.includes(' bundled ')) {
-      phase = '[rebuilt]';
+  const onEvent = eventSink.onBundleToolEvent || (() => {});
+  const optionsKeyFor = options0 => {
+    // Include options in the key on purpose: bundle output may differ by
+    // bundle-source options, and correctness prefers over-segmentation over
+    // accidental cache aliasing.
+    try {
+      return JSON.stringify(options0 || {});
+    } catch {
+      return '[unstringifiable-options]';
     }
-    const flattened = args.map(arg =>
-      // Don't print stack traces.
-      arg instanceof Error ? arg.message : arg,
-    );
-    console.log(
-      // Make all messages prefixed and dim.
-      `${styles.dim.open}[bundleTool]${phase}`,
-      ...flattened,
-      styles.dim.close,
-    );
   };
+  const sanitizeName = name => name.replace(/[^a-zA-Z0-9._-]/g, '-');
+  const getTargetNames = (canonicalRootPath, targetName, options0) => {
+    const requestedTargetName =
+      targetName || path.basename(canonicalRootPath, '.js');
+    const optionsKey = optionsKeyFor(options0);
+    const hashKey = createHash('sha256')
+      .update(`${canonicalRootPath}\n${optionsKey}`)
+      .digest('hex')
+      .slice(0, 12);
+    const cacheTargetName = `${sanitizeName(requestedTargetName)}-${hashKey}`;
+    return harden({ requestedTargetName, cacheTargetName, optionsKey });
+  };
+  const log = (...args) => {
+    onEvent({
+      type: 'bundle-source-log',
+      args,
+      phase: inferLogPhase(args),
+      timestamp: now(),
+    });
+  };
+
   const rawCache = await wrappedMaker(
-    dest,
+    destPath,
     { log, ...options },
     loadModule,
     pid,
   );
-  const lockRoot = path.resolve(dest, '.bundle-locks');
+  const lockRoot = path.resolve(destPath, '.bundle-locks');
   /** @type {Map<string, Promise<unknown>>} */
   const inProcessLoads = new Map();
-
-  /**
-   * @param {string} targetName
-   * @param {() => Promise<any>} duringLock
-   */
-  const withBundleLock = async (targetName, duringLock) => {
-    const lockName = `${encodeURIComponent(targetName)}.lock`;
-    const lockPath = path.resolve(lockRoot, lockName);
-    const ownerPath = path.resolve(lockPath, 'owner.json');
-    await mkdir(lockRoot, { recursive: true });
-    const started = performance.now();
-
-    const isPidAlive = lockPid => {
-      if (!Number.isInteger(lockPid) || lockPid <= 0) {
-        return false;
-      }
-      try {
-        process.kill(lockPid, 0);
-        return true;
-      } catch {
-        return false;
-      }
-    };
-
-    const maybeBreakStaleLock = async () => {
-      const ownerTxt = await readFile(ownerPath, 'utf8').catch(() => undefined);
-      let lockInfo;
-      if (typeof ownerTxt === 'string') {
-        try {
-          lockInfo = JSON.parse(ownerTxt);
-        } catch {
-          // Another process may be in the middle of writing owner.json.
-          // Treat malformed data as unknown owner and rely on age checks below.
-          lockInfo = undefined;
-        }
-      }
-      if (
-        lockInfo &&
-        Number.isInteger(lockInfo.pid) &&
-        !isPidAlive(lockInfo.pid)
-      ) {
-        await rm(lockPath, { recursive: true, force: true });
-        return true;
-      }
-      try {
-        const st = await stat(lockPath);
-        const ageMs = Date.now() - st.mtimeMs;
-        if (ageMs >= BUNDLE_STALE_LOCK_MS) {
-          await rm(lockPath, { recursive: true, force: true });
-          return true;
-        }
-      } catch {
-        return true;
-      }
-      return false;
-    };
-
-    // Cross-process lock via mkdir(). Retriable on EEXIST.
-    for (;;) {
-      try {
-        await mkdir(lockPath);
-        await writeFile(
-          ownerPath,
-          JSON.stringify({ pid: process.pid, createdAt: Date.now() }),
-          'utf8',
-        );
-        break;
-      } catch (e) {
-        if (e && typeof e === 'object' && 'code' in e && e.code === 'EEXIST') {
-          const brokeStaleLock = await maybeBreakStaleLock();
-          if (brokeStaleLock) {
-            continue;
-          }
-          const waitedMs = performance.now() - started;
-          if (waitedMs >= BUNDLE_LOCK_ACQUIRE_TIMEOUT_MS) {
-            throw Error(
-              `Timed out waiting for bundle lock ${lockPath} after ${waitedMs}ms`,
-            );
-          }
-          await delay(20);
-          continue;
-        }
-        throw e;
-      }
-    }
-
-    try {
-      return await duringLock();
-    } finally {
-      await rm(lockPath, { recursive: true, force: true });
-    }
-  };
+  const { withLock } = makeDirectoryLock({
+    fs: fsPromises,
+    delayMs,
+    now,
+    pid,
+    isPidAlive,
+    lockRoot,
+    staleLockMs: BUNDLE_STALE_LOCK_MS,
+    acquireTimeoutMs: BUNDLE_LOCK_ACQUIRE_TIMEOUT_MS,
+    onEvent,
+  });
 
   const cache = harden({
     ...rawCache,
     add: (rootPath, targetName, log0, options0) => {
       const canonicalRootPath = canonicalizeSourceSpec(rootPath);
-      const resolvedTargetName =
-        targetName || path.basename(canonicalRootPath, '.js');
-      return withBundleLock(resolvedTargetName, () =>
-        rawCache.add(canonicalRootPath, resolvedTargetName, log0, options0),
+      const { requestedTargetName, cacheTargetName } = getTargetNames(
+        canonicalRootPath,
+        targetName,
+        options0,
+      );
+      return withLock(requestedTargetName, () =>
+        rawCache.add(canonicalRootPath, cacheTargetName, log0, options0),
       );
     },
     validateOrAdd: (rootPath, targetName, log0, options0) => {
       const canonicalRootPath = canonicalizeSourceSpec(rootPath);
-      const resolvedTargetName =
-        targetName || path.basename(canonicalRootPath, '.js');
-      return withBundleLock(resolvedTargetName, () =>
+      const { requestedTargetName, cacheTargetName } = getTargetNames(
+        canonicalRootPath,
+        targetName,
+        options0,
+      );
+      return withLock(requestedTargetName, () =>
         rawCache.validateOrAdd(
           canonicalRootPath,
-          resolvedTargetName,
+          cacheTargetName,
           log0,
           options0,
         ),
@@ -214,23 +276,21 @@ export const makeNodeBundleCache = async (dest, options, loadModule, pid) => {
     },
     load: async (rootPath, targetName, log0, options0) => {
       const canonicalRootPath = canonicalizeSourceSpec(rootPath);
-      const resolvedTargetName =
-        targetName || path.basename(canonicalRootPath, '.js');
-      const key = `${resolvedTargetName}:${canonicalRootPath}:${JSON.stringify(
-        options0 || {},
-      )}`;
+      const { requestedTargetName, cacheTargetName, optionsKey } =
+        getTargetNames(canonicalRootPath, targetName, options0);
+      const key = `${cacheTargetName}:${canonicalRootPath}:${optionsKey}`;
       const found = inProcessLoads.get(key);
       if (found) {
         return found;
       }
-      const pending = withBundleLock(resolvedTargetName, async () => {
+      const pending = withLock(requestedTargetName, async () => {
         const { bundleFileName } = await rawCache.validateOrAdd(
           canonicalRootPath,
-          resolvedTargetName,
+          cacheTargetName,
           log0,
           options0,
         );
-        const bundlePath = path.resolve(dest, bundleFileName);
+        const bundlePath = path.resolve(destPath, bundleFileName);
         return import(bundlePath).then(m => harden(m.default));
       });
       inProcessLoads.set(key, pending);
@@ -250,7 +310,7 @@ export const makeNodeBundleCache = async (dest, options, loadModule, pid) => {
    * @returns {Promise<LoadedRegistryBundles<T>>}
    */
   const loadRegistry = async registry => {
-    const started = performance.now();
+    const started = monotonicNow();
     await null;
     try {
       const loaded = await Promise.all(
@@ -270,11 +330,16 @@ export const makeNodeBundleCache = async (dest, options, loadModule, pid) => {
         harden(Object.fromEntries(loaded))
       );
     } finally {
-      const elapsedMs = performance.now() - started;
-      console.log(
-        `${styles.dim.open}[bundleTool][registry] loaded ${Object.keys(registry).length} bundle(s) in ${elapsedMs}ms`,
-        styles.dim.close,
-      );
+      const elapsedMs = monotonicNow() - started;
+      onEvent({
+        type: 'registry-loaded',
+        phase: '[registry]',
+        args: [
+          `loaded ${Object.keys(registry).length} bundle(s) in ${elapsedMs}ms`,
+        ],
+        elapsedMs,
+        bundleCount: Object.keys(registry).length,
+      });
     }
   };
   return /** @type {BundleCache} */ (
@@ -285,41 +350,14 @@ export const makeNodeBundleCache = async (dest, options, loadModule, pid) => {
   );
 };
 
-/** @type {Map<string, Promise<BundleCache>>} */
-const providedCaches = new Map();
-
 /**
- * Make a new bundle cache for the destination. If there is already one for that
- * destination, return it.
- *
  * @param {string} dest
- * @param {object} options
- * @param {string} [options.format]
- * @param {boolean} [options.dev]
- * @param {number} [options.byteLimit=490_000] - Maximum bundle size in bytes before
- *   falling back to optimizations that may reduce legibility.
- * @param {(id: string) => Promise<any>} loadModule
  * @param {number} [pid]
  * @returns {Promise<BundleCache>}
+ * @alpha
  */
-export const provideBundleCache = (dest, options, loadModule, pid) => {
-  const uniqueDest = [dest, options.format, options.dev].join('-');
-  // store the promise instead of awaiting to prevent a race
-  let bundleCache = providedCaches.get(uniqueDest);
-  if (!bundleCache) {
-    bundleCache = makeNodeBundleCache(dest, options, loadModule, pid);
-    providedCaches.set(uniqueDest, bundleCache);
-  }
-  return bundleCache;
-};
-harden(provideBundleCache);
-
-/**
- * @param {string} dest
- * @returns {Promise<BundleCache>}
- */
-export const unsafeMakeBundleCache = dest =>
-  makeNodeBundleCache(dest, {}, s => import(s));
+export const unsafeMakeBundleCache = (dest, pid = process.pid) =>
+  makeNodeBundleCache(dest, makeAmbientBundleToolPowers({ pid }));
 
 /**
  * This is a hack to share bundles across multiple SwingSet instances. It is
@@ -344,6 +382,7 @@ export const sharedBundleCachePath = fileURLToPath(
  * separately for use in other contexts that need to know where the shared
  * bundles are located, but that also makes it more likely to cause conflicts if
  * used in multiple places at once.
+ * @alpha
  */
 export const unsafeSharedBundleCache = unsafeMakeBundleCache(
   sharedBundleCachePath,